.github/workflows/ci.yml

@@ -1,6 +1,5 @@
 name: CI
 
-
 on:
   # Triggers the workflow on push or pull request events but only for the master branch
   push:
@@ -11,21 +10,41 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-
 jobs:
 
-  testing:
+  basic_install:
-    name: 'install + test-suite'
+    name: 'pip install'
     runs-on: ubuntu-latest
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
+        with:
+          ref: master
 
       - name: Setup python
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: '3.9'
+
+      - name: Install dependencies
+        run: pip install -e . --upgrade-strategy eager -r requirements.txt
+
+      - name: Run piker cli
+        run: piker 
+
+  testing:
+    name: 'test suite'
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.9'
 
       - name: Install dependencies
         run: pip install -U . -r requirements-test.txt -r requirements.txt --upgrade-strategy eager

.gitignore

@@ -97,9 +97,6 @@ ENV/
 # mkdocs documentation
 /site
 
-# extra scripts dir
-/snippets
-
 # mypy
 .mypy_cache/
 .vscode/settings.json

README.rst

@@ -74,57 +74,23 @@ for a development install::
 
 install for tinas
 *****************
-for windows peeps you can start by installing all the prerequisite software:
+for windows peeps you can start by getting `conda installed`_
-
+and the `C++ build toolz`_ on your system.
-- install git with all default settings - https://git-scm.com/download/win
-- install anaconda all default settings - https://www.anaconda.com/products/individual
-- install microsoft build tools (check the box for Desktop development for C++, you might be able to uncheck some optional downloads)  - https://visualstudio.microsoft.com/visual-cpp-build-tools/
-- install visual studio code default settings - https://code.visualstudio.com/download
-
 
 then, `crack a conda shell`_ and run the following commands::
 
-    mkdir code # create code directory
+    conda create piker --python=3.9
-    cd code # change directory to code
+    conda activate piker
-    git clone https://github.com/pikers/piker.git # downloads piker installation package from github
+    conda install pip
-    cd piker # change directory to piker
+    pip install --upgrade setuptools
+    cd dIreCToRieZ\oF\cODez\piker\
+    pip install -r requirements -e .
 
-    conda create -n pikonda # creates conda environment named pikonda
-    conda activate pikonda # activates pikonda
 
-    conda install -c conda-forge python-levenshtein # in case it is not already installed
+in order to look coolio in front of all ur tina friends (and maybe
-    conda install pip # may already be installed
+want to help us with testin, hackzing or configgin), install
-    pip # will show if pip is installed
+`vscode`_ and `setup a coolio tiled wm console`_ so you can start
-    
+living the life of the tech literate..
-    pip install -e . -r requirements.txt # install piker in editable mode
-
-test Piker to see if it is working::
-
-    piker -b binance chart btcusdt.binance # formatting for loading a chart
-    piker -b kraken -b binance chart xbtusdt.kraken
-    piker -b kraken -b binance -b ib chart qqq.nasdaq.ib
-    piker -b ib chart tsla.nasdaq.ib
-
-potential error::
-    
-    FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\user\\AppData\\Roaming\\piker\\brokers.toml'
-    
-solution:
-
-- navigate to file directory above (may be different on your machine, location should be listed in the error code)
-- copy and paste file from 'C:\\Users\\user\\code\\data/brokers.toml' or create a blank file using notepad at the location above
-
-Visual Studio Code setup:
-
-- now that piker is installed we can set up vscode as the default terminal for running piker and editing the code
-- open Visual Studio Code
-- file --> Add Folder to Workspace --> C:\Users\user\code\piker (adds piker directory where all piker files are located)
-- file --> Save Workspace As --> save it wherever you want and call it whatever you want, this is going to be your default workspace for running and editing piker code
-- ctrl + shift + p --> start typing Python: Select Interpetter --> when the option comes up select it --> Select at the workspace level --> select the one that shows ('pikonda')
-- change the default terminal to cmd.exe instead of powershell (default)
-- now when you create a new terminal VScode should automatically activate you conda env so that piker can be run as the first command after a new terminal is created
-
-also, try out fancyzones as part of powertoyz for a decent tiling windows manager to manage all the cool new software you are going to be running.
 
 .. _conda installed: https://
 .. _C++ build toolz: https://
@@ -138,7 +104,7 @@ provider support
 ****************
 for live data feeds the in-progress set of supported brokers is:
 
-- IB_ via ``ib_insync``, also see our `container docs`_
+- IB_ via ``ib_insync``
 - binance_ and kraken_ for crypto over their public websocket API
 - questrade_ (ish) which comes with effectively free L1
 
@@ -150,7 +116,6 @@ coming soon...
 if you want your broker supported and they have an API let us know.
 
 .. _IB: https://interactivebrokers.github.io/tws-api/index.html
-.. _container docs: https://github.com/pikers/piker/tree/master/dockering/ib
 .. _questrade: https://www.questrade.com/api/documentation
 .. _kraken: https://www.kraken.com/features/api#public-market-data
 .. _binance: https://github.com/pikers/piker/pull/182

config/brokers.toml

@@ -8,45 +8,20 @@ expires_at = 1616095326.355846
 
 [kraken]
 key_descr = "api_0"
-api_key = ""
+public_key = ""
-secret = ""
+private_key = ""
 
 [ib]
-hosts = [
+host = "127.0.0.1"
-    "127.0.0.1",
-]
-# XXX: the order in which ports will be scanned
-# (by the `brokerd` daemon-actor)
-# is determined # by the line order here.
-# TODO: when we eventually spawn gateways in our
-# container, we can just dynamically allocate these
-# using IBC.
-ports = [
-    4002,  # gw
-    7497,  # tws
-]
 
-# XXX: for a paper account the flex web query service
+ports.gw = 4002
-# is not supported so you have to manually download
+ports.tws = 7497
-# and XML report and put it in a location that can be
+ports.order = ["gw", "tws",]
-# accessed by the ``brokerd.ib`` backend code for parsing.
-flex_token = '666666666666666666666666'
-flex_trades_query_id = '666666'  # live account
 
-# when clients are being scanned this determines
+accounts.margin = "X0000000"
-# which clients are preferred to be used for data
+accounts.ira = "X0000000"
-# feeds based on the order of account names, if
+accounts.paper = "XX0000000"
-# detected as active on an API client.
-prefer_data_account = [
-    'paper',
-    'margin',
-    'ira',
-]
 
-[ib.accounts]
+# the order in which accounts will be selected (if found through
-# the order in which accounts will be selectable
+# `brokerd`) when a new symbol is loaded
-# in the order mode UI (if found via clients during
+accounts_order = ['paper', 'margin', 'ira']
-# API-app scanning)when a new symbol is loaded.
-paper = "XX0000000"
-margin = "X0000000"
-ira = "X0000000"

dockering/ib/README.rst

@@ -1,30 +0,0 @@
-running ``ib`` gateway in ``docker``
-------------------------------------
-We have a config based on the (now defunct)
-image from "waytrade":
-
-https://github.com/waytrade/ib-gateway-docker
-
-To startup this image with our custom settings
-simply run the command::
-
-    docker compose up
-
-And you should have the following socket-available services:
-
-- ``x11vnc1@127.0.0.1:3003``
-- ``ib-gw@127.0.0.1:4002``
-
-You can attach to the container via a VNC client
-without password auth.
-
-SECURITY STUFF!?!?!
--------------------
-Though "``ib``" claims they host filter connections outside
-localhost (aka ``127.0.0.1``) it's probably better if you filter
-the socket at the OS level using a stateless firewall rule::
-
-    ip rule add not unicast iif lo to 0.0.0.0/0 dport 4002
-
-We will soon have this baked into our own custom image but for
-now you'll have to do it urself dawgy.

dockering/ib/docker-compose.yml

@@ -1,64 +0,0 @@
-# rework from the original @
-# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
-version: "3.5"
-
-services:
-  ib-gateway:
-    # other image tags available:
-    # https://github.com/waytrade/ib-gateway-docker#supported-tags
-    image: waytrade/ib-gateway:981.3j
-    restart: always
-    network_mode: 'host'
-
-    volumes:
-      - type: bind
-        source: ./jts.ini
-        target: /root/Jts/jts.ini
-        # don't let IBC clobber this file for
-        # the main reason of not having a stupid
-        # timezone set..
-        read_only: true
-
-      # force our own IBC config
-      - type: bind
-        source: ./ibc.ini
-        target: /root/ibc/config.ini
-
-      # force our noop script - socat isn't needed in host mode.
-      - type: bind
-        source: ./fork_ports_delayed.sh
-        target: /root/scripts/fork_ports_delayed.sh
-
-      # force our noop script - socat isn't needed in host mode.
-      - type: bind
-        source: ./run_x11_vnc.sh
-        target: /root/scripts/run_x11_vnc.sh
-        read_only: true
-
-    # NOTE:to fill these out, define an `.env` file in the same dir as
-    # this compose file which looks something like:
-    # TWS_USERID='myuser'
-    # TWS_PASSWORD='guest'
-    # TRADING_MODE=paper (or live)
-    # VNC_SERVER_PASSWORD='diggity'
-
-    environment:
-      TWS_USERID: ${TWS_USERID}
-      TWS_PASSWORD: ${TWS_PASSWORD}
-      TRADING_MODE: ${TRADING_MODE:-paper}
-      VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
-
-    # ports:
-    #   - target: 4002
-    #     host_ip: 127.0.0.1
-    #     published: 4002
-    #     protocol: tcp
-
-      # original mappings for use in non-host-mode
-      # which we won't really need going forward since
-      # ideally we just pick the port to have ib-gw listen
-      # on **when** we spawn the container - i.e. everything
-      # will be driven by a ``brokers.toml`` def.
-      # - "127.0.0.1:4001:4001"
-      # - "127.0.0.1:4002:4002"
-      # - "127.0.0.1:5900:5900"

dockering/ib/fork_ports_delayed.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-# we now just set this is to a noop script
-# since we can just run the container in
-# `network_mode: 'host'` and get literally
-# the exact same behaviour XD

dockering/ib/ibc.ini

@@ -1,711 +0,0 @@
-# Note that in the comments in this file, TWS refers to both the Trader
-# Workstation and the IB Gateway, unless explicitly stated otherwise.
-#
-# When referred to below, the default value for a setting is the value
-# assumed if either the setting is included but no value is specified, or
-# the setting is not included at all.
-#
-# IBC may also be used to start the FIX CTCI Gateway. All settings
-# relating to this have names prefixed with FIX.
-#
-# The IB API Gateway and the FIX CTCI Gateway share the same code. Which
-# gateway actually runs is governed by an option on the initial gateway
-# login screen. The FIX setting described under IBC Startup
-# Settings below controls this.
-
-
-
-# =============================================================================
-# 1.   IBC Startup Settings
-# =============================================================================
-
-
-# IBC may be used to start the IB Gateway for the FIX CTCI. This
-# setting must be set to 'yes' if you want to run the FIX CTCI gateway. The
-# default is 'no'.
-
-FIX=no
-
-
-
-# =============================================================================
-# 2.   Authentication Settings
-# =============================================================================
-
-# TWS and the IB API gateway require a single username and password.
-# You may specify the username and password using the following settings:
-#
-#	IbLoginId
-#	IbPassword
-#
-# Alternatively, you can specify the username and password in the command
-# files used to start TWS or the Gateway, but this is not recommended for
-# security reasons.
-#
-# If you don't specify them, you will be prompted for them in the usual
-# login dialog when TWS starts (but whatever you have specified will be
-# included in the dialog automatically: for example you may specify the
-# username but not the password, and then you will be prompted for the
-# password via the login dialog). Note that if you specify either
-# the username or the password (or both) in the command file, then
-# IbLoginId and IbPassword settings defined in this file are ignored.
-#
-#
-# The FIX CTCI gateway requires one username and password for FIX order
-# routing, and optionally a separate username and password for market
-# data connections. You may specify the usernames and passwords using
-# the following settings:
-#
-#	FIXLoginId
-#	FIXPassword
-#	IbLoginId	(optional - for market data connections)
-#	IbPassword	(optional - for market data connections)
-#
-# Alternatively you can specify the FIX username and password in the
-# command file used to start the FIX CTCI Gateway, but this is not
-# recommended for security reasons.
-#
-# If you don't specify them, you will be prompted for them in the usual
-# login dialog when FIX CTCI gateway starts (but whatever you have
-# specified will be included in the dialog automatically: for example
-# you may specify the usernames but not the passwords, and then you will
-# be prompted for the passwords via the login dialog). Note that if you
-# specify either the FIX username or the FIX password (or both) on the
-# command line, then FIXLoginId and FIXPassword settings defined in this
-# file are ignored; he same applies to the market data username and
-# password.
-
-# IB API Authentication Settings
-# ------------------------------
-
-# Your TWS username:
-
-IbLoginId=
-
-
-# Your TWS password:
-
-IbPassword=
-
-
-# FIX CTCI Authentication Settings
-# --------------------------------
-
-# Your FIX CTCI username:
-
-FIXLoginId=
-
-
-# Your FIX CTCI password:
-
-FIXPassword=
-
-
-# Second Factor Authentication Settings
-# -------------------------------------
-
-# If you have enabled more than one second factor authentication
-# device, TWS presents a list from which you must select the device
-# you want to use for this login. You can use this setting to
-# instruct IBC to select a particular item in the list on your
-# behalf. Note that you must spell this value exactly as it appears
-# in the list. If no value is set, you must manually select the
-# relevant list entry.
-
-SecondFactorDevice=
-
-
-# If you use the IBKR Mobile app for second factor authentication,
-# and you fail to complete the process before the time limit imposed
-# by IBKR, you can use this setting to tell IBC to exit: arrangements
-# can then be made to automatically restart IBC in order to initiate
-# the login sequence afresh. Otherwise, manual intervention at TWS's
-# Second Factor Authentication dialog is needed to complete the
-# login.
-#
-# Permitted values are 'yes' and 'no'. The default is 'no'.
-#
-# Note that the scripts provided with the IBC zips for Windows and
-# Linux provide options to automatically restart in these
-# circumstances, but only if this setting is also set to 'yes'.
-
-ExitAfterSecondFactorAuthenticationTimeout=no
-
-
-# This setting is only relevant if
-# ExitAfterSecondFactorAuthenticationTimeout is set to 'yes'.
-#
-# It controls how long (in seconds) IBC waits for login to complete
-# after the user acknowledges the second factor authentication
-# alert at the IBKR Mobile app. If login has not completed after
-# this time, IBC terminates.
-# The default value is 40.
-
-SecondFactorAuthenticationExitInterval=
-
-
-# Trading Mode
-# ------------
-#
-# TWS 955 introduced a new Trading Mode combo box on its login
-# dialog. This indicates whether the live account or the paper
-# trading account corresponding to the supplied credentials is
-# to be used. The allowed values are 'live' (the default) and
-# 'paper'. For earlier versions of TWS this setting has no
-# effect.
-
-TradingMode=
-
-
-# Paper-trading Account Warning
-# -----------------------------
-#
-# Logging in to a paper-trading account results in TWS displaying
-# a dialog asking the user to confirm that they are aware that this
-# is not a brokerage account. Until this dialog has been accepted,
-# TWS will not allow API connections to succeed. Setting this
-# to 'yes' (the default) will cause IBC to automatically
-# confirm acceptance. Setting it to 'no' will leave the dialog
-# on display, and the user will have to deal with it manually.
-
-AcceptNonBrokerageAccountWarning=yes
-
-
-# Login Dialog Display Timeout
-#-----------------------------
-#
-# In some circumstances, starting TWS may result in failure to display
-# the login dialog. Restarting TWS may help to resolve this situation,
-# and IBC does this automatically.
-#
-# This setting controls how long (in seconds) IBC waits for the login
-# dialog to appear before restarting TWS.
-#
-# Note that in normal circumstances with a reasonably specified 
-# computer the time to displaying the login dialog is typically less
-# than 20 seconds, and frequently much less. However many factors can
-# influence this, and it is unwise to set this value too low.
-#
-# The default value is 60.
-
-LoginDialogDisplayTimeout = 60
-
-
-
-# =============================================================================
-# 3.   TWS Startup Settings
-# =============================================================================
-
-# Path to settings store
-# ----------------------
-#
-# Path to the directory where TWS should store its settings. This is
-# normally the folder in which TWS is installed. However you may set
-# it to some other location if you wish (for example if you want to
-# run multiple instances of TWS with different settings).
-#
-# It is recommended for clarity that you use an absolute path. The
-# effect of using a relative path is undefined.
-#
-# Linux and macOS users should use the appropriate path syntax.
-#
-# Note that, for Windows users, you MUST use double separator
-# characters to separate the elements of the folder path: for
-# example, IbDir=C:\\IBLiveSettings is valid, but
-# IbDir=C:\IBLiveSettings is NOT valid and will give unexpected
-# results. Linux and macOS users need not use double separators,
-# but they are acceptable.
-#
-# The default is the current working directory when IBC is
-# started.
-
-IbDir=/root/Jts
-
-
-# Store settings on server
-# ------------------------
-#
-# If you wish to store a copy of your TWS settings on IB's
-# servers as well as locally on your computer, set this to
-# 'yes': this enables you to run TWS on different computers
-# with the same configuration, market data lines, etc. If set
-# to 'no', running TWS on different computers will not share the
-# same settings. If no value is specified, TWS will obtain its
-# settings from the same place as the last time this user logged
-# in (whether manually or using IBC).
-
-StoreSettingsOnServer=
-
-
-# Minimize TWS on startup
-# -----------------------
-#
-# Set to 'yes' to minimize TWS when it starts:
-
-MinimizeMainWindow=no
-
-
-# Existing Session Detected Action
-# --------------------------------
-#
-# When a user logs on to an IBKR account for trading purposes by any means, the
-# IBKR account server checks to see whether the account is already logged in
-# elsewhere. If so, a dialog is displayed to both the users that enables them
-# to determine what happens next. The 'ExistingSessionDetectedAction' setting
-# instructs TWS how to proceed when it displays this dialog:
-#
-#   * If the new TWS session is set to 'secondary', the existing session continues
-#     and the new session terminates. Thus a secondary TWS session can never
-#     override any other session.
-#
-#   * If the existing TWS session is set to 'primary', the existing session
-#     continues and the new session terminates (even if the new session is also
-#     set to primary). Thus a primary TWS session can never be overridden by
-#     any new session).
-#
-#   * If both the existing and the new TWS sessions are set to 'primaryoverride',
-#     the existing session terminates and the new session proceeds.
-#
-#   * If the existing TWS session is set to 'manual', the user must handle the
-#     dialog.
-#
-# The difference between 'primary' and 'primaryoverride' is that a
-# 'primaryoverride' session can be overriden over by a new 'primary' session,
-# but a 'primary' session cannot be overriden by any other session.
-#
-# When set to 'primary', if another TWS session is started and manually told to
-# end the 'primary' session, the 'primary' session is automatically reconnected.
-#
-# The default is 'manual'.
-
-ExistingSessionDetectedAction=primary
-
-
-# Override TWS API Port Number
-# ----------------------------
-#
-# If OverrideTwsApiPort is set to an integer, IBC changes the
-# 'Socket port' in TWS's API configuration to that number shortly
-# after startup. Leaving the setting blank will make no change to
-# the current setting. This setting is only intended for use in
-# certain specialized situations where the port number needs to
-# be set dynamically at run-time: most users will never need it,
-# so don't use it unless you know you need it.
-
-OverrideTwsApiPort=4002
-
-
-# Read-only Login
-# ---------------
-#
-# If ReadOnlyLogin is set to 'yes', and the user is enrolled in IB's
-# account security programme, the user will not be asked to perform
-# the second factor authentication action, and login to TWS will
-# occur automatically in read-only mode: in this mode, placing or
-# managing orders is not allowed. If set to 'no', and the user is
-# enrolled in IB's account security programme, the user must perform
-# the relevant second factor authentication action to complete the
-# login. 
-
-# If the user is not enrolled in IB's account security programme,
-# this setting is ignored. The default is 'no'.
-
-ReadOnlyLogin=no
-
-
-# Read-only API
-# -------------
-#
-# If ReadOnlyApi is set to 'yes', API programs cannot submit, modify
-# or cancel orders. If set to 'no', API programs can do these things.
-# If not set, the existing TWS/Gateway configuration is unchanged.
-# NB: this setting is really only supplied for the benefit of new TWS
-# or Gateway instances that are being automatically installed and
-# started without user intervention (eg Docker containers). Where
-# a user is involved, they should use the Global Configuration to
-# set the relevant checkbox (this only needs to be done once) and
-# not provide a value for this setting.
-
-ReadOnlyApi=no
-
-
-# Market data size for US stocks - lots or shares
-# -----------------------------------------------
-#
-# Since IB introduced the option of market data for US stocks showing
-# bid, ask and last sizes in shares rather than lots, TWS and Gateway
-# display a dialog immediately after login notifying the user about
-# this and requiring user input before allowing market data to be
-# accessed. The user can request that the dialog not be shown again.
-#
-# It is recommended that the user should handle this dialog manually
-# rather than using these settings, which are provided for situations 
-# where the user interface is not easily accessible, or where user
-# settings are not preserved between sessions (eg some Docker images).
-#
-# - If this setting is set to 'accept', the dialog will be handled
-#   automatically and the option to not show it again will be
-#   selected.
-#
-#   Note that in this case, the only way to allow the dialog to be
-#   displayed again is to manually enable the 'Bid, Ask and Last
-#   Size Display Update' message in the 'Messages' section of the TWS
-#   configuration dialog. So you should only use 'Accept' if you are
-#   sure you really don't want the dialog to be displayed again, or
-#   you have easy access to the user interface.
-#
-# - If set to 'defer', the dialog will be handled automatically (so
-#   that market data will start), but the option to not show it again
-#   will not be selected, and it will be shown again after the next
-#   login.
-#
-# - If set to 'ignore', the user has to deal with the dialog manually.
-#
-# The default value is 'ignore'.
-#
-# Note if set to 'accept' or 'defer', TWS also automatically sets
-# the API settings checkbox labelled 'Send market data in lots for
-# US stocks for dual-mode API clients'. IBC cannot prevent this.
-# However you can change this immmediately by setting
-# SendMarketDataInLotsForUSstocks (see below) to 'no' .
-
-AcceptBidAskLastSizeDisplayUpdateNotification=accept
-
-
-# This setting determines whether the API settings checkbox labelled
-# 'Send market data in lots for US stocks for dual-mode API clients'
-# is set or cleared. If set to 'yes', the checkbox is set. If set to
-# 'no' the checkbox is cleared. If defaulted, the checkbox is
-# unchanged.
-
-SendMarketDataInLotsForUSstocks=
-
-
-
-# =============================================================================
-# 4.   TWS Auto-Closedown
-# =============================================================================
-#
-# IMPORTANT NOTE: Starting with TWS 974, this setting no longer
-# works properly, because IB have changed the way TWS handles its
-# autologoff mechanism. 
-#
-# You should now configure the TWS autologoff time to something
-# convenient for you, and restart IBC each day.
-#
-# Alternatively, discontinue use of IBC and use the auto-relogin
-# mechanism within TWS 974 and later versions (note that the 
-# auto-relogin mechanism provided by IB is not available if you
-# use IBC).
-
-# Set to yes or no (lower case).
-#
-#   yes   means allow TWS to shut down automatically at its
-# 	  specified shutdown time, which is set via the TWS
-#	  configuration menu.
-#
-#   no    means TWS never shuts down automatically.
-#
-# NB: IB recommends that you do not keep TWS running
-# continuously. If you set this setting to 'no', you may
-# experience incorrect TWS operation.
-#
-# NB: the default for this setting is 'no'. Since this will
-# only work properly with TWS versions earlier than 974, you
-# should explicitly set this to 'yes' for version 974 and later.
-
-IbAutoClosedown=yes
-
-
-
-# =============================================================================
-# 5.   TWS Tidy Closedown Time
-# =============================================================================
-#
-# NB: starting with TWS 974 this is no longer a useful option
-# because both TWS and Gateway now have the same auto-logoff
-# mechanism, and IBC can no longer avoid this.
-#
-# Note that giving this setting a value does not change TWS's
-# auto-logoff in any way: any setting will be additional to the
-# TWS auto-logoff.
-#
-# To tell IBC to tidily close TWS at a specified time every
-# day, set this value to <hh:mm>, for example:
-# ClosedownAt=22:00
-#
-# To tell IBC to tidily close TWS at a specified day and time
-# each week, set this value to <dayOfWeek hh:mm>, for example:
-# ClosedownAt=Friday 22:00
-#
-# Note that the day of the week must be specified using your
-# default locale. Also note that Java will only accept
-# characters encoded to ISO 8859-1 (Latin-1). This means that
-# if the day name in your default locale uses any non-Latin-1
-# characters you need to encode them using Unicode escapes
-# (see http://java.sun.com/docs/books/jls/third_edition/html/lexical.html#3.3
-# for details). For example, to tidily close TWS at 12:00 on
-# Saturday where the default locale is Simplified Chinese,
-# use the following:
-# #ClosedownAt=\u661F\u671F\u516D 12:00
-
-ClosedownAt=
-
-
-
-# =============================================================================
-# 6.   Other TWS Settings
-# =============================================================================
-
-# Accept Incoming Connection
-# --------------------------
-#
-# If set to 'accept', IBC automatically accepts incoming
-# API connection dialogs. If set to 'reject', IBC
-# automatically rejects incoming API connection dialogs. If
-# set to 'manual', the user must decide whether to accept or reject
-# incoming API connection dialogs. The default is 'manual'.
-# NB: it is recommended to set this to 'reject', and to explicitly
-# configure which IP addresses can connect to the API in TWS's API
-# configuration page, as this is much more secure (in this case, no
-# incoming API connection dialogs will occur for those IP addresses).
-
-AcceptIncomingConnectionAction=reject
-
-
-# Allow Blind Trading
-# -------------------
-#
-# If you attempt to place an order for a contract for which
-# you have no market data subscription, TWS displays a dialog
-# to warn you against such blind trading.
-#
-#   yes   means the dialog is dismissed as though the user had
-# 	  clicked the 'Ok' button: this means that you accept
-# 	  the risk and want the order to be submitted.
-#
-#   no    means the dialog remains on display and must be
-#         handled by the user.
-
-AllowBlindTrading=yes
-
-
-# Save Settings on a Schedule
-# ---------------------------
-#
-# You can tell TWS to automatically save its settings on a schedule
-# of your choosing. You can specify one or more specific times,
-# like this:
-#
-# SaveTwsSettingsAt=HH:MM [ HH:MM]...
-#
-# for example:
-# SaveTwsSettingsAt=08:00   12:30 17:30
-#
-# Or you can specify an interval at which settings are to be saved,
-# optionally starting at a specific time and continuing until another
-# time, like this:
-#
-#SaveTwsSettingsAt=Every n [{mins | hours}] [hh:mm] [hh:mm]
-#
-# where the first hh:mm is the start time and the second is the end
-# time. If you don't specify the end time, settings are saved regularly
-# from the start time till midnight. If you don't specify the start time.
-# settings are saved regularly all day, beginning at 00:00. Note that
-# settings will always be saved at the end time, even if that is not
-# exactly one interval later than the previous time. If neither 'mins'
-# nor 'hours' is specified, 'mins' is assumed. Examples:
-#
-# To save every 30 minutes all day starting at 00:00
-#SaveTwsSettingsAt=Every 30
-#SaveTwsSettingsAt=Every 30 mins
-#
-# To save every hour starting at 08:00 and ending at midnight
-#SaveTwsSettingsAt=Every 1 hours 08:00
-#SaveTwsSettingsAt=Every 1 hours 08:00 00:00
-#
-# To save every 90 minutes starting at 08:00 up to and including 17:43
-#SaveTwsSettingsAt=Every 90 08:00 17:43
-
-SaveTwsSettingsAt=
-
-
-
-# =============================================================================
-# 7.   Settings Specific to Indian Versions of TWS
-# =============================================================================
-
-# Indian versions of TWS may display a password expiry
-# notification dialog and a NSE Compliance dialog. These can be
-# dismissed by setting the following to yes. By default the
-# password expiry notice is not dismissed, but the NSE Compliance
-# notice is dismissed.
-
-# Warning: setting DismissPasswordExpiryWarning=yes will mean
-# you will not be notified when your password is about to expire.
-# You must then take other measures to ensure that your password
-# is changed within the expiry period, otherwise IBC will
-# not be able to login successfully.
-
-DismissPasswordExpiryWarning=no
-DismissNSEComplianceNotice=yes
-
-
-
-# =============================================================================
-# 8.   IBC Command Server Settings
-# =============================================================================
-
-# Do NOT CHANGE THE FOLLOWING SETTINGS unless you
-# intend to issue commands to IBC (for example
-# using telnet). Note that these settings have nothing to
-# do with running programs that use the TWS API.
-
-# Command Server Port Number
-# --------------------------
-#
-# The port number that IBC listens on for commands
-# such as "STOP". DO NOT set this to the port number
-# used for TWS API connections. There is no good reason
-# to change this setting unless the port is used by
-# some other application (typically another instance of
-# IBC). The default value is 0, which tells IBC not to
-# start the command server
-
-#CommandServerPort=7462
-
-
-# Permitted Command Sources
-# -------------------------
-#
-# A comma separated list of IP addresses, or host names,
-# which are allowed addresses for sending commands to
-# IBC.  Commands can always be sent from the
-# same host as IBC is running on.
-
-ControlFrom=127.0.0.1
-
-
-# Address for Receiving Commands
-# ------------------------------
-#
-# Specifies the IP address on which the Command Server
-# is so listen. For a multi-homed host, this can be used
-# to specify that connection requests are only to be
-# accepted on the specified address. The default is to
-# accept connection requests on all local addresses.
-
-BindAddress=127.0.0.1
-
-
-# Command Prompt
-# --------------
-#
-# The specified string is output by the server when
-# the connection is first opened and after the completion
-# of each command. This can be useful if sending commands
-# using an interactive program such as telnet. The default
-# is that no prompt is output.
-# For example:
-#
-# CommandPrompt=>
-
-CommandPrompt=
-
-
-# Suppress Command Server Info Messages
-# -------------------------------------
-#
-# Some commands can return intermediate information about
-# their progress. This setting controls whether such
-# information is sent. The default is that such information
-# is not sent.
-
-SuppressInfoMessages=no
-
-
-
-# =============================================================================
-# 9.   Diagnostic Settings
-# =============================================================================
-#
-# IBC can log information about the structure of windows
-# displayed by TWS. This information is useful when adding
-# new features to IBC or when behaviour is not as expected. 
-#
-# The logged information shows the hierarchical organisation
-# of all the components of the window, and includes the
-# current values of text boxes and labels.
-#
-# Note that this structure logging has a small performance
-# impact, and depending on the settings can cause the logfile
-# size to be significantly increased. It is therefore
-# recommended that the LogStructureWhen setting be set to
-# 'never' (the default) unless there is a specific reason
-# that this information is needed.
-
-
-# Scope of Structure Logging
-# --------------------------
-#
-# The LogStructureScope setting indicates which windows are
-# eligible for structure logging:
-#
-#    - if set to 'known', only windows that IBC recognizes
-#      are eligible - these are windows that IBC has some
-#      interest in monitoring, usually to take some action
-#      on the user's behalf;
-#
-#    - if set to 'unknown', only windows that IBC does not
-#      recognize are eligible. Most windows displayed by
-#      TWS fall into this category;
-#
-#    - if set to 'untitled', only windows that IBC does not
-#      recognize and that have no title are eligible. These
-#      are usually message boxes or similar small windows,
-#
-#    - if set to 'all', then every window displayed by TWS
-#      is eligible.
-#
-# The default value is 'known'.
-
-LogStructureScope=all
-
-
-# When to Log Window Structure
-# ----------------------------
-#
-# The LogStructureWhen setting specifies the circumstances
-# when eligible TWS windows have their structure logged:
-#
-#     - if set to 'open' or 'yes' or 'true', IBC logs the
-#       structure of an eligible window the first time it
-#       is encountered;
-#
-#    - if set to 'activate', the structure is logged every
-#      time an eligible window is made active;
-#
-#    - if set to 'never' or 'no' or 'false', structure
-#      information is never logged.
-#
-# The default value is 'never'.
-
-LogStructureWhen=never
-
-
-# DEPRECATED SETTING
-# ------------------
-#
-# LogComponents - THIS SETTING WILL BE REMOVED IN A FUTURE
-# RELEASE
-#
-# If LogComponents is set to any value, this is equivalent
-# to setting LogStructureWhen to that same value and
-# LogStructureScope to 'all': the actual values of those
-# settings are ignored. The default is that the values
-# of LogStructureScope and LogStructureWhen are honoured.
-
-#LogComponents=
-
-
-

dockering/ib/jts.ini

@@ -1,33 +0,0 @@
-[IBGateway]
-ApiOnly=true
-LocalServerPort=4002
-# NOTE: must be set if using IBC's "reject" mode
-TrustedIPs=127.0.0.1
-; RemoteHostOrderRouting=ndc1.ibllc.com
-; WriteDebug=true
-; RemotePortOrderRouting=4001
-; useRemoteSettings=false
-; tradingMode=p
-; Steps=8
-; colorPalletName=dark
-
-# window geo, this may be useful for sending `xdotool` commands?
-; MainWindow.Width=1986
-; screenHeight=3960
-
-
-[Logon]
-Locale=en
-# most markets are oriented around this zone
-# so might as well hard code it.
-TimeZone=America/New_York
-UseSSL=true
-displayedproxymsg=1
-os_titlebar=true
-s3store=true
-useRemoteSettings=false
-
-[Communication]
-ctciAutoEncrypt=true
-Region=usr
-; Peer=cdc1.ibllc.com:4001

dockering/ib/run_x11_vnc.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-# start VNC server
-x11vnc \
-    -ncache_cr \
-    -listen localhost \
-    -display :1 \
-    -forever \
-    -shared \
-    -logappend /var/log/x11vnc.log \
-    -bg \
-    -noipv6 \
-    -autoport 3003 \
-    # can't use this because of ``asyncvnc`` issue:
-    # https://github.com/barneygale/asyncvnc/issues/1
-    # -passwd 'ibcansmbz'

piker/__init__.py

@@ -18,3 +18,10 @@
 piker: trading gear for hackers.
 
 """
+import msgpack  # noqa
+
+# TODO: remove this now right?
+import msgpack_numpy
+
+# patch msgpack for numpy arrays
+msgpack_numpy.patch()

piker/_daemon.py

@@ -19,7 +19,7 @@ Structured, daemon tree service management.
 
 """
 from typing import Optional, Union, Callable, Any
-from contextlib import asynccontextmanager as acm
+from contextlib import asynccontextmanager
 from collections import defaultdict
 
 from pydantic import BaseModel
@@ -34,11 +34,9 @@ from .brokers import get_brokermod
 log = get_logger(__name__)
 
 _root_dname = 'pikerd'
-
-_registry_addr = ('127.0.0.1', 6116)
 _tractor_kwargs: dict[str, Any] = {
     # use a different registry addr then tractor's default
-    'arbiter_addr': _registry_addr
+    'arbiter_addr':  ('127.0.0.1', 6116),
 }
 _root_modules = [
     __name__,
@@ -80,6 +78,7 @@ class Services(BaseModel):
         ) -> Any:
 
             with trio.CancelScope() as cs:
+
                 async with portal.open_context(
                     target,
                     **kwargs,
@@ -88,21 +87,19 @@ class Services(BaseModel):
 
                     # unblock once the remote context has started
                     task_status.started((cs, first))
-                    log.info(
+
-                        f'`pikerd` service {name} started with value {first}'
-                    )
-                    try:
                     # wait on any context's return value
                     ctx_res = await ctx.result()
-                    except tractor.ContextCancelled:
+                    log.info(
-                        return await self.cancel_service(name)
+                        f'`pikerd` service {name} started with value {ctx_res}'
-                    else:
+                    )
+
                 # wait on any error from the sub-actor
-                        # NOTE: this will block indefinitely until
+                # NOTE: this will block indefinitely until cancelled
-                        # cancelled either by error from the target
+                # either by error from the target context function or
-                        # context function or by being cancelled here by
+                # by being cancelled here by the surroundingn cancel
-                        # the surrounding cancel scope
+                # scope
-                        return (await portal.result(), ctx_res)
+                return await (portal.result(), ctx_res)
 
         cs, first = await self.service_n.start(open_context_in_task)
 
@@ -112,17 +109,14 @@ class Services(BaseModel):
 
         return cs, first
 
-    # TODO: per service cancellation by scope, we aren't using this
-    # anywhere right?
     async def cancel_service(
         self,
         name: str,
+
     ) -> Any:
+
         log.info(f'Cancelling `pikerd` service {name}')
         cs, portal = self.service_tasks[name]
-        # XXX: not entirely sure why this is required,
-        # and should probably be better fine tuned in
-        # ``tractor``?
         cs.cancel()
         return await portal.cancel_actor()
 
@@ -130,7 +124,7 @@ class Services(BaseModel):
 _services: Optional[Services] = None
 
 
-@acm
+@asynccontextmanager
 async def open_pikerd(
     start_method: str = 'trio',
     loglevel: Optional[str] = None,
@@ -156,7 +150,7 @@ async def open_pikerd(
         tractor.open_root_actor(
 
             # passed through to ``open_root_actor``
-            arbiter_addr=_registry_addr,
+            arbiter_addr=_tractor_kwargs['arbiter_addr'],
             name=_root_dname,
             loglevel=loglevel,
             debug_mode=debug_mode,
@@ -185,48 +179,7 @@ async def open_pikerd(
             yield _services
 
 
-@acm
+@asynccontextmanager
-async def open_piker_runtime(
-    name: str,
-    enable_modules: list[str] = [],
-    start_method: str = 'trio',
-    loglevel: Optional[str] = None,
-
-    # XXX: you should pretty much never want debug mode
-    # for data daemons when running in production.
-    debug_mode: bool = False,
-
-) -> Optional[tractor._portal.Portal]:
-    '''
-    Start a piker actor who's runtime will automatically
-    sync with existing piker actors in local network
-    based on configuration.
-
-    '''
-    global _services
-    assert _services is None
-
-    # XXX: this may open a root actor as well
-    async with (
-        tractor.open_root_actor(
-
-            # passed through to ``open_root_actor``
-            arbiter_addr=_registry_addr,
-            name=name,
-            loglevel=loglevel,
-            debug_mode=debug_mode,
-            start_method=start_method,
-
-            # TODO: eventually we should be able to avoid
-            # having the root have more then permissions to
-            # spawn other specialized daemons I think?
-            enable_modules=_root_modules,
-        ) as _,
-    ):
-        yield tractor.current_actor()
-
-
-@acm
 async def maybe_open_runtime(
     loglevel: Optional[str] = None,
     **kwargs,
@@ -249,7 +202,7 @@ async def maybe_open_runtime(
         yield
 
 
-@acm
+@asynccontextmanager
 async def maybe_open_pikerd(
     loglevel: Optional[str] = None,
     **kwargs,
@@ -300,36 +253,7 @@ class Brokerd:
     locks = defaultdict(trio.Lock)
 
 
-@acm
+@asynccontextmanager
-async def find_service(
-    service_name: str,
-) -> Optional[tractor.Portal]:
-
-    log.info(f'Scanning for service `{service_name}`')
-    # attach to existing daemon by name if possible
-    async with tractor.find_actor(
-        service_name,
-        arbiter_sockaddr=_registry_addr,
-    ) as maybe_portal:
-        yield maybe_portal
-
-
-async def check_for_service(
-    service_name: str,
-
-) -> bool:
-    '''
-    Service daemon "liveness" predicate.
-
-    '''
-    async with tractor.query_actor(
-        service_name,
-        arbiter_sockaddr=_registry_addr,
-    ) as sockaddr:
-        return sockaddr
-
-
-@acm
 async def maybe_spawn_daemon(
 
     service_name: str,
@@ -339,7 +263,7 @@ async def maybe_spawn_daemon(
     **kwargs,
 
 ) -> tractor.Portal:
-    '''
+    """
     If no ``service_name`` daemon-actor can be found,
     spawn one in a local subactor and return a portal to it.
 
@@ -350,7 +274,7 @@ async def maybe_spawn_daemon(
     This can be seen as a service starting api for remote-actor
     clients.
 
-    '''
+    """
     if loglevel:
         get_console_log(loglevel)
 
@@ -359,14 +283,13 @@ async def maybe_spawn_daemon(
     lock = Brokerd.locks[service_name]
     await lock.acquire()
 
-    async with find_service(service_name) as portal:
+    # attach to existing daemon by name if possible
+    async with tractor.find_actor(service_name) as portal:
         if portal is not None:
             lock.release()
             yield portal
             return
 
-    log.warning(f"Couldn't find any existing {service_name}")
-
     # ask root ``pikerd`` daemon to spawn the daemon we need if
     # pikerd is not live we now become the root of the
     # process tree
@@ -402,7 +325,6 @@ async def maybe_spawn_daemon(
         async with tractor.wait_for_actor(service_name) as portal:
             lock.release()
             yield portal
-            await portal.cancel_actor()
 
 
 async def spawn_brokerd(
@@ -426,19 +348,9 @@ async def spawn_brokerd(
 
     # ask `pikerd` to spawn a new sub-actor and manage it under its
     # actor nursery
-    modpath = brokermod.__name__
-    broker_enable = [modpath]
-    for submodname in getattr(
-        brokermod,
-        '__enable_modules__',
-        [],
-    ):
-        subpath = f'{modpath}.{submodname}'
-        broker_enable.append(subpath)
-
     portal = await _services.actor_n.start_actor(
         dname,
-        enable_modules=_data_mods + broker_enable,
+        enable_modules=_data_mods + [brokermod.__name__],
         loglevel=loglevel,
         debug_mode=_services.debug_mode,
         **tractor_kwargs
@@ -456,7 +368,7 @@ async def spawn_brokerd(
     return True
 
 
-@acm
+@asynccontextmanager
 async def maybe_spawn_brokerd(
 
     brokername: str,
@@ -464,9 +376,7 @@ async def maybe_spawn_brokerd(
     **kwargs,
 
 ) -> tractor.Portal:
-    '''
+    '''Helper to spawn a brokerd service.
-    Helper to spawn a brokerd service *from* a client
-    who wishes to use the sub-actor-daemon.
 
     '''
     async with maybe_spawn_daemon(
@@ -518,7 +428,7 @@ async def spawn_emsd(
     return True
 
 
-@acm
+@asynccontextmanager
 async def maybe_open_emsd(
 
     brokername: str,
@@ -537,25 +447,3 @@ async def maybe_open_emsd(
 
     ) as portal:
         yield portal
-
-
-# TODO: ideally we can start the tsdb "on demand" but it's
-# probably going to require "rootless" docker, at least if we don't
-# want to expect the user to start ``pikerd`` with root perms all the
-# time.
-# async def maybe_open_marketstored(
-#     loglevel: Optional[str] = None,
-#     **kwargs,
-
-# ) -> tractor._portal.Portal:  # noqa
-
-#     async with maybe_spawn_daemon(
-
-#         'marketstored',
-#         service_task_target=spawn_emsd,
-#         spawn_args={'loglevel': loglevel},
-#         loglevel=loglevel,
-#         **kwargs,
-
-#     ) as portal:
-#         yield portal

piker/_profile.py

@@ -21,10 +21,7 @@ Profiling wrappers for internal libs.
 import time
 from functools import wraps
 
-# NOTE: you can pass a flag to enable this:
+_pg_profile: bool = True
-# ``piker chart <args> --profile``.
-_pg_profile: bool = False
-ms_slower_then: float = 0
 
 
 def pg_profile_enabled() -> bool:

piker/brokers/_util.py

@@ -33,49 +33,13 @@ class SymbolNotFound(BrokerError):
 
 
 class NoData(BrokerError):
-    '''
+    "Symbol data not permitted"
-    Symbol data not permitted or no data
-    for time range found.
-
-    '''
-    def __init__(
-        self,
-        *args,
-        frame_size: int = 1000,
-
-    ) -> None:
-        super().__init__(*args)
-
-        # when raised, machinery can check if the backend
-        # set a "frame size" for doing datetime calcs.
-        self.frame_size: int = 1000
-
-
-class DataUnavailable(BrokerError):
-    '''
-    Signal storage requests to terminate.
-
-    '''
-    # TODO: add in a reason that can be displayed in the
-    # UI (for eg. `kraken` is bs and you should complain
-    # to them that you can't pull more OHLC data..)
-
-
-class DataThrottle(BrokerError):
-    '''
-    Broker throttled request rate for data.
-
-    '''
-    # TODO: add in throttle metrics/feedback
-
 
 
 def resproc(
     resp: asks.response_objects.Response,
     log: logging.Logger,
-    return_json: bool = True,
+    return_json: bool = True
-    log_resp: bool = False,
-
 ) -> asks.response_objects.Response:
     """Process response and return its json content.
 
@@ -84,12 +48,11 @@ def resproc(
     if not resp.status_code == 200:
         raise BrokerError(resp.body)
     try:
-        msg = resp.json()
+        json = resp.json()
     except json.decoder.JSONDecodeError:
         log.exception(f"Failed to process {resp}:\n{resp.text}")
         raise BrokerError(resp.text)
+    else:
+        log.debug(f"Received json contents:\n{colorize_json(json)}")
 
-    if log_resp:
+    return json if return_json else resp
-        log.debug(f"Received json contents:\n{colorize_json(msg)}")
-
-    return msg if return_json else resp

piker/brokers/binance.py

@@ -18,17 +18,13 @@
 Binance backend
 
 """
-from contextlib import asynccontextmanager as acm
+from contextlib import asynccontextmanager
-from datetime import datetime
+from typing import List, Dict, Any, Tuple, Union, Optional, AsyncGenerator
-from typing import (
-    Any, Union, Optional,
-    AsyncGenerator, Callable,
-)
 import time
 
 import trio
 from trio_typing import TaskStatus
-import pendulum
+import arrow
 import asks
 from fuzzywuzzy import process as fuzzy
 import numpy as np
@@ -92,7 +88,7 @@ class Pair(BaseModel):
     baseCommissionPrecision: int
     quoteCommissionPrecision: int
 
-    orderTypes: list[str]
+    orderTypes: List[str]
 
     icebergAllowed: bool
     ocoAllowed: bool
@@ -100,8 +96,8 @@ class Pair(BaseModel):
     isSpotTradingAllowed: bool
     isMarginTradingAllowed: bool
 
-    filters: list[dict[str, Union[str, int, float]]]
+    filters: List[Dict[str, Union[str, int, float]]]
-    permissions: list[str]
+    permissions: List[str]
 
 
 @dataclass
@@ -133,7 +129,7 @@ class OHLC:
     bar_wap: float = 0.0
 
 
-# convert datetime obj timestamp to unixtime in milliseconds
+# convert arrow timestamp to unixtime in miliseconds
 def binance_timestamp(when):
     return int((when.timestamp() * 1000) + (when.microsecond / 1000))
 
@@ -149,7 +145,7 @@ class Client:
         self,
         method: str,
         params: dict,
-    ) -> dict[str, Any]:
+    ) -> Dict[str, Any]:
         resp = await self._sesh.get(
             path=f'/api/v3/{method}',
             params=params,
@@ -204,7 +200,7 @@ class Client:
         self,
         pattern: str,
         limit: int = None,
-    ) -> dict[str, Any]:
+    ) -> Dict[str, Any]:
         if self._pairs is not None:
             data = self._pairs
         else:
@@ -222,22 +218,20 @@ class Client:
     async def bars(
         self,
         symbol: str,
-        start_dt: Optional[datetime] = None,
+        start_time: int = None,
-        end_dt: Optional[datetime] = None,
+        end_time: int = None,
         limit: int = 1000,  # <- max allowed per query
         as_np: bool = True,
 
     ) -> dict:
 
-        if end_dt is None:
+        if start_time is None:
-            end_dt = pendulum.now('UTC')
+            start_time = binance_timestamp(
+                arrow.utcnow().floor('minute').shift(minutes=-limit)
+            )
 
-        if start_dt is None:
+        if end_time is None:
-            start_dt = end_dt.start_of(
+            end_time = binance_timestamp(arrow.utcnow())
-                'minute').subtract(minutes=limit)
-
-        start_time = binance_timestamp(start_dt)
-        end_time = binance_timestamp(end_dt)
 
         # https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
         bars = await self._api(
@@ -279,7 +273,7 @@ class Client:
         return array
 
 
-@acm
+@asynccontextmanager
 async def get_client() -> Client:
     client = Client()
     await client.cache_symbols()
@@ -359,7 +353,7 @@ async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]:
             }
 
 
-def make_sub(pairs: list[str], sub_name: str, uid: int) -> dict[str, str]:
+def make_sub(pairs: List[str], sub_name: str, uid: int) -> Dict[str, str]:
     """Create a request subscription packet dict.
 
     https://binance-docs.github.io/apidocs/spot/en/#live-subscribing-unsubscribing-to-streams
@@ -374,37 +368,6 @@ def make_sub(pairs: list[str], sub_name: str, uid: int) -> dict[str, str]:
     }
 
 
-@acm
-async def open_history_client(
-    symbol: str,
-
-) -> tuple[Callable, int]:
-
-    # TODO implement history getter for the new storage layer.
-    async with open_cached_client('binance') as client:
-
-        async def get_ohlc(
-            end_dt: Optional[datetime] = None,
-            start_dt: Optional[datetime] = None,
-
-        ) -> tuple[
-            np.ndarray,
-            datetime,  # start
-            datetime,  # end
-        ]:
-
-            array = await client.bars(
-                symbol,
-                start_dt=start_dt,
-                end_dt=end_dt,
-            )
-            start_dt = pendulum.from_timestamp(array[0]['time'])
-            end_dt = pendulum.from_timestamp(array[-1]['time'])
-            return array, start_dt, end_dt
-
-        yield get_ohlc, {'erlangs': 3, 'rate': 3}
-
-
 async def backfill_bars(
     sym: str,
     shm: ShmArray,  # type: ignore # noqa
@@ -422,12 +385,13 @@ async def backfill_bars(
 async def stream_quotes(
 
     send_chan: trio.abc.SendChannel,
-    symbols: list[str],
+    symbols: List[str],
+    shm: ShmArray,
     feed_is_live: trio.Event,
     loglevel: str = None,
 
     # startup sync
-    task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
+    task_status: TaskStatus[Tuple[Dict, Dict]] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
     # XXX: required to propagate ``tractor`` loglevel to piker logging
@@ -452,8 +416,8 @@ async def stream_quotes(
 
             # XXX: after manually inspecting the response format we
             # just directly pick out the info we need
-            si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
+            si['price_tick_size'] = syminfo.filters[0]['tickSize']
-            si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
+            si['lot_tick_size'] = syminfo.filters[2]['stepSize']
             si['asset_type'] = 'crypto'
 
         symbol = symbols[0]
@@ -464,11 +428,10 @@ async def stream_quotes(
             symbol: {
                 'symbol_info': sym_infos[sym],
                 'shm_write_opts': {'sum_tick_vml': False},
-                'fqsn': sym,
             },
         }
 
-        @acm
+        @asynccontextmanager
         async def subscribe(ws: wsproto.WSConnection):
             # setup subs
 
@@ -518,7 +481,8 @@ async def stream_quotes(
                 # TODO: use ``anext()`` when it lands in 3.10!
                 typ, quote = await msg_gen.__anext__()
 
-            task_status.started((init_msgs,  quote))
+            first_quote = {quote['symbol'].lower(): quote}
+            task_status.started((init_msgs,  first_quote))
 
             # signal to caller feed is ready for consumption
             feed_is_live.set()

piker/brokers/cli.py

@@ -23,6 +23,7 @@ from operator import attrgetter
 from operator import itemgetter
 
 import click
+import pandas as pd
 import trio
 import tractor
 
@@ -46,10 +47,8 @@ _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
 @click.argument('kwargs', nargs=-1)
 @click.pass_obj
 def api(config, meth, kwargs, keys):
-    '''
+    """Make a broker-client API method call
-    Make a broker-client API method call
+    """
-
-    '''
     # global opts
     broker = config['brokers'][0]
 
@@ -80,13 +79,13 @@ def api(config, meth, kwargs, keys):
 
 
 @cli.command()
+@click.option('--df-output', '-df', flag_value=True,
+              help='Output in `pandas.DataFrame` format')
 @click.argument('tickers', nargs=-1, required=True)
 @click.pass_obj
-def quote(config, tickers):
+def quote(config, tickers, df_output):
-    '''
+    """Print symbol quotes to the console
-    Print symbol quotes to the console
+    """
-
-    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -101,19 +100,28 @@ def quote(config, tickers):
             if ticker not in syms:
                 brokermod.log.warn(f"Could not find symbol {ticker}?")
 
+    if df_output:
+        cols = next(filter(bool, quotes)).copy()
+        cols.pop('symbol')
+        df = pd.DataFrame(
+            (quote or {} for quote in quotes),
+            columns=cols,
+        )
+        click.echo(df)
+    else:
         click.echo(colorize_json(quotes))
 
 
 @cli.command()
+@click.option('--df-output', '-df', flag_value=True,
+              help='Output in `pandas.DataFrame` format')
 @click.option('--count', '-c', default=1000,
               help='Number of bars to retrieve')
 @click.argument('symbol', required=True)
 @click.pass_obj
-def bars(config, symbol, count):
+def bars(config, symbol, count, df_output):
-    '''
+    """Retreive 1m bars for symbol and print on the console
-    Retreive 1m bars for symbol and print on the console
+    """
-
-    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -125,7 +133,7 @@ def bars(config, symbol, count):
             brokermod,
             symbol,
             count=count,
-            as_np=False,
+            as_np=df_output
         )
     )
 
@@ -133,6 +141,9 @@ def bars(config, symbol, count):
         log.error(f"No quotes could be found for {symbol}?")
         return
 
+    if df_output:
+        click.echo(pd.DataFrame(bars))
+    else:
         click.echo(colorize_json(bars))
 
 
@@ -145,10 +156,8 @@ def bars(config, symbol, count):
 @click.argument('name', nargs=1, required=True)
 @click.pass_obj
 def record(config, rate, name, dhost, filename):
-    '''
+    """Record client side quotes to a file on disk
-    Record client side quotes to a file on disk
+    """
-
-    '''
     # global opts
     brokermod = config['brokermods'][0]
     loglevel = config['loglevel']
@@ -186,10 +195,8 @@ def record(config, rate, name, dhost, filename):
 @click.argument('symbol', required=True)
 @click.pass_context
 def contracts(ctx, loglevel, broker, symbol, ids):
-    '''
+    """Get list of all option contracts for symbol
-    Get list of all option contracts for symbol
+    """
-
-    '''
     brokermod = get_brokermod(broker)
     get_console_log(loglevel)
 
@@ -206,14 +213,14 @@ def contracts(ctx, loglevel, broker, symbol, ids):
 
 
 @cli.command()
+@click.option('--df-output', '-df', flag_value=True,
+              help='Output in `pandas.DataFrame` format')
 @click.option('--date', '-d', help='Contracts expiry date')
 @click.argument('symbol', required=True)
 @click.pass_obj
-def optsquote(config, symbol, date):
+def optsquote(config, symbol, df_output, date):
-    '''
+    """Retreive symbol option quotes on the console
-    Retreive symbol option quotes on the console
+    """
-
-    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -226,6 +233,13 @@ def optsquote(config, symbol, date):
         log.error(f"No option quotes could be found for {symbol}?")
         return
 
+    if df_output:
+        df = pd.DataFrame(
+            (quote.values() for quote in quotes),
+            columns=quotes[0].keys(),
+        )
+        click.echo(df)
+    else:
         click.echo(colorize_json(quotes))
 
 
@@ -233,10 +247,8 @@ def optsquote(config, symbol, date):
 @click.argument('tickers', nargs=-1, required=True)
 @click.pass_obj
 def symbol_info(config, tickers):
-    '''
+    """Print symbol quotes to the console
-    Print symbol quotes to the console
+    """
-
-    '''
     # global opts
     brokermod = config['brokermods'][0]
 
@@ -258,10 +270,8 @@ def symbol_info(config, tickers):
 @click.argument('pattern', required=True)
 @click.pass_obj
 def search(config, pattern):
-    '''
+    """Search for symbols from broker backend(s).
-    Search for symbols from broker backend(s).
+    """
-
-    '''
     # global opts
     brokermods = config['brokermods']
 

piker/brokers/core.py

@@ -142,23 +142,15 @@ async def symbol_search(
     brokermods: list[ModuleType],
     pattern: str,
     **kwargs,
-
 ) -> Dict[str, Dict[str, Dict[str, Any]]]:
-    '''
+    """Return symbol info from broker.
-    Return symbol info from broker.
+    """
-
-    '''
     results = []
 
-    async def search_backend(
+    async def search_backend(brokername: str) -> None:
-        brokermod: ModuleType
-    ) -> None:
-
-        brokername: str = mod.name
 
         async with maybe_spawn_brokerd(
-            mod.name,
+            brokername,
-            infect_asyncio=getattr(mod, '_infect_asyncio', False),
         ) as portal:
 
             results.append((

piker/brokers/ib/__init__.py

@@ -1,67 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Interactive Brokers API backend.
-
-Sub-modules within break into the core functionalities:
-
-- ``broker.py`` part for orders / trading endpoints
-- ``data.py`` for real-time data feed endpoints
-
-- ``client.py`` for the core API machinery which is ``trio``-ized
-  wrapping around ``ib_insync``.
-
-- ``report.py`` for the hackery to build manual pp calcs
-  to avoid ib's absolute bullshit FIFO style position
-  tracking..
-
-"""
-from .api import (
-    get_client,
-)
-from .feed import (
-    open_history_client,
-    open_symbol_search,
-    stream_quotes,
-)
-from .broker import trades_dialogue
-
-__all__ = [
-    'get_client',
-    'trades_dialogue',
-    'open_history_client',
-    'open_symbol_search',
-    'stream_quotes',
-]
-
-
-# tractor RPC enable arg
-__enable_modules__: list[str] = [
-    'api',
-    'feed',
-    'broker',
-]
-
-# passed to ``tractor.ActorNursery.start_actor()``
-_spawn_kwargs = {
-    'infect_asyncio': True,
-}
-
-# annotation to let backend agnostic code
-# know if ``brokerd`` should be spawned with
-# ``tractor``'s aio mode.
-_infect_asyncio: bool = True

piker/brokers/ib/broker.py

@@ -1,590 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Order and trades endpoints for use with ``piker``'s EMS.
-
-"""
-from __future__ import annotations
-from dataclasses import asdict
-from functools import partial
-from pprint import pformat
-import time
-from typing import (
-    Any,
-    Optional,
-    AsyncIterator,
-)
-
-import trio
-from trio_typing import TaskStatus
-import tractor
-from ib_insync.contract import (
-    Contract,
-    Option,
-)
-from ib_insync.order import (
-    Trade,
-    OrderStatus,
-)
-from ib_insync.objects import (
-    Fill,
-    Execution,
-)
-from ib_insync.objects import Position
-
-from piker import config
-from piker.log import get_console_log
-from piker.clearing._messages import (
-    BrokerdOrder,
-    BrokerdOrderAck,
-    BrokerdStatus,
-    BrokerdPosition,
-    BrokerdCancel,
-    BrokerdFill,
-    BrokerdError,
-)
-from .api import (
-    _accounts2clients,
-    _adhoc_futes_set,
-    log,
-    get_config,
-    open_client_proxies,
-    Client,
-)
-
-
-def pack_position(
-    pos: Position
-
-) -> dict[str, Any]:
-    con = pos.contract
-
-    if isinstance(con, Option):
-        # TODO: option symbol parsing and sane display:
-        symbol = con.localSymbol.replace(' ', '')
-
-    else:
-        # TODO: lookup fqsn even for derivs.
-        symbol = con.symbol.lower()
-
-    exch = (con.primaryExchange or con.exchange).lower()
-    symkey = '.'.join((symbol, exch))
-    if not exch:
-        # attempt to lookup the symbol from our
-        # hacked set..
-        for sym in _adhoc_futes_set:
-            if symbol in sym:
-                symkey = sym
-                break
-
-    expiry = con.lastTradeDateOrContractMonth
-    if expiry:
-        symkey += f'.{expiry}'
-
-    # TODO: options contracts into a sane format..
-
-    return BrokerdPosition(
-        broker='ib',
-        account=pos.account,
-        symbol=symkey,
-        currency=con.currency,
-        size=float(pos.position),
-        avg_price=float(pos.avgCost) / float(con.multiplier or 1.0),
-    )
-
-
-async def handle_order_requests(
-
-    ems_order_stream: tractor.MsgStream,
-    accounts_def: dict[str, str],
-
-) -> None:
-
-    request_msg: dict
-    async for request_msg in ems_order_stream:
-        log.info(f'Received order request {request_msg}')
-
-        action = request_msg['action']
-        account = request_msg['account']
-
-        acct_number = accounts_def.get(account)
-        if not acct_number:
-            log.error(
-                f'An IB account number for name {account} is not found?\n'
-                'Make sure you have all TWS and GW instances running.'
-            )
-            await ems_order_stream.send(BrokerdError(
-                oid=request_msg['oid'],
-                symbol=request_msg['symbol'],
-                reason=f'No account found: `{account}` ?',
-            ).dict())
-            continue
-
-        client = _accounts2clients.get(account)
-        if not client:
-            log.error(
-                f'An IB client for account name {account} is not found.\n'
-                'Make sure you have all TWS and GW instances running.'
-            )
-            await ems_order_stream.send(BrokerdError(
-                oid=request_msg['oid'],
-                symbol=request_msg['symbol'],
-                reason=f'No api client loaded for account: `{account}` ?',
-            ).dict())
-            continue
-
-        if action in {'buy', 'sell'}:
-            # validate
-            order = BrokerdOrder(**request_msg)
-
-            # call our client api to submit the order
-            reqid = client.submit_limit(
-                oid=order.oid,
-                symbol=order.symbol,
-                price=order.price,
-                action=order.action,
-                size=order.size,
-                account=acct_number,
-
-                # XXX: by default 0 tells ``ib_insync`` methods that
-                # there is no existing order so ask the client to create
-                # a new one (which it seems to do by allocating an int
-                # counter - collision prone..)
-                reqid=order.reqid,
-            )
-            if reqid is None:
-                await ems_order_stream.send(BrokerdError(
-                    oid=request_msg['oid'],
-                    symbol=request_msg['symbol'],
-                    reason='Order already active?',
-                ).dict())
-
-            # deliver ack that order has been submitted to broker routing
-            await ems_order_stream.send(
-                BrokerdOrderAck(
-                    # ems order request id
-                    oid=order.oid,
-                    # broker specific request id
-                    reqid=reqid,
-                    time_ns=time.time_ns(),
-                    account=account,
-                ).dict()
-            )
-
-        elif action == 'cancel':
-            msg = BrokerdCancel(**request_msg)
-            client.submit_cancel(reqid=msg.reqid)
-
-        else:
-            log.error(f'Unknown order command: {request_msg}')
-
-
-async def recv_trade_updates(
-
-    client: Client,
-    to_trio: trio.abc.SendChannel,
-
-) -> None:
-    """Stream a ticker using the std L1 api.
-    """
-    client.inline_errors(to_trio)
-
-    # sync with trio task
-    to_trio.send_nowait(None)
-
-    def push_tradesies(eventkit_obj, obj, fill=None):
-        """Push events to trio task.
-
-        """
-        if fill is not None:
-            # execution details event
-            item = ('fill', (obj, fill))
-
-        elif eventkit_obj.name() == 'positionEvent':
-            item = ('position', obj)
-
-        else:
-            item = ('status', obj)
-
-        log.info(f'eventkit event ->\n{pformat(item)}')
-
-        try:
-            to_trio.send_nowait(item)
-        except trio.BrokenResourceError:
-            log.exception(f'Disconnected from {eventkit_obj} updates')
-            eventkit_obj.disconnect(push_tradesies)
-
-    # hook up to the weird eventkit object - event stream api
-    for ev_name in [
-        'orderStatusEvent',  # all order updates
-        'execDetailsEvent',  # all "fill" updates
-        'positionEvent',  # avg price updates per symbol per account
-
-        # 'commissionReportEvent',
-        # XXX: ugh, it is a separate event from IB and it's
-        # emitted as follows:
-        # self.ib.commissionReportEvent.emit(trade, fill, report)
-
-        # XXX: not sure yet if we need these
-        # 'updatePortfolioEvent',
-
-        # XXX: these all seem to be weird ib_insync intrernal
-        # events that we probably don't care that much about
-        # given the internal design is wonky af..
-        # 'newOrderEvent',
-        # 'orderModifyEvent',
-        # 'cancelOrderEvent',
-        # 'openOrderEvent',
-    ]:
-        eventkit_obj = getattr(client.ib, ev_name)
-        handler = partial(push_tradesies, eventkit_obj)
-        eventkit_obj.connect(handler)
-
-    # let the engine run and stream
-    await client.ib.disconnectedEvent
-
-
-@tractor.context
-async def trades_dialogue(
-
-    ctx: tractor.Context,
-    loglevel: str = None,
-
-) -> AsyncIterator[dict[str, Any]]:
-
-    # XXX: required to propagate ``tractor`` loglevel to piker logging
-    get_console_log(loglevel or tractor.current_actor().loglevel)
-
-    accounts_def = config.load_accounts(['ib'])
-
-    global _client_cache
-
-    # deliver positions to subscriber before anything else
-    all_positions = []
-    accounts = set()
-    clients: list[tuple[Client, trio.MemoryReceiveChannel]] = []
-
-    async with (
-        trio.open_nursery() as nurse,
-        open_client_proxies() as (proxies, aioclients),
-    ):
-        for account, proxy in proxies.items():
-
-            client = aioclients[account]
-
-            async def open_stream(
-                task_status: TaskStatus[
-                    trio.abc.ReceiveChannel
-                ] = trio.TASK_STATUS_IGNORED,
-            ):
-                # each api client has a unique event stream
-                async with tractor.to_asyncio.open_channel_from(
-                    recv_trade_updates,
-                    client=client,
-                ) as (first, trade_event_stream):
-
-                    task_status.started(trade_event_stream)
-                    await trio.sleep_forever()
-
-            trade_event_stream = await nurse.start(open_stream)
-
-            clients.append((client, trade_event_stream))
-
-            assert account in accounts_def
-            accounts.add(account)
-
-        for client in aioclients.values():
-            for pos in client.positions():
-
-                msg = pack_position(pos)
-                msg.account = accounts_def.inverse[msg.account]
-
-                assert msg.account in accounts, (
-                    f'Position for unknown account: {msg.account}')
-
-                all_positions.append(msg.dict())
-
-        trades: list[dict] = []
-        for proxy in proxies.values():
-            trades.append(await proxy.trades())
-
-        log.info(f'Loaded {len(trades)} from this session')
-        # TODO: write trades to local ``trades.toml``
-        # - use above per-session trades data and write to local file
-        # - get the "flex reports" working and pull historical data and
-        # also save locally.
-
-        await ctx.started((
-            all_positions,
-            tuple(name for name in accounts_def if name in accounts),
-        ))
-
-        async with (
-            ctx.open_stream() as ems_stream,
-            trio.open_nursery() as n,
-        ):
-            # start order request handler **before** local trades event loop
-            n.start_soon(handle_order_requests, ems_stream, accounts_def)
-
-            # allocate event relay tasks for each client connection
-            for client, stream in clients:
-                n.start_soon(
-                    deliver_trade_events,
-                    stream,
-                    ems_stream,
-                    accounts_def
-                )
-
-            # block until cancelled
-            await trio.sleep_forever()
-
-
-async def deliver_trade_events(
-
-    trade_event_stream: trio.MemoryReceiveChannel,
-    ems_stream: tractor.MsgStream,
-    accounts_def: dict[str, str],
-
-) -> None:
-    '''Format and relay all trade events for a given client to the EMS.
-
-    '''
-    action_map = {'BOT': 'buy', 'SLD': 'sell'}
-
-    # TODO: for some reason we can receive a ``None`` here when the
-    # ib-gw goes down? Not sure exactly how that's happening looking
-    # at the eventkit code above but we should probably handle it...
-    async for event_name, item in trade_event_stream:
-
-        log.info(f'ib sending {event_name}:\n{pformat(item)}')
-
-        # TODO: templating the ib statuses in comparison with other
-        # brokers is likely the way to go:
-        # https://interactivebrokers.github.io/tws-api/interfaceIBApi_1_1EWrapper.html#a17f2a02d6449710b6394d0266a353313
-        # short list:
-        # - PendingSubmit
-        # - PendingCancel
-        # - PreSubmitted (simulated orders)
-        # - ApiCancelled (cancelled by client before submission
-        #                 to routing)
-        # - Cancelled
-        # - Filled
-        # - Inactive (reject or cancelled but not by trader)
-
-        # XXX: here's some other sucky cases from the api
-        # - short-sale but securities haven't been located, in this
-        #   case we should probably keep the order in some kind of
-        #   weird state or cancel it outright?
-
-        # status='PendingSubmit', message=''),
-        # status='Cancelled', message='Error 404,
-        #   reqId 1550: Order held while securities are located.'),
-        # status='PreSubmitted', message='')],
-
-        if event_name == 'status':
-
-            # XXX: begin normalization of nonsense ib_insync internal
-            # object-state tracking representations...
-
-            # unwrap needed data from ib_insync internal types
-            trade: Trade = item
-            status: OrderStatus = trade.orderStatus
-
-            # skip duplicate filled updates - we get the deats
-            # from the execution details event
-            msg = BrokerdStatus(
-
-                reqid=trade.order.orderId,
-                time_ns=time.time_ns(),  # cuz why not
-                account=accounts_def.inverse[trade.order.account],
-
-                # everyone doin camel case..
-                status=status.status.lower(),  # force lower case
-
-                filled=status.filled,
-                reason=status.whyHeld,
-
-                # this seems to not be necessarily up to date in the
-                # execDetails event.. so we have to send it here I guess?
-                remaining=status.remaining,
-
-                broker_details={'name': 'ib'},
-            )
-
-        elif event_name == 'fill':
-
-            # for wtv reason this is a separate event type
-            # from IB, not sure why it's needed other then for extra
-            # complexity and over-engineering :eyeroll:.
-            # we may just end up dropping these events (or
-            # translating them to ``Status`` msgs) if we can
-            # show the equivalent status events are no more latent.
-
-            # unpack ib_insync types
-            # pep-0526 style:
-            # https://www.python.org/dev/peps/pep-0526/#global-and-local-variable-annotations
-            trade: Trade
-            fill: Fill
-            trade, fill = item
-            execu: Execution = fill.execution
-
-            # TODO: normalize out commissions details?
-            details = {
-                'contract': asdict(fill.contract),
-                'execution': asdict(fill.execution),
-                'commissions': asdict(fill.commissionReport),
-                'broker_time': execu.time,   # supposedly server fill time
-                'name': 'ib',
-            }
-
-            msg = BrokerdFill(
-                # should match the value returned from `.submit_limit()`
-                reqid=execu.orderId,
-                time_ns=time.time_ns(),  # cuz why not
-
-                action=action_map[execu.side],
-                size=execu.shares,
-                price=execu.price,
-
-                broker_details=details,
-                # XXX: required by order mode currently
-                broker_time=details['broker_time'],
-
-            )
-
-        elif event_name == 'error':
-
-            err: dict = item
-
-            # f$#$% gawd dammit insync..
-            con = err['contract']
-            if isinstance(con, Contract):
-                err['contract'] = asdict(con)
-
-            if err['reqid'] == -1:
-                log.error(f'TWS external order error:\n{pformat(err)}')
-
-            # TODO: what schema for this msg if we're going to make it
-            # portable across all backends?
-            # msg = BrokerdError(**err)
-            continue
-
-        elif event_name == 'position':
-            msg = pack_position(item)
-            msg.account = accounts_def.inverse[msg.account]
-
-        elif event_name == 'event':
-
-            # it's either a general system status event or an external
-            # trade event?
-            log.info(f"TWS system status: \n{pformat(item)}")
-
-            # TODO: support this again but needs parsing at the callback
-            # level...
-            # reqid = item.get('reqid', 0)
-            # if getattr(msg, 'reqid', 0) < -1:
-            # log.info(f"TWS triggered trade\n{pformat(msg.dict())}")
-
-            continue
-
-            # msg.reqid = 'tws-' + str(-1 * reqid)
-
-            # mark msg as from "external system"
-            # TODO: probably something better then this.. and start
-            # considering multiplayer/group trades tracking
-            # msg.broker_details['external_src'] = 'tws'
-
-        # XXX: we always serialize to a dict for msgpack
-        # translations, ideally we can move to an msgspec (or other)
-        # encoder # that can be enabled in ``tractor`` ahead of
-        # time so we can pass through the message types directly.
-        await ems_stream.send(msg.dict())
-
-
-def load_flex_trades(
-    path: Optional[str] = None,
-
-) -> dict[str, str]:
-
-    from pprint import pprint
-    from ib_insync import flexreport, util
-
-    conf = get_config()
-
-    if not path:
-        # load ``brokers.toml`` and try to get the flex
-        # token and query id that must be previously defined
-        # by the user.
-        token = conf.get('flex_token')
-        if not token:
-            raise ValueError(
-               'You must specify a ``flex_token`` field in your'
-               '`brokers.toml` in order load your trade log, see our'
-               'intructions for how to set this up here:\n'
-               'PUT LINK HERE!'
-            )
-
-        qid = conf['flex_trades_query_id']
-
-        # TODO: hack this into our logging
-        # system like we do with the API client..
-        util.logToConsole()
-
-        # TODO: rewrite the query part of this with async..httpx?
-        report = flexreport.FlexReport(
-            token=token,
-            queryId=qid,
-        )
-
-    else:
-        # XXX: another project we could potentially look at,
-        # https://pypi.org/project/ibflex/
-        report = flexreport.FlexReport(path=path)
-
-    trade_entries = report.extract('Trade')
-    trades = {
-        # XXX: LOL apparently ``toml`` has a bug
-        # where a section key error will show up in the write
-        # if you leave this as an ``int``?
-        str(t.__dict__['tradeID']): t.__dict__
-        for t in trade_entries
-    }
-
-    ln = len(trades)
-    log.info(f'Loaded {ln} trades from flex query')
-
-    trades_by_account = {}
-    for tid, trade in trades.items():
-        trades_by_account.setdefault(
-            # oddly for some so-called "BookTrade" entries
-            # this field seems to be blank, no cuckin clue.
-            # trade['ibExecID']
-            str(trade['accountId']), {}
-        )[tid] = trade
-
-    section = {'ib': trades_by_account}
-    pprint(section)
-
-    # TODO: load the config first and append in
-    # the new trades loaded here..
-    try:
-        config.write(section, 'trades')
-    except KeyError:
-        import pdbpp; pdbpp.set_trace()  # noqa
-
-
-if __name__ == '__main__':
-    load_flex_trades()

piker/brokers/ib/feed.py

@@ -1,938 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Data feed endpoints pre-wrapped and ready for use with ``tractor``/``trio``.
-
-"""
-from __future__ import annotations
-import asyncio
-from contextlib import asynccontextmanager as acm
-from dataclasses import asdict
-from datetime import datetime
-from math import isnan
-import time
-from typing import (
-    Callable,
-    Optional,
-    Awaitable,
-)
-
-from async_generator import aclosing
-from fuzzywuzzy import process as fuzzy
-import numpy as np
-import pendulum
-import tractor
-import trio
-from trio_typing import TaskStatus
-
-from piker.data._sharedmem import ShmArray
-from .._util import SymbolNotFound, NoData
-from .api import (
-    _adhoc_futes_set,
-    log,
-    load_aio_clients,
-    ibis,
-    MethodProxy,
-    open_client_proxies,
-    get_preferred_data_client,
-    Ticker,
-    RequestError,
-    Contract,
-)
-
-
-# https://interactivebrokers.github.io/tws-api/tick_types.html
-tick_types = {
-    77: 'trade',
-
-    # a "utrade" aka an off exchange "unreportable" (dark) vlm:
-    # https://interactivebrokers.github.io/tws-api/tick_types.html#rt_volume
-    48: 'dark_trade',
-
-    # standard L1 ticks
-    0: 'bsize',
-    1: 'bid',
-    2: 'ask',
-    3: 'asize',
-    4: 'last',
-    5: 'size',
-    8: 'volume',
-
-    # ``ib_insync`` already packs these into
-    # quotes under the following fields.
-    # 55: 'trades_per_min',  # `'tradeRate'`
-    # 56: 'vlm_per_min',  # `'volumeRate'`
-    # 89: 'shortable',  # `'shortableShares'`
-}
-
-
-@acm
-async def open_data_client() -> MethodProxy:
-    '''
-    Open the first found preferred "data client" as defined in the
-    user's ``brokers.toml`` in the ``ib.prefer_data_account`` variable
-    and deliver that client wrapped in a ``MethodProxy``.
-
-    '''
-    async with (
-        open_client_proxies() as (proxies, clients),
-    ):
-        account_name, client = get_preferred_data_client(clients)
-        proxy = proxies.get(f'ib.{account_name}')
-        if not proxy:
-            raise ValueError(
-                f'No preferred data client could be found for {account_name}!'
-            )
-
-        yield proxy
-
-
-@acm
-async def open_history_client(
-    symbol: str,
-
-) -> tuple[Callable, int]:
-    '''
-    History retreival endpoint - delivers a historical frame callble
-    that takes in ``pendulum.datetime`` and returns ``numpy`` arrays.
-
-    '''
-    async with open_data_client() as proxy:
-
-        async def get_hist(
-            end_dt: Optional[datetime] = None,
-            start_dt: Optional[datetime] = None,
-
-        ) -> tuple[np.ndarray, str]:
-
-            out, fails = await get_bars(proxy, symbol, end_dt=end_dt)
-
-            # TODO: add logic here to handle tradable hours and only grab
-            # valid bars in the range
-            if out is None:
-                # could be trying to retreive bars over weekend
-                log.error(f"Can't grab bars starting at {end_dt}!?!?")
-                raise NoData(
-                    f'{end_dt}',
-                    frame_size=2000,
-                )
-
-            bars, bars_array, first_dt, last_dt = out
-
-            # volume cleaning since there's -ve entries,
-            # wood luv to know what crookery that is..
-            vlm = bars_array['volume']
-            vlm[vlm < 0] = 0
-
-            return bars_array, first_dt, last_dt
-
-        # TODO: it seems like we can do async queries for ohlc
-        # but getting the order right still isn't working and I'm not
-        # quite sure why.. needs some tinkering and probably
-        # a lookthrough of the ``ib_insync`` machinery, for eg. maybe
-        # we have to do the batch queries on the `asyncio` side?
-        yield get_hist, {'erlangs': 1, 'rate': 6}
-
-
-_pacing: str = (
-    'Historical Market Data Service error '
-    'message:Historical data request pacing violation'
-)
-
-
-async def get_bars(
-
-    proxy: MethodProxy,
-    fqsn: str,
-
-    # blank to start which tells ib to look up the latest datum
-    end_dt: str = '',
-
-) -> (dict, np.ndarray):
-    '''
-    Retrieve historical data from a ``trio``-side task using
-    a ``MethoProxy``.
-
-    '''
-    fails = 0
-    bars: Optional[list] = None
-    first_dt: datetime = None
-    last_dt: datetime = None
-
-    if end_dt:
-        last_dt = pendulum.from_timestamp(end_dt.timestamp())
-
-    for _ in range(10):
-        try:
-            out = await proxy.bars(
-                fqsn=fqsn,
-                end_dt=end_dt,
-            )
-            if out:
-                bars, bars_array = out
-
-            else:
-                await tractor.breakpoint()
-
-            if bars_array is None:
-                raise SymbolNotFound(fqsn)
-
-            first_dt = pendulum.from_timestamp(
-                bars[0].date.timestamp())
-
-            last_dt = pendulum.from_timestamp(
-                bars[-1].date.timestamp())
-
-            time = bars_array['time']
-            assert time[-1] == last_dt.timestamp()
-            assert time[0] == first_dt.timestamp()
-            log.info(
-                f'{len(bars)} bars retreived for {first_dt} -> {last_dt}'
-            )
-
-            return (bars, bars_array, first_dt, last_dt), fails
-
-        except RequestError as err:
-            msg = err.message
-            # why do we always need to rebind this?
-            # _err = err
-
-            if 'No market data permissions for' in msg:
-                # TODO: signalling for no permissions searches
-                raise NoData(
-                    f'Symbol: {fqsn}',
-                )
-
-            elif (
-                err.code == 162
-                and 'HMDS query returned no data' in err.message
-            ):
-                # XXX: this is now done in the storage mgmt layer
-                # and we shouldn't implicitly decrement the frame dt
-                # index since the upper layer may be doing so
-                # concurrently and we don't want to be delivering frames
-                # that weren't asked for.
-                log.warning(
-                    f'NO DATA found ending @ {end_dt}\n'
-                )
-
-                # try to decrement start point and look further back
-                # end_dt = last_dt = last_dt.subtract(seconds=2000)
-
-                raise NoData(
-                    f'Symbol: {fqsn}',
-                    frame_size=2000,
-                )
-
-            elif _pacing in msg:
-
-                log.warning(
-                    'History throttle rate reached!\n'
-                    'Resetting farms with `ctrl-alt-f` hack\n'
-                )
-                # TODO: we might have to put a task lock around this
-                # method..
-                hist_ev = proxy.status_event(
-                    'HMDS data farm connection is OK:ushmds'
-                )
-
-                # XXX: other event messages we might want to try and
-                # wait for but i wasn't able to get any of this
-                # reliable..
-                # reconnect_start = proxy.status_event(
-                #     'Market data farm is connecting:usfuture'
-                # )
-                # live_ev = proxy.status_event(
-                #     'Market data farm connection is OK:usfuture'
-                # )
-
-                # try to wait on the reset event(s) to arrive, a timeout
-                # will trigger a retry up to 6 times (for now).
-                tries: int = 2
-                timeout: float = 10
-
-                # try 3 time with a data reset then fail over to
-                # a connection reset.
-                for i in range(1, tries):
-
-                    log.warning('Sending DATA RESET request')
-                    await data_reset_hack(reset_type='data')
-
-                    with trio.move_on_after(timeout) as cs:
-                        for name, ev in [
-                            # TODO: not sure if waiting on other events
-                            # is all that useful here or not. in theory
-                            # you could wait on one of the ones above
-                            # first to verify the reset request was
-                            # sent?
-                            ('history', hist_ev),
-                        ]:
-                            await ev.wait()
-                            log.info(f"{name} DATA RESET")
-                            break
-
-                    if cs.cancelled_caught:
-                        fails += 1
-                        log.warning(
-                            f'Data reset {name} timeout, retrying {i}.'
-                        )
-
-                        continue
-                else:
-
-                    log.warning('Sending CONNECTION RESET')
-                    await data_reset_hack(reset_type='connection')
-
-                    with trio.move_on_after(timeout) as cs:
-                        for name, ev in [
-                            # TODO: not sure if waiting on other events
-                            # is all that useful here or not. in theory
-                            # you could wait on one of the ones above
-                            # first to verify the reset request was
-                            # sent?
-                            ('history', hist_ev),
-                        ]:
-                            await ev.wait()
-                            log.info(f"{name} DATA RESET")
-
-                    if cs.cancelled_caught:
-                        fails += 1
-                        log.warning('Data CONNECTION RESET timeout!?')
-
-            else:
-                raise
-
-    return None, None
-    # else:  # throttle wasn't fixed so error out immediately
-    #     raise _err
-
-
-async def backfill_bars(
-
-    fqsn: str,
-    shm: ShmArray,  # type: ignore # noqa
-
-    # TODO: we want to avoid overrunning the underlying shm array buffer
-    # and we should probably calc the number of calls to make depending
-    # on that until we have the `marketstore` daemon in place in which
-    # case the shm size will be driven by user config and available sys
-    # memory.
-    count: int = 16,
-
-    task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
-
-) -> None:
-    '''
-    Fill historical bars into shared mem / storage afap.
-
-    TODO: avoid pacing constraints:
-    https://github.com/pikers/piker/issues/128
-
-    '''
-    # last_dt1 = None
-    last_dt = None
-
-    with trio.CancelScope() as cs:
-
-        async with open_data_client() as proxy:
-
-            out, fails = await get_bars(proxy, fqsn)
-
-            if out is None:
-                raise RuntimeError("Could not pull currrent history?!")
-
-            (first_bars, bars_array, first_dt, last_dt) = out
-            vlm = bars_array['volume']
-            vlm[vlm < 0] = 0
-            last_dt = first_dt
-
-            # write historical data to buffer
-            shm.push(bars_array)
-
-            task_status.started(cs)
-
-            i = 0
-            while i < count:
-
-                out, fails = await get_bars(proxy, fqsn, end_dt=first_dt)
-
-                if out is None:
-                    # could be trying to retreive bars over weekend
-                    # TODO: add logic here to handle tradable hours and
-                    # only grab valid bars in the range
-                    log.error(f"Can't grab bars starting at {first_dt}!?!?")
-
-                    # XXX: get_bars() should internally decrement dt by
-                    # 2k seconds and try again.
-                    continue
-
-                (first_bars, bars_array, first_dt, last_dt) = out
-                # last_dt1 = last_dt
-                # last_dt = first_dt
-
-                # volume cleaning since there's -ve entries,
-                # wood luv to know what crookery that is..
-                vlm = bars_array['volume']
-                vlm[vlm < 0] = 0
-
-                # TODO we should probably dig into forums to see what peeps
-                # think this data "means" and then use it as an indicator of
-                # sorts? dinkus has mentioned that $vlms for the day dont'
-                # match other platforms nor the summary stat tws shows in
-                # the monitor - it's probably worth investigating.
-
-                shm.push(bars_array, prepend=True)
-                i += 1
-
-
-asset_type_map = {
-    'STK': 'stock',
-    'OPT': 'option',
-    'FUT': 'future',
-    'CONTFUT': 'continuous_future',
-    'CASH': 'forex',
-    'IND': 'index',
-    'CFD': 'cfd',
-    'BOND': 'bond',
-    'CMDTY': 'commodity',
-    'FOP': 'futures_option',
-    'FUND': 'mutual_fund',
-    'WAR': 'warrant',
-    'IOPT': 'warran',
-    'BAG': 'bag',
-    # 'NEWS': 'news',
-}
-
-
-_quote_streams: dict[str, trio.abc.ReceiveStream] = {}
-
-
-async def _setup_quote_stream(
-
-    from_trio: asyncio.Queue,
-    to_trio: trio.abc.SendChannel,
-
-    symbol: str,
-    opts: tuple[int] = (
-        '375',  # RT trade volume (excludes utrades)
-        '233',  # RT trade volume (includes utrades)
-        '236',  # Shortable shares
-
-        # these all appear to only be updated every 25s thus
-        # making them mostly useless and explains why the scanner
-        # is always slow XD
-        # '293',  # Trade count for day
-        '294',  # Trade rate / minute
-        '295',  # Vlm rate / minute
-    ),
-    contract: Optional[Contract] = None,
-
-) -> trio.abc.ReceiveChannel:
-    '''
-    Stream a ticker using the std L1 api.
-
-    This task is ``asyncio``-side and must be called from
-    ``tractor.to_asyncio.open_channel_from()``.
-
-    '''
-    global _quote_streams
-
-    to_trio.send_nowait(None)
-
-    async with load_aio_clients() as accts2clients:
-        caccount_name, client = get_preferred_data_client(accts2clients)
-        contract = contract or (await client.find_contract(symbol))
-        ticker: Ticker = client.ib.reqMktData(contract, ','.join(opts))
-
-        # NOTE: it's batch-wise and slow af but I guess could
-        # be good for backchecking? Seems to be every 5s maybe?
-        # ticker: Ticker = client.ib.reqTickByTickData(
-        #     contract, 'Last',
-        # )
-
-        # # define a simple queue push routine that streams quote packets
-        # # to trio over the ``to_trio`` memory channel.
-        # to_trio, from_aio = trio.open_memory_channel(2**8)  # type: ignore
-        def teardown():
-            ticker.updateEvent.disconnect(push)
-            log.error(f"Disconnected stream for `{symbol}`")
-            client.ib.cancelMktData(contract)
-
-            # decouple broadcast mem chan
-            _quote_streams.pop(symbol, None)
-
-        def push(t: Ticker) -> None:
-            """
-            Push quotes to trio task.
-
-            """
-            # log.debug(t)
-            try:
-                to_trio.send_nowait(t)
-
-            except (
-                trio.BrokenResourceError,
-
-                # XXX: HACK, not sure why this gets left stale (probably
-                # due to our terrible ``tractor.to_asyncio``
-                # implementation for streams.. but if the mem chan
-                # gets left here and starts blocking just kill the feed?
-                # trio.WouldBlock,
-            ):
-                # XXX: eventkit's ``Event.emit()`` for whatever redic
-                # reason will catch and ignore regular exceptions
-                # resulting in tracebacks spammed to console..
-                # Manually do the dereg ourselves.
-                teardown()
-            except trio.WouldBlock:
-                log.warning(
-                    f'channel is blocking symbol feed for {symbol}?'
-                    f'\n{to_trio.statistics}'
-                )
-
-            # except trio.WouldBlock:
-            #     # for slow debugging purposes to avoid clobbering prompt
-            #     # with log msgs
-            #     pass
-
-        ticker.updateEvent.connect(push)
-        try:
-            await asyncio.sleep(float('inf'))
-        finally:
-            teardown()
-
-        # return from_aio
-
-
-@acm
-async def open_aio_quote_stream(
-
-    symbol: str,
-    contract: Optional[Contract] = None,
-
-) -> trio.abc.ReceiveStream:
-
-    from tractor.trionics import broadcast_receiver
-    global _quote_streams
-
-    from_aio = _quote_streams.get(symbol)
-    if from_aio:
-
-        # if we already have a cached feed deliver a rx side clone to consumer
-        async with broadcast_receiver(
-            from_aio,
-            2**6,
-        ) as from_aio:
-            yield from_aio
-            return
-
-    async with tractor.to_asyncio.open_channel_from(
-        _setup_quote_stream,
-        symbol=symbol,
-        contract=contract,
-
-    ) as (first, from_aio):
-
-        # cache feed for later consumers
-        _quote_streams[symbol] = from_aio
-
-        yield from_aio
-
-
-# TODO: cython/mypyc/numba this!
-def normalize(
-    ticker: Ticker,
-    calc_price: bool = False
-
-) -> dict:
-
-    # should be real volume for this contract by default
-    calc_price = False
-
-    # check for special contract types
-    con = ticker.contract
-    if type(con) in (
-        ibis.Commodity,
-        ibis.Forex,
-    ):
-        # commodities and forex don't have an exchange name and
-        # no real volume so we have to calculate the price
-        suffix = con.secType
-        # no real volume on this tract
-        calc_price = True
-
-    else:
-        suffix = con.primaryExchange
-        if not suffix:
-            suffix = con.exchange
-
-        # append a `.<suffix>` to the returned symbol
-        # key for derivatives that normally is the expiry
-        # date key.
-        expiry = con.lastTradeDateOrContractMonth
-        if expiry:
-            suffix += f'.{expiry}'
-
-    # convert named tuples to dicts so we send usable keys
-    new_ticks = []
-    for tick in ticker.ticks:
-        if tick and not isinstance(tick, dict):
-            td = tick._asdict()
-            td['type'] = tick_types.get(
-                td['tickType'],
-                'n/a',
-            )
-
-            new_ticks.append(td)
-
-            tbt = ticker.tickByTicks
-            if tbt:
-                print(f'tickbyticks:\n {ticker.tickByTicks}')
-
-    ticker.ticks = new_ticks
-
-    # some contracts don't have volume so we may want to calculate
-    # a midpoint price based on data we can acquire (such as bid / ask)
-    if calc_price:
-        ticker.ticks.append(
-            {'type': 'trade', 'price': ticker.marketPrice()}
-        )
-
-    # serialize for transport
-    data = asdict(ticker)
-
-    # generate fqsn with possible specialized suffix
-    # for derivatives, note the lowercase.
-    data['symbol'] = data['fqsn'] = '.'.join(
-        (con.symbol, suffix)
-    ).lower()
-
-    # convert named tuples to dicts for transport
-    tbts = data.get('tickByTicks')
-    if tbts:
-        data['tickByTicks'] = [tbt._asdict() for tbt in tbts]
-
-    # add time stamps for downstream latency measurements
-    data['brokerd_ts'] = time.time()
-
-    # stupid stupid shit...don't even care any more..
-    # leave it until we do a proper latency study
-    # if ticker.rtTime is not None:
-    #     data['broker_ts'] = data['rtTime_s'] = float(
-    #         ticker.rtTime.timestamp) / 1000.
-    data.pop('rtTime')
-
-    return data
-
-
-async def stream_quotes(
-
-    send_chan: trio.abc.SendChannel,
-    symbols: list[str],
-    feed_is_live: trio.Event,
-    loglevel: str = None,
-
-    # startup sync
-    task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
-
-) -> None:
-    '''
-    Stream symbol quotes.
-
-    This is a ``trio`` callable routine meant to be invoked
-    once the brokerd is up.
-
-    '''
-    # TODO: support multiple subscriptions
-    sym = symbols[0]
-    log.info(f'request for real-time quotes: {sym}')
-
-    async with open_data_client() as proxy:
-
-        con, first_ticker, details = await proxy.get_sym_details(symbol=sym)
-        first_quote = normalize(first_ticker)
-        # print(f'first quote: {first_quote}')
-
-        def mk_init_msgs() -> dict[str, dict]:
-            '''
-            Collect a bunch of meta-data useful for feed startup and
-            pack in a `dict`-msg.
-
-            '''
-            # pass back some symbol info like min_tick, trading_hours, etc.
-            syminfo = asdict(details)
-            syminfo.update(syminfo['contract'])
-
-            # nested dataclass we probably don't need and that won't IPC
-            # serialize
-            syminfo.pop('secIdList')
-
-            # TODO: more consistent field translation
-            atype = syminfo['asset_type'] = asset_type_map[syminfo['secType']]
-
-            # for stocks it seems TWS reports too small a tick size
-            # such that you can't submit orders with that granularity?
-            min_tick = 0.01 if atype == 'stock' else 0
-
-            syminfo['price_tick_size'] = max(syminfo['minTick'], min_tick)
-
-            # for "traditional" assets, volume is normally discreet, not
-            # a float
-            syminfo['lot_tick_size'] = 0.0
-
-            ibclient = proxy._aio_ns.ib.client
-            host, port = ibclient.host, ibclient.port
-
-            # TODO: for loop through all symbols passed in
-            init_msgs = {
-                # pass back token, and bool, signalling if we're the writer
-                # and that history has been written
-                sym: {
-                    'symbol_info': syminfo,
-                    'fqsn': first_quote['fqsn'],
-                },
-                'status': {
-                    'data_ep': f'{host}:{port}',
-                },
-
-            }
-            return init_msgs
-
-        init_msgs = mk_init_msgs()
-
-        # TODO: we should instead spawn a task that waits on a feed to start
-        # and let it wait indefinitely..instead of this hard coded stuff.
-        with trio.move_on_after(1):
-            contract, first_ticker, details = await proxy.get_quote(symbol=sym)
-
-        # it might be outside regular trading hours so see if we can at
-        # least grab history.
-        if isnan(first_ticker.last):
-            task_status.started((init_msgs, first_quote))
-
-            # it's not really live but this will unblock
-            # the brokerd feed task to tell the ui to update?
-            feed_is_live.set()
-
-            # block and let data history backfill code run.
-            await trio.sleep_forever()
-            return  # we never expect feed to come up?
-
-        async with open_aio_quote_stream(
-            symbol=sym,
-            contract=con,
-        ) as stream:
-
-            # ugh, clear ticks since we've consumed them
-            # (ahem, ib_insync is stateful trash)
-            first_ticker.ticks = []
-
-            task_status.started((init_msgs, first_quote))
-
-            async with aclosing(stream):
-                if type(first_ticker.contract) not in (
-                    ibis.Commodity,
-                    ibis.Forex
-                ):
-                    # wait for real volume on feed (trading might be closed)
-                    while True:
-                        ticker = await stream.receive()
-
-                        # for a real volume contract we rait for the first
-                        # "real" trade to take place
-                        if (
-                            # not calc_price
-                            # and not ticker.rtTime
-                            not ticker.rtTime
-                        ):
-                            # spin consuming tickers until we get a real
-                            # market datum
-                            log.debug(f"New unsent ticker: {ticker}")
-                            continue
-                        else:
-                            log.debug("Received first real volume tick")
-                            # ugh, clear ticks since we've consumed them
-                            # (ahem, ib_insync is truly stateful trash)
-                            ticker.ticks = []
-
-                            # XXX: this works because we don't use
-                            # ``aclosing()`` above?
-                            break
-
-                    quote = normalize(ticker)
-                    log.debug(f"First ticker received {quote}")
-
-                # tell caller quotes are now coming in live
-                feed_is_live.set()
-
-                # last = time.time()
-                async for ticker in stream:
-                    quote = normalize(ticker)
-                    await send_chan.send({quote['fqsn']: quote})
-
-                    # ugh, clear ticks since we've consumed them
-                    ticker.ticks = []
-                    # last = time.time()
-
-
-async def data_reset_hack(
-    reset_type: str = 'data',
-
-) -> None:
-    '''
-    Run key combos for resetting data feeds and yield back to caller
-    when complete.
-
-    This is a linux-only hack around:
-
-    https://interactivebrokers.github.io/tws-api/historical_limitations.html#pacing_violations
-
-    TODOs:
-        - a return type that hopefully determines if the hack was
-          successful.
-        - other OS support?
-        - integration with ``ib-gw`` run in docker + Xorg?
-
-    '''
-
-    async def vnc_click_hack(
-        reset_type: str = 'data'
-    ) -> None:
-        '''
-        Reset the data or netowork connection for the VNC attached
-        ib gateway using magic combos.
-
-        '''
-        key = {'data': 'f', 'connection': 'r'}[reset_type]
-
-        import asyncvnc
-
-        async with asyncvnc.connect(
-            'localhost',
-            port=3003,
-            # password='ibcansmbz',
-        ) as client:
-
-            # move to middle of screen
-            # 640x1800
-            client.mouse.move(
-                x=500,
-                y=500,
-            )
-            client.mouse.click()
-            client.keyboard.press('Ctrl', 'Alt', key)  # keys are stacked
-
-    await tractor.to_asyncio.run_task(vnc_click_hack)
-
-    # we don't really need the ``xdotool`` approach any more B)
-    return True
-
-
-@tractor.context
-async def open_symbol_search(
-    ctx: tractor.Context,
-
-) -> None:
-
-    # TODO: load user defined symbol set locally for fast search?
-    await ctx.started({})
-
-    async with open_data_client() as proxy:
-        async with ctx.open_stream() as stream:
-
-            last = time.time()
-
-            async for pattern in stream:
-                log.debug(f'received {pattern}')
-                now = time.time()
-
-                assert pattern, 'IB can not accept blank search pattern'
-
-                # throttle search requests to no faster then 1Hz
-                diff = now - last
-                if diff < 1.0:
-                    log.debug('throttle sleeping')
-                    await trio.sleep(diff)
-                    try:
-                        pattern = stream.receive_nowait()
-                    except trio.WouldBlock:
-                        pass
-
-                if not pattern or pattern.isspace():
-                    log.warning('empty pattern received, skipping..')
-
-                    # TODO: *BUG* if nothing is returned here the client
-                    # side will cache a null set result and not showing
-                    # anything to the use on re-searches when this query
-                    # timed out. We probably need a special "timeout" msg
-                    # or something...
-
-                    # XXX: this unblocks the far end search task which may
-                    # hold up a multi-search nursery block
-                    await stream.send({})
-
-                    continue
-
-                log.debug(f'searching for {pattern}')
-
-                last = time.time()
-
-                # async batch search using api stocks endpoint and module
-                # defined adhoc symbol set.
-                stock_results = []
-
-                async def stash_results(target: Awaitable[list]):
-                    stock_results.extend(await target)
-
-                async with trio.open_nursery() as sn:
-                    sn.start_soon(
-                        stash_results,
-                        proxy.search_symbols(
-                            pattern=pattern,
-                            upto=5,
-                        ),
-                    )
-
-                    # trigger async request
-                    await trio.sleep(0)
-
-                    # match against our ad-hoc set immediately
-                    adhoc_matches = fuzzy.extractBests(
-                        pattern,
-                        list(_adhoc_futes_set),
-                        score_cutoff=90,
-                    )
-                    log.info(f'fuzzy matched adhocs: {adhoc_matches}')
-                    adhoc_match_results = {}
-                    if adhoc_matches:
-                        # TODO: do we need to pull contract details?
-                        adhoc_match_results = {i[0]: {} for i in adhoc_matches}
-
-                log.debug(f'fuzzy matching stocks {stock_results}')
-                stock_matches = fuzzy.extractBests(
-                    pattern,
-                    stock_results,
-                    score_cutoff=50,
-                )
-
-                matches = adhoc_match_results | {
-                    item[0]: {} for item in stock_matches
-                }
-                # TODO: we used to deliver contract details
-                # {item[2]: item[0] for item in stock_matches}
-
-                log.debug(f"sending matches: {matches.keys()}")
-                await stream.send(matches)

piker/brokers/kraken.py

@@ -14,20 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-'''
+"""
 Kraken backend.
 
-'''
+"""
-from contextlib import asynccontextmanager as acm
+from contextlib import asynccontextmanager
 from dataclasses import asdict, field
-from datetime import datetime
+from typing import List, Dict, Any, Tuple, Optional
-from pprint import pformat
-from typing import Any, Optional, AsyncIterator, Callable, Union
 import time
 
 from trio_typing import TaskStatus
 import trio
-import pendulum
+import arrow
 import asks
 from fuzzywuzzy import process as fuzzy
 import numpy as np
@@ -35,30 +33,12 @@ import tractor
 from pydantic.dataclasses import dataclass
 from pydantic import BaseModel
 import wsproto
-import urllib.parse
-import hashlib
-import hmac
-import base64
 
-from .. import config
 from .._cacheables import open_cached_client
-from ._util import (
+from ._util import resproc, SymbolNotFound, BrokerError
-    resproc,
-    SymbolNotFound,
-    BrokerError,
-    DataThrottle,
-    DataUnavailable,
-)
 from ..log import get_logger, get_console_log
 from ..data import ShmArray
-from ..data._web_bs import open_autorecon_ws, NoBsWs
+from ..data._web_bs import open_autorecon_ws
-from ..clearing._paper_engine import PaperBoi
-from ..clearing._messages import (
-    BrokerdPosition, BrokerdOrder, BrokerdStatus,
-    BrokerdOrderAck, BrokerdError, BrokerdCancel,
-    BrokerdFill,
-)
-
 
 log = get_logger(__name__)
 
@@ -87,7 +67,7 @@ ohlc_dtype = np.dtype(_ohlc_dtype)
 _show_wap_in_history = True
 
 
-_symbol_info_translation: dict[str, str] = {
+_symbol_info_translation: Dict[str, str] = {
     'tick_decimals': 'pair_decimals',
 }
 
@@ -109,16 +89,16 @@ class Pair(BaseModel):
     lot_multiplier: float
 
     # array of leverage amounts available when buying
-    leverage_buy: list[int]
+    leverage_buy: List[int]
     # array of leverage amounts available when selling
-    leverage_sell: list[int]
+    leverage_sell: List[int]
 
     # fee schedule array in [volume, percent fee] tuples
-    fees: list[tuple[int, float]]
+    fees: List[Tuple[int, float]]
 
     # maker fee schedule array in [volume, percent fee] tuples (if on
     # maker/taker)
-    fees_maker: list[tuple[int, float]]
+    fees_maker: List[Tuple[int, float]]
 
     fee_volume_currency: str  # volume discount currency
     margin_call: str  # margin call level
@@ -126,27 +106,13 @@ class Pair(BaseModel):
     ordermin: float  # minimum order volume for pair
 
 
-class Trade(BaseModel):
-    '''
-    Trade class that helps parse and validate ownTrades stream
-
-    '''
-    reqid: str  # kraken order transaction id
-    action: str  # buy or sell
-    price: str  # price of asset
-    size: str  # vol of asset
-    broker_time: str  # e.g GTC, GTD
-
-
 @dataclass
 class OHLC:
-    '''
+    """Description of the flattened OHLC quote format.
-    Description of the flattened OHLC quote format.
 
     For schema details see:
         https://docs.kraken.com/websockets/#message-ohlc
-
+    """
-    '''
     chan_id: int  # internal kraken id
     chan_name: str  # eg. ohlc-1  (name-interval)
     pair: str  # fx pair
@@ -160,54 +126,12 @@ class OHLC:
     volume: float  # Accumulated volume **within interval**
     count: int  # Number of trades within interval
     # (sampled) generated tick data
-    ticks: list[Any] = field(default_factory=list)
+    ticks: List[Any] = field(default_factory=list)
-
-
-def get_config() -> dict[str, Any]:
-
-    conf, path = config.load()
-    section = conf.get('kraken')
-
-    if section is None:
-        log.warning(f'No config section found for kraken in {path}')
-        return {}
-
-    return section
-
-
-def get_kraken_signature(
-    urlpath: str,
-    data: dict[str, Any],
-    secret: str
-) -> str:
-    postdata = urllib.parse.urlencode(data)
-    encoded = (str(data['nonce']) + postdata).encode()
-    message = urlpath.encode() + hashlib.sha256(encoded).digest()
-
-    mac = hmac.new(base64.b64decode(secret), message, hashlib.sha512)
-    sigdigest = base64.b64encode(mac.digest())
-    return sigdigest.decode()
-
-
-class InvalidKey(ValueError):
-    '''
-    EAPI:Invalid key
-    This error is returned when the API key used for the call is
-    either expired or disabled, please review the API key in your
-    Settings -> API tab of account management or generate a new one
-    and update your application.
-
-    '''
 
 
 class Client:
 
-    def __init__(
+    def __init__(self) -> None:
-        self,
-        name: str = '',
-        api_key: str = '',
-        secret: str = ''
-    ) -> None:
         self._sesh = asks.Session(connections=4)
         self._sesh.base_location = _url
         self._sesh.headers.update({
@@ -215,12 +139,9 @@ class Client:
                 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
         })
         self._pairs: list[str] = []
-        self._name = name
-        self._api_key = api_key
-        self._secret = secret
 
     @property
-    def pairs(self) -> dict[str, Any]:
+    def pairs(self) -> Dict[str, Any]:
         if self._pairs is None:
             raise RuntimeError(
                 "Make sure to run `cache_symbols()` on startup!"
@@ -233,7 +154,7 @@ class Client:
         self,
         method: str,
         data: dict,
-    ) -> dict[str, Any]:
+    ) -> Dict[str, Any]:
         resp = await self._sesh.post(
             path=f'/public/{method}',
             json=data,
@@ -241,112 +162,6 @@ class Client:
         )
         return resproc(resp, log)
 
-    async def _private(
-        self,
-        method: str,
-        data: dict,
-        uri_path: str
-    ) -> dict[str, Any]:
-        headers = {
-            'Content-Type':
-                'application/x-www-form-urlencoded',
-            'API-Key':
-                self._api_key,
-            'API-Sign':
-                get_kraken_signature(uri_path, data, self._secret)
-        }
-        resp = await self._sesh.post(
-            path=f'/private/{method}',
-            data=data,
-            headers=headers,
-            timeout=float('inf')
-        )
-        return resproc(resp, log)
-
-    async def endpoint(
-        self,
-        method: str,
-        data: dict[str, Any]
-    ) -> dict[str, Any]:
-        uri_path = f'/0/private/{method}'
-        data['nonce'] = str(int(1000*time.time()))
-        return await self._private(method, data, uri_path)
-
-    async def get_trades(
-        self,
-        data: dict[str, Any] = {}
-    ) -> dict[str, Any]:
-        data['ofs'] = 0
-        # Grab all trade history
-        # https://docs.kraken.com/rest/#operation/getTradeHistory
-        # Kraken uses 'ofs' to refer to the offset
-        while True:
-            resp = await self.endpoint('TradesHistory', data)
-            # grab the first 50 trades
-            if data['ofs'] == 0:
-                trades = resp['result']['trades']
-            # load the next 50 trades using dict constructor
-            # for speed
-            elif data['ofs'] == 50:
-                trades = dict(trades, **resp['result']['trades'])
-            # catch the end of the trades
-            elif resp['result']['trades'] == {}:
-                count = resp['result']['count']
-                break
-            # update existing dict if num trades exceeds 100
-            else:
-                trades.update(resp['result']['trades'])
-            # increment the offset counter
-            data['ofs'] += 50
-            # To avoid exceeding API rate limit in case of a lot of trades
-            await trio.sleep(1)
-
-        # make sure you grabbed all the trades
-        assert count == len(trades.values())
-
-        return trades
-
-    async def submit_limit(
-        self,
-        symbol: str,
-        price: float,
-        action: str,
-        size: float,
-        reqid: str = None,
-        validate: bool = False  # set True test call without a real submission
-    ) -> dict:
-        '''
-        Place an order and return integer request id provided by client.
-
-        '''
-        # Build common data dict for common keys from both endpoints
-        data = {
-            "pair": symbol,
-            "price": str(price),
-            "validate": validate
-        }
-        if reqid is None:
-            # Build order data for kraken api
-            data |= {
-                "ordertype": "limit", "type": action, "volume": str(size)
-            }
-            return await self.endpoint('AddOrder', data)
-        else:
-            # Edit order data for kraken api
-            data["txid"] = reqid
-            return await self.endpoint('EditOrder', data)
-
-    async def submit_cancel(
-        self,
-        reqid: str,
-    ) -> dict:
-        '''
-        Send cancel request for order id ``reqid``.
-
-        '''
-        # txid is a transaction id given by kraken
-        return await self.endpoint('CancelOrder', {"txid": reqid})
-
     async def symbol_info(
         self,
         pair: Optional[str] = None,
@@ -382,7 +197,7 @@ class Client:
         self,
         pattern: str,
         limit: int = None,
-    ) -> dict[str, Any]:
+    ) -> Dict[str, Any]:
         if self._pairs is not None:
             data = self._pairs
         else:
@@ -399,26 +214,17 @@ class Client:
     async def bars(
         self,
         symbol: str = 'XBTUSD',
-
         # UTC 2017-07-02 12:53:20
-        since: Optional[Union[int, datetime]] = None,
+        since: int = None,
         count: int = 720,  # <- max allowed per query
         as_np: bool = True,
-
     ) -> dict:
-
         if since is None:
-            since = pendulum.now('UTC').start_of('minute').subtract(
+            since = arrow.utcnow().floor('minute').shift(
-                minutes=count).timestamp()
+                minutes=-count).timestamp()
-
-        elif isinstance(since, int):
-            since = pendulum.from_timestamp(since).timestamp()
-
-        else:  # presumably a pendulum datetime
-            since = since.timestamp()
 
         # UTC 2017-07-02 12:53:20 is oldest seconds value
-        since = str(max(1499000000, int(since)))
+        since = str(max(1499000000, since))
         json = await self._public(
             'OHLC',
             data={
@@ -462,29 +268,11 @@ class Client:
             array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars
             return array
         except KeyError:
-            errmsg = json['error'][0]
+            raise SymbolNotFound(json['error'][0] + f': {symbol}')
-
-            if 'not found' in errmsg:
-                raise SymbolNotFound(errmsg + f': {symbol}')
-
-            elif 'Too many requests' in errmsg:
-                raise DataThrottle(f'{symbol}')
-
-            else:
-                raise BrokerError(errmsg)
 
 
-@acm
+@asynccontextmanager
 async def get_client() -> Client:
-
-    section = get_config()
-    if section:
-        client = Client(
-            name=section['key_descr'],
-            api_key=section['api_key'],
-            secret=section['secret']
-        )
-    else:
     client = Client()
 
     # at startup, load all symbols locally for fast search
@@ -493,382 +281,8 @@ async def get_client() -> Client:
     yield client
 
 
-def pack_positions(
+async def stream_messages(ws):
-    acc: str,
-    trades: dict
-) -> list[Any]:
-    positions: dict[str, float] = {}
-    vols: dict[str, float] = {}
-    costs: dict[str, float] = {}
-    position_msgs: list[Any] = []
 
-    for trade in trades.values():
-        sign = -1 if trade['type'] == 'sell' else 1
-        pair = trade['pair']
-        vol = float(trade['vol'])
-        vols[pair] = vols.get(pair, 0) + sign * vol
-        costs[pair] = costs.get(pair, 0) + sign * float(trade['cost'])
-        positions[pair] = costs[pair] / vols[pair] if vols[pair] else 0
-
-    for ticker, pos in positions.items():
-        vol = float(vols[ticker])
-        if not vol:
-            continue
-        norm_sym = normalize_symbol(ticker)
-        msg = BrokerdPosition(
-            broker='kraken',
-            account=acc,
-            symbol=norm_sym,
-            currency=norm_sym[-3:],
-            size=vol,
-            avg_price=float(pos),
-        )
-        position_msgs.append(msg.dict())
-
-    return position_msgs
-
-
-def normalize_symbol(
-    ticker: str
-) -> str:
-    # This is to convert symbol names from what kraken
-    # uses to the traditional 3x3 pair symbol syntax
-    symlen = len(ticker)
-    if symlen == 6:
-        return ticker.lower()
-    else:
-        for sym in ['XXBT', 'XXMR', 'ZEUR']:
-            if sym in ticker:
-                ticker = ticker.replace(sym, sym[1:])
-        return ticker.lower()
-
-
-def make_auth_sub(data: dict[str, Any]) -> dict[str, str]:
-    '''
-    Create a request subscription packet dict.
-
-    ## TODO: point to the auth urls
-    https://docs.kraken.com/websockets/#message-subscribe
-
-    '''
-    # eg. specific logic for this in kraken's sync client:
-    # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
-    return {
-        'event': 'subscribe',
-        'subscription': data,
-    }
-
-
-async def handle_order_requests(
-
-        client: Client,
-        ems_order_stream: tractor.MsgStream,
-
-) -> None:
-
-    request_msg: dict
-    order: BrokerdOrder
-
-    async for request_msg in ems_order_stream:
-        log.info(
-            'Received order request:\n'
-            f'{pformat(request_msg)}'
-        )
-
-        action = request_msg['action']
-
-        if action in {'buy', 'sell'}:
-
-            account = request_msg['account']
-            if account != 'kraken.spot':
-                log.error(
-                    'This is a kraken account, \
-                    only a `kraken.spot` selection is valid'
-                )
-                await ems_order_stream.send(BrokerdError(
-                    oid=request_msg['oid'],
-                    symbol=request_msg['symbol'],
-
-                    # reason=f'Kraken only, No account found: `{account}` ?',
-                    reason=(
-                        'Kraken only, order mode disabled due to '
-                        'https://github.com/pikers/piker/issues/299'
-                    ),
-
-                ).dict())
-                continue
-
-            # validate
-            order = BrokerdOrder(**request_msg)
-            # call our client api to submit the order
-            resp = await client.submit_limit(
-                symbol=order.symbol,
-                price=order.price,
-                action=order.action,
-                size=order.size,
-                reqid=order.reqid,
-            )
-
-            err = resp['error']
-            if err:
-                oid = order.oid
-                log.error(f'Failed to submit order: {oid}')
-
-                await ems_order_stream.send(
-                    BrokerdError(
-                        oid=order.oid,
-                        reqid=order.reqid,
-                        symbol=order.symbol,
-                        reason="Failed order submission",
-                        broker_details=resp
-                    ).dict()
-                )
-            else:
-                # TODO: handle multiple orders (cancels?)
-                #       txid is an array of strings
-                if order.reqid is None:
-                    reqid = resp['result']['txid'][0]
-                else:
-                    # update the internal pairing of oid to krakens
-                    # txid with the new txid that is returned on edit
-                    reqid = resp['result']['txid']
-
-                # deliver ack that order has been submitted to broker routing
-                await ems_order_stream.send(
-                    BrokerdOrderAck(
-
-                        # ems order request id
-                        oid=order.oid,
-
-                        # broker specific request id
-                        reqid=reqid,
-
-                        # account the made the order
-                        account=order.account
-
-                    ).dict()
-                )
-
-        elif action == 'cancel':
-            msg = BrokerdCancel(**request_msg)
-
-            # Send order cancellation to kraken
-            resp = await client.submit_cancel(
-                reqid=msg.reqid
-            )
-
-            # Check to make sure there was no error returned by
-            # the kraken endpoint. Assert one order was cancelled.
-            try:
-                result = resp['result']
-                count = result['count']
-
-            # check for 'error' key if we received no 'result'
-            except KeyError:
-                error = resp.get('error')
-
-                await ems_order_stream.send(
-                    BrokerdError(
-                        oid=msg.oid,
-                        reqid=msg.reqid,
-                        symbol=msg.symbol,
-                        reason="Failed order cancel",
-                        broker_details=resp
-                    ).dict()
-                )
-
-                if not error:
-                    raise BrokerError(f'Unknown order cancel response: {resp}')
-
-            else:
-                if not count:  # no orders were cancelled?
-
-                    # XXX: what exactly is this from and why would we care?
-                    # there doesn't seem to be any docs here?
-                    # https://docs.kraken.com/rest/#operation/cancelOrder
-
-                    # Check to make sure the cancellation is NOT pending,
-                    # then send the confirmation to the ems order stream
-                    pending = result.get('pending')
-                    if pending:
-                        log.error(f'Order {oid} cancel was not yet successful')
-
-                        await ems_order_stream.send(
-                            BrokerdError(
-                                oid=msg.oid,
-                                reqid=msg.reqid,
-                                symbol=msg.symbol,
-                                # TODO: maybe figure out if pending
-                                # cancels will eventually get cancelled
-                                reason="Order cancel is still pending?",
-                                broker_details=resp
-                            ).dict()
-                        )
-
-                else:  # order cancel success case.
-
-                    await ems_order_stream.send(
-                        BrokerdStatus(
-                            reqid=msg.reqid,
-                            account=msg.account,
-                            time_ns=time.time_ns(),
-                            status='cancelled',
-                            reason='Order cancelled',
-                            broker_details={'name': 'kraken'}
-                        ).dict()
-                    )
-    else:
-        log.error(f'Unknown order command: {request_msg}')
-
-
-@tractor.context
-async def trades_dialogue(
-    ctx: tractor.Context,
-    loglevel: str = None,
-) -> AsyncIterator[dict[str, Any]]:
-
-    # XXX: required to propagate ``tractor`` loglevel to piker logging
-    get_console_log(loglevel or tractor.current_actor().loglevel)
-
-    @acm
-    async def subscribe(ws: wsproto.WSConnection, token: str):
-        # XXX: setup subs
-        # https://docs.kraken.com/websockets/#message-subscribe
-        # specific logic for this in kraken's shitty sync client:
-        # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
-        trades_sub = make_auth_sub(
-            {'name': 'ownTrades', 'token': token}
-        )
-
-        # TODO: we want to eventually allow unsubs which should
-        # be completely fine to request from a separate task
-        # since internally the ws methods appear to be FIFO
-        # locked.
-        await ws.send_msg(trades_sub)
-
-        yield
-
-        # unsub from all pairs on teardown
-        await ws.send_msg({
-            'event': 'unsubscribe',
-            'subscription': ['ownTrades'],
-        })
-
-        # XXX: do we need to ack the unsub?
-        # await ws.recv_msg()
-
-    # Authenticated block
-    async with get_client() as client:
-        if not client._api_key:
-            log.error('Missing Kraken API key: Trades WS connection failed')
-            await ctx.started(({}, ['paper']))
-
-            async with (
-                ctx.open_stream() as ems_stream,
-                trio.open_nursery() as n,
-            ):
-
-                client = PaperBoi(
-                    'kraken',
-                    ems_stream,
-                    _buys={},
-                    _sells={},
-
-                    _reqids={},
-
-                    # TODO: load paper positions from ``positions.toml``
-                    _positions={},
-                )
-
-                # TODO: maybe add multiple accounts
-                n.start_soon(handle_order_requests, client, ems_stream)
-
-        acc_name = 'kraken.' + client._name
-        trades = await client.get_trades()
-
-        position_msgs = pack_positions(acc_name, trades)
-
-        await ctx.started((position_msgs, (acc_name,)))
-
-        # Get websocket token for authenticated data stream
-        # Assert that a token was actually received.
-        resp = await client.endpoint('GetWebSocketsToken', {})
-
-        # lol wtf is this..
-        assert resp['error'] == []
-
-        token = resp['result']['token']
-
-        async with (
-            ctx.open_stream() as ems_stream,
-            trio.open_nursery() as n,
-        ):
-            # TODO: maybe add multiple accounts
-            n.start_soon(handle_order_requests, client, ems_stream)
-
-            # Process trades msg stream of ws
-            async with open_autorecon_ws(
-                'wss://ws-auth.kraken.com/',
-                fixture=subscribe,
-                token=token,
-            ) as ws:
-                async for msg in process_trade_msgs(ws):
-                    for trade in msg:
-                        # check the type of packaged message
-                        assert type(trade) == Trade
-
-                        # prepare and send a filled status update
-                        filled_msg = BrokerdStatus(
-                            reqid=trade.reqid,
-                            time_ns=time.time_ns(),
-
-                            account='kraken.spot',
-                            status='filled',
-                            filled=float(trade.size),
-                            reason='Order filled by kraken',
-                            broker_details={
-                                'name': 'kraken',
-                                'broker_time': trade.broker_time
-                            },
-
-                            # TODO: figure out if kraken gives a count
-                            # of how many units of underlying were
-                            # filled. Alternatively we can decrement
-                            # this value ourselves by associating and
-                            # calcing from the diff with the original
-                            # client-side request, see:
-                            # https://github.com/pikers/piker/issues/296
-                            remaining=0,
-                        )
-
-                        await ems_stream.send(filled_msg.dict())
-
-                        # send a fill msg for gui update
-                        fill_msg = BrokerdFill(
-                            reqid=trade.reqid,
-                            time_ns=time.time_ns(),
-
-                            action=trade.action,
-                            size=float(trade.size),
-                            price=float(trade.price),
-                            # TODO: maybe capture more msg data i.e fees?
-                            broker_details={'name': 'kraken'},
-                            broker_time=float(trade.broker_time)
-                        )
-
-                        await ems_stream.send(fill_msg.dict())
-
-
-async def stream_messages(
-    ws: NoBsWs,
-):
-    '''
-    Message stream parser and heartbeat handler.
-
-    Deliver ws subscription messages as well as handle heartbeat logic
-    though a single async generator.
-
-    '''
     too_slow_count = last_hb = 0
 
     while True:
@@ -906,18 +320,6 @@ async def stream_messages(
             if err:
                 raise BrokerError(err)
         else:
-            yield msg
-
-
-async def process_data_feed_msgs(
-    ws: NoBsWs,
-):
-    '''
-    Parse and pack data feed messages.
-
-    '''
-    async for msg in stream_messages(ws):
-
             chan_id, *payload_array, chan_name, pair = msg
 
             if 'ohlc' in chan_name:
@@ -947,54 +349,10 @@ async def process_data_feed_msgs(
 
             else:
                 print(f'UNHANDLED MSG: {msg}')
-            yield msg
-
-
-async def process_trade_msgs(
-    ws: NoBsWs,
-):
-    '''
-    Parse and pack data feed messages.
-
-    '''
-    sequence_counter = 0
-    async for msg in stream_messages(ws):
-
-        try:
-            # check that we are on the ownTrades stream and that msgs
-            # are arriving in sequence with kraken For clarification the
-            # kraken ws api docs for this stream:
-            # https://docs.kraken.com/websockets/#message-ownTrades
-            assert msg[1] == 'ownTrades'
-            assert msg[2]['sequence'] > sequence_counter
-            sequence_counter += 1
-            raw_msgs = msg[0]
-            trade_msgs = []
-
-            # Check that we are only processing new trades
-            if msg[2]['sequence'] != 1:
-                # check if its a new order or an update msg
-                for trade_msg in raw_msgs:
-                    trade = list(trade_msg.values())[0]
-                    order_msg = Trade(
-                        reqid=trade['ordertxid'],
-                        action=trade['type'],
-                        price=trade['price'],
-                        size=trade['vol'],
-                        broker_time=trade['time']
-                    )
-                    trade_msgs.append(order_msg)
-
-            yield trade_msgs
-
-        except AssertionError:
-            print(f'UNHANDLED MSG: {msg}')
-            yield msg
 
 
 def normalize(
     ohlc: OHLC,
-
 ) -> dict:
     quote = asdict(ohlc)
     quote['broker_ts'] = quote['time']
@@ -1012,13 +370,12 @@ def normalize(
     return topic, quote
 
 
-def make_sub(pairs: list[str], data: dict[str, Any]) -> dict[str, str]:
+def make_sub(pairs: List[str], data: Dict[str, Any]) -> Dict[str, str]:
-    '''
+    """Create a request subscription packet dict.
-    Create a request subscription packet dict.
 
     https://docs.kraken.com/websockets/#message-subscribe
 
-    '''
+    """
     # eg. specific logic for this in kraken's sync client:
     # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
     return {
@@ -1028,55 +385,6 @@ def make_sub(pairs: list[str], data: dict[str, Any]) -> dict[str, str]:
     }
 
 
-@acm
-async def open_history_client(
-    symbol: str,
-
-) -> tuple[Callable, int]:
-
-    # TODO implement history getter for the new storage layer.
-    async with open_cached_client('kraken') as client:
-
-        # lol, kraken won't send any more then the "last"
-        # 720 1m bars.. so we have to just ignore further
-        # requests of this type..
-        queries: int = 0
-
-        async def get_ohlc(
-            end_dt: Optional[datetime] = None,
-            start_dt: Optional[datetime] = None,
-
-        ) -> tuple[
-            np.ndarray,
-            datetime,  # start
-            datetime,  # end
-        ]:
-
-            nonlocal queries
-            if queries > 0:
-                raise DataUnavailable
-
-            count = 0
-            while count <= 3:
-                try:
-                    array = await client.bars(
-                        symbol,
-                        since=end_dt,
-                    )
-                    count += 1
-                    queries += 1
-                    break
-                except DataThrottle:
-                    log.warning(f'kraken OHLC throttle for {symbol}')
-                    await trio.sleep(1)
-
-            start_dt = pendulum.from_timestamp(array[0]['time'])
-            end_dt = pendulum.from_timestamp(array[-1]['time'])
-            return array, start_dt, end_dt
-
-        yield get_ohlc, {'erlangs': 1, 'rate': 1}
-
-
 async def backfill_bars(
 
     sym: str,
@@ -1085,9 +393,8 @@ async def backfill_bars(
     task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
-    '''
+    """Fill historical bars into shared mem / storage afap.
-    Fill historical bars into shared mem / storage afap.
+    """
-    '''
     with trio.CancelScope() as cs:
         async with open_cached_client('kraken') as client:
             bars = await client.bars(symbol=sym)
@@ -1098,7 +405,8 @@ async def backfill_bars(
 async def stream_quotes(
 
     send_chan: trio.abc.SendChannel,
-    symbols: list[str],
+    symbols: List[str],
+    shm: ShmArray,
     feed_is_live: trio.Event,
     loglevel: str = None,
 
@@ -1106,15 +414,13 @@ async def stream_quotes(
     sub_type: str = 'ohlc',
 
     # startup sync
-    task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
+    task_status: TaskStatus[Tuple[Dict, Dict]] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
-    '''
+    """Subscribe for ohlc stream of quotes for ``pairs``.
-    Subscribe for ohlc stream of quotes for ``pairs``.
 
     ``pairs`` must be formatted <crypto_symbol>/<fiat_symbol>.
-
+    """
-    '''
     # XXX: required to propagate ``tractor`` loglevel to piker logging
     get_console_log(loglevel or tractor.current_actor().loglevel)
 
@@ -1145,11 +451,10 @@ async def stream_quotes(
             symbol: {
                 'symbol_info': sym_infos[sym],
                 'shm_write_opts': {'sum_tick_vml': False},
-                'fqsn': sym,
             },
         }
 
-        @acm
+        @asynccontextmanager
         async def subscribe(ws: wsproto.WSConnection):
             # XXX: setup subs
             # https://docs.kraken.com/websockets/#message-subscribe
@@ -1187,23 +492,23 @@ async def stream_quotes(
             # XXX: do we need to ack the unsub?
             # await ws.recv_msg()
 
-        # see the tips on reconnection logic:
+        # see the tips on reonnection logic:
         # https://support.kraken.com/hc/en-us/articles/360044504011-WebSocket-API-unexpected-disconnections-from-market-data-feeds
-        ws: NoBsWs
         async with open_autorecon_ws(
             'wss://ws.kraken.com/',
             fixture=subscribe,
         ) as ws:
 
             # pull a first quote and deliver
-            msg_gen = process_data_feed_msgs(ws)
+            msg_gen = stream_messages(ws)
 
             # TODO: use ``anext()`` when it lands in 3.10!
             typ, ohlc_last = await msg_gen.__anext__()
 
             topic, quote = normalize(ohlc_last)
 
-            task_status.started((init_msgs,  quote))
+            first_quote = {topic: quote}
+            task_status.started((init_msgs,  first_quote))
 
             # lol, only "closes" when they're margin squeezing clients ;P
             feed_is_live.set()
@@ -1254,7 +559,6 @@ async def stream_quotes(
 @tractor.context
 async def open_symbol_search(
     ctx: tractor.Context,
-
 ) -> Client:
     async with open_cached_client('kraken') as client:
 

piker/brokers/questrade.py

@@ -19,6 +19,7 @@ Questrade API backend.
 """
 from __future__ import annotations
 import inspect
+import contextlib
 import time
 from datetime import datetime
 from functools import partial
@@ -31,10 +32,11 @@ from typing import (
     Callable,
 )
 
-import pendulum
+import arrow
 import trio
 import tractor
 from async_generator import asynccontextmanager
+import pandas as pd
 import numpy as np
 import wrapt
 import asks
@@ -44,6 +46,7 @@ from .._cacheables import open_cached_client, async_lifo_cache
 from .. import config
 from ._util import resproc, BrokerError, SymbolNotFound
 from ..log import get_logger, colorize_json, get_console_log
+from . import get_brokermod
 
 
 log = get_logger(__name__)
@@ -598,16 +601,12 @@ class Client:
         sid = sids[symbol]
 
         # get last market open end time
-        est_end = now = pendulum.now('UTC').in_timezoe(
+        est_end = now = arrow.utcnow().to('US/Eastern').floor('minute')
-            'America/New_York').start_of('minute')
-
         # on non-paid feeds we can't retreive the first 15 mins
         wd = now.isoweekday()
         if wd > 5:
             quotes = await self.quote([symbol])
-            est_end = pendulum.parse(
+            est_end = arrow.get(quotes[0]['lastTradeTime'])
-                quotes[0]['lastTradeTime']
-            )
             if est_end.hour == 0:
                 # XXX don't bother figuring out extended hours for now
                 est_end = est_end.replace(hour=17)
@@ -668,7 +667,7 @@ def get_OHLCV(
     """
     del bar['end']
     del bar['VWAP']
-    bar['start'] = pendulum.from_timestamp(bar['start']) / 10**9
+    bar['start'] = pd.Timestamp(bar['start']).value/10**9
     return tuple(bar.values())
 
 

piker/clearing/_allocate.py

@@ -29,8 +29,7 @@ from ._messages import BrokerdPosition, Status
 
 
 class Position(BaseModel):
-    '''
+    '''Basic pp (personal position) model with attached fills history.
-    Basic pp (personal position) model with attached fills history.
 
     This type should be IPC wire ready?
 
@@ -62,15 +61,6 @@ class Position(BaseModel):
         self.avg_price = avg_price
         self.size = size
 
-    @property
-    def dsize(self) -> float:
-        '''
-        The "dollar" size of the pp, normally in trading (fiat) unit
-        terms.
-
-        '''
-        return self.avg_price * self.size
-
 
 _size_units = bidict({
     'currency': '$ size',
@@ -124,8 +114,7 @@ class Allocator(BaseModel):
     def step_sizes(
         self,
     ) -> (float, float):
-        '''
+        '''Return the units size for each unit type as a tuple.
-        Return the units size for each unit type as a tuple.
 
         '''
         slots = self.slots
@@ -153,8 +142,7 @@ class Allocator(BaseModel):
         action: str,
 
     ) -> dict:
-        '''
+        '''Generate order request info for the "next" submittable order
-        Generate order request info for the "next" submittable order
         depending on position / order entry config.
 
         '''
@@ -178,9 +166,7 @@ class Allocator(BaseModel):
             l_sub_pp = (self.currency_limit - live_cost_basis) / price
 
         else:
-            raise ValueError(
+            raise ValueError(f"Not valid size unit '{size}'")
-                f"Not valid size unit '{size_unit}'"
-            )
 
         # an entry (adding-to or starting a pp)
         if (
@@ -264,8 +250,7 @@ class Allocator(BaseModel):
         pp: Position,
 
     ) -> float:
-        '''
+        '''Calc and return the number of slots used by this ``Position``.
-        Calc and return the number of slots used by this ``Position``.
 
         '''
         abs_pp_size = abs(pp.size)
@@ -284,14 +269,6 @@ class Allocator(BaseModel):
         return round(prop * self.slots)
 
 
-_derivs = (
-    'future',
-    'continuous_future',
-    'option',
-    'futures_option',
-)
-
-
 def mk_allocator(
 
     symbol: Symbol,
@@ -300,7 +277,7 @@ def mk_allocator(
     # default allocation settings
     defaults: dict[str, float] = {
         'account': None,  # select paper by default
-        'size_unit': 'currency',
+        'size_unit': 'currency', #_size_units['currency'],
         'units_limit': 400,
         'currency_limit': 5e3,
         'slots': 4,
@@ -328,9 +305,11 @@ def mk_allocator(
 
     asset_type = symbol.type_key
 
+
     # specific configs by asset class / type
 
-    if asset_type in _derivs:
+    if asset_type in ('future', 'option', 'futures_option'):
+
         # since it's harder to know how currency "applies" in this case
         # given leverage properties
         alloc.size_unit = '# units'
@@ -353,7 +332,7 @@ def mk_allocator(
         if startup_size > alloc.units_limit:
             alloc.units_limit = startup_size
 
-            if asset_type in _derivs:
+            if asset_type in ('future', 'option', 'futures_option'):
                 alloc.slots = alloc.units_limit
 
     return alloc

piker/clearing/_client.py

@@ -18,7 +18,7 @@
 Orders and execution client API.
 
 """
-from contextlib import asynccontextmanager as acm
+from contextlib import asynccontextmanager
 from typing import Dict
 from pprint import pformat
 from dataclasses import dataclass, field
@@ -27,6 +27,7 @@ import trio
 import tractor
 from tractor.trionics import broadcast_receiver
 
+from ..data._source import Symbol
 from ..log import get_logger
 from ._ems import _emsd_main
 from .._daemon import maybe_open_emsd
@@ -155,19 +156,16 @@ async def relay_order_cmds_from_sync_code(
                 await to_ems_stream.send(cmd)
 
 
-@acm
+@asynccontextmanager
 async def open_ems(
-    fqsn: str,
+    broker: str,
+    symbol: Symbol,
 
-) -> (
+) -> (OrderBook, tractor.MsgStream, dict):
-    OrderBook,
+    """Spawn an EMS daemon and begin sending orders and receiving
-    tractor.MsgStream,
-    dict,
-):
-    '''
-    Spawn an EMS daemon and begin sending orders and receiving
     alerts.
 
+
     This EMS tries to reduce most broker's terrible order entry apis to
     a very simple protocol built on a few easy to grok and/or
     "rantsy" premises:
@@ -196,22 +194,21 @@ async def open_ems(
     - 'dark_executed', 'broker_executed'
     - 'broker_filled'
 
-    '''
+    """
     # wait for service to connect back to us signalling
     # ready for order commands
     book = get_orders()
 
-    from ..data._source import unpack_fqsn
-    broker, symbol, suffix = unpack_fqsn(fqsn)
-
     async with maybe_open_emsd(broker) as portal:
 
         async with (
+
             # connect to emsd
             portal.open_context(
 
                 _emsd_main,
-                fqsn=fqsn,
+                broker=broker,
+                symbol=symbol.key,
 
             ) as (ctx, (positions, accounts)),
 
@@ -221,7 +218,7 @@ async def open_ems(
             async with trio.open_nursery() as n:
                 n.start_soon(
                     relay_order_cmds_from_sync_code,
-                    fqsn,
+                    symbol.key,
                     trades_stream
                 )
 

piker/clearing/_ems.py

@@ -20,6 +20,7 @@ In da suit parlances: "Execution management systems"
 """
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
+from math import isnan
 from pprint import pformat
 import time
 from typing import AsyncIterator, Callable
@@ -53,8 +54,7 @@ def mk_check(
     action: str,
 
 ) -> Callable[[float, float], bool]:
-    '''
+    """Create a predicate for given ``exec_price`` based on last known
-    Create a predicate for given ``exec_price`` based on last known
     price, ``known_last``.
 
     This is an automatic alert level thunk generator based on where the
@@ -62,7 +62,7 @@ def mk_check(
     interest is; pick an appropriate comparison operator based on
     avoiding the case where the a predicate returns true immediately.
 
-    '''
+    """
     # str compares:
     # https://stackoverflow.com/questions/46708708/compare-strings-in-numba-compiled-function
 
@@ -80,9 +80,7 @@ def mk_check(
 
         return check_lt
 
-    raise ValueError(
+    raise ValueError('trigger: {trigger_price}, last: {known_last}')
-        f'trigger: {trigger_price}, last: {known_last}'
-    )
 
 
 @dataclass
@@ -114,8 +112,8 @@ class _DarkBook:
 
     # tracks most recent values per symbol each from data feed
     lasts: dict[
-        str,
+        tuple[str, str],
-        float,
+        float
     ] = field(default_factory=dict)
 
     # mapping of piker ems order ids to current brokerd order flow message
@@ -136,42 +134,40 @@ async def clear_dark_triggers(
     ems_client_order_stream: tractor.MsgStream,
     quote_stream: tractor.ReceiveMsgStream,  # noqa
     broker: str,
-    fqsn: str,
+    symbol: str,
 
     book: _DarkBook,
 
 ) -> None:
-    '''
+    """Core dark order trigger loop.
-    Core dark order trigger loop.
 
     Scan the (price) data feed and submit triggered orders
     to broker.
 
-    '''
+    """
+    # this stream may eventually contain multiple symbols
     # XXX: optimize this for speed!
-    # TODO:
-    # - numba all this!
-    # - this stream may eventually contain multiple symbols
     async for quotes in quote_stream:
+
+        # TODO: numba all this!
+
         # start = time.time()
         for sym, quote in quotes.items():
-            execs = book.orders.get(sym, {})
+
+            execs = book.orders.get(sym, None)
+            if execs is None:
+                continue
+
             for tick in iterticks(
                 quote,
                 # dark order price filter(s)
-                types=(
+                types=('ask', 'bid', 'trade', 'last')
-                    'ask',
-                    'bid',
-                    'trade',
-                    'last',
-                    # 'dark_trade',  # TODO: should allow via config?
-                )
             ):
                 price = tick.get('price')
                 ttype = tick['type']
 
                 # update to keep new cmds informed
-                book.lasts[sym] = price
+                book.lasts[(broker, symbol)] = price
 
                 for oid, (
                     pred,
@@ -182,6 +178,7 @@ async def clear_dark_triggers(
                 ) in (
                     tuple(execs.items())
                 ):
+
                     if (
                         not pred or
                         ttype not in tf or
@@ -196,7 +193,6 @@ async def clear_dark_triggers(
 
                     action: str = cmd['action']
                     symbol: str = cmd['symbol']
-                    bfqsn: str = symbol.replace(f'.{broker}', '')
 
                     if action == 'alert':
                         # nothing to do but relay a status
@@ -226,7 +222,7 @@ async def clear_dark_triggers(
                             # order-request and instead create a new one.
                             reqid=None,
 
-                            symbol=bfqsn,
+                            symbol=sym,
                             price=submit_price,
                             size=cmd['size'],
                         )
@@ -248,9 +244,12 @@ async def clear_dark_triggers(
                         oid=oid,  # ems order id
                         resp=resp,
                         time_ns=time.time_ns(),
-                        symbol=fqsn,
+
+                        symbol=symbol,
                         trigger_price=price,
+
                         broker_details={'name': broker},
+
                         cmd=cmd,  # original request message
 
                     ).dict()
@@ -263,20 +262,12 @@ async def clear_dark_triggers(
                             f'pred for {oid} was already removed!?'
                         )
 
-                    try:
                     await ems_client_order_stream.send(msg)
-                    except (
-                        trio.ClosedResourceError,
-                    ):
-                        log.warning(
-                            f'client {ems_client_order_stream} stream is broke'
-                        )
-                        break
 
                 else:  # condition scan loop complete
                     log.debug(f'execs are {execs}')
                     if execs:
-                        book.orders[fqsn] = execs
+                        book.orders[symbol] = execs
 
         # print(f'execs scan took: {time.time() - start}')
 
@@ -299,8 +290,7 @@ class TradesRelay:
 
 
 class Router(BaseModel):
-    '''
+    '''Order router which manages and tracks per-broker dark book,
-    Order router which manages and tracks per-broker dark book,
     alerts, clearing and related data feed management.
 
     A singleton per ``emsd`` actor.
@@ -388,8 +378,7 @@ async def open_brokerd_trades_dialogue(
     task_status: TaskStatus[TradesRelay] = trio.TASK_STATUS_IGNORED,
 
 ) -> tuple[dict, tractor.MsgStream]:
-    '''
+    '''Open and yield ``brokerd`` trades dialogue context-stream if none
-    Open and yield ``brokerd`` trades dialogue context-stream if none
     already exists.
 
     '''
@@ -426,7 +415,8 @@ async def open_brokerd_trades_dialogue(
             # actor to simulate the real IPC load it'll have when also
             # pulling data from feeds
             open_trades_endpoint = paper.open_paperboi(
-                fqsn='.'.join([symbol, broker]),
+                broker=broker,
+                symbol=symbol,
                 loglevel=loglevel,
             )
 
@@ -464,13 +454,12 @@ async def open_brokerd_trades_dialogue(
                 # locally cache and track positions per account.
                 pps = {}
                 for msg in positions:
-                    log.info(f'loading pp: {msg}')
 
                     account = msg['account']
                     assert account in accounts
 
                     pps.setdefault(
-                        f'{msg["symbol"]}.{broker}',
+                        msg['symbol'],
                         {}
                     )[account] = msg
 
@@ -500,9 +489,7 @@ async def open_brokerd_trades_dialogue(
         finally:
             # parent context must have been closed
             # remove from cache so next client will respawn if needed
-            relay = _router.relays.pop(broker, None)
+            _router.relays.pop(broker)
-            if not relay:
-                log.warning(f'Relay for {broker} was already removed!?')
 
 
 @tractor.context
@@ -534,8 +521,7 @@ async def translate_and_relay_brokerd_events(
     router: Router,
 
 ) -> AsyncIterator[dict]:
-    '''
+    '''Trades update loop - receive updates from ``brokerd`` trades
-    Trades update loop - receive updates from ``brokerd`` trades
     endpoint, convert to EMS response msgs, transmit **only** to
     ordering client(s).
 
@@ -563,10 +549,7 @@ async def translate_and_relay_brokerd_events(
 
         name = brokerd_msg['name']
 
-        log.info(
+        log.info(f'Received broker trade event:\n{pformat(brokerd_msg)}')
-            f'Received broker trade event:\n'
-            f'{pformat(brokerd_msg)}'
-        )
 
         if name == 'position':
 
@@ -574,28 +557,14 @@ async def translate_and_relay_brokerd_events(
 
             # XXX: this will be useful for automatic strats yah?
             # keep pps per account up to date locally in ``emsd`` mem
-            sym, broker = pos_msg['symbol'], pos_msg['broker']
+            relay.positions.setdefault(pos_msg['symbol'], {}).setdefault(
-
-            relay.positions.setdefault(
-                # NOTE: translate to a FQSN!
-                f'{sym}.{broker}',
-                {}
-            ).setdefault(
                 pos_msg['account'], {}
             ).update(pos_msg)
 
             # fan-out-relay position msgs immediately by
             # broadcasting updates on all client streams
-            for client_stream in router.clients.copy():
+            for client_stream in router.clients:
-                try:
                 await client_stream.send(pos_msg)
-                except(
-                    trio.ClosedResourceError,
-                    trio.BrokenResourceError,
-                ):
-                    router.clients.remove(client_stream)
-                    log.warning(
-                        f'client for {client_stream} was already closed?')
 
             continue
 
@@ -618,28 +587,19 @@ async def translate_and_relay_brokerd_events(
             # packed at submission since we already know it ahead of
             # time
             paper = brokerd_msg['broker_details'].get('paper_info')
-            ext = brokerd_msg['broker_details'].get('external')
             if paper:
                 # paperboi keeps the ems id up front
                 oid = paper['oid']
 
-            elif ext:
+            else:
                 # may be an order msg specified as "external" to the
                 # piker ems flow (i.e. generated by some other
                 # external broker backend client (like tws for ib)
+                ext = brokerd_msg['broker_details'].get('external')
+                if ext:
                     log.error(f"External trade event {ext}")
 
                 continue
-
-            else:
-                # something is out of order, we don't have an oid for
-                # this broker-side message.
-                log.error(
-                    'Unknown oid:{oid} for msg:\n'
-                    f'{pformat(brokerd_msg)}'
-                    'Unable to relay message to client side!?'
-                )
-
         else:
             # check for existing live flow entry
             entry = book._ems_entries.get(oid)
@@ -837,9 +797,7 @@ async def process_client_order_cmds(
                 if reqid:
 
                     # send cancel to brokerd immediately!
-                    log.info(
+                    log.info("Submitting cancel for live order {reqid}")
-                        f'Submitting cancel for live order {reqid}'
-                    )
 
                     await brokerd_order_stream.send(msg.dict())
 
@@ -876,15 +834,11 @@ async def process_client_order_cmds(
 
             msg = Order(**cmd)
 
-            fqsn = msg.symbol
+            sym = msg.symbol
             trigger_price = msg.price
             size = msg.size
             exec_mode = msg.exec_mode
             broker = msg.brokers[0]
-            # remove the broker part before creating a message
-            # to send to the specific broker since they probably
-            # aren't expectig their own name, but should they?
-            sym = fqsn.replace(f'.{broker}', '')
 
             if exec_mode == 'live' and action in ('buy', 'sell',):
 
@@ -942,7 +896,7 @@ async def process_client_order_cmds(
                 # price received from the feed, instead of being
                 # like every other shitty tina platform that makes
                 # the user choose the predicate operator.
-                last = dark_book.lasts[fqsn]
+                last = dark_book.lasts[(broker, sym)]
                 pred = mk_check(trigger_price, last, action)
 
                 spread_slap: float = 5
@@ -973,7 +927,7 @@ async def process_client_order_cmds(
                 # dark book entry if the order id already exists
 
                 dark_book.orders.setdefault(
-                    fqsn, {}
+                    sym, {}
                 )[oid] = (
                     pred,
                     tickfilter,
@@ -1000,8 +954,8 @@ async def process_client_order_cmds(
 async def _emsd_main(
 
     ctx: tractor.Context,
-    fqsn: str,
+    broker: str,
-
+    symbol: str,
     _exec_mode: str = 'dark',  # ('paper', 'dark', 'live')
     loglevel: str = 'info',
 
@@ -1043,8 +997,6 @@ async def _emsd_main(
     global _router
     assert _router
 
-    from ..data._source import unpack_fqsn
-    broker, symbol, suffix = unpack_fqsn(fqsn)
     dark_book = _router.get_dark_book(broker)
 
     # TODO: would be nice if in tractor we can require either a ctx arg,
@@ -1057,16 +1009,18 @@ async def _emsd_main(
     # spawn one task per broker feed
     async with (
         maybe_open_feed(
-            [fqsn],
+            broker,
+            [symbol],
             loglevel=loglevel,
-        ) as (feed, quote_stream),
+        ) as (feed, stream),
     ):
 
         # XXX: this should be initial price quote from target provider
-        first_quote = feed.first_quotes[fqsn]
+        first_quote = feed.first_quotes[symbol]
 
         book = _router.get_dark_book(broker)
-        book.lasts[fqsn] = first_quote['last']
+        last = book.lasts[(broker, symbol)] = first_quote['last']
+        assert not isnan(last)  # ib is a cucker but we've fixed it in the backend
 
         # open a stream with the brokerd backend for order
         # flow dialogue
@@ -1090,13 +1044,13 @@ async def _emsd_main(
 
             # flatten out collected pps from brokerd for delivery
             pp_msgs = {
-                fqsn: list(pps.values())
+                sym: list(pps.values())
-                for fqsn, pps in relay.positions.items()
+                for sym, pps in relay.positions.items()
             }
 
             # signal to client that we're started and deliver
             # all known pps and accounts for this ``brokerd``.
-            await ems_ctx.started((pp_msgs, list(relay.accounts)))
+            await ems_ctx.started((pp_msgs, relay.accounts))
 
             # establish 2-way stream with requesting order-client and
             # begin handling inbound order requests and updates
@@ -1108,9 +1062,9 @@ async def _emsd_main(
 
                     brokerd_stream,
                     ems_client_order_stream,
-                    quote_stream,
+                    stream,
                     broker,
-                    fqsn,  # form: <name>.<venue>.<suffix>.<broker>
+                    symbol,
                     book
                 )
 
@@ -1126,7 +1080,7 @@ async def _emsd_main(
                         # relay.brokerd_dialogue,
                         brokerd_stream,
 
-                        fqsn,
+                        symbol,
                         feed,
                         dark_book,
                         _router,

piker/clearing/_messages.py

@@ -155,11 +155,8 @@ class BrokerdOrder(BaseModel):
 
 
 class BrokerdOrderAck(BaseModel):
-    '''
+    '''Immediate reponse to a brokerd order request providing
-    Immediate reponse to a brokerd order request providing the broker
+    the broker specifci unique order id.
-    specific unique order id so that the EMS can associate this
-    (presumably differently formatted broker side ID) with our own
-    ``.oid`` (which is a uuid4).
 
     '''
     name: str = 'ack'
@@ -184,7 +181,7 @@ class BrokerdStatus(BaseModel):
     # {
     #   'submitted',
     #   'cancelled',
-    #   'filled',
+    #   'executed',
     # }
     status: str
 
@@ -206,8 +203,7 @@ class BrokerdStatus(BaseModel):
 
 
 class BrokerdFill(BaseModel):
-    '''
+    '''A single message indicating a "fill-details" event from the broker
-    A single message indicating a "fill-details" event from the broker
     if avaiable.
 
     '''
@@ -231,18 +227,16 @@ class BrokerdFill(BaseModel):
 
 
 class BrokerdError(BaseModel):
-    '''
+    '''Optional error type that can be relayed to emsd for error handling.
-    Optional error type that can be relayed to emsd for error handling.
 
     This is still a TODO thing since we're not sure how to employ it yet.
-
     '''
     name: str = 'error'
     oid: str
 
     # if no brokerd order request was actually submitted (eg. we errored
     # at the ``pikerd`` layer) then there will be ``reqid`` allocated.
-    reqid: Optional[Union[int, str]] = None
+    reqid: Union[int, str] = ''
 
     symbol: str
     reason: str

piker/clearing/_paper_engine.py

@@ -32,7 +32,6 @@ from dataclasses import dataclass
 
 from .. import data
 from ..data._normalize import iterticks
-from ..data._source import unpack_fqsn
 from ..log import get_logger
 from ._messages import (
     BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
@@ -390,7 +389,7 @@ async def handle_order_requests(
             account = request_msg['account']
             if account != 'paper':
                 log.error(
-                    'This is a paper account, only a `paper` selection is valid'
+                    'On a paper account, only a `paper` selection is valid'
                 )
                 await ems_order_stream.send(BrokerdError(
                     oid=request_msg['oid'],
@@ -447,7 +446,7 @@ async def trades_dialogue(
 
     ctx: tractor.Context,
     broker: str,
-    fqsn: str,
+    symbol: str,
     loglevel: str = None,
 
 ) -> None:
@@ -456,15 +455,15 @@ async def trades_dialogue(
     async with (
 
         data.open_feed(
-            [fqsn],
+            broker,
+            [symbol],
             loglevel=loglevel,
         ) as feed,
 
     ):
         # TODO: load paper positions per broker from .toml config file
         # and pass as symbol to position data mapping: ``dict[str, dict]``
-        # await ctx.started(all_positions)
+        await ctx.started(({}, ['paper']))
-        await ctx.started(({}, {'paper',}))
 
         async with (
             ctx.open_stream() as ems_stream,
@@ -491,16 +490,15 @@ async def trades_dialogue(
 
 @asynccontextmanager
 async def open_paperboi(
-    fqsn: str,
+    broker: str,
+    symbol: str,
     loglevel: str,
 
 ) -> Callable:
-    '''
+    '''Spawn a paper engine actor and yield through access to
-    Spawn a paper engine actor and yield through access to
     its context.
 
     '''
-    broker, symbol, expiry = unpack_fqsn(fqsn)
     service_name = f'paperboi.{broker}'
 
     async with (
@@ -519,7 +517,7 @@ async def open_paperboi(
         async with portal.open_context(
                 trades_dialogue,
                 broker=broker,
-                fqsn=fqsn,
+                symbol=symbol,
                 loglevel=loglevel,
 
         ) as (ctx, first):

piker/cli/__init__.py

@@ -1,25 +1,7 @@
-# piker: trading gear for hackers
+"""
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-'''
 CLI commons.
-
+"""
-'''
 import os
-from pprint import pformat
 
 import click
 import trio
@@ -34,22 +16,29 @@ from .. import config
 log = get_logger('cli')
 DEFAULT_BROKER = 'questrade'
 
+_config_dir = click.get_app_dir('piker')
+_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
+_context_defaults = dict(
+    default_map={
+        # Questrade specific quote poll rates
+        'monitor': {
+            'rate': 3,
+        },
+        'optschain': {
+            'rate': 1,
+        },
+    }
+)
+
 
 @click.command()
 @click.option('--loglevel', '-l', default='warning', help='Logging level')
 @click.option('--tl', is_flag=True, help='Enable tractor logging')
 @click.option('--pdb', is_flag=True, help='Enable tractor debug mode')
 @click.option('--host', '-h', default='127.0.0.1', help='Host address to bind')
-@click.option(
+def pikerd(loglevel, host, tl, pdb):
-    '--tsdb',
+    """Spawn the piker broker-daemon.
-    is_flag=True,
+    """
-    help='Enable local ``marketstore`` instance'
-)
-def pikerd(loglevel, host, tl, pdb, tsdb):
-    '''
-    Spawn the piker broker-daemon.
-
-    '''
     from .._daemon import open_pikerd
     log = get_console_log(loglevel)
 
@@ -63,38 +52,13 @@ def pikerd(loglevel, host, tl, pdb, tsdb):
         ))
 
     async def main():
-
+        async with open_pikerd(loglevel=loglevel, debug_mode=pdb):
-        async with (
-            open_pikerd(
-                loglevel=loglevel,
-                debug_mode=pdb,
-            ),  # normally delivers a ``Services`` handle
-            trio.open_nursery() as n,
-        ):
-            if tsdb:
-                from piker.data._ahab import start_ahab
-                from piker.data.marketstore import start_marketstore
-
-                log.info('Spawning `marketstore` supervisor')
-                ctn_ready, config, (cid, pid) = await n.start(
-                    start_ahab,
-                    'marketstored',
-                    start_marketstore,
-
-                )
-                log.info(
-                    f'`marketstore` up!\n'
-                    f'`marketstored` pid: {pid}\n'
-                    f'docker container id: {cid}\n'
-                    f'config: {pformat(config)}'
-                )
-
             await trio.sleep_forever()
 
     trio.run(main)
 
 
-@click.group(context_settings=config._context_defaults)
+@click.group(context_settings=_context_defaults)
 @click.option(
     '--brokers', '-b',
     default=[DEFAULT_BROKER],
@@ -123,8 +87,8 @@ def cli(ctx, brokers, loglevel, tl, configdir):
         'loglevel': loglevel,
         'tractorloglevel': None,
         'log': get_console_log(loglevel),
-        'confdir': config._config_dir,
+        'confdir': _config_dir,
-        'wl_path': config._watchlists_data_path,
+        'wl_path': _watchlists_data_path,
     })
 
     # allow enabling same loglevel in ``tractor`` machinery
@@ -145,11 +109,13 @@ def services(config, tl, names):
         ) as portal:
             registry = await portal.run_from_ns('self', 'get_registry')
             json_d = {}
-            for key, socket in registry.items():
+            for uid, socket in registry.items():
-                # name, uuid = uid
+                name, uuid = uid
                 host, port = socket
-                json_d[key] = f'{host}:{port}'
+                json_d[f'{name}.{uuid}'] = f'{host}:{port}'
-            click.echo(f"{colorize_json(json_d)}")
+            click.echo(
+                f"Available `piker` services:\n{colorize_json(json_d)}"
+            )
 
     tractor.run(
         list_services,

piker/config.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship for pikers)
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,10 +16,7 @@
 
 """
 Broker configuration mgmt.
-
 """
-import platform
-import sys
 import os
 from os.path import dirname
 import shutil
@@ -27,106 +24,14 @@ from typing import Optional
 
 from bidict import bidict
 import toml
+import click
 
 from .log import get_logger
 
 log = get_logger('broker-config')
 
-
+_config_dir = click.get_app_dir('piker')
-# taken from ``click`` since apparently they have some
+_file_name = 'brokers.toml'
-# super weirdness with sigint and sudo..no clue
-def get_app_dir(app_name, roaming=True, force_posix=False):
-    r"""Returns the config folder for the application.  The default behavior
-    is to return whatever is most appropriate for the operating system.
-
-    To give you an idea, for an app called ``"Foo Bar"``, something like
-    the following folders could be returned:
-
-    Mac OS X:
-      ``~/Library/Application Support/Foo Bar``
-    Mac OS X (POSIX):
-      ``~/.foo-bar``
-    Unix:
-      ``~/.config/foo-bar``
-    Unix (POSIX):
-      ``~/.foo-bar``
-    Win XP (roaming):
-      ``C:\Documents and Settings\<user>\Local Settings\Application Data\Foo``
-    Win XP (not roaming):
-      ``C:\Documents and Settings\<user>\Application Data\Foo Bar``
-    Win 7 (roaming):
-      ``C:\Users\<user>\AppData\Roaming\Foo Bar``
-    Win 7 (not roaming):
-      ``C:\Users\<user>\AppData\Local\Foo Bar``
-
-    .. versionadded:: 2.0
-
-    :param app_name: the application name.  This should be properly capitalized
-                     and can contain whitespace.
-    :param roaming: controls if the folder should be roaming or not on Windows.
-                    Has no affect otherwise.
-    :param force_posix: if this is set to `True` then on any POSIX system the
-                        folder will be stored in the home folder with a leading
-                        dot instead of the XDG config home or darwin's
-                        application support folder.
-    """
-
-    def _posixify(name):
-        return "-".join(name.split()).lower()
-
-    # if WIN:
-    if platform.system() == 'Windows':
-        key = "APPDATA" if roaming else "LOCALAPPDATA"
-        folder = os.environ.get(key)
-        if folder is None:
-            folder = os.path.expanduser("~")
-        return os.path.join(folder, app_name)
-    if force_posix:
-        return os.path.join(
-            os.path.expanduser("~/.{}".format(_posixify(app_name))))
-    if sys.platform == "darwin":
-        return os.path.join(
-            os.path.expanduser("~/Library/Application Support"), app_name
-        )
-    return os.path.join(
-        os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")),
-        _posixify(app_name),
-    )
-
-
-_config_dir = _click_config_dir = get_app_dir('piker')
-_parent_user = os.environ.get('SUDO_USER')
-
-if _parent_user:
-    non_root_user_dir = os.path.expanduser(
-        f'~{_parent_user}'
-    )
-    root = 'root'
-    _config_dir = (
-        non_root_user_dir +
-        _click_config_dir[
-            _click_config_dir.rfind(root) + len(root):
-        ]
-    )
-
-_conf_names: set[str] = {
-    'brokers',
-    'trades',
-    'watchlists',
-}
-
-_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
-_context_defaults = dict(
-    default_map={
-        # Questrade specific quote poll rates
-        'monitor': {
-            'rate': 3,
-        },
-        'optschain': {
-            'rate': 1,
-        },
-    }
-)
 
 
 def _override_config_dir(
@@ -136,72 +41,41 @@ def _override_config_dir(
     _config_dir = path
 
 
-def _conf_fn_w_ext(
+def get_broker_conf_path():
-    name: str,
-) -> str:
-    # change this if we ever change the config file format.
-    return f'{name}.toml'
-
-
-def get_conf_path(
-    conf_name: str = 'brokers',
-
-) -> str:
     """Return the default config path normally under
     ``~/.config/piker`` on linux.
 
     Contains files such as:
     - brokers.toml
     - watchlists.toml
-    - trades.toml
-
-    # maybe coming soon ;)
     - signals.toml
     - strats.toml
 
     """
-    assert conf_name in _conf_names
+    return os.path.join(_config_dir, _file_name)
-    fn = _conf_fn_w_ext(conf_name)
-    return os.path.join(
-        _config_dir,
-        fn,
-    )
 
 
 def repodir():
-    '''
+    """Return the abspath to the repo directory.
-    Return the abspath to the repo directory.
+    """
-
-    '''
     dirpath = os.path.abspath(
         # we're 3 levels down in **this** module file
-        dirname(dirname(os.path.realpath(__file__)))
+        dirname(dirname(dirname(os.path.realpath(__file__))))
     )
     return dirpath
 
 
 def load(
-    conf_name: str = 'brokers',
     path: str = None
-
 ) -> (dict, str):
-    '''
+    """Load broker config.
-    Load config file by name.
+    """
-
+    path = path or get_broker_conf_path()
-    '''
-    path = path or get_conf_path(conf_name)
     if not os.path.isfile(path):
-        fn = _conf_fn_w_ext(conf_name)
+        shutil.copyfile(
-
+            os.path.join(repodir(), 'data/brokers.toml'),
-        template = os.path.join(
+            path,
-            repodir(),
-            'config',
-            fn
         )
-        # try to copy in a template config to the user's directory
-        # if one exists.
-        if os.path.isfile(template):
-            shutil.copyfile(template, path)
 
     config = toml.load(path)
     log.debug(f"Read config file {path}")
@@ -210,17 +84,13 @@ def load(
 
 def write(
     config: dict,  # toml config as dict
-    name: str = 'brokers',
     path: str = None,
-
 ) -> None:
-    ''''
+    """Write broker config to disk.
-    Write broker config to disk.
 
     Create a ``brokers.ini`` file if one does not exist.
-
+    """
-    '''
+    path = path or get_broker_conf_path()
-    path = path or get_conf_path(name)
     dirname = os.path.dirname(path)
     if not os.path.isdir(dirname):
         log.debug(f"Creating config dir {_config_dir}")
@@ -230,10 +100,7 @@ def write(
         raise ValueError(
             "Watch out you're trying to write a blank config!")
 
-    log.debug(
+    log.debug(f"Writing config file {path}")
-        f"Writing config `{name}` file to:\n"
-        f"{path}"
-    )
     with open(path, 'w') as cf:
         return toml.dump(config, cf)
 
@@ -263,5 +130,4 @@ def load_accounts(
 
     # our default paper engine entry
     accounts['paper'] = None
-
     return accounts

piker/data/_ahab.py

@@ -1,385 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-'''
-Supervisor for docker with included specific-image service helpers.
-
-'''
-import os
-import time
-from typing import (
-    Optional,
-    Callable,
-    Any,
-)
-from contextlib import asynccontextmanager as acm
-
-import trio
-from trio_typing import TaskStatus
-import tractor
-from tractor.msg import NamespacePath
-import docker
-import json
-from docker.models.containers import Container as DockerContainer
-from docker.errors import (
-    DockerException,
-    APIError,
-)
-from requests.exceptions import ConnectionError, ReadTimeout
-
-from ..log import get_logger, get_console_log
-from .. import config
-
-log = get_logger(__name__)
-
-
-class DockerNotStarted(Exception):
-    'Prolly you dint start da daemon bruh'
-
-
-class ContainerError(RuntimeError):
-    'Error reported via app-container logging level'
-
-
-@acm
-async def open_docker(
-    url: Optional[str] = None,
-    **kwargs,
-
-) -> docker.DockerClient:
-
-    client: Optional[docker.DockerClient] = None
-    try:
-        client = docker.DockerClient(
-            base_url=url,
-            **kwargs
-        ) if url else docker.from_env(**kwargs)
-
-        yield client
-
-    except (
-        DockerException,
-        APIError,
-    ) as err:
-
-        def unpack_msg(err: Exception) -> str:
-            args = getattr(err, 'args', None)
-            if args:
-                return args
-            else:
-                return str(err)
-
-        # could be more specific so let's check if it's just perms.
-        if err.args:
-            errs = err.args
-            for err in errs:
-                msg = unpack_msg(err)
-                if 'PermissionError' in msg:
-                    raise DockerException('You dint run as root yo!')
-
-                elif 'FileNotFoundError' in msg:
-                    raise DockerNotStarted('Did you start da service sister?')
-
-        # not perms?
-        raise
-
-    finally:
-        if client:
-            client.close()
-
-
-class Container:
-    '''
-    Wrapper around a ``docker.models.containers.Container`` to include
-    log capture and relay through our native logging system and helper
-    method(s) for cancellation/teardown.
-
-    '''
-    def __init__(
-        self,
-        cntr: DockerContainer,
-    ) -> None:
-
-        self.cntr = cntr
-        # log msg de-duplication
-        self.seen_so_far = set()
-
-    async def process_logs_until(
-        self,
-        patt: str,
-        bp_on_msg: bool = False,
-    ) -> bool:
-        '''
-        Attempt to capture container log messages and relay through our
-        native logging system.
-
-        '''
-        seen_so_far = self.seen_so_far
-
-        while True:
-            logs = self.cntr.logs()
-            entries = logs.decode().split('\n')
-            for entry in entries:
-
-                # ignore null lines
-                if not entry:
-                    continue
-
-                try:
-                    record = json.loads(entry.strip())
-                except json.JSONDecodeError:
-                    if 'Error' in entry:
-                        raise RuntimeError(entry)
-                    raise
-
-                msg = record['msg']
-                level = record['level']
-                if msg and entry not in seen_so_far:
-                    seen_so_far.add(entry)
-                    if bp_on_msg:
-                        await tractor.breakpoint()
-
-                    getattr(log, level, log.error)(f'{msg}')
-
-                    # print(f'level: {level}')
-                    if level in ('error', 'fatal'):
-                        raise ContainerError(msg)
-
-                if patt in msg:
-                    return True
-
-                # do a checkpoint so we don't block if cancelled B)
-                await trio.sleep(0.01)
-
-        return False
-
-    def try_signal(
-        self,
-        signal: str = 'SIGINT',
-
-    ) -> bool:
-        try:
-            # XXX: market store doesn't seem to shutdown nicely all the
-            # time with this (maybe because there are still open grpc
-            # connections?) noticably after client connections have been
-            # made or are in use/teardown. It works just fine if you
-            # just start and stop the container tho?..
-            log.cancel(f'SENDING {signal} to {self.cntr.id}')
-            self.cntr.kill(signal)
-            return True
-
-        except docker.errors.APIError as err:
-            if 'is not running' in err.explanation:
-                return False
-
-    async def cancel(
-        self,
-        stop_msg: str,
-    ) -> None:
-
-        cid = self.cntr.id
-        # first try a graceful cancel
-        log.cancel(
-            f'SIGINT cancelling container: {cid}\n'
-            f'waiting on stop msg: "{stop_msg}"'
-        )
-        self.try_signal('SIGINT')
-
-        start = time.time()
-        for _ in range(30):
-
-            with trio.move_on_after(0.5) as cs:
-                cs.shield = True
-                await self.process_logs_until(stop_msg)
-
-                # if we aren't cancelled on above checkpoint then we
-                # assume we read the expected stop msg and terminated.
-                break
-
-            try:
-                log.info(f'Polling for container shutdown:\n{cid}')
-
-                if self.cntr.status not in {'exited', 'not-running'}:
-                    self.cntr.wait(
-                        timeout=0.1,
-                        condition='not-running',
-                    )
-
-                break
-
-            except (
-                ReadTimeout,
-            ):
-                log.info(f'Still waiting on container:\n{cid}')
-                continue
-
-            except (
-                docker.errors.APIError,
-                ConnectionError,
-            ):
-                log.exception('Docker connection failure')
-                break
-        else:
-            delay = time.time() - start
-            log.error(
-                f'Failed to kill container {cid} after {delay}s\n'
-                'sending SIGKILL..'
-            )
-            # get out the big guns, bc apparently marketstore
-            # doesn't actually know how to terminate gracefully
-            # :eyeroll:...
-            self.try_signal('SIGKILL')
-            self.cntr.wait(
-                timeout=3,
-                condition='not-running',
-            )
-
-        log.cancel(f'Container stopped: {cid}')
-
-
-@tractor.context
-async def open_ahabd(
-    ctx: tractor.Context,
-    endpoint: str,  # ns-pointer str-msg-type
-
-    **kwargs,
-
-) -> None:
-    get_console_log('info', name=__name__)
-
-    async with open_docker() as client:
-
-        # TODO: eventually offer a config-oriented API to do the mounts,
-        # params, etc. passing to ``Containter.run()``?
-        # call into endpoint for container config/init
-        ep_func = NamespacePath(endpoint).load_ref()
-        (
-            dcntr,
-            cntr_config,
-            start_msg,
-            stop_msg,
-        ) = ep_func(client)
-        cntr = Container(dcntr)
-
-        with trio.move_on_after(1):
-            found = await cntr.process_logs_until(start_msg)
-
-            if not found and cntr not in client.containers.list():
-                raise RuntimeError(
-                    'Failed to start `marketstore` check logs deats'
-                )
-
-        await ctx.started((
-            cntr.cntr.id,
-            os.getpid(),
-            cntr_config,
-        ))
-
-        try:
-
-            # TODO: we might eventually want a proxy-style msg-prot here
-            # to allow remote control of containers without needing
-            # callers to have root perms?
-            await trio.sleep_forever()
-
-        finally:
-            with trio.CancelScope(shield=True):
-                await cntr.cancel(stop_msg)
-
-
-async def start_ahab(
-    service_name: str,
-    endpoint: Callable[docker.DockerClient, DockerContainer],
-    task_status: TaskStatus[
-        tuple[
-            trio.Event,
-            dict[str, Any],
-        ],
-    ] = trio.TASK_STATUS_IGNORED,
-
-) -> None:
-    '''
-    Start a ``docker`` container supervisor with given service name.
-
-    Currently the actor calling this task should normally be started
-    with root permissions (until we decide to use something that doesn't
-    require this, like docker's rootless mode or some wrapper project) but
-    te root perms are de-escalated after the docker supervisor sub-actor
-    is started.
-
-    '''
-    cn_ready = trio.Event()
-    try:
-        async with tractor.open_nursery(
-            loglevel='runtime',
-        ) as tn:
-
-            portal = await tn.start_actor(
-                service_name,
-                enable_modules=[__name__]
-            )
-
-            # TODO: we have issues with this on teardown
-            # where ``tractor`` tries to issue ``os.kill()``
-            # and hits perms errors since the root process
-            # doesn't any longer have root perms..
-
-            # de-escalate root perms to the original user
-            # after the docker supervisor actor is spawned.
-            if config._parent_user:
-                import pwd
-                os.setuid(
-                    pwd.getpwnam(
-                        config._parent_user
-                    )[2]  # named user's uid
-                )
-
-            async with portal.open_context(
-                open_ahabd,
-                endpoint=str(NamespacePath.from_ref(endpoint)),
-            ) as (ctx, first):
-
-                cid, pid, cntr_config = first
-
-                task_status.started((
-                    cn_ready,
-                    cntr_config,
-                    (cid, pid),
-                ))
-
-                await trio.sleep_forever()
-
-    # since we demoted root perms in this parent
-    # we'll get a perms error on proc cleanup in
-    # ``tractor`` nursery exit. just make sure
-    # the child is terminated and don't raise the
-    # error if so.
-
-    # TODO: we could also consider adding
-    # a ``tractor.ZombieDetected`` or something that we could raise
-    # if we find the child didn't terminate.
-    except PermissionError:
-        log.warning('Failed to cancel root permsed container')
-
-    except (
-        trio.MultiError,
-    ) as err:
-        for subexc in err.exceptions:
-            if isinstance(subexc, PermissionError):
-                log.warning('Failed to cancel root perms-ed container')
-                return
-        else:
-            raise

piker/data/_normalize.py

@@ -14,67 +14,27 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-'''
+"""
 Stream format enforcement.
+"""
 
-'''
+from typing import AsyncIterator, Optional, Tuple
-from itertools import chain
+
-from typing import AsyncIterator
+import numpy as np
 
 
 def iterticks(
     quote: dict,
-    types: tuple[str] = (
+    types: Tuple[str] = ('trade', 'dark_trade'),
-        'trade',
-        'dark_trade',
-    ),
-    deduplicate_darks: bool = False,
 
 ) -> AsyncIterator:
     '''
     Iterate through ticks delivered per quote cycle.
 
     '''
-    if deduplicate_darks:
-        assert 'dark_trade' in types
-
     # print(f"{quote}\n\n")
     ticks = quote.get('ticks', ())
-    trades = {}
-    darks = {}
-
     if ticks:
-
-        # do a first pass and attempt to remove duplicate dark
-        # trades with the same tick signature.
-        if deduplicate_darks:
-            for tick in ticks:
-                ttype = tick.get('type')
-
-                time = tick.get('time', None)
-                if time:
-                    sig = (
-                        time,
-                        tick['price'],
-                        tick['size']
-                    )
-
-                    if ttype == 'dark_trade':
-                        darks[sig] = tick
-
-                    elif ttype == 'trade':
-                        trades[sig] = tick
-
-            # filter duplicates
-            for sig, tick in trades.items():
-                tick = darks.pop(sig, None)
-                if tick:
-                    ticks.remove(tick)
-                    # print(f'DUPLICATE {tick}')
-
-            # re-insert ticks
-            ticks.extend(list(chain(trades.values(), darks.values())))
-
         for tick in ticks:
             # print(f"{quote['symbol']}: {tick}")
             ttype = tick.get('type')

piker/data/_sampling.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of pikers)
+# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,57 +15,40 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
-Sampling and broadcast machinery for (soft) real-time delivery of
+Data buffers for fast shared humpy.
-financial data flows.
-
 """
-from __future__ import annotations
-from collections import Counter
 import time
-from typing import TYPE_CHECKING, Optional, Union
+from typing import Dict, List
 
 import tractor
 import trio
 from trio_typing import TaskStatus
 
+from ._sharedmem import ShmArray
 from ..log import get_logger
 
-if TYPE_CHECKING:
-    from ._sharedmem import ShmArray
-    from .feed import _FeedsBus
 
 log = get_logger(__name__)
 
 
-class sampler:
-    '''
-    Global sampling engine registry.
-
-    Manages state for sampling events, shm incrementing and
-    sample period logic.
-
-    '''
 # TODO: we could stick these in a composed type to avoid
 # angering the "i hate module scoped variables crowd" (yawn).
-    ohlcv_shms: dict[int, list[ShmArray]] = {}
+_shms: Dict[int, List[ShmArray]] = {}
+_start_increment: Dict[str, trio.Event] = {}
+_incrementers: Dict[int, trio.CancelScope] = {}
+_subscribers: Dict[str, tractor.Context] = {}
 
-    # holds one-task-per-sample-period tasks which are spawned as-needed by
-    # data feed requests with a given detected time step usually from
-    # history loading.
-    incrementers: dict[int, trio.CancelScope] = {}
 
-    # holds all the ``tractor.Context`` remote subscriptions for
+def shm_incrementing(shm_token_name: str) -> trio.Event:
-    # a particular sample period increment event: all subscribers are
+    global _start_increment
-    # notified on a step.
+    return _start_increment.setdefault(shm_token_name, trio.Event())
-    subscribers: dict[int, tractor.Context] = {}
 
 
 async def increment_ohlc_buffer(
     delay_s: int,
     task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
 ):
-    '''
+    """Task which inserts new bars into the provide shared memory array
-    Task which inserts new bars into the provide shared memory array
     every ``delay_s`` seconds.
 
     This task fulfills 2 purposes:
@@ -76,8 +59,8 @@ async def increment_ohlc_buffer(
 
     Note that if **no** actor has initiated this task then **none** of
     the underlying buffers will actually be incremented.
+    """
 
-    '''
     # # wait for brokerd to signal we should start sampling
     # await shm_incrementing(shm_token['shm_name']).wait()
 
@@ -86,18 +69,19 @@ async def increment_ohlc_buffer(
     # to solve this is to make this task aware of the instrument's
     # tradable hours?
 
+    global _incrementers
+
     # adjust delay to compensate for trio processing time
-    ad = min(sampler.ohlcv_shms.keys()) - 0.001
+    ad = min(_shms.keys()) - 0.001
 
     total_s = 0  # total seconds counted
-    lowest = min(sampler.ohlcv_shms.keys())
+    lowest = min(_shms.keys())
-    lowest_shm = sampler.ohlcv_shms[lowest][0]
     ad = lowest - 0.001
 
     with trio.CancelScope() as cs:
 
         # register this time period step as active
-        sampler.incrementers[delay_s] = cs
+        _incrementers[delay_s] = cs
         task_status.started(cs)
 
         while True:
@@ -107,10 +91,8 @@ async def increment_ohlc_buffer(
             total_s += lowest
 
             # increment all subscribed shm arrays
-            # TODO:
+            # TODO: this in ``numba``
-            # - this in ``numba``
+            for delay_s, shms in _shms.items():
-            # - just lookup shms for this step instead of iterating?
-            for delay_s, shms in sampler.ohlcv_shms.items():
                 if total_s % delay_s != 0:
                     continue
 
@@ -135,107 +117,65 @@ async def increment_ohlc_buffer(
                     # write to the buffer
                     shm.push(last)
 
-            await broadcast(delay_s, shm=lowest_shm)
+                # broadcast the buffer index step
+                subs = _subscribers.get(delay_s, ())
 
-
+                for ctx in subs:
-async def broadcast(
-    delay_s: int,
-    shm: Optional[ShmArray] = None,
-
-) -> None:
-    '''
-    Broadcast the given ``shm: ShmArray``'s buffer index step to any
-    subscribers for a given sample period.
-
-    The sent msg will include the first and last index which slice into
-    the buffer's non-empty data.
-
-    '''
-    subs = sampler.subscribers.get(delay_s, ())
-
-    first = last = -1
-
-    if shm is None:
-        periods = sampler.ohlcv_shms.keys()
-        # if this is an update triggered by a history update there
-        # might not actually be any sampling bus setup since there's
-        # no "live feed" active yet.
-        if periods:
-            lowest = min(periods)
-            shm = sampler.ohlcv_shms[lowest][0]
-            first = shm._first.value
-            last = shm._last.value
-
-    for stream in subs:
                     try:
-            await stream.send({
+                        await ctx.send_yield({'index': shm._last.value})
-                'first': first,
-                'last': last,
-                'index': last,
-            })
                     except (
                         trio.BrokenResourceError,
                         trio.ClosedResourceError
                     ):
-            log.error(
+                        log.error(f'{ctx.chan.uid} dropped connection')
-                f'{stream._ctx.chan.uid} dropped connection'
+                        subs.remove(ctx)
-            )
-            try:
-                subs.remove(stream)
-            except ValueError:
-                log.warning(
-                    f'{stream._ctx.chan.uid} sub already removed!?'
-                )
 
 
-@tractor.context
+@tractor.stream
 async def iter_ohlc_periods(
     ctx: tractor.Context,
     delay_s: int,
 
 ) -> None:
-    '''
+    """
     Subscribe to OHLC sampling "step" events: when the time
     aggregation period increments, this event stream emits an index
     event.
 
-    '''
+    """
     # add our subscription
-    subs = sampler.subscribers.setdefault(delay_s, [])
+    global _subscribers
-    await ctx.started()
+    subs = _subscribers.setdefault(delay_s, [])
-    async with ctx.open_stream() as stream:
+    subs.append(ctx)
-        subs.append(stream)
 
     try:
         # stream and block until cancelled
         await trio.sleep_forever()
     finally:
         try:
-                subs.remove(stream)
+            subs.remove(ctx)
         except ValueError:
             log.error(
-                    f'iOHLC step stream was already dropped {ctx.chan.uid}?'
+                f'iOHLC step stream was already dropped for {ctx.chan.uid}?'
             )
 
 
 async def sample_and_broadcast(
 
-    bus: _FeedsBus,  # noqa
+    bus: '_FeedsBus',  # noqa
     shm: ShmArray,
     quote_stream: trio.abc.ReceiveChannel,
-    brokername: str,
     sum_tick_vlm: bool = True,
 
 ) -> None:
 
     log.info("Started shared mem bar writer")
 
-    overruns = Counter()
-
     # iterate stream delivered by broker
     async for quotes in quote_stream:
         # TODO: ``numba`` this!
-        for broker_symbol, quote in quotes.items():
+        for sym, quote in quotes.items():
+
             # TODO: in theory you can send the IPC msg *before* writing
             # to the sharedmem array to decrease latency, however, that
             # will require at least some way to prevent task switching
@@ -299,22 +239,10 @@ async def sample_and_broadcast(
             # end up triggering backpressure which which will
             # eventually block this producer end of the feed and
             # thus other consumers still attached.
-            subs: list[
+            subs = bus._subscribers[sym.lower()]
-                tuple[
-                    Union[tractor.MsgStream, trio.MemorySendChannel],
-                    tractor.Context,
-                    Optional[float],  # tick throttle in Hz
-                ]
-            ] = bus._subscribers[broker_symbol.lower()]
 
-            # NOTE: by default the broker backend doesn't append
+            lags = 0
-            # it's own "name" into the fqsn schema (but maybe it
+            for (stream, tick_throttle) in subs:
-            # should?) so we have to manually generate the correct
-            # key here.
-            bsym = f'{broker_symbol}.{brokername}'
-            lags: int = 0
-
-            for (stream, ctx, tick_throttle) in subs:
 
                 try:
                     with trio.move_on_after(0.2) as cs:
@@ -322,49 +250,21 @@ async def sample_and_broadcast(
                             # this is a send mem chan that likely
                             # pushes to the ``uniform_rate_send()`` below.
                             try:
-                                stream.send_nowait(
+                                stream.send_nowait((sym, quote))
-                                    (bsym, quote)
-                                )
                             except trio.WouldBlock:
-                                chan = ctx.chan
+                                ctx = getattr(sream, '_ctx', None)
                                 if ctx:
                                     log.warning(
                                         f'Feed overrun {bus.brokername} ->'
-                                        f'{chan.uid} !!!'
+                                        f'{ctx.channel.uid} !!!'
                                     )
                                 else:
-                                    key = id(stream)
-                                    overruns[key] += 1
                                     log.warning(
-                                        f'Feed overrun {broker_symbol}'
+                                        f'Feed overrun {bus.brokername} -> '
-                                        '@{bus.brokername} -> '
                                         f'feed @ {tick_throttle} Hz'
                                     )
-                                    if overruns[key] > 6:
-                                        # TODO: should we check for the
-                                        # context being cancelled? this
-                                        # could happen but the
-                                        # channel-ipc-pipe is still up.
-                                        if not chan.connected():
-                                            log.warning(
-                                                'Dropping broken consumer:\n'
-                                                f'{broker_symbol}:'
-                                                f'{ctx.cid}@{chan.uid}'
-                                            )
-                                            await stream.aclose()
-                                            raise trio.BrokenResourceError
                         else:
-                                            log.warning(
+                            await stream.send({sym: quote})
-                                                'Feed getting overrun bro!\n'
-                                                f'{broker_symbol}:'
-                                                f'{ctx.cid}@{chan.uid}'
-                                            )
-                                            continue
-
-                        else:
-                            await stream.send(
-                                {bsym: quote}
-                            )
 
                     if cs.cancelled_caught:
                         lags += 1
@@ -376,29 +276,21 @@ async def sample_and_broadcast(
                     trio.ClosedResourceError,
                     trio.EndOfChannel,
                 ):
-                    chan = ctx.chan
+                    ctx = getattr(stream, '_ctx', None)
                     if ctx:
                         log.warning(
-                            'Dropped `brokerd`-quotes-feed connection:\n'
+                            f'{ctx.chan.uid} dropped  '
-                            f'{broker_symbol}:'
+                            '`brokerd`-quotes-feed connection'
-                            f'{ctx.cid}@{chan.uid}'
                         )
                     if tick_throttle:
-                        assert stream._closed
+                        assert stream.closed()
 
                     # XXX: do we need to deregister here
                     # if it's done in the fee bus code?
                     # so far seems like no since this should all
                     # be single-threaded. Doing it anyway though
                     # since there seems to be some kinda race..
-                    try:
                     subs.remove((stream, tick_throttle))
-                    except ValueError:
-                        log.error(
-                            f'Stream was already removed from subs!?\n'
-                            f'{broker_symbol}:'
-                            f'{ctx.cid}@{chan.uid}'
-                        )
 
 
 # TODO: a less naive throttler, here's some snippets:
@@ -411,8 +303,6 @@ async def uniform_rate_send(
     quote_stream: trio.abc.ReceiveChannel,
     stream: tractor.MsgStream,
 
-    task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
-
 ) -> None:
 
     # TODO: compute the approx overhead latency per cycle
@@ -423,8 +313,6 @@ async def uniform_rate_send(
     last_send = time.time()
     diff = 0
 
-    task_status.started()
-
     while True:
 
         # compute the remaining time to sleep for this throttled cycle
@@ -432,12 +320,7 @@ async def uniform_rate_send(
 
         if left_to_sleep > 0:
             with trio.move_on_after(left_to_sleep) as cs:
-                try:
                 sym, last_quote = await quote_stream.receive()
-                except trio.EndOfChannel:
-                    log.exception(f"feed for {stream} ended?")
-                    break
-
                 diff = time.time() - last_send
 
                 if not first_quote:
@@ -488,7 +371,7 @@ async def uniform_rate_send(
 
             # we have a quote already so send it now.
 
-        # measured_rate = 1 / (time.time() - last_send)
+        measured_rate = 1 / (time.time() - last_send)
         # log.info(
         #     f'`{sym}` throttled send hz: {round(measured_rate, ndigits=1)}'
         # )
@@ -497,20 +380,10 @@ async def uniform_rate_send(
         # rate timing exactly lul
         try:
             await stream.send({sym: first_quote})
-        except (
+        except trio.ClosedResourceError:
-            # NOTE: any of these can be raised by ``tractor``'s IPC
-            # transport-layer and we want to be highly resilient
-            # to consumers which crash or lose network connection.
-            # I.e. we **DO NOT** want to crash and propagate up to
-            # ``pikerd`` these kinds of errors!
-            trio.ClosedResourceError,
-            trio.BrokenResourceError,
-            ConnectionResetError,
-        ):
             # if the feed consumer goes down then drop
             # out of this rate limiter
             log.warning(f'{stream} closed')
-            await stream.aclose()
             return
 
         # reset send cycle state

piker/data/_sharedmem.py

@@ -18,19 +18,17 @@
 NumPy compatible shared memory buffers for real-time IPC streaming.
 
 """
-from __future__ import annotations
+from dataclasses import dataclass, asdict
 from sys import byteorder
-import time
+from typing import List, Tuple, Optional
-from typing import Optional
 from multiprocessing.shared_memory import SharedMemory, _USE_POSIX
+from multiprocessing import resource_tracker as mantracker
 
 if _USE_POSIX:
     from _posixshmem import shm_unlink
 
 import tractor
 import numpy as np
-from pydantic import BaseModel
-from numpy.lib import recfunctions as rfn
 
 from ..log import get_logger
 from ._source import base_iohlc_dtype
@@ -39,19 +37,6 @@ from ._source import base_iohlc_dtype
 log = get_logger(__name__)
 
 
-# how  much is probably dependent on lifestyle
-_secs_in_day = int(60 * 60 * 24)
-# we try for a buncha times, but only on a run-every-other-day kinda week.
-_days_worth = 16
-_default_size = _days_worth * _secs_in_day
-# where to start the new data append index
-_rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
-
-
-def cuckoff_mantracker():
-
-    from multiprocessing import resource_tracker as mantracker
-
 # Tell the "resource tracker" thing to fuck off.
 class ManTracker(mantracker.ResourceTracker):
     def register(self, name, rtype):
@@ -63,19 +48,17 @@ def cuckoff_mantracker():
     def ensure_running(self):
         pass
 
+
 # "know your land and know your prey"
 # https://www.dailymotion.com/video/x6ozzco
 mantracker._resource_tracker = ManTracker()
 mantracker.register = mantracker._resource_tracker.register
 mantracker.ensure_running = mantracker._resource_tracker.ensure_running
-    # ensure_running = mantracker._resource_tracker.ensure_running
+ensure_running = mantracker._resource_tracker.ensure_running
 mantracker.unregister = mantracker._resource_tracker.unregister
 mantracker.getfd = mantracker._resource_tracker.getfd
 
 
-cuckoff_mantracker()
-
-
 class SharedInt:
     """Wrapper around a single entry shared memory array which
     holds an ``int`` value used as an index counter.
@@ -99,42 +82,29 @@ class SharedInt:
         if _USE_POSIX:
             # We manually unlink to bypass all the "resource tracker"
             # nonsense meant for non-SC systems.
-            name = self._shm.name
+            shm_unlink(self._shm.name)
-            try:
-                shm_unlink(name)
-            except FileNotFoundError:
-                # might be a teardown race here?
-                log.warning(f'Shm for {name} already unlinked?')
 
 
-class _Token(BaseModel):
+@dataclass
-    '''
+class _Token:
-    Internal represenation of a shared memory "token"
+    """Internal represenation of a shared memory "token"
     which can be used to key a system wide post shm entry.
-
+    """
-    '''
-    class Config:
-        frozen = True
-
     shm_name: str  # this servers as a "key" value
     shm_first_index_name: str
     shm_last_index_name: str
-    dtype_descr: tuple
+    dtype_descr: List[Tuple[str]]
 
-    @property
+    def __post_init__(self):
-    def dtype(self) -> np.dtype:
+        # np.array requires a list for dtype
-        return np.dtype(list(map(tuple, self.dtype_descr))).descr
+        self.dtype_descr = np.dtype(list(map(tuple, self.dtype_descr))).descr
 
     def as_msg(self):
-        return self.dict()
+        return asdict(self)
 
     @classmethod
-    def from_msg(cls, msg: dict) -> _Token:
+    def from_msg(self, msg: dict) -> '_Token':
-        if isinstance(msg, _Token):
+        return msg if isinstance(msg, _Token) else _Token(**msg)
-            return msg
-
-        msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
-        return _Token(**msg)
 
 
 # TODO: this api?
@@ -157,23 +127,20 @@ def _make_token(
     key: str,
     dtype: Optional[np.dtype] = None,
 ) -> _Token:
-    '''
+    """Create a serializable token that can be used
-    Create a serializable token that can be used
     to access a shared array.
-
+    """
-    '''
     dtype = base_iohlc_dtype if dtype is None else dtype
     return _Token(
-        shm_name=key,
+        key,
-        shm_first_index_name=key + "_first",
+        key + "_first",
-        shm_last_index_name=key + "_last",
+        key + "_last",
-        dtype_descr=np.dtype(dtype).descr
+        np.dtype(dtype).descr
     )
 
 
 class ShmArray:
-    '''
+    """A shared memory ``numpy`` (compatible) array API.
-    A shared memory ``numpy`` (compatible) array API.
 
     An underlying shared memory buffer is allocated based on
     a user specified ``numpy.ndarray``. This fixed size array
@@ -183,7 +150,7 @@ class ShmArray:
     ``SharedInt`` interfaces) values such that multiple processes can
     interact with the same array using a synchronized-index.
 
-    '''
+    """
     def __init__(
         self,
         shmarr: np.ndarray,
@@ -204,21 +171,17 @@ class ShmArray:
         self._post_init: bool = False
 
         # pushing data does not write the index (aka primary key)
-        dtype = shmarr.dtype
-        if dtype.fields:
         self._write_fields = list(shmarr.dtype.fields.keys())[1:]
-        else:
-            self._write_fields = None
 
     # TODO: ringbuf api?
 
     @property
     def _token(self) -> _Token:
         return _Token(
-            shm_name=self._shm.name,
+            self._shm.name,
-            shm_first_index_name=self._first._shm.name,
+            self._first._shm.name,
-            shm_last_index_name=self._last._shm.name,
+            self._last._shm.name,
-            dtype_descr=tuple(self._array.dtype.descr),
+            self._array.dtype.descr,
         )
 
     @property
@@ -234,8 +197,7 @@ class ShmArray:
 
     @property
     def array(self) -> np.ndarray:
-        '''
+        '''Return an up-to-date ``np.ndarray`` view of the
-        Return an up-to-date ``np.ndarray`` view of the
         so-far-written data to the underlying shm buffer.
 
         '''
@@ -254,129 +216,62 @@ class ShmArray:
 
         return a
 
-    def ustruct(
-        self,
-        fields: Optional[list[str]] = None,
-
-        # type that all field values will be cast to
-        # in the returned view.
-        common_dtype: np.dtype = np.float,
-
-    ) -> np.ndarray:
-
-        array = self._array
-
-        if fields:
-            selection = array[fields]
-            # fcount = len(fields)
-        else:
-            selection = array
-            # fcount = len(array.dtype.fields)
-
-        # XXX: manual ``.view()`` attempt that also doesn't work.
-        # uview = selection.view(
-        #     dtype='<f16',
-        # ).reshape(-1, 4, order='A')
-
-        # assert len(selection) == len(uview)
-
-        u = rfn.structured_to_unstructured(
-            selection,
-            # dtype=float,
-            copy=True,
-        )
-
-        # unstruct = np.ndarray(u.shape, dtype=a.dtype, buffer=shm.buf)
-        # array[:] = a[:]
-        return u
-        # return ShmArray(
-        #     shmarr=u,
-        #     first=self._first,
-        #     last=self._last,
-        #     shm=self._shm
-        # )
-
     def last(
         self,
         length: int = 1,
-
     ) -> np.ndarray:
-        '''
-        Return the last ``length``'s worth of ("row") entries from the
-        array.
-
-        '''
         return self.array[-length:]
 
     def push(
         self,
         data: np.ndarray,
 
-        field_map: Optional[dict[str, str]] = None,
         prepend: bool = False,
-        update_first: bool = True,
         start: Optional[int] = None,
 
     ) -> int:
-        '''
+        '''Ring buffer like "push" to append data
-        Ring buffer like "push" to append data
         into the buffer and return updated "last" index.
 
         NB: no actual ring logic yet to give a "loop around" on overflow
         condition, lel.
-
         '''
+        self._post_init = True
         length = len(data)
+        index = start or self._last.value
 
         if prepend:
-            index = (start or self._first.value) - length
+            index = self._first.value - length
 
             if index < 0:
                 raise ValueError(
                     f'Array size of {self._len} was overrun during prepend.\n'
-                    f'You have passed {abs(index)} too many datums.'
+                    'You have passed {abs(index)} too many datums.'
                 )
 
-        else:
-            index = start if start is not None else self._last.value
-
         end = index + length
 
-        if field_map:
+        fields = self._write_fields
-            src_names, dst_names = zip(*field_map.items())
-        else:
-            dst_names = src_names = self._write_fields
 
         try:
-            self._array[
+            self._array[fields][index:end] = data[fields][:]
-                list(dst_names)
-            ][index:end] = data[list(src_names)][:]
 
             # NOTE: there was a race here between updating
             # the first and last indices and when the next reader
             # tries to access ``.array`` (which due to the index
             # overlap will be empty). Pretty sure we've fixed it now
             # but leaving this here as a reminder.
-            if prepend and update_first and length:
+            if prepend:
                 assert index < self._first.value
 
-            if (
+            if index < self._first.value:
-                index < self._first.value
-                and update_first
-            ):
-                assert prepend, 'prepend=True not passed but index decreased?'
                 self._first.value = index
-
+            else:
-            elif not prepend:
                 self._last.value = end
 
-            self._post_init = True
             return end
 
         except ValueError as err:
-            if field_map:
-                raise
-
             # should raise if diff detected
             self.diff_err_fields(data)
             raise err
@@ -403,7 +298,6 @@ class ShmArray:
                 f"Input array has unknown field(s): {only_in_theirs}"
             )
 
-    # TODO: support "silent" prepends that don't update ._first.value?
     def prepend(
         self,
         data: np.ndarray,
@@ -430,6 +324,12 @@ class ShmArray:
         ...
 
 
+# how  much is probably dependent on lifestyle
+_secs_in_day = int(60 * 60 * 24)
+# we try for 3 times but only on a run-every-other-day kinda week.
+_default_size = 3 * _secs_in_day
+
+
 def open_shm_array(
 
     key: Optional[str] = None,
@@ -454,11 +354,7 @@ def open_shm_array(
         create=True,
         size=a.nbytes
     )
-    array = np.ndarray(
+    array = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf)
-        a.shape,
-        dtype=a.dtype,
-        buffer=shm.buf
-    )
     array[:] = a[:]
     array.setflags(write=int(not readonly))
 
@@ -484,24 +380,7 @@ def open_shm_array(
         )
     )
 
-    # start the "real-time" updated section after 3-days worth of 1s
+    last.value = first.value = int(_secs_in_day)
-    # sampled OHLC. this allows appending up to a days worth from
-    # tick/quote feeds before having to flush to a (tsdb) storage
-    # backend, and looks something like,
-    # -------------------------
-    # |              |        i
-    # _________________________
-    # <-------------> <------->
-    #  history         real-time
-    #
-    # Once fully "prepended", the history section will leave the
-    # ``ShmArray._start.value: int = 0`` and the yet-to-be written
-    # real-time section will start at ``ShmArray.index: int``.
-
-    # this sets the index to 3/4 of the length of the buffer
-    # leaving a "days worth of second samples" for the real-time
-    # section.
-    last.value = first.value = _rt_buffer_start
 
     shmarr = ShmArray(
         array,
@@ -523,48 +402,27 @@ def open_shm_array(
 
 
 def attach_shm_array(
-    token: tuple[str, str, tuple[str, str]],
+    token: Tuple[str, str, Tuple[str, str]],
     size: int = _default_size,
     readonly: bool = True,
-
 ) -> ShmArray:
-    '''
+    """Attach to an existing shared memory array previously
-    Attach to an existing shared memory array previously
     created by another process using ``open_shared_array``.
 
     No new shared mem is allocated but wrapper types for read/write
     access are constructed.
-
+    """
-    '''
     token = _Token.from_msg(token)
     key = token.shm_name
 
     if key in _known_tokens:
         assert _Token.from_msg(_known_tokens[key]) == token, "WTF"
 
-    # XXX: ugh, looks like due to the ``shm_open()`` C api we can't
-    # actually place files in a subdir, see discussion here:
-    # https://stackoverflow.com/a/11103289
-
     # attach to array buffer and view as per dtype
-    _err: Optional[Exception] = None
+    shm = SharedMemory(name=key)
-    for _ in range(3):
-        try:
-            shm = SharedMemory(
-                name=key,
-                create=False,
-            )
-            break
-        except OSError as oserr:
-            _err = oserr
-            time.sleep(0.1)
-    else:
-        if _err:
-            raise _err
-
     shmarr = np.ndarray(
         (size,),
-        dtype=token.dtype,
+        dtype=token.dtype_descr,
         buffer=shm.buf
     )
     shmarr.setflags(write=int(not readonly))
@@ -612,10 +470,8 @@ def maybe_open_shm_array(
     key: str,
     dtype: Optional[np.dtype] = None,
     **kwargs,
-
+) -> Tuple[ShmArray, bool]:
-) -> tuple[ShmArray, bool]:
+    """Attempt to attach to a shared memory block using a "key" lookup
-    '''
-    Attempt to attach to a shared memory block using a "key" lookup
     to registered blocks in the users overall "system" registry
     (presumes you don't have the block's explicit token).
 
@@ -629,8 +485,7 @@ def maybe_open_shm_array(
 
     If you know the explicit ``_Token`` for your memory segment instead
     use ``attach_shm_array``.
-
+    """
-    '''
     try:
         # see if we already know this key
         token = _known_tokens[key]

piker/data/_source.py

@@ -17,13 +17,12 @@
 """
 numpy data source coversion helpers.
 """
-from __future__ import annotations
+from typing import Dict, Any, List
-from typing import Any
 import decimal
 
-from bidict import bidict
 import numpy as np
-from pydantic import BaseModel
+import pandas as pd
+from pydantic import BaseModel, validate_arguments
 # from numba import from_dtype
 
 
@@ -33,7 +32,7 @@ ohlc_fields = [
     ('high', float),
     ('low', float),
     ('close', float),
-    ('volume', float),
+    ('volume', int),
     ('bar_wap', float),
 ]
 
@@ -48,29 +47,16 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
 # https://github.com/numba/numba/issues/4511
 # numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
 
-# map time frame "keys" to seconds values
+# map time frame "keys" to minutes values
-tf_in_1s = bidict({
+tf_in_1m = {
-    1: '1s',
+    '1m': 1,
-    60: '1m',
+    '5m':  5,
-    60*5: '5m',
+    '15m': 15,
-    60*15: '15m',
+    '30m':  30,
-    60*30: '30m',
+    '1h': 60,
-    60*60: '1h',
+    '4h': 240,
-    60*60*24: '1d',
+    '1d': 1440,
-})
+}
-
-
-def mk_fqsn(
-    provider: str,
-    symbol: str,
-
-) -> str:
-    '''
-    Generate a "fully qualified symbol name" which is
-    a reverse-hierarchical cross broker/provider symbol
-
-    '''
-    return '.'.join([symbol, provider]).lower()
 
 
 def float_digits(
@@ -92,113 +78,31 @@ def ohlc_zeros(length: int) -> np.ndarray:
     return np.zeros(length, dtype=base_ohlc_dtype)
 
 
-def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
-    '''
-    Unpack a fully-qualified-symbol-name to ``tuple``.
-
-    '''
-    venue = ''
-    suffix = ''
-
-    # TODO: probably reverse the order of all this XD
-    tokens = fqsn.split('.')
-    if len(tokens) < 3:
-        # probably crypto
-        symbol, broker = tokens
-        return (
-            broker,
-            symbol,
-            '',
-        )
-
-    elif len(tokens) > 3:
-        symbol, venue, suffix, broker = tokens
-    else:
-        symbol, venue, broker = tokens
-        suffix = ''
-
-    # head, _, broker = fqsn.rpartition('.')
-    # symbol, _, suffix = head.rpartition('.')
-    return (
-        broker,
-        '.'.join([symbol, venue]),
-        suffix,
-    )
-
-
 class Symbol(BaseModel):
-    '''
+    """I guess this is some kinda container thing for dealing with
-    I guess this is some kinda container thing for dealing with
     all the different meta-data formats from brokers?
 
-    '''
+    Yah, i guess dats what it izz.
+    """
     key: str
-    tick_size: float = 0.01
+    type_key: str  # {'stock', 'forex', 'future', ... etc.}
-    lot_tick_size: float = 0.0  # "volume" precision as min step value
+    tick_size: float
-    tick_size_digits: int = 2
+    lot_tick_size: float  # "volume" precision as min step value
-    lot_size_digits: int = 0
+    tick_size_digits: int
-    suffix: str = ''
+    lot_size_digits: int
-    broker_info: dict[str, dict[str, Any]] = {}
+    broker_info: Dict[str, Dict[str, Any]] = {}
 
     # specifies a "class" of financial instrument
     # ex. stock, futer, option, bond etc.
 
-    # @validate_arguments
-    @classmethod
-    def from_broker_info(
-        cls,
-        broker: str,
-        symbol: str,
-        info: dict[str, Any],
-        suffix: str = '',
-
-    # XXX: like wtf..
-    # ) -> 'Symbol':
-    ) -> None:
-
-        tick_size = info.get('price_tick_size', 0.01)
-        lot_tick_size = info.get('lot_tick_size', 0.0)
-
-        return Symbol(
-            key=symbol,
-            tick_size=tick_size,
-            lot_tick_size=lot_tick_size,
-            tick_size_digits=float_digits(tick_size),
-            lot_size_digits=float_digits(lot_tick_size),
-            suffix=suffix,
-            broker_info={broker: info},
-        )
-
-    @classmethod
-    def from_fqsn(
-        cls,
-        fqsn: str,
-        info: dict[str, Any],
-
-    # XXX: like wtf..
-    # ) -> 'Symbol':
-    ) -> None:
-        broker, key, suffix = unpack_fqsn(fqsn)
-        return cls.from_broker_info(
-            broker,
-            key,
-            info=info,
-            suffix=suffix,
-        )
-
     @property
-    def type_key(self) -> str:
+    def brokers(self) -> List[str]:
-        return list(self.broker_info.values())[0]['asset_type']
-
-    @property
-    def brokers(self) -> list[str]:
         return list(self.broker_info.keys())
 
     def nearest_tick(self, value: float) -> float:
-        '''
+        """Return the nearest tick value based on mininum increment.
-        Return the nearest tick value based on mininum increment.
 
-        '''
+        """
         mult = 1 / self.tick_size
         return round(value * mult) / mult
 
@@ -214,44 +118,85 @@ class Symbol(BaseModel):
             self.key,
         )
 
-    def tokens(self) -> tuple[str]:
-        broker, key = self.front_feed()
-        if self.suffix:
-            return (key, self.suffix, broker)
-        else:
-            return (key, broker)
 
-    def front_fqsn(self) -> str:
+@validate_arguments
-        '''
+def mk_symbol(
-        fqsn = "fully qualified symbol name"
 
-        Basically the idea here is for all client-ish code (aka programs/actors
+    key: str,
-        that ask the provider agnostic layers in the stack for data) should be
+    type_key: str,
-        able to tell which backend / venue / derivative each data feed/flow is
+    tick_size: float = 0.01,
-        from by an explicit string key of the current form:
+    lot_tick_size: float = 0,
+    broker_info: dict[str, Any] = {},
 
-        <instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
+) -> Symbol:
-
+    '''Create and return an instrument description for the
-        TODO: I have thoughts that we should actually change this to be
+    "symbol" named as ``key``.
-        more like an "attr lookup" (like how the web should have done
-        urls, but marketting peeps ruined it etc. etc.):
-
-        <broker>.<venue>.<instrumentname>.<suffixwithmetadata>
 
     '''
-        tokens = self.tokens()
+    return Symbol(
-        fqsn = '.'.join(tokens)
+        key=key,
-        return fqsn
+        type_key=type_key,
+        tick_size=tick_size,
+        lot_tick_size=lot_tick_size,
+        tick_size_digits=float_digits(tick_size),
+        lot_size_digits=float_digits(lot_tick_size),
+        broker_info=broker_info,
+    )
 
-    def iterfqsns(self) -> list[str]:
-        keys = []
-        for broker in self.broker_info.keys():
-            fqsn = mk_fqsn(self.key, broker)
-            if self.suffix:
-                fqsn += f'.{self.suffix}'
-            keys.append(fqsn)
 
-        return keys
+def from_df(
+
+    df: pd.DataFrame,
+    source=None,
+    default_tf=None
+
+) -> np.recarray:
+    """Convert OHLC formatted ``pandas.DataFrame`` to ``numpy.recarray``.
+
+    """
+    df.reset_index(inplace=True)
+
+    # hackery to convert field names
+    date = 'Date'
+    if 'date' in df.columns:
+        date = 'date'
+
+    # convert to POSIX time
+    df[date] = [d.timestamp() for d in df[date]]
+
+    # try to rename from some camel case
+    columns = {
+        'Date': 'time',
+        'date': 'time',
+        'Open': 'open',
+        'High': 'high',
+        'Low': 'low',
+        'Close': 'close',
+        'Volume': 'volume',
+
+        # most feeds are providing this over sesssion anchored
+        'vwap': 'bar_wap',
+
+        # XXX: ib_insync calls this the "wap of the bar"
+        # but no clue what is actually is...
+        # https://github.com/pikers/piker/issues/119#issuecomment-729120988
+        'average': 'bar_wap',
+    }
+
+    df = df.rename(columns=columns)
+
+    for name in df.columns:
+        # if name not in base_ohlc_dtype.names[1:]:
+        if name not in base_ohlc_dtype.names:
+            del df[name]
+
+    # TODO: it turns out column access on recarrays is actually slower:
+    # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
+    # it might make sense to make these structured arrays?
+    array = df.to_records(index=False)
+    _nan_to_closest_num(array)
+
+    return array
 
 
 def _nan_to_closest_num(array: np.ndarray):

piker/data/_web_bs.py

@@ -53,13 +53,11 @@ class NoBsWs:
     def __init__(
         self,
         url: str,
-        token: str,
         stack: AsyncExitStack,
         fixture: Callable,
         serializer: ModuleType = json,
     ):
         self.url = url
-        self.token = token
         self.fixture = fixture
         self._stack = stack
         self._ws: 'WebSocketConnection' = None  # noqa
@@ -83,15 +81,9 @@ class NoBsWs:
                     trio_websocket.open_websocket_url(self.url)
                 )
                 # rerun user code fixture
-                if self.token == '':
                 ret = await self._stack.enter_async_context(
                     self.fixture(self)
                 )
-                else:
-                    ret = await self._stack.enter_async_context(
-                        self.fixture(self, self.token)
-                    )
-
                 assert ret is None
 
                 log.info(f'Connection success: {self.url}')
@@ -135,14 +127,12 @@ async def open_autorecon_ws(
 
     # TODO: proper type annot smh
     fixture: Callable,
-    # used for authenticated websockets
-    token: str = '',
 ) -> AsyncGenerator[tuple[...],  NoBsWs]:
     """Apparently we can QoS for all sorts of reasons..so catch em.
 
     """
     async with AsyncExitStack() as stack:
-        ws = NoBsWs(url, token, stack, fixture=fixture)
+        ws = NoBsWs(url, stack, fixture=fixture)
         await ws._connect()
 
         try:

piker/data/cli.py

@@ -16,34 +16,26 @@
 
 """
 marketstore cli.
-
 """
+from typing import List
 from functools import partial
 from pprint import pformat
 
-from anyio_marketstore import open_marketstore_client
 import trio
 import tractor
 import click
-import numpy as np
 
 from .marketstore import (
     get_client,
-    # stream_quotes,
+    stream_quotes,
     ingest_quote_stream,
-    # _url,
+    _url,
     _tick_tbk_ids,
     mk_tbk,
 )
 from ..cli import cli
 from .. import watchlists as wl
 from ..log import get_logger
-from ._sharedmem import (
-    maybe_open_shm_array,
-)
-from ._source import (
-    base_iohlc_dtype,
-)
 
 
 log = get_logger(__name__)
@@ -57,58 +49,51 @@ log = get_logger(__name__)
 )
 @click.argument('names', nargs=-1)
 @click.pass_obj
-def ms_stream(
+def ms_stream(config: dict, names: List[str], url: str):
-    config: dict,
+    """Connect to a marketstore time bucket stream for (a set of) symbols(s)
-    names: list[str],
-    url: str,
-) -> None:
-    '''
-    Connect to a marketstore time bucket stream for (a set of) symbols(s)
     and print to console.
-
+    """
-    '''
     async def main():
-        # async for quote in stream_quotes(symbols=names):
+        async for quote in stream_quotes(symbols=names):
-        #    log.info(f"Received quote:\n{quote}")
+            log.info(f"Received quote:\n{quote}")
-        ...
 
     trio.run(main)
 
 
-# @cli.command()
+@cli.command()
-# @click.option(
+@click.option(
-#     '--url',
+    '--url',
-#     default=_url,
+    default=_url,
-#     help='HTTP URL of marketstore instance'
+    help='HTTP URL of marketstore instance'
-# )
+)
-# @click.argument('names', nargs=-1)
+@click.argument('names', nargs=-1)
-# @click.pass_obj
+@click.pass_obj
-# def ms_destroy(config: dict, names: list[str], url: str) -> None:
+def ms_destroy(config: dict, names: List[str], url: str) -> None:
-#     """Destroy symbol entries in the local marketstore instance.
+    """Destroy symbol entries in the local marketstore instance.
-#     """
+    """
-#     async def main():
+    async def main():
-#         nonlocal names
+        nonlocal names
-#         async with get_client(url) as client:
+        async with get_client(url) as client:
-# 
+
-#             if not names:
+            if not names:
-#                 names = await client.list_symbols()
+                names = await client.list_symbols()
-# 
+
-#             # default is to wipe db entirely.
+            # default is to wipe db entirely.
-#             answer = input(
+            answer = input(
-#                 "This will entirely wipe you local marketstore db @ "
+                "This will entirely wipe you local marketstore db @ "
-#                 f"{url} of the following symbols:\n {pformat(names)}"
+                f"{url} of the following symbols:\n {pformat(names)}"
-#                 "\n\nDelete [N/y]?\n")
+                "\n\nDelete [N/y]?\n")
-# 
+
-#             if answer == 'y':
+            if answer == 'y':
-#                 for sym in names:
+                for sym in names:
-#                     # tbk = _tick_tbk.format(sym)
+                    # tbk = _tick_tbk.format(sym)
-#                     tbk = tuple(sym, *_tick_tbk_ids)
+                    tbk = tuple(sym, *_tick_tbk_ids)
-#                     print(f"Destroying {tbk}..")
+                    print(f"Destroying {tbk}..")
-#                     await client.destroy(mk_tbk(tbk))
+                    await client.destroy(mk_tbk(tbk))
-#             else:
+            else:
-#                 print("Nothing deleted.")
+                print("Nothing deleted.")
-# 
+
-#     tractor.run(main)
+    tractor.run(main)
 
 
 @cli.command()
@@ -117,53 +102,41 @@ def ms_stream(
     is_flag=True,
     help='Enable tractor logging')
 @click.option(
-    '--host',
+    '--url',
-    default='localhost'
+    default=_url,
+    help='HTTP URL of marketstore instance'
 )
-@click.option(
+@click.argument('name', nargs=1, required=True)
-    '--port',
-    default=5993
-)
-@click.argument('symbols', nargs=-1)
 @click.pass_obj
-def storesh(
+def ms_shell(config, name, tl, url):
-    config,
+    """Start an IPython shell ready to query the local marketstore db.
-    tl,
+    """
-    host,
-    port,
-    symbols: list[str],
-):
-    '''
-    Start an IPython shell ready to query the local marketstore db.
-
-    '''
-    from piker.data.marketstore import tsdb_history_update
-    from piker._daemon import open_piker_runtime
-
     async def main():
-        nonlocal symbols
+        async with get_client(url) as client:
+            query = client.query  # noqa
+            # TODO: write magics to query marketstore
+            from IPython import embed
+            embed()
 
-        async with open_piker_runtime(
+    tractor.run(main)
-            'storesh',
-            enable_modules=['piker.data._ahab'],
-        ):
-            symbol = symbols[0]
-            await tsdb_history_update(symbol)
-
-    trio.run(main)
 
 
 @cli.command()
 @click.option('--test-file', '-t', help='Test quote stream file')
 @click.option('--tl', is_flag=True, help='Enable tractor logging')
+@click.option('--tl', is_flag=True, help='Enable tractor logging')
+@click.option(
+    '--url',
+    default=_url,
+    help='HTTP URL of marketstore instance'
+)
 @click.argument('name', nargs=1, required=True)
 @click.pass_obj
-def ingest(config, name, test_file, tl):
+def ingest(config, name, test_file, tl, url):
-    '''
+    """Ingest real-time broker quotes and ticks to a marketstore instance.
-    Ingest real-time broker quotes and ticks to a marketstore instance.
+    """
-
-    '''
     # global opts
+    brokermod = config['brokermod']
     loglevel = config['loglevel']
     tractorloglevel = config['tractorloglevel']
     # log = config['log']
@@ -172,25 +145,15 @@ def ingest(config, name, test_file, tl):
     watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins)
     symbols = watchlists[name]
 
-    grouped_syms = {}
+    tractor.run(
-    for sym in symbols:
+        partial(
-        symbol, _, provider = sym.rpartition('.')
-        if provider not in grouped_syms:
-            grouped_syms[provider] = []
-
-        grouped_syms[provider].append(symbol)
-
-    async def entry_point():
-        async with tractor.open_nursery() as n:
-            for provider, symbols in grouped_syms.items(): 
-                await n.run_in_actor(
             ingest_quote_stream,
-                    name='ingest_marketstore',
+            symbols,
-                    symbols=symbols,
+            brokermod.name,
-                    brokername=provider,
             tries=1,
-                    actorloglevel=loglevel,
+            loglevel=loglevel,
-                    loglevel=tractorloglevel
+        ),
+        name='ingest_marketstore',
+        loglevel=tractorloglevel,
+        debug_mode=True,
     )
-
-    tractor.run(entry_point)

piker/data/marketstore.py

@@ -14,200 +14,36 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-'''
+"""
 ``marketstore`` integration.
 
 - client management routines
 - ticK data ingest routines
 - websocket client for subscribing to write triggers
 - todo: tick sequence stream-cloning for testing
-
+- todo: docker container management automation
-'''
+"""
-from __future__ import annotations
+from contextlib import asynccontextmanager
-from contextlib import asynccontextmanager as acm
+from typing import Dict, Any, List, Callable, Tuple
-from datetime import datetime
-from pprint import pformat
-from typing import (
-    Any,
-    Optional,
-    Union,
-    TYPE_CHECKING,
-)
 import time
 from math import isnan
 
-from bidict import bidict
 import msgpack
-import pyqtgraph as pg
 import numpy as np
+import pandas as pd
+import pymarketstore as pymkts
 import tractor
 from trio_websocket import open_websocket_url
-from anyio_marketstore import (
-    open_marketstore_client,
-    MarketstoreClient,
-    Params,
-)
-import pendulum
-import purerpc
 
-if TYPE_CHECKING:
-    import docker
-    from ._ahab import DockerContainer
-
-from .feed import maybe_open_feed
 from ..log import get_logger, get_console_log
+from ..data import open_feed
 
 
 log = get_logger(__name__)
 
-
+_tick_tbk_ids: Tuple[str, str] = ('1Sec', 'TICK')
-# container level config
-_config = {
-    'grpc_listen_port': 5995,
-    'ws_listen_port': 5993,
-    'log_level': 'debug',
-}
-
-_yaml_config = '''
-# piker's ``marketstore`` config.
-
-# mount this config using:
-# sudo docker run --mount \
-# type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
-# 5993:5993 alpacamarkets/marketstore:latest
-
-root_directory: data
-listen_port: {ws_listen_port}
-grpc_listen_port: {grpc_listen_port}
-log_level: {log_level}
-queryable: true
-stop_grace_period: 0
-wal_rotate_interval: 5
-stale_threshold: 5
-enable_add: true
-enable_remove: false
-
-triggers:
-  - module: ondiskagg.so
-    on: "*/1Sec/OHLCV"
-    config:
-        # filter: "nasdaq"
-        destinations:
-            - 1Min
-            - 5Min
-            - 15Min
-            - 1H
-            - 1D
-
-  - module: stream.so
-    on: '*/*/*'
-    # config:
-    #     filter: "nasdaq"
-
-'''.format(**_config)
-
-
-def start_marketstore(
-    client: docker.DockerClient,
-
-    **kwargs,
-
-) -> tuple[DockerContainer, dict[str, Any]]:
-    '''
-    Start and supervise a marketstore instance with its config bind-mounted
-    in from the piker config directory on the system.
-
-    The equivalent cli cmd to this code is:
-
-        sudo docker run --mount \
-        type=bind,source="$HOME/.config/piker/",target="/etc" -i -p \
-        5993:5993 alpacamarkets/marketstore:latest
-
-    '''
-    import os
-    import docker
-    from .. import config
-    get_console_log('info', name=__name__)
-
-    mktsdir = os.path.join(config._config_dir, 'marketstore')
-
-    # create when dne
-    if not os.path.isdir(mktsdir):
-        os.mkdir(mktsdir)
-
-    yml_file = os.path.join(mktsdir, 'mkts.yml')
-    if not os.path.isfile(yml_file):
-        log.warning(
-            f'No `marketstore` config exists?: {yml_file}\n'
-            'Generating new file from template:\n'
-            f'{_yaml_config}\n'
-        )
-        with open(yml_file, 'w') as yf:
-            yf.write(_yaml_config)
-
-    # create a mount from user's local piker config dir into container
-    config_dir_mnt = docker.types.Mount(
-        target='/etc',
-        source=mktsdir,
-        type='bind',
-    )
-
-    # create a user config subdir where the marketstore
-    # backing filesystem database can be persisted.
-    persistent_data_dir = os.path.join(
-        mktsdir, 'data',
-    )
-    if not os.path.isdir(persistent_data_dir):
-        os.mkdir(persistent_data_dir)
-
-    data_dir_mnt = docker.types.Mount(
-        target='/data',
-        source=persistent_data_dir,
-        type='bind',
-    )
-
-    dcntr: DockerContainer = client.containers.run(
-        'alpacamarkets/marketstore:latest',
-        # do we need this for cmds?
-        # '-i',
-
-        # '-p 5993:5993',
-        ports={
-            '5993/tcp': 5993,  # jsonrpc / ws?
-            '5995/tcp': 5995,  # grpc
-        },
-        mounts=[
-            config_dir_mnt,
-            data_dir_mnt,
-        ],
-        detach=True,
-        # stop_signal='SIGINT',
-        init=True,
-        # remove=True,
-    )
-    return (
-        dcntr,
-        _config,
-
-        # expected startup and stop msgs
-        "launching tcp listener for all services...",
-        "exiting...",
-    )
-
-
-_tick_tbk_ids: tuple[str, str] = ('1Sec', 'TICK')
 _tick_tbk: str = '{}/' + '/'.join(_tick_tbk_ids)
-
+_url: str = 'http://localhost:5993/rpc'
-_tick_dt = [
-    # these two are required for as a "primary key"
-    ('Epoch', 'i8'),
-    ('Nanoseconds', 'i4'),
-    ('IsTrade', 'i1'),
-    ('IsBid', 'i1'),
-    ('Price', 'f4'),
-    ('Size', 'f4')
-]
-
 _quote_dt = [
     # these two are required for as a "primary key"
     ('Epoch', 'i8'),
@@ -225,7 +61,6 @@ _quote_dt = [
     # ('brokerd_ts', 'i64'),
     # ('VWAP', 'f4')
 ]
-
 _quote_tmp = {}.fromkeys(dict(_quote_dt).keys(), np.nan)
 _tick_map = {
     'Up': 1,
@@ -234,52 +69,31 @@ _tick_map = {
     None: np.nan,
 }
 
-_ohlcv_dt = [
-    # these two are required for as a "primary key"
-    ('Epoch', 'i8'),
-    # ('Nanoseconds', 'i4'),
 
-    # ohlcv sampling
+class MarketStoreError(Exception):
-    ('Open', 'f4'),
+    "Generic marketstore client error"
-    ('High', 'f4'),
-    ('Low', 'f4'),
-    ('Close', 'f4'),
-    ('Volume', 'f4'),
-]
 
 
-ohlc_key_map = bidict({
+def err_on_resp(response: dict) -> None:
-    'Epoch': 'time',
+    """Raise any errors found in responses from client request.
-    'Open': 'open',
+    """
-    'High': 'high',
+    responses = response['responses']
-    'Low': 'low',
+    if responses is not None:
-    'Close': 'close',
+        for r in responses:
-    'Volume': 'volume',
+            err = r['error']
-})
+            if err:
-
+                raise MarketStoreError(err)
-
-def mk_tbk(keys: tuple[str, str, str]) -> str:
-    '''
-    Generate a marketstore table key from a tuple.
-    Converts,
-        ``('SPY', '1Sec', 'TICK')`` -> ``"SPY/1Sec/TICK"```
-
-    '''
-    return '/'.join(keys)
 
 
 def quote_to_marketstore_structarray(
-    quote: dict[str, Any],
+    quote: Dict[str, Any],
-    last_fill: Optional[float]
+    last_fill: str,
-
 ) -> np.array:
-    '''
+    """Return marketstore writeable structarray from quote ``dict``.
-    Return marketstore writeable structarray from quote ``dict``.
+    """
-
-    '''
     if last_fill:
         # new fill bby
-        now = int(pendulum.parse(last_fill).timestamp)
+        now = timestamp(last_fill)
     else:
         # this should get inserted upstream by the broker-client to
         # subtract from IPC latency
@@ -287,7 +101,7 @@ def quote_to_marketstore_structarray(
 
     secs, ns = now / 10**9, now % 10**9
 
-    # pack into list[tuple[str, Any]]
+    # pack into List[Tuple[str, Any]]
     array_input = []
 
     # insert 'Epoch' entry first and then 'Nanoseconds'.
@@ -309,467 +123,146 @@ def quote_to_marketstore_structarray(
     return np.array([tuple(array_input)], dtype=_quote_dt)
 
 
-@acm
+def timestamp(datestr: str) -> int:
+    """Return marketstore compatible 'Epoch' integer in nanoseconds
+    from a date formatted str.
+    """
+    return int(pd.Timestamp(datestr).value)
+
+
+def mk_tbk(keys: Tuple[str, str, str]) -> str:
+    """Generate a marketstore table key from a tuple.
+
+    Converts,
+        ``('SPY', '1Sec', 'TICK')`` -> ``"SPY/1Sec/TICK"```
+    """
+    return '{}/' + '/'.join(keys)
+
+
+class Client:
+    """Async wrapper around the alpaca ``pymarketstore`` sync client.
+
+    This will server as the shell for building out a proper async client
+    that isn't horribly documented and un-tested..
+    """
+    def __init__(self, url: str):
+        self._client = pymkts.Client(url)
+
+    async def _invoke(
+        self,
+        meth: Callable,
+        *args,
+        **kwargs,
+    ) -> Any:
+        return err_on_resp(meth(*args, **kwargs))
+
+    async def destroy(
+        self,
+        tbk: Tuple[str, str, str],
+    ) -> None:
+        return await self._invoke(self._client.destroy, mk_tbk(tbk))
+
+    async def list_symbols(
+        self,
+        tbk: str,
+    ) -> List[str]:
+        return await self._invoke(self._client.list_symbols, mk_tbk(tbk))
+
+    async def write(
+        self,
+        symbol: str,
+        array: np.ndarray,
+    ) -> None:
+        start = time.time()
+        await self._invoke(
+            self._client.write,
+            array,
+            _tick_tbk.format(symbol),
+            isvariablelength=True
+        )
+        log.debug(f"{symbol} write time (s): {time.time() - start}")
+
+    def query(
+        self,
+        symbol,
+        tbk: Tuple[str, str] = _tick_tbk_ids,
+    ) -> pd.DataFrame:
+        # XXX: causes crash
+        # client.query(pymkts.Params(symbol, '*', 'OHCLV'
+        result = self._client.query(
+            pymkts.Params(symbol, *tbk),
+        )
+        return result.first().df()
+
+
+@asynccontextmanager
 async def get_client(
-    host: str = 'localhost',
+    url: str = _url,
-    port: int = 5995
+) -> Client:
-
+    yield Client(url)
-) -> MarketstoreClient:
-    '''
-    Load a ``anyio_marketstore`` grpc client connected
-    to an existing ``marketstore`` server.
-
-    '''
-    async with open_marketstore_client(
-        host,
-        port
-    ) as client:
-        yield client
-
-
-class MarketStoreError(Exception):
-    "Generic marketstore client error"
-
-
-# def err_on_resp(response: dict) -> None:
-#     """Raise any errors found in responses from client request.
-#     """
-#     responses = response['responses']
-#     if responses is not None:
-#         for r in responses:
-#             err = r['error']
-#             if err:
-#                 raise MarketStoreError(err)
-
-
-# map of seconds ints to "time frame" accepted keys
-tf_in_1s = bidict({
-    1: '1Sec',
-    60: '1Min',
-    60*5: '5Min',
-    60*15: '15Min',
-    60*30: '30Min',
-    60*60: '1H',
-    60*60*24: '1D',
-})
-
-
-class Storage:
-    '''
-    High level storage api for both real-time and historical ingest.
-
-    '''
-    def __init__(
-        self,
-        client: MarketstoreClient,
-
-    ) -> None:
-        # TODO: eventually this should be an api/interface type that
-        # ensures we can support multiple tsdb backends.
-        self.client = client
-
-        # series' cache from tsdb reads
-        self._arrays: dict[str, np.ndarray] = {}
-
-    async def list_keys(self) -> list[str]:
-        return await self.client.list_symbols()
-
-    async def search_keys(self, pattern: str) -> list[str]:
-        '''
-        Search for time series key in the storage backend.
-
-        '''
-        ...
-
-    async def write_ticks(self, ticks: list) -> None:
-        ...
-
-    async def load(
-        self,
-        fqsn: str,
-
-    ) -> tuple[
-        dict[int, np.ndarray],  # timeframe (in secs) to series
-        Optional[datetime],  # first dt
-        Optional[datetime],  # last dt
-    ]:
-
-        first_tsdb_dt, last_tsdb_dt = None, None
-        tsdb_arrays = await self.read_ohlcv(
-            fqsn,
-            # on first load we don't need to pull the max
-            # history per request size worth.
-            limit=3000,
-        )
-        log.info(f'Loaded tsdb history {tsdb_arrays}')
-
-        if tsdb_arrays:
-            fastest = list(tsdb_arrays.values())[0]
-            times = fastest['Epoch']
-            first, last = times[0], times[-1]
-            first_tsdb_dt, last_tsdb_dt = map(
-                pendulum.from_timestamp, [first, last]
-            )
-
-        return tsdb_arrays, first_tsdb_dt, last_tsdb_dt
-
-    async def read_ohlcv(
-        self,
-        fqsn: str,
-        timeframe: Optional[Union[int, str]] = None,
-        end: Optional[int] = None,
-        limit: int = int(800e3),
-
-    ) -> tuple[
-        MarketstoreClient,
-        Union[dict, np.ndarray]
-    ]:
-        client = self.client
-        syms = await client.list_symbols()
-
-        if fqsn not in syms:
-            return {}
-
-        tfstr = tf_in_1s[1]
-
-        params = Params(
-            symbols=fqsn,
-            timeframe=tfstr,
-            attrgroup='OHLCV',
-            end=end,
-            # limit_from_start=True,
-
-            # TODO: figure the max limit here given the
-            # ``purepc`` msg size limit of purerpc: 33554432
-            limit=limit,
-        )
-
-        if timeframe is None:
-            log.info(f'starting {fqsn} tsdb granularity scan..')
-            # loop through and try to find highest granularity
-            for tfstr in tf_in_1s.values():
-                try:
-                    log.info(f'querying for {tfstr}@{fqsn}')
-                    params.set('timeframe', tfstr)
-                    result = await client.query(params)
-                    break
-
-                except purerpc.grpclib.exceptions.UnknownError:
-                    # XXX: this is already logged by the container and
-                    # thus shows up through `marketstored` logs relay.
-                    # log.warning(f'{tfstr}@{fqsn} not found')
-                    continue
-            else:
-                return {}
-
-        else:
-            result = await client.query(params)
-
-        # TODO: it turns out column access on recarrays is actually slower:
-        # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
-        # it might make sense to make these structured arrays?
-        # Fill out a `numpy` array-results map
-        arrays = {}
-        for fqsn, data_set in result.by_symbols().items():
-            arrays.setdefault(fqsn, {})[
-                tf_in_1s.inverse[data_set.timeframe]
-            ] = data_set.array
-
-        return arrays[fqsn][timeframe] if timeframe else arrays[fqsn]
-
-    async def delete_ts(
-        self,
-        key: str,
-        timeframe: Optional[Union[int, str]] = None,
-
-    ) -> bool:
-
-        client = self.client
-        syms = await client.list_symbols()
-        print(syms)
-        # if key not in syms:
-        #     raise KeyError(f'`{fqsn}` table key not found?')
-
-        return await client.destroy(tbk=key)
-
-    async def write_ohlcv(
-        self,
-        fqsn: str,
-        ohlcv: np.ndarray,
-        append_and_duplicate: bool = True,
-        limit: int = int(800e3),
-
-    ) -> None:
-        # build mkts schema compat array for writing
-        mkts_dt = np.dtype(_ohlcv_dt)
-        mkts_array = np.zeros(
-            len(ohlcv),
-            dtype=mkts_dt,
-        )
-        # copy from shm array (yes it's this easy):
-        # https://numpy.org/doc/stable/user/basics.rec.html#assignment-from-other-structured-arrays
-        mkts_array[:] = ohlcv[[
-            'time',
-            'open',
-            'high',
-            'low',
-            'close',
-            'volume',
-        ]]
-
-        m, r = divmod(len(mkts_array), limit)
-
-        for i in range(m, 1):
-            to_push = mkts_array[i-1:i*limit]
-
-            # write to db
-            resp = await self.client.write(
-                to_push,
-                tbk=f'{fqsn}/1Sec/OHLCV',
-
-                # NOTE: will will append duplicates
-                # for the same timestamp-index.
-                # TODO: pre deduplicate?
-                isvariablelength=append_and_duplicate,
-            )
-
-            log.info(
-                f'Wrote {mkts_array.size} datums to tsdb\n'
-            )
-
-            for resp in resp.responses:
-                err = resp.error
-                if err:
-                    raise MarketStoreError(err)
-
-        if r:
-            to_push = mkts_array[m*limit:]
-
-            # write to db
-            resp = await self.client.write(
-                to_push,
-                tbk=f'{fqsn}/1Sec/OHLCV',
-
-                # NOTE: will will append duplicates
-                # for the same timestamp-index.
-                # TODO: pre deduplicate?
-                isvariablelength=append_and_duplicate,
-            )
-
-            log.info(
-                f'Wrote {mkts_array.size} datums to tsdb\n'
-            )
-
-            for resp in resp.responses:
-                err = resp.error
-                if err:
-                    raise MarketStoreError(err)
-
-    # XXX: currently the only way to do this is through the CLI:
-
-    # sudo ./marketstore connect --dir ~/.config/piker/data
-    # >> \show mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
-    # and this seems to block and use up mem..
-    # >> \trim mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
-
-    # relevant source code for this is here:
-    # https://github.com/alpacahq/marketstore/blob/master/cmd/connect/session/trim.go#L14
-    # def delete_range(self, start_dt, end_dt) -> None:
-    #     ...
-
-@acm
-async def open_storage_client(
-    fqsn: str,
-    period: Optional[Union[int, str]] = None,  # in seconds
-
-) -> tuple[Storage, dict[str, np.ndarray]]:
-    '''
-    Load a series by key and deliver in ``numpy`` struct array format.
-
-    '''
-    async with (
-        # eventually a storage backend endpoint
-        get_client() as client,
-    ):
-        # slap on our wrapper api
-        yield Storage(client)
-
-
-async def tsdb_history_update(
-    fqsn: Optional[str] = None,
-
-) -> list[str]:
-
-    # TODO: real-time dedicated task for ensuring
-    # history consistency between the tsdb, shm and real-time feed..
-
-    # update sequence design notes:
-
-    # - load existing highest frequency data from mkts
-    #   * how do we want to offer this to the UI?
-    #    - lazy loading?
-    #    - try to load it all and expect graphics caching/diffing
-    #      to  hide extra bits that aren't in view?
-
-    # - compute the diff between latest data from broker and shm
-    #   * use sql api in mkts to determine where the backend should
-    #     start querying for data?
-    #   * append any diff with new shm length
-    #   * determine missing (gapped) history by scanning
-    #   * how far back do we look?
-
-    # - begin rt update ingest and aggregation
-    #   * could start by always writing ticks to mkts instead of
-    #     worrying about a shm queue for now.
-    #   * we have a short list of shm queues worth groking:
-    #     - https://github.com/pikers/piker/issues/107
-    #   * the original data feed arch blurb:
-    #     - https://github.com/pikers/piker/issues/98
-    #
-    profiler = pg.debug.Profiler(
-        disabled=False,  # not pg_profile_enabled(),
-        delayed=False,
-    )
-
-    async with (
-        open_storage_client(fqsn) as storage,
-
-        maybe_open_feed(
-            [fqsn],
-            start_stream=False,
-
-        ) as (feed, stream),
-    ):
-        profiler(f'opened feed for {fqsn}')
-
-        to_append = feed.shm.array
-        to_prepend = None
-
-        if fqsn:
-            symbol = feed.symbols.get(fqsn)
-            if symbol:
-                fqsn = symbol.front_fqsn()
-
-            # diff db history with shm and only write the missing portions
-            ohlcv = feed.shm.array
-
-            # TODO: use pg profiler
-            tsdb_arrays = await storage.read_ohlcv(fqsn)
-            # hist diffing
-            if tsdb_arrays:
-                for secs in (1, 60):
-                    ts = tsdb_arrays.get(secs)
-                    if ts is not None and len(ts):
-                        # these aren't currently used but can be referenced from
-                        # within the embedded ipython shell below.
-                        to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
-                        to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
-
-            profiler('Finished db arrays diffs')
-
-        syms = await storage.client.list_symbols()
-        log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
-        profiler(f'listed symbols {syms}')
-
-        # TODO: ask if user wants to write history for detected
-        # available shm buffers?
-        from tractor.trionics import ipython_embed
-        await ipython_embed()
-
-        # for array in [to_append, to_prepend]:
-        #     if array is None:
-        #         continue
-
-        #     log.info(
-        #         f'Writing datums {array.size} -> to tsdb from shm\n'
-        #     )
-        #     await storage.write_ohlcv(fqsn, array)
-
-        # profiler('Finished db writes')
 
 
 async def ingest_quote_stream(
-    symbols: list[str],
+    symbols: List[str],
     brokername: str,
     tries: int = 1,
     loglevel: str = None,
-
 ) -> None:
-    '''
+    """Ingest a broker quote stream into marketstore in (sampled) tick format.
-    Ingest a broker quote stream into a ``marketstore`` tsdb.
+    """
+    async with open_feed(
+        brokername,
+        symbols,
+        loglevel=loglevel,
+    ) as (first_quotes, qstream):
 
-    '''
+        quote_cache = first_quotes.copy()
-    async with (
+
-        maybe_open_feed(brokername, symbols, loglevel=loglevel) as feed,
+        async with get_client() as ms_client:
-        get_client() as ms_client,
+
-    ):
+            # start ingest to marketstore
-        async for quotes in feed.stream:
+            async for quotes in qstream:
                 log.info(quotes)
                 for symbol, quote in quotes.items():
-                for tick in quote.get('ticks', ()):
-                    ticktype = tick.get('type', 'n/a')
 
-            # techtonic tick write
+                    # remap tick strs to ints
-            array = quote_to_marketstore_structarray({
+                    quote['tick'] = _tick_map[quote.get('tick', 'Equal')]
-                'IsTrade': 1 if ticktype == 'trade' else 0,
-                'IsBid': 1 if ticktype in ('bid', 'bsize') else 0,
-                'Price': tick.get('price'),
-                'Size': tick.get('size')
-            }, last_fill=quote.get('broker_ts', None))
 
-            await ms_client.write(array, _tick_tbk)
+                    # check for volume update (i.e. did trades happen
+                    # since last quote)
+                    new_vol = quote.get('volume', None)
+                    if new_vol is None:
+                        log.debug(f"No fills for {symbol}")
+                        if new_vol == quote_cache.get('volume'):
+                            # should never happen due to field diffing
+                            # on sender side
+                            log.error(
+                                f"{symbol}: got same volume as last quote?")
 
-            # LEGACY WRITE LOOP (using old tick dt)
+                    quote_cache.update(quote)
-            # quote_cache = {
-            #     'size': 0,
-            #     'tick': 0
-            # }
 
-            # async for quotes in qstream:
+                    a = quote_to_marketstore_structarray(
-            #     log.info(quotes)
+                        quote,
-            #     for symbol, quote in quotes.items():
+                        # TODO: check this closer to the broker query api
-
+                        last_fill=quote.get('fill_time', '')
-            #         # remap tick strs to ints
+                    )
-            #         quote['tick'] = _tick_map[quote.get('tick', 'Equal')]
+                    await ms_client.write(symbol, a)
-
-            #         # check for volume update (i.e. did trades happen
-            #         # since last quote)
-            #         new_vol = quote.get('volume', None)
-            #         if new_vol is None:
-            #             log.debug(f"No fills for {symbol}")
-            #             if new_vol == quote_cache.get('volume'):
-            #                 # should never happen due to field diffing
-            #                 # on sender side
-            #                 log.error(
-            #                     f"{symbol}: got same volume as last quote?")
-
-            #         quote_cache.update(quote)
-
-            #         a = quote_to_marketstore_structarray(
-            #             quote,
-            #             # TODO: check this closer to the broker query api
-            #             last_fill=quote.get('fill_time', '')
-            #         )
-            #         await ms_client.write(symbol, a)
 
 
 async def stream_quotes(
-    symbols: list[str],
+    symbols: List[str],
     host: str = 'localhost',
     port: int = 5993,
     diff_cached: bool = True,
     loglevel: str = None,
-
 ) -> None:
-    '''
+    """Open a symbol stream from a running instance of marketstore and
-    Open a symbol stream from a running instance of marketstore and
     log to console.
-
+    """
-    '''
     # XXX: required to propagate ``tractor`` loglevel to piker logging
     get_console_log(loglevel or tractor.current_actor().loglevel)
 
-    tbks: dict[str, str] = {sym: f"{sym}/*/*" for sym in symbols}
+    tbks: Dict[str, str] = {sym: f"{sym}/*/*" for sym in symbols}
 
     async with open_websocket_url(f'ws://{host}:{port}/ws') as ws:
         # send subs topics to server
@@ -778,7 +271,7 @@ async def stream_quotes(
         )
         log.info(resp)
 
-        async def recv() -> dict[str, Any]:
+        async def recv() -> Dict[str, Any]:
             return msgpack.loads((await ws.get_message()), encoding='utf-8')
 
         streams = (await recv())['streams']

piker/fsp/_api.py

@@ -40,8 +40,6 @@ from tractor.msg import NamespacePath
 from ..data._sharedmem import (
     ShmArray,
     maybe_open_shm_array,
-    attach_shm_array,
-    _Token,
 )
 from ..log import get_logger
 
@@ -74,13 +72,6 @@ class Fsp:
     # - custom function wrappers,
     # https://wrapt.readthedocs.io/en/latest/wrappers.html#custom-function-wrappers
 
-    # actor-local map of source flow shm tokens
-    # + the consuming fsp *to* the consumers output
-    # shm flow.
-    _flow_registry: dict[
-        tuple[_Token, str], _Token,
-    ] = {}
-
     def __init__(
         self,
         func: Callable[..., Awaitable],
@@ -102,7 +93,7 @@ class Fsp:
         self.config: dict[str, Any] = config
 
         # register with declared set.
-        _fsp_registry[self.ns_path] = self
+        _fsp_registry[self.ns_path] = func
 
     @property
     def name(self) -> str:
@@ -120,24 +111,6 @@ class Fsp:
     ):
         return self.func(*args, **kwargs)
 
-    # TODO: lru_cache this? prettty sure it'll work?
-    def get_shm(
-        self,
-        src_shm: ShmArray,
-
-    ) -> ShmArray:
-        '''
-        Provide access to allocated shared mem array
-        for this "instance" of a signal processor for
-        the given ``key``.
-
-        '''
-        dst_token = self._flow_registry[
-            (src_shm._token, self.name)
-        ]
-        shm = attach_shm_array(dst_token)
-        return shm
-
 
 def fsp(
     wrapped=None,
@@ -159,27 +132,18 @@ def fsp(
     return Fsp(wrapped, outputs=(wrapped.__name__,))
 
 
-def mk_fsp_shm_key(
-    sym: str,
-    target: Fsp
-
-) -> str:
-    uid = tractor.current_actor().uid
-    return f'{sym}.fsp.{target.name}.{".".join(uid)}'
-
-
 def maybe_mk_fsp_shm(
     sym: str,
-    target: Fsp,
+    target: fsp,
     readonly: bool = True,
 
-) -> (str, ShmArray, bool):
+) -> (ShmArray, bool):
     '''
     Allocate a single row shm array for an symbol-fsp pair if none
     exists, otherwise load the shm already existing for that token.
 
     '''
-    assert isinstance(sym, str), '`sym` should be file-name-friendly `str`'
+    uid = tractor.current_actor().uid
 
     # TODO: load output types from `Fsp`
     # - should `index` be a required internal field?
@@ -188,7 +152,7 @@ def maybe_mk_fsp_shm(
         [(field_name, float) for field_name in target.outputs]
     )
 
-    key = mk_fsp_shm_key(sym, target)
+    key = f'{sym}.fsp.{target.name}.{".".join(uid)}'
 
     shm, opened = maybe_open_shm_array(
         key,
@@ -196,4 +160,4 @@ def maybe_mk_fsp_shm(
         dtype=fsp_dtype,
         readonly=True,
     )
-    return key, shm, opened
+    return shm, opened

piker/fsp/_engine.py

@@ -20,10 +20,7 @@ core task logic for processing chains
 '''
 from dataclasses import dataclass
 from functools import partial
-from typing import (
+from typing import AsyncIterator, Callable, Optional
-    AsyncIterator, Callable, Optional,
-    Union,
-)
 
 import numpy as np
 import pyqtgraph as pg
@@ -37,12 +34,8 @@ from .. import data
 from ..data import attach_shm_array
 from ..data.feed import Feed
 from ..data._sharedmem import ShmArray
-from ..data._source import Symbol
+from ._api import Fsp
-from ._api import (
+from ._api import _load_builtins
-    Fsp,
-    _load_builtins,
-    _Token,
-)
 
 log = get_logger(__name__)
 
@@ -76,7 +69,8 @@ async def filter_quotes_by_sym(
 
 async def fsp_compute(
 
-    symbol: Symbol,
+    ctx: tractor.Context,
+    symbol: str,
     feed: Feed,
     quote_stream: trio.abc.ReceiveChannel,
 
@@ -85,7 +79,7 @@ async def fsp_compute(
 
     func: Callable,
 
-    # attach_stream: bool = False,
+    attach_stream: bool = False,
     task_status: TaskStatus[None] = trio.TASK_STATUS_IGNORED,
 
 ) -> None:
@@ -95,81 +89,40 @@ async def fsp_compute(
         disabled=True
     )
 
-    fqsn = symbol.front_fqsn()
     out_stream = func(
 
         # TODO: do we even need this if we do the feed api right?
         # shouldn't a local stream do this before we get a handle
         # to the async iterable? it's that or we do some kinda
         # async itertools style?
-        filter_quotes_by_sym(fqsn, quote_stream),
+        filter_quotes_by_sym(symbol, quote_stream),
-
-        # XXX: currently the ``ohlcv`` arg
         feed.shm,
     )
 
     # Conduct a single iteration of fsp with historical bars input
     # and get historical output
-    history_output: Union[
-        dict[str, np.ndarray],  # multi-output case
-        np.ndarray,  # single output case
-    ]
     history_output = await out_stream.__anext__()
 
     func_name = func.__name__
     profiler(f'{func_name} generated history')
 
     # build struct array with an 'index' field to push as history
-
-    # TODO: push using a[['f0', 'f1', .., 'fn']] = .. syntax no?
-    # if the output array is multi-field then push
-    # each respective field.
-    fields = getattr(dst.array.dtype, 'fields', None).copy()
-    fields.pop('index')
-    history: Optional[np.ndarray] = None  # TODO: nptyping here!
-
-    if fields and len(fields) > 1 and fields:
-        if not isinstance(history_output, dict):
-            raise ValueError(
-                f'`{func_name}` is a multi-output FSP and should yield a '
-                '`dict[str, np.ndarray]` for history'
-            )
-
-        for key in fields.keys():
-            if key in history_output:
-                output = history_output[key]
-
-                if history is None:
-
-                    if output is None:
-                        length = len(src.array)
-                    else:
-                        length = len(output)
-
-                    # using the first output, determine
-                    # the length of the struct-array that
-                    # will be pushed to shm.
-                    history = np.zeros(
-                        length,
-                        dtype=dst.array.dtype
-                    )
-
-                if output is None:
-                    continue
-
-                history[key] = output
-
-    # single-key output stream
-    else:
-        if not isinstance(history_output, np.ndarray):
-            raise ValueError(
-                f'`{func_name}` is a single output FSP and should yield an '
-                '`np.ndarray` for history'
-            )
     history = np.zeros(
         len(history_output),
         dtype=dst.array.dtype
     )
+
+    # TODO: push using a[['f0', 'f1', .., 'fn']] = .. syntax no?
+    # if the output array is multi-field then push
+    # each respective field.
+    fields = getattr(history.dtype, 'fields', None)
+    if fields:
+        for key in fields.keys():
+            if key in history.dtype.fields:
+                history[func_name] = history_output
+
+    # single-key output stream
+    else:
         history[func_name] = history_output
 
     # TODO: XXX:
@@ -192,23 +145,22 @@ async def fsp_compute(
     profiler(f'{func_name} pushed history')
     profiler.finish()
 
+    # TODO: UGH, what is the right way to do something like this?
+    if not ctx._started_called:
+        await ctx.started(index)
+
     # setup a respawn handle
     with trio.CancelScope() as cs:
-
-        # TODO: might be better to just make a "restart" method where
-        # the target task is spawned implicitly and then the event is
-        # set via some higher level api? At that poing we might as well
-        # be writing a one-cancels-one nursery though right?
         tracker = TaskTracker(trio.Event(), cs)
         task_status.started((tracker, index))
-
         profiler(f'{func_name} yield last index')
 
         # import time
         # last = time.time()
 
         try:
-
+            # rt stream
+            async with ctx.open_stream() as stream:
                 async for processed in out_stream:
 
                     log.debug(f"{func_name}: {processed}")
@@ -219,14 +171,8 @@ async def fsp_compute(
                     # NOTE: for now we aren't streaming this to the consumer
                     # stream latest array index entry which basically just acts
                     # as trigger msg to tell the consumer to read from shm
-                # TODO: further this should likely be implemented much
+                    if attach_stream:
-                # like our `Feed` api where there is one background
+                        await stream.send(index)
-                # "service" task which computes output and then sends to
-                # N-consumers who subscribe for the real-time output,
-                # which we'll likely want to implement using local-mem
-                # chans for the fan out?
-                # if attach_stream:
-                #     await client_stream.send(index)
 
                     # period = time.time() - last
                     # hz = 1/period if period else float('nan')
@@ -243,15 +189,14 @@ async def cascade(
     ctx: tractor.Context,
 
     # data feed key
-    fqsn: str,
+    brokername: str,
+    symbol: str,
 
     src_shm_token: dict,
     dst_shm_token: tuple[str, np.dtype],
 
     ns_path: NamespacePath,
 
-    shm_registry: dict[str, _Token],
-
     zero_on_step: bool = False,
     loglevel: Optional[str] = None,
 
@@ -261,10 +206,7 @@ async def cascade(
     destination shm array buffer.
 
     '''
-    profiler = pg.debug.Profiler(
+    profiler = pg.debug.Profiler(delayed=False, disabled=False)
-        delayed=False,
-        disabled=False
-    )
 
     if loglevel:
         get_console_log(loglevel)
@@ -277,21 +219,9 @@ async def cascade(
     log.info(
         f'Registered FSP set:\n{lines}'
     )
-
+    func: Fsp = reg.get(
-    # update actorlocal flows table which registers
-    # readonly "instances" of this fsp for symbol/source
-    # so that consumer fsps can look it up by source + fsp.
-    # TODO: ugh i hate this wind/unwind to list over the wire
-    # but not sure how else to do it.
-    for (token, fsp_name, dst_token) in shm_registry:
-        Fsp._flow_registry[
-            (_Token.from_msg(token), fsp_name)
-        ] = _Token.from_msg(dst_token)
-
-    fsp: Fsp = reg.get(
         NamespacePath(ns_path)
     )
-    func = fsp.func
 
     if not func:
         # TODO: assume it's a func target path
@@ -299,7 +229,8 @@ async def cascade(
 
     # open a data feed stream with requested broker
     async with data.feed.maybe_open_feed(
-        [fqsn],
+        brokername,
+        [symbol],
 
         # TODO throttle tick outputs from *this* daemon since
         # it'll emit tons of ticks due to the throttle only
@@ -308,7 +239,6 @@ async def cascade(
         # tick_throttle=60,
 
     ) as (feed, quote_stream):
-        symbol = feed.symbols[fqsn]
 
         profiler(f'{func}: feed up')
 
@@ -323,6 +253,7 @@ async def cascade(
             fsp_target = partial(
 
                 fsp_compute,
+                ctx=ctx,
                 symbol=symbol,
                 feed=feed,
                 quote_stream=quote_stream,
@@ -331,7 +262,7 @@ async def cascade(
                 src=src,
                 dst=dst,
 
-                # target
+                # func_name=func_name,
                 func=func
             )
 
@@ -343,39 +274,13 @@ async def cascade(
 
             profiler(f'{func_name}: fsp up')
 
-            # sync client
+            async def resync(tracker: TaskTracker) -> tuple[TaskTracker, int]:
-            await ctx.started(index)
-
-            # XXX:  rt stream with client which we MUST
-            # open here (and keep it open) in order to make
-            # incremental "updates" as history prepends take
-            # place.
-            async with ctx.open_stream() as client_stream:
-
-                # TODO: these likely should all become
-                # methods of this ``TaskLifetime`` or wtv
-                # abstraction..
-                async def resync(
-                    tracker: TaskTracker,
-
-                ) -> tuple[TaskTracker, int]:
                 # TODO: adopt an incremental update engine/approach
                 # where possible here eventually!
-                    log.debug(f're-syncing fsp {func_name} to source')
+                log.warning(f're-syncing fsp {func_name} to source')
                 tracker.cs.cancel()
                 await tracker.complete.wait()
-                    tracker, index = await n.start(fsp_target)
+                return await n.start(fsp_target)
-
-                    # always trigger UI refresh after history update,
-                    # see ``piker.ui._fsp.FspAdmin.open_chain()`` and
-                    # ``piker.ui._display.trigger_update()``.
-                    await client_stream.send({
-                        'fsp_update': {
-                            'key': dst_shm_token,
-                            'first': dst._first.value,
-                            'last': dst._last.value,
-                    }})
-                    return tracker, index
 
             def is_synced(
                 src: ShmArray,
@@ -415,21 +320,14 @@ async def cascade(
 
             s, step, ld = is_synced(src, dst)
 
-                # detect sample period step for subscription to increment
-                # signal
-                times = src.array['time']
-                delay_s = times[-1] - times[times != times[-1]][-1]
-
             # Increment the underlying shared memory buffer on every
             # "increment" msg received from the underlying data feed.
-                async with feed.index_stream(
+            async with feed.index_stream() as stream:
-                    int(delay_s)
-                ) as istream:
 
                 profiler(f'{func_name}: sample stream up')
                 profiler.finish()
 
-                    async for _ in istream:
+                async for msg in stream:
 
                     # respawn the compute task if the source
                     # array has been updated such that we compute

piker/fsp/_momo.py

@@ -167,36 +167,9 @@ def _wma(
 
     assert length == len(weights)
 
-    # lol, for long sequences this is nutso slow and expensive..
     return np.convolve(signal, weights, 'valid')
 
 
-@fsp
-async def wma(
-
-    source,  #: AsyncStream[np.ndarray],
-    length: int,
-    ohlcv: np.ndarray,  # price time-frame "aware"
-
-) -> AsyncIterator[np.ndarray]:  # maybe something like like FspStream?
-    '''
-    Streaming weighted moving average.
-
-    ``weights`` is a sequence of already scaled values. As an example
-    for the WMA often found in "techincal analysis":
-    ``weights = np.arange(1, N) * N*(N-1)/2``.
-
-    '''
-    # deliver historical output as "first yield"
-    yield _wma(ohlcv.array['close'], length)
-
-    # begin real-time section
-
-    async for quote in source:
-        for tick in iterticks(quote, type='trade'):
-            yield _wma(ohlcv.last(length))
-
-
 @fsp
 async def rsi(
 
@@ -251,3 +224,29 @@ async def rsi(
                 down_ema_last=last_down_ema_close,
             )
             yield rsi_out[-1:]
+
+
+@fsp
+async def wma(
+
+    source,  #: AsyncStream[np.ndarray],
+    length: int,
+    ohlcv: np.ndarray,  # price time-frame "aware"
+
+) -> AsyncIterator[np.ndarray]:  # maybe something like like FspStream?
+    '''
+    Streaming weighted moving average.
+
+    ``weights`` is a sequence of already scaled values. As an example
+    for the WMA often found in "techincal analysis":
+    ``weights = np.arange(1, N) * N*(N-1)/2``.
+
+    '''
+    # deliver historical output as "first yield"
+    yield _wma(ohlcv.array['close'], length)
+
+    # begin real-time section
+
+    async for quote in source:
+        for tick in iterticks(quote, type='trade'):
+            yield _wma(ohlcv.last(length))

piker/fsp/_volume.py

@@ -22,25 +22,17 @@ from tractor.trionics._broadcast import AsyncReceiver
 from ._api import fsp
 from ..data._normalize import iterticks
 from ..data._sharedmem import ShmArray
-from ._momo import _wma
-from ..log import get_logger
-
-log = get_logger(__name__)
 
 
-# NOTE: is the same as our `wma` fsp, and if so which one is faster?
-# Ohhh, this is an IIR style i think? So it has an anchor point
-# effectively instead of a moving window/FIR style?
 def wap(
 
     signal: np.ndarray,
     weights: np.ndarray,
 
 ) -> np.ndarray:
-    '''
+    """Weighted average price from signal and weights.
-    Weighted average price from signal and weights.
 
-    '''
+    """
     cum_weights = np.cumsum(weights)
     cum_weighted_input = np.cumsum(signal * weights)
 
@@ -97,10 +89,7 @@ async def tina_vwap(
     # vwap_tot = h_vwap[-1]
 
     async for quote in source:
-        for tick in iterticks(
+        for tick in iterticks(quote, types=['trade']):
-            quote,
-            types=['trade'],
-        ):
 
             # c, h, l, v = ohlcv.array[-1][
             #     ['closes', 'high', 'low', 'volume']
@@ -118,12 +107,8 @@ async def tina_vwap(
 
 
 @fsp(
-    outputs=(
+    outputs=('dolla_vlm', 'dark_vlm'),
-        'dolla_vlm',
+    ohlc=False,
-        'dark_vlm',
-        'trade_count',
-        'dark_trade_count',
-    ),
     curve_style='step',
 )
 async def dolla_vlm(
@@ -147,24 +132,14 @@ async def dolla_vlm(
     v = a['volume']
 
     # on first iteration yield history
-    yield {
+    yield chl3 * v
-        'dolla_vlm': chl3 * v,
-        'dark_vlm': None,
-    }
 
     i = ohlcv.index
-    dvlm = vlm = 0
+    output = vlm = 0
-    dark_trade_count = trade_count = 0
+    dvlm = 0
 
     async for quote in source:
-        for tick in iterticks(
+        for tick in iterticks(quote):
-            quote,
-            types=(
-                'trade',
-                'dark_trade',
-            ),
-            deduplicate_darks=True,
-        ):
 
             # this computes tick-by-tick weightings from here forward
             size = tick['size']
@@ -173,30 +148,24 @@ async def dolla_vlm(
             li = ohlcv.index
             if li > i:
                 i = li
-                trade_count = dark_trade_count = dvlm = vlm = 0
+                vlm = 0
+                dvlm = 0
 
             # TODO: for marginned instruments (futes, etfs?) we need to
             # show the margin $vlm by multiplying by whatever multiplier
             # is reported in the sym info.
 
             ttype = tick.get('type')
-
             if ttype == 'dark_trade':
+                print(f'dark_trade: {tick}')
+                key = 'dark_vlm'
                 dvlm += price * size
-                yield 'dark_vlm', dvlm
+                output = dvlm
-
-                dark_trade_count += 1
-                yield 'dark_trade_count', dark_trade_count
-
-                # print(f'{dark_trade_count}th dark_trade: {tick}')
 
             else:
-                # print(f'vlm: {tick}')
+                key = 'dolla_vlm'
                 vlm += price * size
-                yield 'dolla_vlm', vlm
+                output = vlm
-
-                trade_count += 1
-                yield 'trade_count', trade_count
 
             # TODO: plot both to compare?
             # c, h, l, v = ohlcv.last()[
@@ -205,154 +174,4 @@ async def dolla_vlm(
             # tina_lvlm = c+h+l/3 * v
             # print(f' tinal vlm: {tina_lvlm}')
 
-
+            yield key, output
-@fsp(
-    # TODO: eventually I guess we should support some kinda declarative
-    # graphics config syntax per output yah? That seems like a clean way
-    # to let users configure things? Not sure how exactly to offer that
-    # api as well as how to expose such a thing *inside* the body?
-    outputs=(
-        # pulled verbatim from `ib` for now
-        '1m_trade_rate',
-        '1m_vlm_rate',
-
-        # our own instantaneous rate calcs which are all
-        # parameterized by a samples count (bars) period
-        'trade_rate',
-        'dark_trade_rate',
-
-        'dvlm_rate',
-        'dark_dvlm_rate',
-    ),
-    curve_style='line',
-)
-async def flow_rates(
-    source: AsyncReceiver[dict],
-    ohlcv: ShmArray,  # OHLC sampled history
-
-    # TODO (idea): a dynamic generic / boxing type that can be updated by other
-    # FSPs, user input, and possibly any general event stream in
-    # real-time. Hint: ideally implemented with caching until mutated
-    # ;)
-    period: 'Param[int]' = 6,  # noqa
-
-    # TODO: support other means by providing a map
-    # to weights `partial()`-ed with `wma()`?
-    mean_type: str = 'arithmetic',
-
-    # TODO (idea): a generic for declaring boxed fsps much like ``pytest``
-    # fixtures? This probably needs a lot of thought if we want to offer
-    # a higher level composition syntax eventually (oh right gotta make
-    # an issue for that).
-    # ideas for how to allow composition / intercalling:
-    # - offer a `Fsp.get_history()` to do the first yield output?
-    #  * err wait can we just have shm access directly?
-    # - how would it work if some consumer fsp wanted to dynamically
-    # change params which are input to the callee fsp? i guess we could
-    # lazy copy in that case?
-    # dvlm: 'Fsp[dolla_vlm]'
-
-) -> AsyncIterator[
-    tuple[str, Union[np.ndarray, float]],
-]:
-    # generally no history available prior to real-time calcs
-    yield {
-        # from ib
-        '1m_trade_rate': None,
-        '1m_vlm_rate': None,
-
-        'trade_rate': None,
-        'dark_trade_rate': None,
-
-        'dvlm_rate': None,
-        'dark_dvlm_rate': None,
-    }
-
-    # TODO: 3.10 do ``anext()``
-    quote = await source.__anext__()
-
-    # ltr = 0
-    # lvr = 0
-    tr = quote.get('tradeRate')
-    yield '1m_trade_rate', tr or 0
-    vr = quote.get('volumeRate')
-    yield '1m_vlm_rate', vr or 0
-
-    yield 'trade_rate', 0
-    yield 'dark_trade_rate', 0
-    yield 'dvlm_rate', 0
-    yield 'dark_dvlm_rate', 0
-
-    # NOTE: in theory we could dynamically allocate a cascade based on
-    # this call but not sure if that's too "dynamic" in terms of
-    # validating cascade flows from message typing perspective.
-
-    # attach to ``dolla_vlm`` fsp running
-    # on this same source flow.
-    dvlm_shm = dolla_vlm.get_shm(ohlcv)
-
-    # precompute arithmetic mean weights (all ones)
-    seq = np.full((period,), 1)
-    weights = seq / seq.sum()
-
-    async for quote in source:
-        if not quote:
-            log.error("OH WTF NO QUOTE IN FSP")
-            continue
-
-        # dvlm_wma = _wma(
-        #     dvlm_shm.array['dolla_vlm'],
-        #     period,
-        #     weights=weights,
-        # )
-        # yield 'dvlm_rate', dvlm_wma[-1]
-
-        if period > 1:
-            trade_rate_wma = _wma(
-                dvlm_shm.array['trade_count'][-period:],
-                period,
-                weights=weights,
-            )
-            trade_rate = trade_rate_wma[-1]
-            # print(trade_rate)
-            yield 'trade_rate', trade_rate
-        else:
-            # instantaneous rate per sample step
-            count = dvlm_shm.array['trade_count'][-1]
-            yield 'trade_rate', count
-
-        # TODO: skip this if no dark vlm is declared
-        # by symbol info (eg. in crypto$)
-        # dark_dvlm_wma = _wma(
-        #     dvlm_shm.array['dark_vlm'],
-        #     period,
-        #     weights=weights,
-        # )
-        # yield 'dark_dvlm_rate', dark_dvlm_wma[-1]
-
-        if period > 1:
-            dark_trade_rate_wma = _wma(
-                dvlm_shm.array['dark_trade_count'][-period:],
-                period,
-                weights=weights,
-            )
-            yield 'dark_trade_rate', dark_trade_rate_wma[-1]
-        else:
-            # instantaneous rate per sample step
-            dark_count = dvlm_shm.array['dark_trade_count'][-1]
-            yield 'dark_trade_rate', dark_count
-
-        # XXX: ib specific schema we should
-        # probably pre-pack ourselves.
-
-        # tr = quote.get('tradeRate')
-        # if tr is not None and tr != ltr:
-        #     # print(f'trade rate: {tr}')
-        #     yield '1m_trade_rate', tr
-        #     ltr = tr
-
-        # vr = quote.get('volumeRate')
-        # if vr is not None and vr != lvr:
-        #     # print(f'vlm rate: {vr}')
-        #     yield '1m_vlm_rate', vr
-        #     lvr = vr

piker/log.py

@@ -25,13 +25,10 @@ from pygments import highlight, lexers, formatters
 
 # Makes it so we only see the full module name when using ``__name__``
 # without the extra "piker." prefix.
-_proj_name: str = 'piker'
+_proj_name = 'piker'
 
 
-def get_logger(
+def get_logger(name: str = None) -> logging.Logger:
-    name: str = None,
-
-) -> logging.Logger:
     '''Return the package log or a sub-log for `name` if provided.
     '''
     return tractor.log.get_logger(name=name, _root_name=_proj_name)

piker/ui/_anchors.py

@@ -25,10 +25,39 @@ from PyQt5.QtCore import QPointF
 from PyQt5.QtWidgets import QGraphicsPathItem
 
 if TYPE_CHECKING:
+    from ._axes import PriceAxis
     from ._chart import ChartPlotWidget
     from ._label import Label
 
 
+def marker_right_points(
+    chart: ChartPlotWidget,  # noqa
+    marker_size: int = 20,
+
+) -> (float, float, float):
+    '''
+    Return x-dimension, y-axis-aware, level-line marker oriented scene
+    values.
+
+    X values correspond to set the end of a level line, end of
+    a paried level line marker, and the right most side of the "right"
+    axis respectively.
+
+    '''
+    # TODO: compute some sensible maximum value here
+    # and use a humanized scheme to limit to that length.
+    l1_len = chart._max_l1_line_len
+    ryaxis = chart.getAxis('right')
+
+    r_axis_x = ryaxis.pos().x()
+    up_to_l1_sc = r_axis_x - l1_len - 10
+
+    marker_right = up_to_l1_sc - (1.375 * 2 * marker_size)
+    line_end = marker_right - (6/16 * marker_size)
+
+    return line_end, marker_right, r_axis_x
+
+
 def vbr_left(
     label: Label,
 

piker/ui/_annotate.py

@@ -26,6 +26,8 @@ from PyQt5.QtWidgets import QGraphicsPathItem
 from pyqtgraph import Point, functions as fn, Color
 import numpy as np
 
+from ._anchors import marker_right_points
+
 
 def mk_marker_path(
 
@@ -114,7 +116,7 @@ class LevelMarker(QGraphicsPathItem):
 
         self.get_level = get_level
         self._on_paint = on_paint
-        self.scene_x = lambda: chart.marker_right_points()[1]
+        self.scene_x = lambda: marker_right_points(chart)[1]
         self.level: float = 0
         self.keep_in_view = keep_in_view
 
@@ -167,7 +169,7 @@ class LevelMarker(QGraphicsPathItem):
         vr = view.state['viewRange']
         ymn, ymx = vr[1]
 
-        # _, marker_right, _ = line._chart.marker_right_points()
+        # _, marker_right, _ = marker_right_points(line._chart)
         x = self.scene_x()
 
         if self.style == '>|':  # short style, points "down-to" line

piker/ui/_axes.py

@@ -19,10 +19,10 @@ Chart axes graphics and behavior.
 
 """
 from functools import lru_cache
-from typing import Optional, Callable
+from typing import List, Tuple, Optional, Callable
 from math import floor
 
-import numpy as np
+import pandas as pd
 import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QPointF
@@ -44,14 +44,10 @@ class Axis(pg.AxisItem):
         self,
         linkedsplits,
         typical_max_str: str = '100 000.000',
-        text_color: str = 'bracket',
         **kwargs
 
     ) -> None:
-        super().__init__(
+        super().__init__(**kwargs)
-            # textPen=textPen,
-            **kwargs
-        )
 
         # XXX: pretty sure this makes things slower
         # self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
@@ -78,32 +74,19 @@ class Axis(pg.AxisItem):
         })
 
         self.setTickFont(_font.font)
-
         # NOTE: this is for surrounding "border"
         self.setPen(_axis_pen)
-
         # this is the text color
-        # self.setTextPen(pg.mkPen(hcolor(text_color)))
+        self.setTextPen(_axis_pen)
-        self.text_color = text_color
-
         self.typical_br = _font._qfm.boundingRect(typical_max_str)
 
         # size the pertinent axis dimension to a "typical value"
         self.size_to_values()
 
-    @property
-    def text_color(self) -> str:
-        return self._text_color
-
-    @text_color.setter
-    def text_color(self, text_color: str) -> None:
-        self.setTextPen(pg.mkPen(hcolor(text_color)))
-        self._text_color = text_color
-
     def size_to_values(self) -> None:
         pass
 
-    def txt_offsets(self) -> tuple[int, int]:
+    def txt_offsets(self) -> Tuple[int, int]:
         return tuple(self.style['tickTextOffset'])
 
 
@@ -126,8 +109,7 @@ class PriceAxis(Axis):
     def set_title(
         self,
         title: str,
-        view: Optional[ChartView] = None,
+        view: Optional[ChartView] = None
-        color: Optional[str] = None,
 
     ) -> Label:
         '''
@@ -141,7 +123,7 @@ class PriceAxis(Axis):
         label = self.title = Label(
             view=view or self.linkedView(),
             fmt_str=title,
-            color=color or self.text_color,
+            color='bracket',
             parent=self,
             # update_on_range_change=False,
         )
@@ -218,14 +200,13 @@ class DynamicDateAxis(Axis):
 
     def _indexes_to_timestrs(
         self,
-        indexes: list[int],
+        indexes: List[int],
 
-    ) -> list[str]:
+    ) -> List[str]:
 
         chart = self.linkedsplits.chart
-        flow = chart._flows[chart.name]
+        bars = chart._arrays[chart.name]
-        shm = flow.shm
+        shm = self.linkedsplits.chart._shm
-        bars = shm.array
         first = shm._first.value
 
         bars_len = len(bars)
@@ -242,17 +223,10 @@ class DynamicDateAxis(Axis):
         )]
 
         # TODO: **don't** have this hard coded shift to EST
-        # delay = times[-1] - times[-2]
+        dts = pd.to_datetime(epochs, unit='s')  # - 4*pd.offsets.Hour()
-        dts = np.array(epochs, dtype='datetime64[s]')
 
-        # see units listing:
+        delay = times[-1] - times[-2]
-        # https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
+        return dts.strftime(self.tick_tpl[delay])
-        return list(np.datetime_as_string(dts))
-
-        # TODO: per timeframe formatting?
-        # - we probably need this based on zoom now right?
-        # prec = self.np_dt_precision[delay]
-        # return dts.strftime(self.tick_tpl[delay])
 
     def tickStrings(
         self,
@@ -438,7 +412,7 @@ class XAxisLabel(AxisLabel):
         | QtCore.Qt.AlignCenter
     )
 
-    def size_hint(self) -> tuple[float, float]:
+    def size_hint(self) -> Tuple[float, float]:
         # size to parent axis height
         return self._parent.height(), None
 
@@ -452,11 +426,11 @@ class XAxisLabel(AxisLabel):
 
         timestrs = self._parent._indexes_to_timestrs([int(value)])
 
-        if not len(timestrs):
+        if not timestrs.any():
             return
 
         pad = 1*' '
-        self.label_str = pad + str(timestrs[0]) + pad
+        self.label_str = pad + timestrs[0] + pad
 
         _, y_offset = self._parent.txt_offsets()
 
@@ -517,7 +491,7 @@ class YAxisLabel(AxisLabel):
         if getattr(self._parent, 'txt_offsets', False):
             self.x_offset, y_offset = self._parent.txt_offsets()
 
-    def size_hint(self) -> tuple[float, float]:
+    def size_hint(self) -> Tuple[float, float]:
         # size to parent axis width(-ish)
         wsh = self._dpifont.boundingRect(' ').height() / 2
         return (

piker/ui/_chart.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,14 +19,10 @@ High level chart-widget apis.
 
 '''
 from __future__ import annotations
-from typing import Optional, TYPE_CHECKING
+from typing import Optional
 
 from PyQt5 import QtCore, QtWidgets
-from PyQt5.QtCore import (
+from PyQt5.QtCore import Qt
-    Qt,
-    QLineF,
-    # QPointF,
-)
 from PyQt5.QtWidgets import (
     QFrame,
     QWidget,
@@ -47,30 +43,25 @@ from ._cursor import (
     Cursor,
     ContentsLabel,
 )
-from ..data._sharedmem import ShmArray
 from ._l1 import L1Labels
 from ._ohlc import BarItems
-from ._curve import (
+from ._curve import FastAppendCurve
-    Curve,
-    StepCurve,
-)
 from ._style import (
     hcolor,
     CHART_MARGINS,
     _xaxis_at,
     _min_points_to_show,
+    _bars_from_right_in_follow_mode,
+    _bars_to_left_in_follow_mode,
 )
 from ..data.feed import Feed
 from ..data._source import Symbol
+from ..data._sharedmem import ShmArray
 from ..log import get_logger
 from ._interaction import ChartView
 from ._forms import FieldsForm
-from .._profile import pg_profile_enabled, ms_slower_then
 from ._overlay import PlotItemOverlay
-from ._flows import Flow
 
-if TYPE_CHECKING:
-    from ._display import DisplayState
 
 log = get_logger(__name__)
 
@@ -120,9 +111,6 @@ class GodWidget(QWidget):
         # assigned in the startup func `_async_main()`
         self._root_n: trio.Nursery = None
 
-        self._widgets: dict[str, QWidget] = {}
-        self._resizing: bool = False
-
     # def init_timeframes_ui(self):
     #     self.tf_layout = QHBoxLayout()
     #     self.tf_layout.setSpacing(0)
@@ -238,23 +226,16 @@ class GodWidget(QWidget):
             # chart is already in memory so just focus it
             linkedsplits.show()
             linkedsplits.focus()
-            linkedsplits.graphics_cycle()
             await trio.sleep(0)
 
             # resume feeds *after* rendering chart view asap
             chart.resume_all_feeds()
 
-            # TODO: we need a check to see if the chart
-            # last had the xlast in view, if so then shift so it's
-            # still in view, if the user was viewing history then
-            # do nothing yah?
-            chart.default_view()
-
         self.linkedsplits = linkedsplits
         symbol = linkedsplits.symbol
         if symbol is not None:
             self.window.setWindowTitle(
-                f'{symbol.front_fqsn()} '
+                f'{symbol.key}@{symbol.brokers} '
                 f'tick:{symbol.tick_size}'
             )
 
@@ -277,16 +258,7 @@ class GodWidget(QWidget):
         Where we do UX magic to make things not suck B)
 
         '''
-        if self._resizing:
+        log.debug('god widget resize')
-            return
-
-        self._resizing = True
-
-        log.info('God widget resize')
-        for name, widget in self._widgets.items():
-            widget.on_resize()
-
-        self._resizing = False
 
 
 class ChartnPane(QFrame):
@@ -361,19 +333,8 @@ class LinkedSplits(QWidget):
         self.layout.setContentsMargins(0, 0, 0, 0)
         self.layout.addWidget(self.splitter)
 
-        # chart-local graphics state that can be passed to
-        # a ``graphic_update_cycle()`` call by any task wishing to
-        # update the UI for a given "chart instance".
-        self.display_state: Optional[DisplayState] = None
-
         self._symbol: Symbol = None
 
-    def graphics_cycle(self, **kwargs) -> None:
-        from . import _display
-        ds = self.display_state
-        if ds:
-            return _display.graphics_update_cycle(ds, **kwargs)
-
     @property
     def symbol(self) -> Symbol:
         return self._symbol
@@ -388,15 +349,12 @@ class LinkedSplits(QWidget):
         '''
         ln = len(self.subplots)
 
-        # proportion allocated to consumer subcharts
         if not prop:
-            prop = 3/8*5/8
+            # proportion allocated to consumer subcharts
-
+            if ln < 2:
-            # if ln < 2:
+                prop = 1/3
-            #     prop = 3/8*5/8
+            elif ln >= 2:
-
+                prop = 3/8
-            # elif ln >= 2:
-            #     prop = 3/8
 
         major = 1 - prop
         min_h_ind = int((self.height() * prop) / ln)
@@ -418,7 +376,7 @@ class LinkedSplits(QWidget):
         self,
 
         symbol: Symbol,
-        shm: ShmArray,
+        array: np.ndarray,
         sidepane: FieldsForm,
 
         style: str = 'bar',
@@ -443,7 +401,7 @@ class LinkedSplits(QWidget):
         self.chart = self.add_plot(
 
             name=symbol.key,
-            shm=shm,
+            array=array,
             style=style,
             _is_main=True,
 
@@ -471,7 +429,7 @@ class LinkedSplits(QWidget):
         self,
 
         name: str,
-        shm: ShmArray,
+        array: np.ndarray,
 
         array_key: Optional[str] = None,
         style: str = 'line',
@@ -515,6 +473,7 @@ class LinkedSplits(QWidget):
             name=name,
             data_key=array_key or name,
 
+            array=array,
             parent=qframe,
             linkedsplits=self,
             axisItems=axes,
@@ -578,7 +537,7 @@ class LinkedSplits(QWidget):
 
             graphics, data_key = cpw.draw_ohlc(
                 name,
-                shm,
+                array,
                 array_key=array_key
             )
             self.cursor.contents_labels.add_label(
@@ -592,7 +551,7 @@ class LinkedSplits(QWidget):
             add_label = True
             graphics, data_key = cpw.draw_curve(
                 name,
-                shm,
+                array,
                 array_key=array_key,
                 color='default_light',
             )
@@ -601,7 +560,7 @@ class LinkedSplits(QWidget):
             add_label = True
             graphics, data_key = cpw.draw_curve(
                 name,
-                shm,
+                array,
                 array_key=array_key,
                 step_mode=True,
                 color='davies',
@@ -655,6 +614,18 @@ class LinkedSplits(QWidget):
                 cpw.sidepane.setMinimumWidth(sp_w)
                 cpw.sidepane.setMaximumWidth(sp_w)
 
+# import pydantic
+
+# class Graphics(pydantic.BaseModel):
+#     '''
+#     Data-AGGRegate: high level API onto multiple (categorized)
+#     ``ShmArray``s with high level processing routines for
+#     graphics computations and display.
+
+#     '''
+#     arrays: dict[str, np.ndarray] = {}
+#     graphics: dict[str, pg.GraphicsObject] = {}
+
 
 class ChartPlotWidget(pg.PlotWidget):
     '''
@@ -689,6 +660,7 @@ class ChartPlotWidget(pg.PlotWidget):
 
         # the "data view" we generate graphics from
         name: str,
+        array: np.ndarray,
         data_key: str,
         linkedsplits: LinkedSplits,
 
@@ -741,9 +713,16 @@ class ChartPlotWidget(pg.PlotWidget):
         self._max_l1_line_len: float = 0
 
         # self.setViewportMargins(0, 0, 0, 0)
+        # self._ohlc = array  # readonly view of ohlc data
 
+        # TODO: move to Aggr above XD
+        # readonly view of data arrays
+        self._arrays = {
+            self.data_key: array,
+        }
+        self._graphics = {}  # registry of underlying graphics
         # registry of overlay curve names
-        self._flows: dict[str, Flow] = {}
+        self._overlays: dict[str, ShmArray] = {}
 
         self._feeds: dict[Symbol, Feed] = {}
 
@@ -756,6 +735,7 @@ class ChartPlotWidget(pg.PlotWidget):
         # show background grid
         self.showGrid(x=False, y=True, alpha=0.3)
 
+        self.default_view()
         self.cv.enable_auto_yrange()
 
         self.pi_overlay: PlotItemOverlay = PlotItemOverlay(self.plotItem)
@@ -800,138 +780,39 @@ class ChartPlotWidget(pg.PlotWidget):
         return int(vr.left()), int(vr.right())
 
     def bars_range(self) -> tuple[int, int, int, int]:
-        '''
+        """Return a range tuple for the bars present in view.
-        Return a range tuple for the bars present in view.
+        """
-
+        l, r = self.view_range()
-        '''
+        array = self._arrays[self.name]
-        main_flow = self._flows[self.name]
+        lbar = max(l, array[0]['index'])
-        ifirst, l, lbar, rbar, r, ilast = main_flow.datums_range()
+        rbar = min(r, array[-1]['index'])
         return l, lbar, rbar, r
 
-    def curve_width_pxs(
-        self,
-    ) -> float:
-        _, lbar, rbar, _ = self.bars_range()
-        return self.view.mapViewToDevice(
-            QLineF(lbar, 0, rbar, 0)
-        ).length()
-
-    def pre_l1_xs(self) -> tuple[float, float]:
-        '''
-        Return the view x-coord for the value just before
-        the L1 labels on the y-axis as well as the length
-        of that L1 label from the y-axis.
-
-        '''
-        line_end, marker_right, yaxis_x = self.marker_right_points()
-        view = self.view
-        line = view.mapToView(
-            QLineF(line_end, 0, yaxis_x, 0)
-        )
-        return line.x1(), line.length()
-
-    def marker_right_points(
-        self,
-        marker_size: int = 20,
-
-    ) -> (float, float, float):
-        '''
-        Return x-dimension, y-axis-aware, level-line marker oriented scene
-        values.
-
-        X values correspond to set the end of a level line, end of
-        a paried level line marker, and the right most side of the "right"
-        axis respectively.
-
-        '''
-        # TODO: compute some sensible maximum value here
-        # and use a humanized scheme to limit to that length.
-        l1_len = self._max_l1_line_len
-        ryaxis = self.getAxis('right')
-
-        r_axis_x = ryaxis.pos().x()
-        up_to_l1_sc = r_axis_x - l1_len - 10
-
-        marker_right = up_to_l1_sc - (1.375 * 2 * marker_size)
-        line_end = marker_right - (6/16 * marker_size)
-
-        return line_end, marker_right, r_axis_x
-
     def default_view(
         self,
-        bars_from_y: int = 3000,
+        index: int = -1,
-
     ) -> None:
-        '''
+        """Set the view box to the "default" startup view of the scene.
-        Set the view box to the "default" startup view of the scene.
 
-        '''
+        """
-        flow = self._flows.get(self.name)
+        xlast = self._arrays[self.name][index]['index']
-        if not flow:
+        begin = xlast - _bars_to_left_in_follow_mode
-            log.warning(f'`Flow` for {self.name} not loaded yet?')
+        end = xlast + _bars_from_right_in_follow_mode
-            return
-
-        index = flow.shm.array['index']
-        xfirst, xlast = index[0], index[-1]
-        l, lbar, rbar, r = self.bars_range()
-        view = self.view
-
-        if (
-            rbar < 0
-            or l < xfirst
-            or l < 0
-            or (rbar - lbar) < 6
-        ):
-            # TODO: set fixed bars count on screen that approx includes as
-            # many bars as possible before a downsample line is shown.
-            begin = xlast - bars_from_y
-            view.setXRange(
-                min=begin,
-                max=xlast,
-                padding=0,
-            )
-            # re-get range
-            l, lbar, rbar, r = self.bars_range()
-
-        # we get the L1 spread label "length" in view coords
-        # terms now that we've scaled either by user control
-        # or to the default set of bars as per the immediate block
-        # above.
-        marker_pos, l1_len = self.pre_l1_xs()
-        end = xlast + l1_len + 1
-        begin = end - (r - l)
-
-        # for debugging
-        # print(
-        #     # f'bars range: {brange}\n'
-        #     f'xlast: {xlast}\n'
-        #     f'marker pos: {marker_pos}\n'
-        #     f'l1 len: {l1_len}\n'
-        #     f'begin: {begin}\n'
-        #     f'end: {end}\n'
-        # )
 
         # remove any custom user yrange setttings
         if self._static_yrange == 'axis':
             self._static_yrange = None
 
+        view = self.view
         view.setXRange(
             min=begin,
             max=end,
             padding=0,
         )
-        self.view.maybe_downsample_graphics()
         view._set_yrange()
-        try:
-            self.linked.graphics_cycle()
-        except IndexError:
-            pass
 
     def increment_view(
         self,
-        steps: int = 1,
-        vb: Optional[ChartView] = None,
-
     ) -> None:
         """
         Increment the data view one step to the right thus "following"
@@ -939,10 +820,9 @@ class ChartPlotWidget(pg.PlotWidget):
 
         """
         l, r = self.view_range()
-        view = vb or self.view
+        self.view.setXRange(
-        view.setXRange(
+            min=l + 1,
-            min=l + steps,
+            max=r + 1,
-            max=r + steps,
 
             # TODO: holy shit, wtf dude... why tf would this not be 0 by
             # default... speechless.
@@ -951,8 +831,9 @@ class ChartPlotWidget(pg.PlotWidget):
 
     def draw_ohlc(
         self,
+
         name: str,
-        shm: ShmArray,
+        data: np.ndarray,
 
         array_key: Optional[str] = None,
 
@@ -962,26 +843,19 @@ class ChartPlotWidget(pg.PlotWidget):
 
         '''
         graphics = BarItems(
-            self.linked,
             self.plotItem,
-            pen_color=self.pen_color,
+            pen_color=self.pen_color
-            name=name,
         )
 
         # adds all bar/candle graphics objects for each data point in
         # the np array buffer to be drawn on next render cycle
         self.plotItem.addItem(graphics)
 
+        # draw after to allow self.scene() to work...
+        graphics.draw_from_data(data)
+
         data_key = array_key or name
-
+        self._graphics[data_key] = graphics
-        self._flows[data_key] = Flow(
-            name=name,
-            plot=self.plotItem,
-            _shm=shm,
-            is_ohlc=True,
-            graphics=graphics,
-        )
-
         self._add_sticky(name, bg_color='davies')
 
         return graphics, data_key
@@ -1022,7 +896,6 @@ class ChartPlotWidget(pg.PlotWidget):
         )
         pi.hideButtons()
 
-        # cv.enable_auto_yrange(self.view)
         cv.enable_auto_yrange()
 
         # compose this new plot's graphics with the current chart's
@@ -1047,21 +920,19 @@ class ChartPlotWidget(pg.PlotWidget):
         self,
 
         name: str,
-        shm: ShmArray,
+        data: np.ndarray,
 
         array_key: Optional[str] = None,
         overlay: bool = False,
         color: Optional[str] = None,
         add_label: bool = True,
-        pi: Optional[pg.PlotItem] = None,
-        step_mode: bool = False,
 
         **pdi_kwargs,
 
     ) -> (pg.PlotDataItem, str):
         '''
         Draw a "curve" (line plot graphics) for the provided data in
-        the input shm array ``shm``.
+        the input array ``data``.
 
         '''
         color = color or self.pen_color or 'default_light'
@@ -1071,37 +942,54 @@ class ChartPlotWidget(pg.PlotWidget):
 
         data_key = array_key or name
 
-        curve_type = {
+        # yah, we wrote our own B)
-            None: Curve,
+        curve = FastAppendCurve(
-            'step': StepCurve,
+            y=data[data_key],
-            # TODO:
+            x=data['index'],
-            # 'bars': BarsItems
+            # antialias=True,
-        }['step' if step_mode else None]
-
-        curve = curve_type(
             name=name,
+
+            # XXX: pretty sure this is just more overhead
+            # on data reads and makes graphics rendering no faster
+            # clipToView=True,
+
+            # TODO: see how this handles with custom ohlcv bars graphics
+            # and/or if we can implement something similar for OHLC graphics
+            # autoDownsample=True,
+            # downsample=60,
+            # downsampleMethod='subsample',
+
             **pdi_kwargs,
         )
 
-        pi = pi or self.plotItem
+        # XXX: see explanation for different caching modes:
+        # https://stackoverflow.com/a/39410081
+        # seems to only be useful if we don't re-generate the entire
+        # QPainterPath every time
+        # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
-        self._flows[data_key] = Flow(
+        # don't ever use this - it's a colossal nightmare of artefacts
-            name=name,
+        # and is disastrous for performance.
-            plot=pi,
+        # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
-            _shm=shm,
+
-            is_ohlc=False,
+        # register curve graphics and backing array for name
-            # register curve graphics with this flow
+        self._graphics[name] = curve
-            graphics=curve,
+        self._arrays[data_key] = data
-        )
+
+        pi = self.plotItem
 
         # TODO: this probably needs its own method?
         if overlay:
+            # anchor_at = ('bottom', 'left')
+            self._overlays[name] = None
+
             if isinstance(overlay, pg.PlotItem):
                 if overlay not in self.pi_overlay.overlays:
                     raise RuntimeError(
                             f'{overlay} must be from `.plotitem_overlay()`'
                         )
                 pi = overlay
+
         else:
             # anchor_at = ('top', 'left')
 
@@ -1109,17 +997,7 @@ class ChartPlotWidget(pg.PlotWidget):
             # (we need something that avoids clutter on x-axis).
             self._add_sticky(name, bg_color=color)
 
-        # NOTE: this is more or less the RENDER call that tells Qt to
-        # start showing the generated graphics-curves. This is kind of
-        # of edge-triggered call where once added any
-        # ``QGraphicsItem.update()`` calls are automatically displayed.
-        # Our internal graphics objects have their own "update from
-        # data" style method API that allows for real-time updates on
-        # the next render cycle; just note a lot of the real-time
-        # updates are implicit and require a bit of digging to
-        # understand.
         pi.addItem(curve)
-
         return curve, data_key
 
     # TODO: make this a ctx mngr
@@ -1151,11 +1029,11 @@ class ChartPlotWidget(pg.PlotWidget):
         )
         return last
 
-    def update_graphics_from_flow(
+    def update_ohlc_from_array(
         self,
-        graphics_name: str,
-        array_key: Optional[str] = None,
 
+        graphics_name: str,
+        array: np.ndarray,
         **kwargs,
 
     ) -> pg.GraphicsObject:
@@ -1163,12 +1041,50 @@ class ChartPlotWidget(pg.PlotWidget):
         Update the named internal graphics from ``array``.
 
         '''
-        flow = self._flows[array_key or graphics_name]
+        self._arrays[self.name] = array
-        return flow.update_graphics(
+        graphics = self._graphics[graphics_name]
-            array_key=array_key,
+        graphics.update_from_array(array, **kwargs)
+        return graphics
+
+    def update_curve_from_array(
+        self,
+
+        graphics_name: str,
+        array: np.ndarray,
+        array_key: Optional[str] = None,
         **kwargs,
+
+    ) -> pg.GraphicsObject:
+        '''
+        Update the named internal graphics from ``array``.
+
+        '''
+        assert len(array)
+        data_key = array_key or graphics_name
+
+        if graphics_name not in self._overlays:
+            self._arrays[self.name] = array
+        else:
+            self._arrays[data_key] = array
+
+        curve = self._graphics[graphics_name]
+
+        # NOTE: back when we weren't implementing the curve graphics
+        # ourselves you'd have updates using this method:
+        # curve.setData(y=array[graphics_name], x=array['index'], **kwargs)
+
+        # NOTE: graphics **must** implement a diff based update
+        # operation where an internal ``FastUpdateCurve._xrange`` is
+        # used to determine if the underlying path needs to be
+        # pre/ap-pended.
+        curve.update_from_array(
+            x=array['index'],
+            y=array[data_key],
+            **kwargs
         )
 
+        return curve
+
     # def _label_h(self, yhigh: float, ylow: float) -> float:
     #     # compute contents label "height" in view terms
     #     # to avoid having data "contents" overlap with them
@@ -1198,9 +1114,6 @@ class ChartPlotWidget(pg.PlotWidget):
 
     #     print(f"bounds (ylow, yhigh): {(ylow, yhigh)}")
 
-    # TODO: pretty sure we can just call the cursor
-    # directly not? i don't wee why we need special "signal proxies"
-    # for this lul..
     def enterEvent(self, ev):  # noqa
         # pg.PlotWidget.enterEvent(self, ev)
         self.sig_mouse_enter.emit(self)
@@ -1214,7 +1127,7 @@ class ChartPlotWidget(pg.PlotWidget):
 
         # TODO: this should go onto some sort of
         # data-view thinger..right?
-        ohlc = self._flows[self.name].shm.array
+        ohlc = self._shm.array
 
         # XXX: not sure why the time is so off here
         # looks like we're gonna have to do some fixing..
@@ -1225,28 +1138,10 @@ class ChartPlotWidget(pg.PlotWidget):
         else:
             return ohlc['index'][-1]
 
-    def in_view(
-        self,
-        array: np.ndarray,
-
-    ) -> np.ndarray:
-        '''
-        Slice an input struct array providing only datums
-        "in view" of this chart.
-
-        '''
-        l, lbar, rbar, r = self.bars_range()
-        ifirst = array[0]['index']
-        # slice data by offset from the first index
-        # available in the passed datum set.
-        return array[lbar - ifirst:(rbar - ifirst) + 1]
-
     def maxmin(
         self,
         name: Optional[str] = None,
-        bars_range: Optional[tuple[
+        bars_range: Optional[tuple[int, int, int, int]] = None,
-            int, int, int, int, int, int
-        ]] = None,
 
     ) -> tuple[float, float]:
         '''
@@ -1255,43 +1150,40 @@ class ChartPlotWidget(pg.PlotWidget):
         If ``bars_range`` is provided use that range.
 
         '''
-        # print(f'Chart[{self.name}].maxmin()')
+        l, lbar, rbar, r = bars_range or self.bars_range()
-        profiler = pg.debug.Profiler(
+        # TODO: logic to check if end of bars in view
-            msg=f'`{str(self)}.maxmin(name={name})`: `{self.name}`',
+        # extra = view_len - _min_points_to_show
-            disabled=not pg_profile_enabled(),
+        # begin = self._arrays['ohlc'][0]['index'] - extra
-            ms_threshold=ms_slower_then,
+        # # end = len(self._arrays['ohlc']) - 1 + extra
-            delayed=True,
+        # end = self._arrays['ohlc'][-1]['index'] - 1 + extra
-        )
+
+        # bars_len = rbar - lbar
+        # log.debug(
+        #     f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n"
+        #     f"view_len: {view_len}, bars_len: {bars_len}\n"
+        #     f"begin: {begin}, end: {end}, extra: {extra}"
+        # )
+
+        a = self._arrays[name or self.name]
+        ifirst = a[0]['index']
+        bars = a[lbar - ifirst:rbar - ifirst + 1]
+
+        if not len(bars):
+            # likely no data loaded yet or extreme scrolling?
+            log.error(f"WTF bars_range = {lbar}:{rbar}")
+            return
 
-        # TODO: here we should instead look up the ``Flow.shm.array``
-        # and read directly from shm to avoid copying to memory first
-        # and then reading it again here.
-        flow_key = name or self.name
-        flow = self._flows.get(flow_key)
         if (
-            flow is None
+            self.data_key == self.linked.symbol.key
         ):
-            log.error(f"flow {flow_key} doesn't exist in chart {self.name} !?")
+            # ohlc sampled bars hi/lo lookup
-            key = res = 0, 0
+            ylow = np.nanmin(bars['low'])
+            yhigh = np.nanmax(bars['high'])
 
         else:
-            (
+            view = bars[name or self.data_key]
-                first,
+            ylow = np.nanmin(view)
-                l,
+            yhigh = np.nanmax(view)
-                lbar,
-                rbar,
-                r,
-                last,
-            ) = bars_range or flow.datums_range()
-            profiler(f'{self.name} got bars range')
 
-            key = round(lbar), round(rbar)
+        # print(f'{(ylow, yhigh)}')
-            res = flow.maxmin(*key)
+        return ylow, yhigh
-            if res == (None, None):
-                log.error(
-                    f"{flow_key} no mxmn for bars_range => {key} !?"
-                )
-                res = 0, 0
-
-        profiler(f'yrange mxmn: {key} -> {res}')
-        return res

piker/ui/_compression.py

@@ -1,318 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-'''
-Graphics related downsampling routines for compressing to pixel
-limits on the display device.
-
-'''
-import math
-from typing import Optional
-
-import numpy as np
-from numpy.lib import recfunctions as rfn
-from numba import (
-    jit,
-    # float64, optional, int64,
-)
-
-from ..log import get_logger
-
-
-log = get_logger(__name__)
-
-
-def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
-    '''
-    Convert a OHLC struct-array containing 'high'/'low' columns
-    to a "joined" max/min 1-d array.
-
-    '''
-    index = ohlc['index']
-    hls = ohlc[[
-        'low',
-        'high',
-    ]]
-
-    mxmn = np.empty(2*hls.size, dtype=np.float64)
-    x = np.empty(2*hls.size, dtype=np.float64)
-    trace_hl(hls, mxmn, x, index[0])
-    x = x + index[0]
-
-    return mxmn, x
-
-
-@jit(
-    # TODO: the type annots..
-    # float64[:](float64[:],),
-    nopython=True,
-)
-def trace_hl(
-    hl: 'np.ndarray',
-    out: np.ndarray,
-    x: np.ndarray,
-    start: int,
-
-    # the "offset" values in the x-domain which
-    # place the 2 output points around each ``int``
-    # master index.
-    margin: float = 0.43,
-
-) -> None:
-    '''
-    "Trace" the outline of the high-low values of an ohlc sequence
-    as a line such that the maximum deviation (aka disperaion) between
-    bars if preserved.
-
-    This routine is expected to modify input arrays in-place.
-
-    '''
-    last_l = hl['low'][0]
-    last_h = hl['high'][0]
-
-    for i in range(hl.size):
-        row = hl[i]
-        l, h = row['low'], row['high']
-
-        up_diff = h - last_l
-        down_diff = last_h - l
-
-        if up_diff > down_diff:
-            out[2*i + 1] = h
-            out[2*i] = last_l
-        else:
-            out[2*i + 1] = l
-            out[2*i] = last_h
-
-        last_l = l
-        last_h = h
-
-        x[2*i] = int(i) - margin
-        x[2*i + 1] = int(i) + margin
-
-    return out
-
-
-def ohlc_flatten(
-    ohlc: np.ndarray,
-    use_mxmn: bool = True,
-
-) -> tuple[np.ndarray, np.ndarray]:
-    '''
-    Convert an OHLCV struct-array into a flat ready-for-line-plotting
-    1-d array that is 4 times the size with x-domain values distributed
-    evenly (by 0.5 steps) over each index.
-
-    '''
-    index = ohlc['index']
-
-    if use_mxmn:
-        # traces a line optimally over highs to lows
-        # using numba. NOTE: pretty sure this is faster
-        # and looks about the same as the below output.
-        flat, x = hl2mxmn(ohlc)
-
-    else:
-        flat = rfn.structured_to_unstructured(
-            ohlc[['open', 'high', 'low', 'close']]
-        ).flatten()
-
-        x = np.linspace(
-            start=index[0] - 0.5,
-            stop=index[-1] + 0.5,
-            num=len(flat),
-        )
-    return x, flat
-
-
-def ds_m4(
-    x: np.ndarray,
-    y: np.ndarray,
-    # units-per-pixel-x(dimension)
-    uppx: float,
-
-    # XXX: troll zone / easter egg..
-    # want to mess with ur pal, pass in the actual
-    # pixel width here instead of uppx-proper (i.e. pass
-    # in our ``pg.GraphicsObject`` derivative's ``.px_width()``
-    # gto mega-trip-out ur bud). Hint, it used to be implemented
-    # (wrongly) using "pixel width", so check the git history ;)
-
-    xrange: Optional[float] = None,
-
-) -> tuple[int, np.ndarray, np.ndarray]:
-    '''
-    Downsample using the M4 algorithm.
-
-    This is more or less an OHLC style sampling of a line-style series.
-
-    '''
-    # NOTE: this method is a so called "visualization driven data
-    # aggregation" approach. It gives error-free line chart
-    # downsampling, see
-    # further scientific paper resources:
-    # - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
-    # - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
-
-    # Details on implementation of this algo are based in,
-    # https://github.com/pikers/piker/issues/109
-
-    # XXX: from infinite on downsampling viewable graphics:
-    # "one thing i remembered about the binning - if you are
-    # picking a range within your timeseries the start and end bin
-    # should be one more bin size outside the visual range, then
-    # you get better visual fidelity at the edges of the graph"
-    # "i didn't show it in the sample code, but it's accounted for
-    # in the start and end indices and number of bins"
-
-    # should never get called unless actually needed
-    assert uppx > 1
-
-    # NOTE: if we didn't pre-slice the data to downsample
-    # you could in theory pass these as the slicing params,
-    # do we care though since we can always just pre-slice the
-    # input?
-    x_start = x[0]  # x value start/lowest in domain
-
-    if xrange is None:
-        x_end = x[-1]  # x end value/highest in domain
-        xrange = (x_end - x_start)
-
-    # XXX: always round up on the input pixels
-    # lnx = len(x)
-    # uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
-
-    pxw = math.ceil(xrange / uppx)
-
-    # scale up the frame "width" directly with uppx
-    w = uppx
-
-    # ensure we make more then enough
-    # frames (windows) for the output pixel
-    frames = pxw
-
-    # if we have more and then exact integer's
-    # (uniform quotient output) worth of datum-domain-points
-    # per windows-frame, add one more window to ensure
-    # we have room for all output down-samples.
-    pts_per_pixel, r = divmod(xrange, frames)
-    if r:
-        # while r:
-        frames += 1
-        pts_per_pixel, r = divmod(xrange, frames)
-
-    # print(
-    #     f'uppx: {uppx}\n'
-    #     f'xrange: {xrange}\n'
-    #     f'pxw: {pxw}\n'
-    #     f'frames: {frames}\n'
-    # )
-    assert frames >= (xrange / uppx)
-
-    # call into ``numba``
-    nb, i_win, y_out = _m4(
-        x,
-        y,
-
-        frames,
-
-        # TODO: see func below..
-        # i_win,
-        # y_out,
-
-        # first index in x data to start at
-        x_start,
-        # window size for each "frame" of data to downsample (normally
-        # scaled by the ratio of pixels on screen to data in x-range).
-        w,
-    )
-
-    # filter out any overshoot in the input allocation arrays by
-    # removing zero-ed tail entries which should start at a certain
-    # index.
-    i_win = i_win[i_win != 0]
-    y_out = y_out[:i_win.size]
-
-    return nb, i_win, y_out
-
-
-@jit(
-    nopython=True,
-    nogil=True,
-)
-def _m4(
-
-    xs: np.ndarray,
-    ys: np.ndarray,
-
-    frames: int,
-
-    # TODO: using this approach by having the ``.zeros()`` alloc lines
-    # below, in put python was causing segs faults and alloc crashes..
-    # we might need to see how it behaves with shm arrays and consider
-    # allocating them once at startup?
-
-    # pre-alloc array of x indices mapping to the start
-    # of each window used for downsampling in y.
-    # i_win: np.ndarray,
-    # pre-alloc array of output downsampled y values
-    # y_out: np.ndarray,
-
-    x_start: int,
-    step: float,
-
-) -> int:
-    # nbins = len(i_win)
-    # count = len(xs)
-
-    # these are pre-allocated and mutated by ``numba``
-    # code in-place.
-    y_out = np.zeros((frames, 4), ys.dtype)
-    i_win = np.zeros(frames, xs.dtype)
-
-    bincount = 0
-    x_left = x_start
-
-    # Find the first window's starting value which *includes* the
-    # first value in the x-domain array, i.e. the first
-    # "left-side-of-window" **plus** the downsampling step,
-    # creates a window which includes the first x **value**.
-    while xs[0] >= x_left + step:
-        x_left += step
-
-    # set all bins in the left-most entry to the starting left-most x value
-    # (aka a row broadcast).
-    i_win[bincount] = x_left
-    # set all y-values to the first value passed in.
-    y_out[bincount] = ys[0]
-
-    for i in range(len(xs)):
-        x = xs[i]
-        y = ys[i]
-        if x < x_left + step:   # the current window "step" is [bin, bin+1)
-            y_out[bincount, 1] = min(y, y_out[bincount, 1])
-            y_out[bincount, 2] = max(y, y_out[bincount, 2])
-            y_out[bincount, 3] = y
-        else:
-            # Find the next bin
-            while x >= x_left + step:
-                x_left += step
-
-            bincount += 1
-            i_win[bincount] = x_left
-            y_out[bincount] = y
-
-    return bincount, i_win, y_out

piker/ui/_cursor.py

@@ -43,8 +43,8 @@ log = get_logger(__name__)
 # latency (in terms of perceived lag in cross hair) so really be sure
 # there's an improvement if you want to change it!
 
-_mouse_rate_limit = 60  # TODO; should we calc current screen refresh rate?
+_mouse_rate_limit = 120  # TODO; should we calc current screen refresh rate?
-_debounce_delay = 0
+_debounce_delay = 1 / 40
 _ch_label_opac = 1
 
 
@@ -95,33 +95,25 @@ class LineDot(pg.CurvePoint):
 
     def event(
         self,
+
         ev: QtCore.QEvent,
 
-    ) -> bool:
+    ) -> None:
-
+        if not isinstance(
-        if (
+            ev, QtCore.QDynamicPropertyChangeEvent
-            not isinstance(ev, QtCore.QDynamicPropertyChangeEvent)
+        ) or self.curve() is None:
-            or self.curve() is None
-        ):
             return False
 
-        # TODO: get rid of this ``.getData()`` and
+        (x, y) = self.curve().getData()
-        # make a more pythonic api to retreive backing
+        index = self.property('index')
-        # numpy arrays...
+        # first = self._plot._arrays['ohlc'][0]['index']
-        # (x, y) = self.curve().getData()
+        # first = x[0]
-        # index = self.property('index')
+        # i = index - first
-        # # first = self._plot._arrays['ohlc'][0]['index']
+        i = index - x[0]
-        # # first = x[0]
+        if i > 0 and i < len(y):
-        # # i = index - first
+            newPos = (index, y[i])
-        # if index:
+            QtWidgets.QGraphicsItem.setPos(self, *newPos)
-        #     i = round(index - x[0])
+            return True
-        #     if i > 0 and i < len(y):
-        #         newPos = (index, y[i])
-        #         QtWidgets.QGraphicsItem.setPos(
-        #             self,
-        #             *newPos,
-        #         )
-        #         return True
 
         return False
 
@@ -196,9 +188,6 @@ class ContentsLabel(pg.LabelItem):
 
         self.setText(
             "<b>i</b>:{index}<br/>"
-            # NB: these fields must be indexed in the correct order via
-            # the slice syntax below.
-            "<b>epoch</b>:{}<br/>"
             "<b>O</b>:{}<br/>"
             "<b>H</b>:{}<br/>"
             "<b>L</b>:{}<br/>"
@@ -206,15 +195,7 @@ class ContentsLabel(pg.LabelItem):
             "<b>V</b>:{}<br/>"
             "<b>wap</b>:{}".format(
                 *array[index - first][
-                    [
+                    ['open', 'high', 'low', 'close', 'volume', 'bar_wap']
-                        'time',
-                        'open',
-                        'high',
-                        'low',
-                        'close',
-                        'volume',
-                        'bar_wap',
-                    ]
                 ],
                 name=name,
                 index=index,
@@ -259,21 +240,23 @@ class ContentsLabels:
     def update_labels(
         self,
         index: int,
+        # array_name: str,
 
     ) -> None:
+        # for name, (label, update) in self._labels.items():
         for chart, name, label, update in self._labels:
 
-            flow = chart._flows[name]
+            array = chart._arrays[name]
-            array = flow.shm.array
-
             if not (
                 index >= 0
                 and index < array[-1]['index']
             ):
                 # out of range
-                print('WTF out of range?')
+                print('out of range?')
                 continue
 
+            # array = chart._arrays[name]
+
             # call provided update func with data point
             try:
                 label.show()
@@ -309,8 +292,7 @@ class ContentsLabels:
 
 
 class Cursor(pg.GraphicsObject):
-    '''
+    '''Multi-plot cursor for use on a ``LinkedSplits`` chart (set).
-    Multi-plot cursor for use on a ``LinkedSplits`` chart (set).
 
     '''
     def __init__(
@@ -325,7 +307,7 @@ class Cursor(pg.GraphicsObject):
 
         self.linked = linkedsplits
         self.graphics: dict[str, pg.GraphicsObject] = {}
-        self.plots: list['PlotChartWidget'] = []  # type: ignore # noqa
+        self.plots: List['PlotChartWidget'] = []  # type: ignore # noqa
         self.active_plot = None
         self.digits: int = digits
         self._datum_xy: tuple[int, float] = (0, 0)
@@ -422,7 +404,6 @@ class Cursor(pg.GraphicsObject):
             slot=self.mouseMoved,
             delay=_debounce_delay,
         )
-
         px_enter = pg.SignalProxy(
             plot.sig_mouse_enter,
             rateLimit=_mouse_rate_limit,
@@ -454,10 +435,7 @@ class Cursor(pg.GraphicsObject):
         if plot.linked.xaxis_chart is plot:
             xlabel = self.xaxis_label = XAxisLabel(
                 parent=self.plots[plot_index].getAxis('bottom'),
-                # parent=self.plots[plot_index].pi_overlay.get_axis(
+                # parent=self.plots[plot_index].pi_overlay.get_axis(plot.plotItem, 'bottom'),
-                #     plot.plotItem, 'bottom'
-                # ),
-
                 opacity=_ch_label_opac,
                 bg_color=self.label_color,
             )
@@ -475,12 +453,9 @@ class Cursor(pg.GraphicsObject):
     ) -> LineDot:
         # if this plot contains curves add line dot "cursors" to denote
         # the current sample under the mouse
-        main_flow = plot._flows[plot.name]
-        # read out last index
-        i = main_flow.shm.array[-1]['index']
         cursor = LineDot(
             curve,
-            index=i,
+            index=plot._arrays[plot.name][-1]['index'],
             plot=plot
         )
         plot.addItem(cursor)
@@ -574,20 +549,17 @@ class Cursor(pg.GraphicsObject):
                 for cursor in opts.get('cursors', ()):
                     cursor.setIndex(ix)
 
-                # Update the label on the bottom of the crosshair.
+                # update the label on the bottom of the crosshair
-                # TODO: make this an up-front calc that we update
+                axes = plot.plotItem.axes
-                # on axis-widget resize events instead of on every mouse
-                # update cylce.
 
+                # TODO: make this an up-front calc that we update
+                # on axis-widget resize events.
                 # left axis offset width for calcuating
                 # absolute x-axis label placement.
                 left_axis_width = 0
-                if len(plot.pi_overlay.overlays):
+                left = axes.get('left')
-                    # breakpoint()
+                if left:
-                    lefts = plot.pi_overlay.get_axes('left')
+                    left_axis_width = left['item'].width()
-                    if lefts:
-                        for left in lefts:
-                            left_axis_width += left.width()
 
                 # map back to abs (label-local) coordinates
                 self.xaxis_label.update_label(

piker/ui/_curve.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,306 +18,301 @@
 Fast, smooth, sexy curves.
 
 """
-from contextlib import contextmanager as cm
+from typing import Optional
-from typing import Optional, Callable
 
 import numpy as np
 import pyqtgraph as pg
-from PyQt5 import QtWidgets
+from PyQt5 import QtGui, QtWidgets
-from PyQt5.QtWidgets import QGraphicsItem
 from PyQt5.QtCore import (
-    Qt,
     QLineF,
     QSizeF,
     QRectF,
-    # QRect,
     QPointF,
 )
-from PyQt5.QtGui import (
+
-    QPainter,
+from .._profile import pg_profile_enabled
-    QPainterPath,
-)
-from .._profile import pg_profile_enabled, ms_slower_then
 from ._style import hcolor
-# from ._compression import (
-#     # ohlc_to_m4_line,
-#     ds_m4,
-# )
-from ..log import get_logger
 
 
-log = get_logger(__name__)
+def step_path_arrays_from_1d(
+    x: np.ndarray,
+    y: np.ndarray,
+    include_endpoints: bool = False,
 
-
+) -> (np.ndarray, np.ndarray):
-_line_styles: dict[str, int] = {
-    'solid': Qt.PenStyle.SolidLine,
-    'dash': Qt.PenStyle.DashLine,
-    'dot': Qt.PenStyle.DotLine,
-    'dashdot': Qt.PenStyle.DashDotLine,
-}
-
-
-class Curve(pg.GraphicsObject):
     '''
-    A faster, simpler, append friendly version of
+    Generate a "step mode" curve aligned with OHLC style bars
-    ``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
+    such that each segment spans each bar (aka "centered" style).
-    updates.
-
-    This type is a much stripped down version of a ``pyqtgraph`` style
-    "graphics object" in the sense that the internal lower level
-    graphics which are drawn in the ``.paint()`` method are actually
-    rendered outside of this class entirely and instead are assigned as
-    state (instance vars) here and then drawn during a Qt graphics
-    cycle.
-
-    The main motivation for this more modular, composed design is that
-    lower level graphics data can be rendered in different threads and
-    then read and drawn in this main thread without having to worry
-    about dealing with Qt's concurrency primitives. See
-    ``piker.ui._flows.Renderer`` for details and logic related to lower
-    level path generation and incremental update. The main differences in
-    the path generation code include:
-
-    - avoiding regeneration of the entire historical path where possible
-      and instead only updating the "new" segment(s) via a ``numpy``
-      array diff calc.
-    - here, the "last" graphics datum-segment is drawn independently
-      such that near-term (high frequency) discrete-time-sampled style
-      updates don't trigger a full path redraw.
 
     '''
+    y_out = y.copy()
+    x_out = x.copy()
+    x2 = np.empty(
+        # the data + 2 endpoints on either end for
+        # "termination of the path".
+        (len(x) + 1, 2),
+        # we want to align with OHLC or other sampling style
+        # bars likely so we need fractinal values
+        dtype=float,
+    )
+    x2[0] = x[0] - 0.5
+    x2[1] = x[0] + 0.5
+    x2[1:] = x[:, np.newaxis] + 0.5
 
-    # sub-type customization methods
+    # flatten to 1-d
-    sub_br: Optional[Callable] = None
+    x_out = x2.reshape(x2.size)
-    sub_paint: Optional[Callable] = None
+
-    declare_paintables: Optional[Callable] = None
+    # we create a 1d with 2 extra indexes to
+    # hold the start and (current) end value for the steps
+    # on either end
+    y2 = np.empty((len(y), 2), dtype=y.dtype)
+    y2[:] = y[:, np.newaxis]
+
+    y_out = np.empty(
+        2*len(y) + 2,
+        dtype=y.dtype
+    )
+
+    # flatten and set 0 endpoints
+    y_out[1:-1] = y2.reshape(y2.size)
+    y_out[0] = 0
+    y_out[-1] = 0
+
+    if not include_endpoints:
+        return x_out[:-1], y_out[:-1]
+
+    else:
+        return x_out, y_out
+
+
+# TODO: got a feeling that dropping this inheritance gets us even more speedups
+class FastAppendCurve(pg.PlotCurveItem):
+    '''
+    A faster, append friendly version of ``pyqtgraph.PlotCurveItem``
+    built for real-time data updates.
+
+    The main difference is avoiding regeneration of the entire
+    historical path where possible and instead only updating the "new"
+    segment(s) via a ``numpy`` array diff calc. Further the "last"
+    graphic segment is drawn independently such that near-term (high
+    frequency) discrete-time-sampled style updates don't trigger a full
+    path redraw.
+
+    '''
 
     def __init__(
         self,
         *args,
-
         step_mode: bool = False,
         color: str = 'default_lightest',
         fill_color: Optional[str] = None,
-        style: str = 'solid',
-        name: Optional[str] = None,
-        use_fpath: bool = True,
 
         **kwargs
 
     ) -> None:
 
-        self._name = name
-
-        # brutaaalll, see comments within..
-        self.yData = None
-        self.xData = None
-
-        # self._last_cap: int = 0
-        self.path: Optional[QPainterPath] = None
-
-        # additional path used for appends which tries to avoid
-        # triggering an update/redraw of the presumably larger
-        # historical ``.path`` above.
-        self.use_fpath = use_fpath
-        self.fast_path: Optional[QPainterPath] = None
-
         # TODO: we can probably just dispense with the parent since
         # we're basically only using the pen setting now...
         super().__init__(*args, **kwargs)
 
+        self._xrange: tuple[int, int] = self.dataBounds(ax=0)
+
         # all history of curve is drawn in single px thickness
-        pen = pg.mkPen(hcolor(color))
+        self.setPen(hcolor(color))
-        pen.setStyle(_line_styles[style])
-
-        if 'dash' in style:
-            pen.setDashPattern([8, 3])
-
-        self._pen = pen
 
         # last segment is drawn in 2px thickness for emphasis
-        # self.last_step_pen = pg.mkPen(hcolor(color), width=2)
+        self.last_step_pen = pg.mkPen(hcolor(color), width=2)
-        self.last_step_pen = pg.mkPen(pen, width=2)
+        self._last_line: QLineF = None
-
+        self._last_step_rect: QRectF = None
-        # self._last_line: Optional[QLineF] = None
-        self._last_line = QLineF()
-        self._last_w: float = 1
 
         # flat-top style histogram-like discrete curve
-        # self._step_mode: bool = step_mode
+        self._step_mode: bool = step_mode
 
         # self._fill = True
-        self._brush = pg.functions.mkBrush(hcolor(fill_color or color))
+        self.setBrush(hcolor(fill_color or color))
-
-        # NOTE: this setting seems to mostly prevent redraws on mouse
-        # interaction which is a huge boon for avg interaction latency.
 
         # TODO: one question still remaining is if this makes trasform
         # interactions slower (such as zooming) and if so maybe if/when
         # we implement a "history" mode for the view we disable this in
         # that mode?
-        # don't enable caching by default for the case where the
+        self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-        # only thing drawn is the "last" line segment which can
-        # have a weird artifact where it won't be fully drawn to its
-        # endpoint (something we saw on trade rate curves)
-        self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
 
-        # XXX: see explanation for different caching modes:
+    def update_from_array(
-        # https://stackoverflow.com/a/39410081
+        self,
-        # seems to only be useful if we don't re-generate the entire
+        x: np.ndarray,
-        # QPainterPath every time
+        y: np.ndarray,
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
-        # don't ever use this - it's a colossal nightmare of artefacts
+    ) -> QtGui.QPainterPath:
-        # and is disastrous for performance.
-        # curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
 
-        # allow sub-type customization
+        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-        declare = self.declare_paintables
+        flip_cache = False
-        if declare:
-            declare()
 
-    # TODO: probably stick this in a new parent
+        istart, istop = self._xrange
-    # type which will contain our own version of
+        # print(f"xrange: {self._xrange}")
-    # what ``PlotCurveItem`` had in terms of base
+
-    # functionality? A `FlowGraphic` maybe?
+        # compute the length diffs between the first/last index entry in
-    def x_uppx(self) -> int:
+        # the input data and the last indexes we have on record from the
+        # last time we updated the curve index.
+        prepend_length = istart - x[0]
+        append_length = x[-1] - istop
+
+        # step mode: draw flat top discrete "step"
+        # over the index space for each datum.
+        if self._step_mode:
+            x_out, y_out = step_path_arrays_from_1d(x[:-1], y[:-1])
 
-        px_vecs = self.pixelVectors()[0]
-        if px_vecs:
-            xs_in_px = px_vecs.x()
-            return round(xs_in_px)
         else:
-            return 0
+            # by default we only pull data up to the last (current) index
+            x_out, y_out = x[:-1], y[:-1]
 
-    def px_width(self) -> float:
+        if self.path is None or prepend_length > 0:
+            self.path = pg.functions.arrayToQPath(
+                x_out,
+                y_out,
+                connect='all',
+                finiteCheck=False,
+            )
+            profiler('generate fresh path')
 
-        vb = self.getViewBox()
+            # if self._step_mode:
-        if not vb:
+            #     self.path.closeSubpath()
-            return 0
 
-        vr = self.viewRect()
+        # TODO: get this piecewise prepend working - right now it's
-        l, r = int(vr.left()), int(vr.right())
+        # giving heck on vwap...
+        # if prepend_length:
+        #     breakpoint()
 
-        start, stop = self._xrange
+        #     prepend_path = pg.functions.arrayToQPath(
-        lbar = max(l, start)
+        #         x[0:prepend_length],
-        rbar = min(r, stop)
+        #         y[0:prepend_length],
+        #         connect='all'
+        #     )
 
-        return vb.mapViewToDevice(
+        #     # swap prepend path in "front"
-            QLineF(lbar, 0, rbar, 0)
+        #     old_path = self.path
-        ).length()
+        #     self.path = prepend_path
+        #     # self.path.moveTo(new_x[0], new_y[0])
+        #     self.path.connectPath(old_path)
+
+        elif append_length > 0:
+            if self._step_mode:
+                new_x, new_y = step_path_arrays_from_1d(
+                    x[-append_length - 2:-1],
+                    y[-append_length - 2:-1],
+                )
+                # [1:] since we don't need the vertical line normally at
+                # the beginning of the step curve taking the first (x,
+                # y) poing down to the x-axis **because** this is an
+                # appended path graphic.
+                new_x = new_x[1:]
+                new_y = new_y[1:]
+
+            else:
+                # print(f"append_length: {append_length}")
+                new_x = x[-append_length - 2:-1]
+                new_y = y[-append_length - 2:-1]
+                # print((new_x, new_y))
+
+            append_path = pg.functions.arrayToQPath(
+                new_x,
+                new_y,
+                connect='all',
+                # finiteCheck=False,
+            )
+
+            path = self.path
+
+            # other merging ideas:
+            # https://stackoverflow.com/questions/8936225/how-to-merge-qpainterpaths
+            if self._step_mode:
+                # path.addPath(append_path)
+                self.path.connectPath(append_path)
+
+                # TODO: try out new work from `pyqtgraph` main which
+                # should repair horrid perf:
+                # https://github.com/pyqtgraph/pyqtgraph/pull/2032
+                # ok, nope still horrible XD
+                # if self._fill:
+                #     # XXX: super slow set "union" op
+                #     self.path = self.path.united(append_path).simplified()
+
+                #     # path.addPath(append_path)
+                #     # path.closeSubpath()
+
+            else:
+                # print(f"append_path br: {append_path.boundingRect()}")
+                # self.path.moveTo(new_x[0], new_y[0])
+                # self.path.connectPath(append_path)
+                path.connectPath(append_path)
+
+            # XXX: pretty annoying but, without this there's little
+            # artefacts on the append updates to the curve...
+            self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
+            self.prepareGeometryChange()
+            flip_cache = True
+
+            # print(f"update br: {self.path.boundingRect()}")
 
         # XXX: lol brutal, the internals of `CurvePoint` (inherited by
         # our `LineDot`) required ``.getData()`` to work..
-    def getData(self):
+        self.xData = x
-        return self.xData, self.yData
+        self.yData = y
 
-    def clear(self):
+        x0, x_last = self._xrange = x[0], x[-1]
-        '''
+        y_last = y[-1]
-        Clear internal graphics making object ready for full re-draw.
 
-        '''
+        # draw the "current" step graphic segment so it lines up with
-        # NOTE: original code from ``pg.PlotCurveItem``
+        # the "middle" of the current (OHLC) sample.
-        self.xData = None
+        if self._step_mode:
-        self.yData = None
+            self._last_line = QLineF(
+                x_last - 0.5, 0,
+                x_last + 0.5, 0,
+            )
+            self._last_step_rect = QRectF(
+                x_last - 0.5, 0,
+                x_last + 0.5, y_last
+            )
+        else:
+            self._last_line = QLineF(
+                x[-2], y[-2],
+                x[-1], y_last
+            )
 
-        # XXX: previously, if not trying to leverage `.reserve()` allocs
+        # trigger redraw of path
-        # then you might as well create a new one..
+        # do update before reverting to cache mode
-        # self.path = None
+        self.prepareGeometryChange()
+        self.update()
 
-        # path reservation aware non-mem de-alloc cleaning
+        if flip_cache:
-        if self.path:
+            # XXX: seems to be needed to avoid artifacts (see above).
-            self.path.clear()
+            self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-
-            if self.fast_path:
-                # self.fast_path.clear()
-                self.fast_path = None
-
-    @cm
-    def reset_cache(self) -> None:
-        self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
-        yield
-        self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
 
     def boundingRect(self):
-        '''
-        Compute and then cache our rect.
-        '''
         if self.path is None:
-            return QPainterPath().boundingRect()
+            return QtGui.QPainterPath().boundingRect()
         else:
             # dynamically override this method after initial
             # path is created to avoid requiring the above None check
-            self.boundingRect = self._path_br
+            self.boundingRect = self._br
-            return self._path_br()
+            return self._br()
 
-    def _path_br(self):
+    def _br(self):
-        '''
+        """Post init ``.boundingRect()```.
-        Post init ``.boundingRect()```.
 
-        '''
+        """
-        # hb = self.path.boundingRect()
         hb = self.path.controlPointRect()
         hb_size = hb.size()
-
-        fp = self.fast_path
-        if fp:
-            fhb = fp.controlPointRect()
-            hb_size = fhb.size() + hb_size
-
         # print(f'hb_size: {hb_size}')
 
-        # if self._last_step_rect:
+        w = hb_size.width() + 1
-        #     hb_size += self._last_step_rect.size()
+        h = hb_size.height() + 1
-
-        # if self._line:
-        #     br = self._last_step_rect.bottomRight()
-
-        # tl = QPointF(
-        #     # self._vr[0],
-        #     # hb.topLeft().y(),
-        #     # 0,
-        #     # hb_size.height() + 1
-        # )
-
-        #     br = self._last_step_rect.bottomRight()
-
-        w = hb_size.width()
-        h = hb_size.height()
-
-        sbr = self.sub_br
-        if sbr:
-            w, h = self.sub_br(w, h)
-        else:
-            # assume plain line graphic and use
-            # default unit step in each direction.
-
-            # only on a plane line do we include
-            # and extra index step's worth of width
-            # since in the step case the end of the curve
-            # actually terminates earlier so we don't need
-            # this for the last step.
-            w += self._last_w
-            # ll = self._last_line
-            h += 1  # ll.y2() - ll.y1()
-
-        # br = QPointF(
-        #     self._vr[-1],
-        #     # tl.x() + w,
-        #     tl.y() + h,
-        # )
 
         br = QRectF(
 
             # top left
-            # hb.topLeft()
-            # tl,
             QPointF(hb.topLeft()),
 
-            # br,
             # total size
-            # QSizeF(hb_size)
-            # hb_size,
             QSizeF(w, h)
         )
         # print(f'bounding rect: {br}')
@@ -325,160 +320,38 @@ class Curve(pg.GraphicsObject):
 
     def paint(
         self,
-        p: QPainter,
+        p: QtGui.QPainter,
         opt: QtWidgets.QStyleOptionGraphicsItem,
         w: QtWidgets.QWidget
-
     ) -> None:
 
-        profiler = pg.debug.Profiler(
+        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-            msg=f'Curve.paint(): `{self._name}`',
+        # p.setRenderHint(p.Antialiasing, True)
-            disabled=not pg_profile_enabled(),
-            ms_threshold=ms_slower_then,
-        )
 
-        sub_paint = self.sub_paint
+        if (
-        if sub_paint:
+            self._step_mode
-            sub_paint(p, profiler)
+            and self._last_step_rect
+        ):
+            brush = self.opts['brush']
+            # p.drawLines(*tuple(filter(bool, self._last_step_lines)))
+            # p.drawRect(self._last_step_rect)
+            p.fillRect(self._last_step_rect, brush)
 
+            # p.drawPath(self.path)
+            # profiler('.drawPath()')
+
+        # else:
         p.setPen(self.last_step_pen)
         p.drawLine(self._last_line)
         profiler('.drawLine()')
-        p.setPen(self._pen)
 
-        path = self.path
+        p.setPen(self.opts['pen'])
-        # cap = path.capacity()
+        p.drawPath(self.path)
-        # if cap != self._last_cap:
+        profiler('.drawPath()')
-        #     print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
-        #     self._last_cap = cap
 
-        if path:
+        # TODO: try out new work from `pyqtgraph` main which
-            p.drawPath(path)
+        # should repair horrid perf:
-            profiler(f'.drawPath(path): {path.capacity()}')
-
-        fp = self.fast_path
-        if fp:
-            p.drawPath(fp)
-            profiler('.drawPath(fast_path)')
-
-        # TODO: try out new work from `pyqtgraph` main which should
-        # repair horrid perf (pretty sure i did and it was still
-        # horrible?):
         # https://github.com/pyqtgraph/pyqtgraph/pull/2032
         # if self._fill:
         #     brush = self.opts['brush']
         #     p.fillPath(self.path, brush)
-
-    def draw_last_datum(
-        self,
-        path: QPainterPath,
-        src_data: np.ndarray,
-        render_data: np.ndarray,
-        reset: bool,
-        array_key: str,
-
-    ) -> None:
-        # default line draw last call
-        # with self.reset_cache():
-        x = render_data['index']
-        y = render_data[array_key]
-
-        # draw the "current" step graphic segment so it
-        # lines up with the "middle" of the current
-        # (OHLC) sample.
-        self._last_line = QLineF(
-            x[-2], y[-2],
-            x[-1], y[-1],
-        )
-
-        return x, y
-
-
-# TODO: this should probably be a "downsampled" curve type
-# that draws a bar-style (but for the px column) last graphics
-# element such that the current datum in view can be shown
-# (via it's max / min) even when highly zoomed out.
-class FlattenedOHLC(Curve):
-
-    def draw_last_datum(
-        self,
-        path: QPainterPath,
-        src_data: np.ndarray,
-        render_data: np.ndarray,
-        reset: bool,
-        array_key: str,
-
-    ) -> None:
-        lasts = src_data[-2:]
-        x = lasts['index']
-        y = lasts['close']
-
-        # draw the "current" step graphic segment so it
-        # lines up with the "middle" of the current
-        # (OHLC) sample.
-        self._last_line = QLineF(
-            x[-2], y[-2],
-            x[-1], y[-1]
-        )
-        return x, y
-
-
-class StepCurve(Curve):
-
-    def declare_paintables(
-        self,
-    ) -> None:
-        self._last_step_rect = QRectF()
-
-    def draw_last_datum(
-        self,
-        path: QPainterPath,
-        src_data: np.ndarray,
-        render_data: np.ndarray,
-        reset: bool,
-        array_key: str,
-
-        w: float = 0.5,
-
-    ) -> None:
-
-        # TODO: remove this and instead place all step curve
-        # updating into pre-path data render callbacks.
-        # full input data
-        x = src_data['index']
-        y = src_data[array_key]
-
-        x_last = x[-1]
-        y_last = y[-1]
-
-        # lol, commenting this makes step curves
-        # all "black" for me :eyeroll:..
-        self._last_line = QLineF(
-            x_last - w, 0,
-            x_last + w, 0,
-        )
-        self._last_step_rect = QRectF(
-            x_last - w, 0,
-            x_last + w, y_last,
-        )
-        return x, y
-
-    def sub_paint(
-        self,
-        p: QPainter,
-        profiler: pg.debug.Profiler,
-
-    ) -> None:
-        # p.drawLines(*tuple(filter(bool, self._last_step_lines)))
-        # p.drawRect(self._last_step_rect)
-        p.fillRect(self._last_step_rect, self._brush)
-        profiler('.fillRect()')
-
-    def sub_br(
-        self,
-        path_w: float,
-        path_h: float,
-
-    ) -> (float, float):
-        # passthrough
-        return path_w, path_h

piker/ui/_display.py

@@ -21,20 +21,16 @@ this module ties together quote and computational (fsp) streams with
 graphics update methods via our custom ``pyqtgraph`` charting api.
 
 '''
-from dataclasses import dataclass
 from functools import partial
 import time
-from typing import Optional, Any, Callable
+from typing import Optional
 
 import numpy as np
 import tractor
 import trio
-import pendulum
-import pyqtgraph as pg
 
-# from .. import brokers
+from .. import brokers
-from ..data.feed import open_feed
+from ..data.feed import open_feed, Feed
-from ._axes import YAxisLabel
 from ._chart import (
     ChartPlotWidget,
     LinkedSplits,
@@ -47,23 +43,18 @@ from ._fsp import (
     has_vlm,
     open_vlm_displays,
 )
-from ..data._sharedmem import ShmArray
+from ..data._sharedmem import ShmArray, try_read
-from ..data._source import tf_in_1s
 from ._forms import (
     FieldsForm,
     mk_order_pane_layout,
 )
 from .order_mode import open_order_mode
-from .._profile import (
-    pg_profile_enabled,
-    ms_slower_then,
-)
 from ..log import get_logger
 
 log = get_logger(__name__)
 
 # TODO: load this from a config.toml!
-_quote_throttle_rate: int = 22  # Hz
+_quote_throttle_rate: int = 58  # Hz
 
 
 # a working tick-type-classes template
@@ -74,20 +65,12 @@ _tick_groups = {
 }
 
 
-# TODO: delegate this to each `Flow.maxmin()` which includes
-# caching and further we should implement the following stream based
-# approach, likely with ``numba``:
-# https://arxiv.org/abs/cs/0610046
-# https://github.com/lemire/pythonmaxmin
 def chart_maxmin(
     chart: ChartPlotWidget,
-    ohlcv_shm: ShmArray,
     vlm_chart: Optional[ChartPlotWidget] = None,
 
 ) -> tuple[
-
     tuple[int, int, int, int],
-
     float,
     float,
     float,
@@ -96,57 +79,34 @@ def chart_maxmin(
     Compute max and min datums "in view" for range limits.
 
     '''
+    # TODO: implement this
+    # https://arxiv.org/abs/cs/0610046
+    # https://github.com/lemire/pythonmaxmin
+
+    array = chart._arrays[chart.name]
+    ifirst = array[0]['index']
+
     last_bars_range = chart.bars_range()
-    out = chart.maxmin()
+    l, lbar, rbar, r = last_bars_range
+    in_view = array[lbar - ifirst:rbar - ifirst + 1]
 
-    if out is None:
+    assert in_view.size
-        return (last_bars_range, 0, 0, 0)
 
-    mn, mx = out
+    mx, mn = np.nanmax(in_view['high']), np.nanmin(in_view['low'])
+
+    # TODO: when we start using line charts, probably want to make
+    # this an overloaded call on our `DataView
+    # sym = chart.name
+    # mx, mn = np.nanmax(in_view[sym]), np.nanmin(in_view[sym])
 
     mx_vlm_in_view = 0
     if vlm_chart:
-        out = vlm_chart.maxmin()
+        mx_vlm_in_view = np.max(in_view['volume'])
-        if out:
-            _, mx_vlm_in_view = out
 
-    return (
+    return last_bars_range, mx, max(mn, 0), mx_vlm_in_view
-        last_bars_range,
-        mx,
-        max(mn, 0),  # presuming price can't be negative?
-        mx_vlm_in_view,
-    )
 
 
-@dataclass
+async def update_linked_charts_graphics(
-class DisplayState:
-    '''
-    Chart-local real-time graphics state container.
-
-    '''
-    quotes: dict[str, Any]
-
-    maxmin: Callable
-    ohlcv: ShmArray
-
-    # high level chart handles
-    linked: LinkedSplits
-    chart: ChartPlotWidget
-    vlm_chart: ChartPlotWidget
-
-    # axis labels
-    l1: L1Labels
-    last_price_sticky: YAxisLabel
-    vlm_sticky: YAxisLabel
-
-    # misc state tracking
-    vars: dict[str, Any]
-
-    wap_in_history: bool = False
-
-
-async def graphics_update_loop(
-
     linked: LinkedSplits,
     stream: tractor.MsgStream,
     ohlcv: np.ndarray,
@@ -171,7 +131,6 @@ async def graphics_update_loop(
     #   of copying it from last bar's close
     # - 1-5 sec bar lookback-autocorrection like tws does?
     #   (would require a background history checker task)
-    display_rate = linked.godwidget.window.current_screen().refreshRate()
 
     chart = linked.chart
 
@@ -183,14 +142,12 @@ async def graphics_update_loop(
 
     if vlm_chart:
         vlm_sticky = vlm_chart._ysticks['volume']
+        vlm_view = vlm_chart.view
+
+    maxmin = partial(chart_maxmin, chart, vlm_chart)
+
+    chart.default_view()
 
-    maxmin = partial(
-        chart_maxmin,
-        chart,
-        ohlcv,
-        vlm_chart,
-    )
-    last_bars_range: tuple[float, float]
     (
         last_bars_range,
         last_mx,
@@ -222,9 +179,8 @@ async def graphics_update_loop(
     tick_margin = 3 * tick_size
 
     chart.show()
-    # view = chart.view
+    view = chart.view
     last_quote = time.time()
-    i_last = ohlcv.index
 
     # async def iter_drain_quotes():
     #     # NOTE: all code below this loop is expected to be synchronous
@@ -249,33 +205,8 @@ async def graphics_update_loop(
 
     # async for quotes in iter_drain_quotes():
 
-    ds = linked.display_state = DisplayState(**{
-        'quotes': {},
-        'linked': linked,
-        'maxmin': maxmin,
-        'ohlcv': ohlcv,
-        'chart': chart,
-        'last_price_sticky': last_price_sticky,
-        'vlm_chart': vlm_chart,
-        'vlm_sticky': vlm_sticky,
-        'l1': l1,
-
-        'vars': {
-            'tick_margin': tick_margin,
-            'i_last': i_last,
-            'i_last_append': i_last,
-            'last_mx_vlm': last_mx_vlm,
-            'last_mx': last_mx,
-            'last_mn': last_mn,
-        }
-    })
-
-    chart.default_view()
-
-    # main real-time quotes update loop
     async for quotes in stream:
 
-        ds.quotes = quotes
         quote_period = time.time() - last_quote
         quote_rate = round(
             1/quote_period, 1) if quote_period > 0 else float('inf')
@@ -284,8 +215,7 @@ async def graphics_update_loop(
 
             # in the absolute worst case we shouldn't see more then
             # twice the expected throttle rate right!?
-            # and quote_rate >= _quote_throttle_rate * 2
+            and quote_rate >= _quote_throttle_rate * 1.5
-            and quote_rate >= display_rate
         ):
             log.warning(f'High quote rate {symbol.key}: {quote_rate}')
 
@@ -296,135 +226,45 @@ async def graphics_update_loop(
             chart.pause_all_feeds()
             continue
 
-        ic = chart.view._ic
+        for sym, quote in quotes.items():
-        if ic:
-            chart.pause_all_feeds()
-            await ic.wait()
-            chart.resume_all_feeds()
 
-        # sync call to update all graphics/UX components.
-        graphics_update_cycle(ds)
-
-
-def graphics_update_cycle(
-    ds: DisplayState,
-    wap_in_history: bool = False,
-    trigger_all: bool = False,  # flag used by prepend history updates
-    prepend_update_index: Optional[int] = None,
-
-) -> None:
-    # TODO: eventually optimize this whole graphics stack with ``numba``
-    # hopefully XD
-
-    chart = ds.chart
-
-    profiler = pg.debug.Profiler(
-        msg=f'Graphics loop cycle for: `{chart.name}`',
-        delayed=True,
-        disabled=not pg_profile_enabled(),
-        # disabled=True,
-        ms_threshold=ms_slower_then,
-
-        # ms_threshold=1/12 * 1e3,
-    )
-
-    # unpack multi-referenced components
-    vlm_chart = ds.vlm_chart
-    l1 = ds.l1
-    ohlcv = ds.ohlcv
-    array = ohlcv.array
-    vars = ds.vars
-    tick_margin = vars['tick_margin']
-
-    update_uppx = 16
-
-    for sym, quote in ds.quotes.items():
-
-        # compute the first available graphic's x-units-per-pixel
-        uppx = vlm_chart.view.x_uppx()
-
-        # NOTE: vlm may be written by the ``brokerd`` backend
-        # event though a tick sample is not emitted.
-        # TODO: show dark trades differently
-        # https://github.com/pikers/piker/issues/116
-
-        # NOTE: this used to be implemented in a dedicated
-        # "increment task": ``check_for_new_bars()`` but it doesn't
-        # make sense to do a whole task switch when we can just do
-        # this simple index-diff and all the fsp sub-curve graphics
-        # are diffed on each draw cycle anyway; so updates to the
-        # "curve" length is already automatic.
-
-        # increment the view position by the sample offset.
-        i_step = ohlcv.index
-        i_diff = i_step - vars['i_last']
-        vars['i_last'] = i_step
-
-        append_diff = i_step - vars['i_last_append']
-
-        # update the "last datum" (aka extending the flow graphic with
-        # new data) only if the number of unit steps is >= the number of
-        # such unit steps per pixel (aka uppx). Iow, if the zoom level
-        # is such that a datum(s) update to graphics wouldn't span
-        # to a new pixel, we don't update yet.
-        do_append = (append_diff >= uppx)
-        if do_append:
-            vars['i_last_append'] = i_step
-
-        do_rt_update = uppx < update_uppx
-        # print(
-        #     f'append_diff:{append_diff}\n'
-        #     f'uppx:{uppx}\n'
-        #     f'do_append: {do_append}'
-        # )
-
-        # TODO: we should only run mxmn when we know
-        # an update is due via ``do_append`` above.
             (
                 brange,
                 mx_in_view,
                 mn_in_view,
                 mx_vlm_in_view,
-        ) = ds.maxmin()
+            ) = maxmin()
             l, lbar, rbar, r = brange
             mx = mx_in_view + tick_margin
             mn = mn_in_view - tick_margin
 
-        profiler('`ds.maxmin()` call')
+            # NOTE: vlm may be written by the ``brokerd`` backend
-
+            # event though a tick sample is not emitted.
-        liv = r >= i_step  # the last datum is in view
+            # TODO: show dark trades differently
-
+            # https://github.com/pikers/piker/issues/116
-        if (
+            array = ohlcv.array
-            prepend_update_index is not None
-            and lbar > prepend_update_index
-        ):
-            # on a history update (usually from the FSP subsys)
-            # if the segment of history that is being prepended
-            # isn't in view there is no reason to do a graphics
-            # update.
-            log.debug('Skipping prepend graphics cycle: frame not in view')
-            return
-
-        # don't real-time "shift" the curve to the
-        # left unless we get one of the following:
-        if (
-            (
-                # i_diff > 0  # no new sample step
-                do_append
-                # and uppx < 4  # chart is zoomed out very far
-                and liv
-            )
-            or trigger_all
-        ):
-            # TODO: we should track and compute whether the last
-            # pixel in a curve should show new data based on uppx
-            # and then iff update curves and shift?
-            chart.increment_view(steps=i_diff)
 
             if vlm_chart:
-                vlm_chart.increment_view(steps=i_diff)
+                vlm_chart.update_curve_from_array('volume', array)
+                vlm_sticky.update_from_data(*array[-1][['index', 'volume']])
 
-            profiler('view incremented')
+                if (
+                    mx_vlm_in_view != last_mx_vlm or
+                    mx_vlm_in_view > last_mx_vlm
+                ):
+                    # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}')
+                    vlm_view._set_yrange(
+                        yrange=(0, mx_vlm_in_view * 1.375)
+                    )
+                    last_mx_vlm = mx_vlm_in_view
+
+                for curve_name, shm in vlm_chart._overlays.items():
+                    update_fsp_chart(
+                        vlm_chart,
+                        shm,
+                        curve_name,
+                        array_key=curve_name,
+                    )
 
             ticks_frame = quote.get('ticks', ())
 
@@ -468,24 +308,9 @@ def graphics_update_cycle(
             # current) tick first order as an optimization where we only
             # update from the last tick from each type class.
             # last_clear_updated: bool = False
+            # for typ, tick in reversed(lasts.items()):
 
-        # update ohlc sampled price bars
+            # iterate in FIFO order per frame
-        if (
-            do_rt_update
-            or do_append
-            or trigger_all
-        ):
-            chart.update_graphics_from_flow(
-                chart.name,
-                # do_append=uppx < update_uppx,
-                do_append=do_append,
-            )
-
-        # NOTE: we always update the "last" datum
-        # since the current range should at least be updated
-        # to it's max/min on the last pixel.
-
-        # iterate in FIFO order per tick-frame
             for typ, tick in lasts.items():
 
                 price = tick.get('price')
@@ -495,7 +320,6 @@ def graphics_update_cycle(
                 # tick frames to determine the y-range for chart
                 # auto-scaling.
                 # TODO: we need a streaming minmax algo here, see def above.
-            if liv:
                 mx = max(price + tick_margin, mx)
                 mn = min(price - tick_margin, mn)
 
@@ -514,15 +338,19 @@ def graphics_update_cycle(
 
                     # update price sticky(s)
                     end = array[-1]
-                ds.last_price_sticky.update_from_data(
+                    last_price_sticky.update_from_data(
                         *end[['index', 'close']]
                     )
 
+                    # update ohlc sampled price bars
+                    chart.update_ohlc_from_array(
+                        chart.name,
+                        array,
+                    )
+
                     if wap_in_history:
                         # update vwap overlay line
-                    chart.update_graphics_from_flow(
+                        chart.update_curve_from_array('bar_wap', ohlcv.array)
-                        'bar_wap',
-                    )
 
                 # L1 book label-line updates
                 # XXX: is this correct for ib?
@@ -535,169 +363,136 @@ def graphics_update_cycle(
                         l1.bid_label.fields['level']: l1.bid_label,
                     }.get(price)
 
-                if (
+                    if label is not None:
-                    label is not None
+                        label.update_fields({'level': price, 'size': size})
-                    and liv
-                ):
-                    label.update_fields(
-                        {'level': price, 'size': size}
-                    )
 
                         # TODO: on trades should we be knocking down
                         # the relevant L1 queue?
                         # label.size -= size
 
-            elif (
+                # elif ticktype in ('ask', 'asize'):
-                typ in _tick_groups['asks']
+                elif typ in _tick_groups['asks']:
-                # TODO: instead we could check if the price is in the
-                # y-view-range?
-                and liv
-            ):
                     l1.ask_label.update_fields({'level': price, 'size': size})
 
-            elif (
+                # elif ticktype in ('bid', 'bsize'):
-                typ in _tick_groups['bids']
+                elif typ in _tick_groups['bids']:
-                # TODO: instead we could check if the price is in the
-                # y-view-range?
-                and liv
-            ):
                     l1.bid_label.update_fields({'level': price, 'size': size})
 
             # check for y-range re-size
             if (
-            (mx > vars['last_mx']) or (mn < vars['last_mn'])
+                (mx > last_mx) or (mn < last_mn)
                 and not chart._static_yrange == 'axis'
-            and liv
             ):
-            main_vb = chart.view
+                # print(f'new y range: {(mn, mx)}')
-            if (
+                view._set_yrange(
-                main_vb._ic is None
+                    yrange=(mn, mx),
-                or not main_vb._ic.is_set()
-            ):
-                # print(f'updating range due to mxmn')
-                main_vb._set_yrange(
                     # TODO: we should probably scale
                     # the view margin based on the size
                     # of the true range? This way you can
                     # slap in orders outside the current
                     # L1 (only) book range.
                     # range_margin=0.1,
-                    yrange=(mn, mx),
                 )
 
-        # XXX: update this every draw cycle to make L1-always-in-view work.
+            last_mx, last_mn = mx, mn
-        vars['last_mx'], vars['last_mn'] = mx, mn
 
-        # run synchronous update on all linked flows
+            # run synchronous update on all derived fsp subplots
-        # TODO: should the "main" (aka source) flow be special?
+            for name, subchart in linked.subplots.items():
-        for curve_name, flow in chart._flows.items():
+                update_fsp_chart(
-            # update any overlayed fsp flows
+                    subchart,
-            if curve_name != chart.data_key:
+                    subchart._shm,
+
+                    # XXX: do we really needs seperate names here?
+                    name,
+                    array_key=name,
+                )
+                subchart.cv._set_yrange()
+
+                # TODO: all overlays on all subplots..
+
+            # run synchronous update on all derived overlays
+            for curve_name, shm in chart._overlays.items():
                 update_fsp_chart(
                     chart,
-                    flow,
+                    shm,
                     curve_name,
                     array_key=curve_name,
                 )
+                # chart._set_yrange()
 
-            # even if we're downsampled bigly
+
-            # draw the last datum in the final
+async def check_for_new_bars(
-            # px column to give the user the mx/mn
+    feed: Feed,
-            # range of that set.
+    ohlcv: np.ndarray,
-            if (
+    linkedsplits: LinkedSplits,
-                not do_append
+
-                # and not do_rt_update
+) -> None:
-                and liv
+    '''
-            ):
+    Task which updates from new bars in the shared ohlcv buffer every
-                flow.draw_last(
+    ``delay_s`` seconds.
-                    array_key=curve_name,
+
-                    only_last_uppx=True,
+    '''
+    # TODO: right now we'll spin printing bars if the last time
+    # stamp is before a large period of no market activity.
+    # Likely the best way to solve this is to make this task
+    # aware of the instrument's tradable hours?
+
+    price_chart = linkedsplits.chart
+    price_chart.default_view()
+
+    async with feed.index_stream() as stream:
+        async for index in stream:
+            # update chart historical bars graphics by incrementing
+            # a time step and drawing the history and new bar
+
+            # When appending a new bar, in the time between the insert
+            # from the writing process and the Qt render call, here,
+            # the index of the shm buffer may be incremented and the
+            # (render) call here might read the new flat bar appended
+            # to the buffer (since -1 index read). In that case H==L and the
+            # body will be set as None (not drawn) on what this render call
+            # *thinks* is the curent bar (even though it's reading data from
+            # the newly inserted flat bar.
+            #
+            # HACK: We need to therefore write only the history (not the
+            # current bar) and then either write the current bar manually
+            # or place a cursor for visual cue of the current time step.
+
+            array = ohlcv.array
+            # avoid unreadable race case on backfills
+            while not try_read(array):
+                await trio.sleep(0.01)
+
+            # XXX: this puts a flat bar on the current time step
+            # TODO: if we eventually have an x-axis time-step "cursor"
+            # we can get rid of this since it is extra overhead.
+            price_chart.update_ohlc_from_array(
+                price_chart.name,
+                array,
+                just_history=False,
             )
 
-        # volume chart logic..
+            # main chart overlays
-        # TODO: can we unify this with the above loop?
+            for name in price_chart._overlays:
-        if vlm_chart:
+
-            # always update y-label
+                price_chart.update_curve_from_array(
-            ds.vlm_sticky.update_from_data(
+                    name,
-                *array[-1][['index', 'volume']]
+                    price_chart._arrays[name]
                 )
 
-            if (
+            # each subplot
-                (
+            for name, chart in linkedsplits.subplots.items():
-                    do_rt_update
-                    or do_append
-                    and liv
-                )
-                or trigger_all
-            ):
-                # TODO: make it so this doesn't have to be called
-                # once the $vlm is up?
-                vlm_chart.update_graphics_from_flow(
-                    'volume',
-                    # UGGGh, see ``maxmin()`` impl in `._fsp` for
-                    # the overlayed plotitems... we need a better
-                    # bay to invoke a maxmin per overlay..
-                    render=False,
-                    # XXX: ^^^^ THIS IS SUPER IMPORTANT! ^^^^
-                    # without this, since we disable the
-                    # 'volume' (units) chart after the $vlm starts
-                    # up we need to be sure to enable this
-                    # auto-ranging otherwise there will be no handler
-                    # connected to update accompanying overlay
-                    # graphics..
-                )
-                profiler('`vlm_chart.update_graphics_from_flow()`')
 
-                if (
+                # TODO: do we need the same unreadable guard as for the
-                    mx_vlm_in_view != vars['last_mx_vlm']
+                # price chart (above) here?
-                ):
+                chart.update_curve_from_array(
-                    yrange = (0, mx_vlm_in_view * 1.375)
+                    chart.name,
-                    vlm_chart.view._set_yrange(
+                    chart._shm.array,
-                        yrange=yrange,
+                    array_key=chart.data_key
-                    )
-                    profiler('`vlm_chart.view._set_yrange()`')
-                    # print(f'mx vlm: {last_mx_vlm} -> {mx_vlm_in_view}')
-                    vars['last_mx_vlm'] = mx_vlm_in_view
-
-            for curve_name, flow in vlm_chart._flows.items():
-
-                if (
-                    curve_name != 'volume' and
-                    flow.render and (
-                        liv and
-                        do_rt_update or do_append
-                    )
-                ):
-                    update_fsp_chart(
-                        vlm_chart,
-                        flow,
-                        curve_name,
-                        array_key=curve_name,
-                        # do_append=uppx < update_uppx,
-                        do_append=do_append,
-                    )
-                    # is this even doing anything?
-                    # (pretty sure it's the real-time
-                    # resizing from last quote?)
-                    fvb = flow.plot.vb
-                    fvb._set_yrange(
-                        name=curve_name,
                 )
 
-                elif (
+            # shift the view if in follow mode
-                    curve_name != 'volume'
+            price_chart.increment_view()
-                    and not do_append
-                    and liv
-                    and uppx >= 1
-                    # even if we're downsampled bigly
-                    # draw the last datum in the final
-                    # px column to give the user the mx/mn
-                    # range of that set.
-                ):
-                    # always update the last datum-element
-                    # graphic for all flows
-                    # print(f'drawing last {flow.name}')
-                    flow.draw_last(array_key=curve_name)
 
 
 async def display_symbol_data(
@@ -723,17 +518,17 @@ async def display_symbol_data(
     )
 
     # historical data fetch
-    # brokermod = brokers.get_brokermod(provider)
+    brokermod = brokers.get_brokermod(provider)
 
     # ohlc_status_done = sbar.open_status(
     #     'retreiving OHLC history.. ',
     #     clear_on_next=True,
     #     group_key=loading_sym_key,
     # )
-    fqsn = '.'.join((sym, provider))
 
     async with open_feed(
-        [fqsn],
+            provider,
+            [sym],
             loglevel=loglevel,
 
             # limit to at least display's FPS
@@ -744,23 +539,15 @@ async def display_symbol_data(
         ohlcv: ShmArray = feed.shm
         bars = ohlcv.array
         symbol = feed.symbols[sym]
-        fqsn = symbol.front_fqsn()
-
-        times = bars['time']
-        end = pendulum.from_timestamp(times[-1])
-        start = pendulum.from_timestamp(times[times != times[-1]][-1])
-        step_size_s = (end - start).seconds
-        tf_key = tf_in_1s[step_size_s]
 
         # load in symbol's ohlc data
         godwidget.window.setWindowTitle(
-            f'{fqsn} '
+            f'{symbol.key}@{symbol.brokers} '
             f'tick:{symbol.tick_size}'
-            f'step:{tf_key} '
         )
 
-        linked = godwidget.linkedsplits
+        linkedsplits = godwidget.linkedsplits
-        linked._symbol = symbol
+        linkedsplits._symbol = symbol
 
         # generate order mode side-pane UI
         # A ``FieldsForm`` form to configure order entry
@@ -770,38 +557,38 @@ async def display_symbol_data(
         godwidget.pp_pane = pp_pane
 
         # create main OHLC chart
-        chart = linked.plot_ohlc_main(
+        chart = linkedsplits.plot_ohlc_main(
             symbol,
-            ohlcv,
+            bars,
             sidepane=pp_pane,
         )
-        chart.default_view()
         chart._feeds[symbol.key] = feed
         chart.setFocus()
 
         # plot historical vwap if available
         wap_in_history = False
 
-        # XXX: FOR SOME REASON THIS IS CAUSING HANGZ!?!
+        if brokermod._show_wap_in_history:
-        # if brokermod._show_wap_in_history:
 
-        #     if 'bar_wap' in bars.dtype.fields:
+            if 'bar_wap' in bars.dtype.fields:
-        #         wap_in_history = True
+                wap_in_history = True
-        #         chart.draw_curve(
+                chart.draw_curve(
-        #             name='bar_wap',
+                    name='bar_wap',
-        #             shm=ohlcv,
+                    data=bars,
-        #             color='default_light',
+                    add_label=False,
-        #             add_label=False,
+                )
-        #         )
 
         # size view to data once at outset
         chart.cv._set_yrange()
 
+        # TODO: a data view api that makes this less shit
+        chart._shm = ohlcv
+
         # NOTE: we must immediately tell Qt to show the OHLC chart
         # to avoid a race where the subplots get added/shown to
         # the linked set *before* the main price chart!
-        linked.show()
+        linkedsplits.show()
-        linked.focus()
+        linkedsplits.focus()
         await trio.sleep(0)
 
         vlm_chart: Optional[ChartPlotWidget] = None
@@ -811,7 +598,7 @@ async def display_symbol_data(
             if has_vlm(ohlcv):
                 vlm_chart = await ln.start(
                     open_vlm_displays,
-                    linked,
+                    linkedsplits,
                     ohlcv,
                 )
 
@@ -819,7 +606,7 @@ async def display_symbol_data(
             # from an input config.
             ln.start_soon(
                 start_fsp_displays,
-                linked,
+                linkedsplits,
                 ohlcv,
                 loading_sym_key,
                 loglevel,
@@ -827,37 +614,45 @@ async def display_symbol_data(
 
             # start graphics update loop after receiving first live quote
             ln.start_soon(
-                graphics_update_loop,
+                update_linked_charts_graphics,
-                linked,
+                linkedsplits,
                 feed.stream,
                 ohlcv,
                 wap_in_history,
                 vlm_chart,
             )
 
+            # start sample step incrementer
+            ln.start_soon(
+                check_for_new_bars,
+                feed,
+                ohlcv,
+                linkedsplits
+            )
+
             async with (
                 open_order_mode(
                     feed,
                     chart,
-                    fqsn,
+                    symbol,
+                    provider,
                     order_mode_started
                 )
             ):
                 # let Qt run to render all widgets and make sure the
                 # sidepanes line up vertically.
                 await trio.sleep(0)
-                linked.resize_sidepanes()
+                linkedsplits.resize_sidepanes()
 
                 # NOTE: we pop the volume chart from the subplots set so
                 # that it isn't double rendered in the display loop
                 # above since we do a maxmin calc on the volume data to
                 # determine if auto-range adjustements should be made.
-                # linked.subplots.pop('volume', None)
+                linkedsplits.subplots.pop('volume', None)
 
                 # TODO: make this not so shit XD
                 # close group status
                 sbar._status_groups[loading_sym_key][1]()
 
                 # let the app run.. bby
-                # linked.graphics_cycle()
                 await trio.sleep_forever()

piker/ui/_editors.py

@@ -343,7 +343,7 @@ class SelectRect(QtGui.QGraphicsRectItem):
         nbars = ixmx - ixmn + 1
 
         chart = self._chart
-        data = chart._flows[chart.name].shm.array[ixmn:ixmx]
+        data = chart._arrays[chart.name][ixmn:ixmx]
 
         if len(data):
             std = data['close'].std()

piker/ui/_exec.py

@@ -49,6 +49,10 @@ from . import _style
 log = get_logger(__name__)
 
 # pyqtgraph global config
+# might as well enable this for now?
+pg.useOpenGL = True
+pg.enableExperimental = True
+
 # engage core tweaks that give us better response
 # latency then the average pg user
 _do_overrides()
@@ -57,9 +61,7 @@ _do_overrides()
 # XXX: pretty sure none of this shit works on linux as per:
 # https://bugreports.qt.io/browse/QTBUG-53022
 # it seems to work on windows.. no idea wtf is up.
-is_windows = False
 if platform.system() == "Windows":
-    is_windows = True
 
     # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute
     # must be set before creating the application
@@ -180,8 +182,6 @@ def run_qtractor(
 
     window.main_widget = main_widget
     window.setCentralWidget(instance)
-    if is_windows:
-        window.configure_to_desktop()
 
     # actually render to screen
     window.show()

piker/ui/_feedstatus.py

@@ -1,83 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-"""
-Feed status and controls widget(s) for embedding in a UI-pane.
-
-"""
-
-from __future__ import annotations
-from textwrap import dedent
-from typing import TYPE_CHECKING
-
-# from PyQt5.QtCore import Qt
-
-from ._style import _font, _font_small
-# from ..calc import humanize
-from ._label import FormatLabel
-
-if TYPE_CHECKING:
-    from ._chart import ChartPlotWidget
-    from ..data.feed import Feed
-    from ._forms import FieldsForm
-
-
-def mk_feed_label(
-    form: FieldsForm,
-    feed: Feed,
-    chart: ChartPlotWidget,
-
-) -> FormatLabel:
-    '''
-    Generate a label from feed meta-data to be displayed
-    in a UI sidepane.
-
-    TODO: eventually buttons for changing settings over
-    a feed control protocol.
-
-    '''
-    status = feed.status
-    assert status
-
-    msg = dedent("""
-        actor: **{actor_name}**\n
-        |_ @**{host}:{port}**\n
-    """)
-
-    for key, val in status.items():
-        if key in ('host', 'port', 'actor_name'):
-            continue
-        msg += f'\n|_ {key}: **{{{key}}}**\n'
-
-    feed_label = FormatLabel(
-        fmt_str=msg,
-        # |_ streams: **{symbols}**\n
-        font=_font.font,
-        font_size=_font_small.px_size,
-        font_color='default_lightest',
-    )
-
-    # form.vbox.setAlignment(feed_label, Qt.AlignBottom)
-    # form.vbox.setAlignment(Qt.AlignBottom)
-    _ = chart.height() - (
-        form.height() +
-        form.fill_bar.height()
-        # feed_label.height()
-    )
-
-    feed_label.format(**feed.status)
-
-    return feed_label

piker/ui/_forms.py

@@ -21,7 +21,6 @@ Text entry "forms" widgets (mostly for configuration and UI user input).
 from __future__ import annotations
 from contextlib import asynccontextmanager
 from functools import partial
-from math import floor
 from typing import (
     Optional, Any, Callable, Awaitable
 )
@@ -106,7 +105,7 @@ class Edit(QLineEdit):
         # TODO: somehow this math ain't right?
         chars_w_pxs = dpi_font.boundingRect('0'*self._chars).width()
         scale = round(dpi_font.scale())
-        psh.setWidth(int(chars_w_pxs * scale))
+        psh.setWidth(chars_w_pxs * scale)
         return psh
 
     def set_width_in_chars(
@@ -219,7 +218,7 @@ class Selection(QComboBox):
 
     ) -> None:
         br = _font.boundingRect(str(char))
-        _, h = br.width(), int(br.height())
+        _, h = br.width(), br.height()
 
         # TODO: something better then this monkey patch
         view = self.view()
@@ -331,7 +330,7 @@ class FieldsForm(QWidget):
         self.form.setSpacing(0)
         self.form.setHorizontalSpacing(0)
 
-        self.vbox.addLayout(self.form, stretch=3)
+        self.vbox.addLayout(self.form, stretch=1/3)
 
         self.labels: dict[str, QLabel] = {}
         self.fields: dict[str, QWidget] = {}
@@ -543,7 +542,7 @@ class FillStatusBar(QProgressBar):
 
     '''
     border_px: int = 2
-    slot_margin_px: int = 1
+    slot_margin_px: int = 2
 
     def __init__(
         self,
@@ -554,16 +553,12 @@ class FillStatusBar(QProgressBar):
 
     ) -> None:
         super().__init__(parent=parent)
-
+        self.approx_h = approx_height_px
-        self.approx_h = int(round(approx_height_px))
-        self.setMinimumHeight(self.approx_h)
-        self.setMaximumHeight(self.approx_h)
-
         self.font_size = font_size
 
         self.setFormat('')  # label format
-        self.setMinimumWidth(int(width_px))
+        self.setMinimumWidth(width_px)
-        self.setMaximumWidth(int(width_px))
+        self.setMaximumWidth(width_px)
 
     def set_slots(
         self,
@@ -572,12 +567,17 @@ class FillStatusBar(QProgressBar):
 
     ) -> None:
 
-        self.setOrientation(Qt.Vertical)
+        approx_h = self.approx_h
-        h = self.height()
-
         # TODO: compute "used height" thus far and mostly fill the rest
-        tot_slot_h, r = divmod(h, slots)
+        tot_slot_h, r = divmod(
+            approx_h,
+            slots,
+        )
+        clipped = slots * tot_slot_h + 2*self.border_px
+        self.setMaximumHeight(clipped)
+        slot_height_px = tot_slot_h - 2*self.slot_margin_px
 
+        self.setOrientation(Qt.Vertical)
         self.setStyleSheet(
             f"""
             QProgressBar {{
@@ -592,28 +592,21 @@ class FillStatusBar(QProgressBar):
                 border: {self.border_px}px solid {hcolor('default_light')};
                 border-radius: 2px;
             }}
+
             QProgressBar::chunk {{
 
                 background-color: {hcolor('default_spotlight')};
                 color: {hcolor('bracket')};
 
                 border-radius: 2px;
+
+                margin: {self.slot_margin_px}px;
+                height: {slot_height_px}px;
+
             }}
             """
         )
 
-        # to set a discrete "block" per slot...
-        # XXX: couldn't get the discrete math to work here such
-        # that it was always correctly showing a discretized value
-        # up to the limit; not sure if it's the ``.setRange()``
-        # / ``.setValue()`` api or not but i was able to get something
-        # close screwing with the divmod above above but after so large
-        # a value it would always be less chunks then the correct
-        # value..
-        # margin: {self.slot_margin_px}px;
-        # height: {slot_height_px}px;
-
-
         # margin-bottom: {slot_margin_px*2}px;
         # margin-top: {slot_margin_px*2}px;
         # color: #19232D;
@@ -664,14 +657,14 @@ def mk_fill_status_bar(
     )
     # size according to dpi scaled fonted contents to avoid
     # resizes on magnitude changes (eg. 9 -> 10 %)
-    min_w = int(_font.boundingRect('1000.0M% pnl').width())
+    min_w = _font.boundingRect('1000.0M% pnl').width()
     left_label.setMinimumWidth(min_w)
     left_label.resize(
         min_w,
         left_label.size().height(),
     )
 
-    bar_labels_lhs.addSpacing(int(5/8 * bar_h))
+    bar_labels_lhs.addSpacing(5/8 * bar_h)
 
     bar_labels_lhs.addWidget(
         left_label,
@@ -750,12 +743,12 @@ def mk_order_pane_layout(
         parent=parent,
         fields_schema={
             'account': {
-                'label': '**accnt**:',
+                'label': '**account**:',
                 'type': 'select',
                 'default_value': ['paper'],
             },
             'size_unit': {
-                'label': '**alloc**:',
+                'label': '**allocate**:',
                 'type': 'select',
                 'default_value': [
                     '$ size',
@@ -803,7 +796,7 @@ def mk_order_pane_layout(
     form.top_label = top_label
 
     # add pp fill bar + spacing
-    vbox.addLayout(hbox, stretch=3)
+    vbox.addLayout(hbox, stretch=1/3)
 
     # TODO: handle resize events and appropriately scale this
     # to the sidepane height?

piker/ui/_fsp.py

@@ -22,7 +22,6 @@ Financial signal processing cluster and real-time graphics management.
 '''
 from contextlib import asynccontextmanager as acm
 from functools import partial
-import inspect
 from itertools import cycle
 from typing import Optional, AsyncGenerator, Any
 
@@ -38,7 +37,6 @@ from .._cacheables import maybe_open_context
 from ..calc import humanize
 from ..data._sharedmem import (
     ShmArray,
-    _Token,
     try_read,
 )
 from ._chart import (
@@ -52,11 +50,7 @@ from ._forms import (
 )
 from ..fsp._api import maybe_mk_fsp_shm, Fsp
 from ..fsp import cascade
-from ..fsp._volume import (
+from ..fsp._volume import tina_vwap, dolla_vlm
-    tina_vwap,
-    dolla_vlm,
-    flow_rates,
-)
 from ..log import get_logger
 
 log = get_logger(__name__)
@@ -72,17 +66,12 @@ def has_vlm(ohlcv: ShmArray) -> bool:
 
 def update_fsp_chart(
     chart: ChartPlotWidget,
-    flow,
+    shm: ShmArray,
     graphics_name: str,
     array_key: Optional[str],
-    **kwargs,
 
 ) -> None:
 
-    shm = flow.shm
-    if not shm:
-        return
-
     array = shm.array
     last_row = try_read(array)
 
@@ -94,10 +83,10 @@ def update_fsp_chart(
     # update graphics
     # NOTE: this does a length check internally which allows it
     # staying above the last row check below..
-    chart.update_graphics_from_flow(
+    chart.update_curve_from_array(
         graphics_name,
+        array,
         array_key=array_key or graphics_name,
-        **kwargs,
     )
 
     # XXX: re: ``array_key``: fsp func names must be unique meaning we
@@ -107,6 +96,9 @@ def update_fsp_chart(
     # read from last calculated value and update any label
     last_val_sticky = chart._ysticks.get(graphics_name)
     if last_val_sticky:
+        # array = shm.array[array_key]
+        # if len(array):
+        #     value = array[-1]
         last = last_row[array_key]
         last_val_sticky.update_from_data(-1, last)
 
@@ -161,11 +153,7 @@ async def open_fsp_sidepane(
     sidepane.model = FspConfig()
 
     # just a logger for now until we get fsp configs up and running.
-    async def settings_change(
+    async def settings_change(key: str, value: str) -> bool:
-        key: str,
-        value: str
-
-    ) -> bool:
         print(f'{key}: {value}')
         return True
 
@@ -244,18 +232,21 @@ async def run_fsp_ui(
 
             chart.draw_curve(
                 name=name,
-                shm=shm,
+                data=shm.array,
                 overlay=True,
                 color='default_light',
                 array_key=name,
+                separate_axes=conf.get('separate_axes', False),
                 **conf.get('chart_kwargs', {})
             )
+            # specially store ref to shm for lookup in display loop
+            chart._overlays[name] = shm
 
         else:
             # create a new sub-chart widget for this fsp
             chart = linkedsplits.add_plot(
                 name=name,
-                shm=shm,
+                array=shm.array,
 
                 array_key=name,
                 sidepane=sidepane,
@@ -267,6 +258,11 @@ async def run_fsp_ui(
                 **conf.get('chart_kwargs', {})
             )
 
+            # XXX: ONLY for sub-chart fsps, overlays have their
+            # data looked up from the chart's internal array set.
+            # TODO: we must get a data view api going STAT!!
+            chart._shm = shm
+
             # should **not** be the same sub-chart widget
             assert chart.name != linkedsplits.chart.name
 
@@ -277,7 +273,7 @@ async def run_fsp_ui(
         # first UI update, usually from shm pushed history
         update_fsp_chart(
             chart,
-            chart._flows[array_key],
+            shm,
             name,
             array_key=array_key,
         )
@@ -367,7 +363,6 @@ class FspAdmin:
             tuple,
             tuple[tractor.MsgStream, ShmArray]
         ] = {}
-        self._flow_registry: dict[_Token, str] = {}
         self.src_shm = src_shm
 
     def rr_next_portal(self) -> tractor.Portal:
@@ -380,7 +375,6 @@ class FspAdmin:
         portal: tractor.Portal,
         complete: trio.Event,
         started: trio.Event,
-        fqsn: str,
         dst_shm: ShmArray,
         conf: dict,
         target: Fsp,
@@ -392,6 +386,7 @@ class FspAdmin:
         cluster and sleeps until signalled to exit.
 
         '''
+        brokername, sym = self.linked.symbol.front_feed()
         ns_path = str(target.ns_path)
         async with (
             portal.open_context(
@@ -400,7 +395,8 @@ class FspAdmin:
                 cascade,
 
                 # data feed key
-                fqsn=fqsn,
+                brokername=brokername,
+                symbol=sym,
 
                 # mems
                 src_shm_token=self.src_shm.token,
@@ -411,19 +407,13 @@ class FspAdmin:
 
                 loglevel=loglevel,
                 zero_on_step=conf.get('zero_on_step', False),
-                shm_registry=[
-                    (token.as_msg(), fsp_name, dst_token.as_msg())
-                    for (token, fsp_name), dst_token
-                    in self._flow_registry.items()
-                ],
 
             ) as (ctx, last_index),
             ctx.open_stream() as stream,
         ):
-
             # register output data
             self._registry[
-                (fqsn, ns_path)
+                (brokername, sym, ns_path)
             ] = (
                 stream,
                 dst_shm,
@@ -433,21 +423,6 @@ class FspAdmin:
             started.set()
 
             # wait for graceful shutdown signal
-            async with stream.subscribe() as stream:
-                async for msg in stream:
-                    info = msg.get('fsp_update')
-                    if info:
-                        # if the chart isn't hidden try to update
-                        # the data on screen.
-                        if not self.linked.isHidden():
-                            log.debug(f'Re-syncing graphics for fsp: {ns_path}')
-                            self.linked.graphics_cycle(
-                                trigger_all=True,
-                                prepend_update_index=info['first'],
-                            )
-                    else:
-                        log.info(f'recved unexpected fsp engine msg: {msg}')
-
             await complete.wait()
 
     async def start_engine_task(
@@ -461,18 +436,14 @@ class FspAdmin:
 
     ) -> (ShmArray, trio.Event):
 
-        fqsn = self.linked.symbol.front_fqsn()
+        fqsn = self.linked.symbol.front_feed()
 
         # allocate an output shm array
-        key, dst_shm, opened = maybe_mk_fsp_shm(
+        dst_shm, opened = maybe_mk_fsp_shm(
             fqsn,
             target=target,
             readonly=True,
         )
-        self._flow_registry[
-            (self.src_shm._token, target.name)
-        ] = dst_shm._token
-
         # if not opened:
         #     raise RuntimeError(
         #         f'Already started FSP `{fqsn}:{func_name}`'
@@ -486,7 +457,6 @@ class FspAdmin:
             portal,
             complete,
             started,
-            fqsn,
             dst_shm,
             conf,
             target,
@@ -585,22 +555,15 @@ async def open_vlm_displays(
     be spawned here.
 
     '''
-    sig = inspect.signature(flow_rates.func)
-    params = sig.parameters
-
     async with (
         open_fsp_sidepane(
             linked, {
-                'flows': {
+                'vlm': {
-
-                    # TODO: add support for dynamically changing these
                     'params': {
-                        u'\u03BC' + '_type': {
+                        'price_func': {
-                            'default_value': str(params['mean_type'].default),
+                            'default_value': 'chl3',
-                        },
+                            # tell target ``Edit`` widget to not allow
-                        'period': {
+                            # edits for now.
-                            'default_value': str(params['period'].default),
-                            # make widget un-editable for now.
                             'widget_kwargs': {'readonly': True},
                         },
                     },
@@ -609,18 +572,12 @@ async def open_vlm_displays(
         ) as sidepane,
         open_fsp_admin(linked, ohlcv) as admin,
     ):
-        # TODO: support updates
-        # period_field = sidepane.fields['period']
-        # period_field.setText(
-        #     str(period_param.default)
-        # )
-
         # built-in vlm which we plot ASAP since it's
         # usually data provided directly with OHLC history.
         shm = ohlcv
         chart = linked.add_plot(
             name='volume',
-            shm=shm,
+            array=shm.array,
 
             array_key='volume',
             sidepane=sidepane,
@@ -635,23 +592,22 @@ async def open_vlm_displays(
         )
 
         # force 0 to always be in view
-        def multi_maxmin(
+        def maxmin(
             names: list[str],
 
         ) -> tuple[float, float]:
-
             mx = 0
             for name in names:
-
                 mxmn = chart.maxmin(name=name)
                 if mxmn:
-                    ymax = mxmn[1]
+                    mx = max(mxmn[1], mx)
-                    if ymax > mx:
+
-                        mx = ymax
+            # if mx:
+            #     return 0, mxmn[1]
 
             return 0, mx
 
-        chart.view.maxmin = partial(multi_maxmin, names=['volume'])
+        chart.view._maxmin = partial(maxmin, names=['volume'])
 
         # TODO: fix the x-axis label issue where if you put
         # the axis on the left it's totally not lined up...
@@ -659,6 +615,11 @@ async def open_vlm_displays(
         # chart.hideAxis('right')
         # chart.showAxis('left')
 
+        # XXX: ONLY for sub-chart fsps, overlays have their
+        # data looked up from the chart's internal array set.
+        # TODO: we must get a data view api going STAT!!
+        chart._shm = shm
+
         # send back new chart to caller
         task_status.started(chart)
 
@@ -673,9 +634,9 @@ async def open_vlm_displays(
 
         last_val_sticky.update_from_data(-1, value)
 
-        vlm_curve = chart.update_graphics_from_flow(
+        vlm_curve = chart.update_curve_from_array(
             'volume',
-            # shm.array,
+            shm.array,
         )
 
         # size view to data once at outset
@@ -687,9 +648,8 @@ async def open_vlm_displays(
 
         if dvlm:
 
-            tasks_ready = []
             # spawn and overlay $ vlm on the same subchart
-            dvlm_shm, started = await admin.start_engine_task(
+            shm, started = await admin.start_engine_task(
                 dolla_vlm,
 
                 {  # fsp engine conf
@@ -703,26 +663,11 @@ async def open_vlm_displays(
                 },
                 # loglevel,
             )
-            tasks_ready.append(started)
-
-            # FIXME: we should error on starting the same fsp right
-            # since it might collide with existing shm.. or wait we
-            # had this before??
-            # dolla_vlm,
-
-            tasks_ready.append(started)
             # profiler(f'created shm for fsp actor: {display_name}')
 
-            # wait for all engine tasks to startup
+            await started.wait()
-            async with trio.open_nursery() as n:
-                for event in tasks_ready:
-                    n.start_soon(event.wait)
 
-            # dolla vlm overlay
+            pi = chart.overlay_plotitem(
-            # XXX: the main chart already contains a vlm "units" axis
-            # so here we add an overlay wth a y-range in
-            # $ liquidity-value units (normally a fiat like USD).
-            dvlm_pi = chart.overlay_plotitem(
                 'dolla_vlm',
                 index=0,  # place axis on inside (nearest to chart)
                 axis_title=' $vlm',
@@ -734,155 +679,68 @@ async def open_vlm_displays(
                         digits=2,
                     ),
                 },
-            )
 
-            # all to be overlayed curve names
-            fields = [
-               'dolla_vlm',
-               'dark_vlm',
-            ]
-            # dvlm_rate_fields = [
-            #     'dvlm_rate',
-            #     'dark_dvlm_rate',
-            # ]
-            trade_rate_fields = [
-                'trade_rate',
-                'dark_trade_rate',
-            ]
-
-            group_mxmn = partial(
-                multi_maxmin,
-                # keep both regular and dark vlm in view
-                names=fields,
-                # names=fields + dvlm_rate_fields,
             )
 
             # add custom auto range handler
-            dvlm_pi.vb._maxmin = group_mxmn
+            pi.vb._maxmin = partial(
-
+                maxmin,
-            # use slightly less light (then bracket) gray
+                # keep both regular and dark vlm in view
-            # for volume from "main exchange" and a more "bluey"
+                names=['dolla_vlm', 'dark_vlm'],
-            # gray for "dark" vlm.
+            )
-            vlm_color = 'i3'
-            dark_vlm_color = 'charcoal'
-
-            # add dvlm (step) curves to common view
-            def chart_curves(
-                names: list[str],
-                pi: pg.PlotItem,
-                shm: ShmArray,
-                step_mode: bool = False,
-                style: str = 'solid',
-
-            ) -> None:
-                for name in names:
-                    if 'dark' in name:
-                        color = dark_vlm_color
-                    elif 'rate' in name:
-                        color = vlm_color
-                    else:
-                        color = 'bracket'
 
             curve, _ = chart.draw_curve(
-                        name=name,
+                name='dolla_vlm',
-                        shm=shm,
+                data=shm.array,
-                        array_key=name,
+                array_key='dolla_vlm',
                 overlay=pi,
-                        color=color,
-                        step_mode=step_mode,
-                        style=style,
-                        pi=pi,
-                    )
-
-                    # TODO: we need a better API to do this..
-                    # specially store ref to shm for lookup in display loop
-                    # since only a placeholder of `None` is entered in
-                    # ``.draw_curve()``.
-                    flow = chart._flows[name]
-                    assert flow.plot is pi
-
-            chart_curves(
-                fields,
-                dvlm_pi,
-                dvlm_shm,
                 step_mode=True,
+                # **conf.get('chart_kwargs', {})
             )
-
-            # spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
-            # up since this one depends on it.
-
-            fr_shm, started = await admin.start_engine_task(
-                flow_rates,
-                {  # fsp engine conf
-                    'func_name': 'flow_rates',
-                    'zero_on_step': False,
-                },
-                # loglevel,
-            )
-            await started.wait()
-
-            # chart_curves(
-            #     dvlm_rate_fields,
-            #     dvlm_pi,
-            #     fr_shm,
-            # )
-
             # TODO: is there a way to "sync" the dual axes such that only
             # one curve is needed?
             # hide the original vlm curve since the $vlm one is now
             # displayed and the curves are effectively the same minus
             # liquidity events (well at least on low OHLC periods - 1s).
             vlm_curve.hide()
-            chart.removeItem(vlm_curve)
-            vflow = chart._flows['volume']
-            vflow.render = False
 
-            # avoid range sorting on volume once disabled
+            # TODO: we need a better API to do this..
-            chart.view.disable_auto_yrange()
+            # specially store ref to shm for lookup in display loop
+            # since only a placeholder of `None` is entered in
+            # ``.draw_curve()``.
+            chart._overlays['dolla_vlm'] = shm
 
-            # Trade rate overlay
+            curve, _ = chart.draw_curve(
-            # XXX: requires an additional overlay for
-            # a trades-per-period (time) y-range.
-            tr_pi = chart.overlay_plotitem(
-                'trade_rates',
-
-                # TODO: dynamically update period (and thus this axis?)
-                # title from user input.
-                axis_title='clears',
-
-                axis_side='left',
-                axis_kwargs={
-                    'typical_max_str': ' 10.0 M ',
-                    'formatter': partial(
-                        humanize,
-                        digits=2,
-                    ),
-                    'text_color': vlm_color,
-                },
 
+                name='dark_vlm',
+                data=shm.array,
+                array_key='dark_vlm',
+                overlay=pi,
+                color='charcoal',  # darker theme hue
+                step_mode=True,
+                # **conf.get('chart_kwargs', {})
             )
-            # add custom auto range handler
+            chart._overlays['dark_vlm'] = shm
-            tr_pi.vb.maxmin = partial(
+            # XXX: old dict-style config before it was moved into the
-                multi_maxmin,
+            # helper task
-                # keep both regular and dark vlm in view
+            #     'dolla_vlm': {
-                names=trade_rate_fields,
+            #         'func_name': 'dolla_vlm',
-            )
+            #         'zero_on_step': True,
+            #         'overlay': 'volume',
+            #         'separate_axes': True,
+            #         'params': {
+            #             'price_func': {
+            #                 'default_value': 'chl3',
+            #                 # tell target ``Edit`` widget to not allow
+            #                 # edits for now.
+            #                 'widget_kwargs': {'readonly': True},
+            #             },
+            #         },
+            #         'chart_kwargs': {'step_mode': True}
+            #     },
 
-            chart_curves(
+            # }
-                trade_rate_fields,
-                tr_pi,
-                fr_shm,
-                # step_mode=True,
 
-                # dashed line to represent "individual trades" being
-                # more "granular" B)
-                style='dash',
-            )
-
-            for pi in (
-                dvlm_pi,
-                tr_pi,
-            ):
             for name, axis_info in pi.axes.items():
                 # lol this sux XD
                 axis = axis_info['item']
@@ -891,10 +749,10 @@ async def open_vlm_displays(
 
         # built-in vlm fsps
         for target, conf in {
-            # tina_vwap: {
+            tina_vwap: {
-            #     'overlay': 'ohlc',  # overlays with OHLCV (main) chart
+                'overlay': 'ohlc',  # overlays with OHLCV (main) chart
-            #     'anchor': 'session',
+                'anchor': 'session',
-            # },
+            },
         }.items():
             started = await admin.open_fsp_chart(
                 target,

piker/ui/_interaction.py

@@ -33,8 +33,7 @@ import numpy as np
 import trio
 
 from ..log import get_logger
-from .._profile import pg_profile_enabled, ms_slower_then
+from ._style import _min_points_to_show
-# from ._style import _min_points_to_show
 from ._editors import SelectRect
 from . import _event
 
@@ -319,7 +318,6 @@ async def handle_viewmode_mouse(
         ):
             # when in order mode, submit execution
             # msg.event.accept()
-            # breakpoint()
             view.order_mode.submit_order()
 
 
@@ -344,7 +342,7 @@ class ChartView(ViewBox):
     wheelEventRelay = QtCore.Signal(object, object, object)
 
     event_relay_source: 'Optional[ViewBox]' = None
-    relays: dict[str, QtCore.Signal] = {}
+    relays: dict[str, Signal] = {}
 
     def __init__(
         self,
@@ -358,13 +356,13 @@ class ChartView(ViewBox):
     ):
         super().__init__(
             parent=parent,
-            name=name,
             # TODO: look into the default view padding
             # support that might replace somem of our
             # ``ChartPlotWidget._set_yrange()`
             # defaultPadding=0.,
             **kwargs
         )
+
         # for "known y-range style"
         self._static_yrange = static_yrange
         self._maxmin = None
@@ -386,34 +384,6 @@ class ChartView(ViewBox):
         self.order_mode: bool = False
 
         self.setFocusPolicy(QtCore.Qt.StrongFocus)
-        self._ic = None
-
-    def start_ic(
-        self,
-    ) -> None:
-        '''
-        Signal the beginning of a click-drag interaction
-        to any interested task waiters.
-
-        '''
-        if self._ic is None:
-            self.chart.pause_all_feeds()
-            self._ic = trio.Event()
-
-    def signal_ic(
-        self,
-        *args,
-
-    ) -> None:
-        '''
-        Signal the end of a click-drag interaction
-        to any waiters.
-
-        '''
-        if self._ic:
-            self._ic.set()
-            self._ic = None
-            self.chart.resume_all_feeds()
 
     @asynccontextmanager
     async def open_async_input_handler(
@@ -451,22 +421,13 @@ class ChartView(ViewBox):
         if self._maxmin is None:
             self._maxmin = chart.maxmin
 
-    @property
-    def maxmin(self) -> Callable:
-        return self._maxmin
-
-    @maxmin.setter
-    def maxmin(self, callback: Callable) -> None:
-        self._maxmin = callback
-
     def wheelEvent(
         self,
         ev,
         axis=None,
         relayed_from: ChartView = None,
     ):
-        '''
+        '''Override "center-point" location for scrolling.
-        Override "center-point" location for scrolling.
 
         This is an override of the ``ViewBox`` method simply changing
         the center of the zoom to be the y-axis.
@@ -484,18 +445,15 @@ class ChartView(ViewBox):
 
         # don't zoom more then the min points setting
         l, lbar, rbar, r = chart.bars_range()
-        # vl = r - l
+        vl = r - l
 
-        # if ev.delta() > 0 and vl <= _min_points_to_show:
+        if ev.delta() > 0 and vl <= _min_points_to_show:
-        #     log.debug("Max zoom bruh...")
+            log.debug("Max zoom bruh...")
-        #     return
+            return
 
-        # if (
+        if ev.delta() < 0 and vl >= len(chart._arrays[chart.name]) + 666:
-        #     ev.delta() < 0
+            log.debug("Min zoom bruh...")
-        #     and vl >= len(chart._flows[chart.name].shm.array) + 666
+            return
-        # ):
-        #     log.debug("Min zoom bruh...")
-        #     return
 
         # actual scaling factor
         s = 1.015 ** (ev.delta() * -1 / 20)  # self.state['wheelScaleFactor'])
@@ -516,11 +474,7 @@ class ChartView(ViewBox):
             # lastPos = ev.lastPos()
             # dif = pos - lastPos
             # dif = dif * -1
-            center = Point(
+            center = Point(fn.invertQTransform(self.childGroup.transform()).map(ev.pos()))
-                fn.invertQTransform(
-                    self.childGroup.transform()
-                ).map(ev.pos())
-            )
             # scale_y = 1.3 ** (center.y() * -1 / 20)
             self.scaleBy(s, center)
 
@@ -569,24 +523,7 @@ class ChartView(ViewBox):
 
             self._resetTarget()
             self.scaleBy(s, focal)
-
-            # XXX: the order of the next 2 lines i'm pretty sure
-            # matters, we want the resize to trigger before the graphics
-            # update, but i gotta feelin that because this one is signal
-            # based (and thus not necessarily sync invoked right away)
-            # that calling the resize method manually might work better.
             self.sigRangeChangedManually.emit(mask)
-
-            # XXX: without this is seems as though sometimes
-            # when zooming in from far out (and maybe vice versa?)
-            # the signal isn't being fired enough since if you pan
-            # just after you'll see further downsampling code run
-            # (pretty noticeable on the OHLC ds curve) but with this
-            # that never seems to happen? Only question is how much this
-            # "double work" is causing latency when these missing event
-            # fires don't happen?
-            self.maybe_downsample_graphics()
-
             ev.accept()
 
     def mouseDragEvent(
@@ -669,11 +606,6 @@ class ChartView(ViewBox):
                 # XXX: WHY
                 ev.accept()
 
-                self.start_ic()
-                # if self._ic is None:
-                #     self.chart.pause_all_feeds()
-                #     self._ic = trio.Event()
-
                 if axis == 1:
                     self.chart._static_yrange = 'axis'
 
@@ -691,12 +623,6 @@ class ChartView(ViewBox):
 
                 self.sigRangeChangedManually.emit(self.state['mouseEnabled'])
 
-                if ev.isFinish():
-                    self.signal_ic()
-                    # self._ic.set()
-                    # self._ic = None
-                    # self.chart.resume_all_feeds()
-
         # WEIRD "RIGHT-CLICK CENTER ZOOM" MODE
         elif button & QtCore.Qt.RightButton:
 
@@ -747,8 +673,8 @@ class ChartView(ViewBox):
 
         # flag to prevent triggering sibling charts from the same linked
         # set from recursion errors.
-        autoscale_linked_plots: bool = False,
+        autoscale_linked_plots: bool = True,
-        name: Optional[str] = None,
+        autoscale_overlays: bool = False,
 
     ) -> None:
         '''
@@ -759,14 +685,6 @@ class ChartView(ViewBox):
         data set.
 
         '''
-        name = self.name
-        # print(f'YRANGE ON {name}')
-        profiler = pg.debug.Profiler(
-            msg=f'`ChartView._set_yrange()`: `{name}`',
-            disabled=not pg_profile_enabled(),
-            ms_threshold=ms_slower_then,
-            delayed=True,
-        )
         set_range = True
         chart = self._chart
 
@@ -790,22 +708,30 @@ class ChartView(ViewBox):
         elif yrange is not None:
             ylow, yhigh = yrange
 
+        # calculate max, min y values in viewable x-range from data.
+        # Make sure min bars/datums on screen is adhered.
+        else:
+            br = bars_range or chart.bars_range()
+
+            # TODO: maybe should be a method on the
+            # chart widget/item?
+            if autoscale_linked_plots:
+                # avoid recursion by sibling plots
+                linked = self.linkedsplits
+                plots = list(linked.subplots.copy().values())
+                main = linked.chart
+                if main:
+                    plots.append(main)
+
+                for chart in plots:
+                    if chart and not chart._static_yrange:
+                        chart.cv._set_yrange(
+                            bars_range=br,
+                            autoscale_linked_plots=False,
+                        )
+
         if set_range:
-
+            ylow, yhigh = self._maxmin()
-            # XXX: only compute the mxmn range
-            # if none is provided as input!
-            if not yrange:
-                # flow = chart._flows[name]
-                yrange = self._maxmin()
-
-                if yrange is None:
-                    log.warning(f'No yrange provided for {name}!?')
-                    print(f"WTF NO YRANGE {name}")
-                    return
-
-            ylow, yhigh = yrange
-
-            profiler(f'callback ._maxmin(): {yrange}')
 
             # view margins: stay within a % of the "true range"
             diff = yhigh - ylow
@@ -820,13 +746,9 @@ class ChartView(ViewBox):
                 yMax=yhigh,
             )
             self.setYRange(ylow, yhigh)
-            profiler(f'set limits: {(ylow, yhigh)}')
-
-        profiler.finish()
 
     def enable_auto_yrange(
-        self,
+        vb: ChartView,
-        src_vb: Optional[ChartView] = None,
 
     ) -> None:
         '''
@@ -834,114 +756,13 @@ class ChartView(ViewBox):
         based on data contents and ``ViewBox`` state.
 
         '''
-        if src_vb is None:
+        vb.sigXRangeChanged.connect(vb._set_yrange)
-            src_vb = self
-
-        # splitter(s) resizing
-        src_vb.sigResized.connect(self._set_yrange)
-
-        # TODO: a smarter way to avoid calling this needlessly?
-        # 2 things i can think of:
-        # - register downsample-able graphics specially and only
-        #   iterate those.
-        # - only register this when certain downsampleable graphics are
-        #   "added to scene".
-        src_vb.sigRangeChangedManually.connect(
-            self.maybe_downsample_graphics
-        )
-
         # mouse wheel doesn't emit XRangeChanged
-        src_vb.sigRangeChangedManually.connect(self._set_yrange)
+        vb.sigRangeChangedManually.connect(vb._set_yrange)
+        vb.sigResized.connect(vb._set_yrange)  # splitter(s) resizing
 
-        # src_vb.sigXRangeChanged.connect(self._set_yrange)
+    def disable_auto_yrange(
-        # src_vb.sigXRangeChanged.connect(
-        #     self.maybe_downsample_graphics
-        # )
-
-    def disable_auto_yrange(self) -> None:
-
-        self.sigResized.disconnect(
-            self._set_yrange,
-        )
-        self.sigRangeChangedManually.disconnect(
-            self.maybe_downsample_graphics
-        )
-        self.sigRangeChangedManually.disconnect(
-            self._set_yrange,
-        )
-
-        # self.sigXRangeChanged.disconnect(self._set_yrange)
-        # self.sigXRangeChanged.disconnect(
-        #     self.maybe_downsample_graphics
-        # )
-
-    def x_uppx(self) -> float:
-        '''
-        Return the "number of x units" within a single
-        pixel currently being displayed for relevant
-        graphics items which are our children.
-
-        '''
-        graphics = [f.graphics for f in self._chart._flows.values()]
-        if not graphics:
-            return 0
-
-        for graphic in graphics:
-            xvec = graphic.pixelVectors()[0]
-            if xvec:
-                return xvec.x()
-        else:
-            return 0
-
-    def maybe_downsample_graphics(
         self,
-        autoscale_overlays: bool = True,
+    ) -> None:
-    ):
 
-        profiler = pg.debug.Profiler(
+        self._chart._static_yrange = 'axis'
-            msg=f'ChartView.maybe_downsample_graphics() for {self.name}',
-            disabled=not pg_profile_enabled(),
-
-            # XXX: important to avoid not seeing underlying
-            # ``.update_graphics_from_flow()`` nested profiling likely
-            # due to the way delaying works and garbage collection of
-            # the profiler in the delegated method calls.
-            ms_threshold=6,
-            # ms_threshold=ms_slower_then,
-        )
-
-        # TODO: a faster single-loop-iterator way of doing this XD
-        chart = self._chart
-        linked = self.linkedsplits
-        plots = linked.subplots | {chart.name: chart}
-        for chart_name, chart in plots.items():
-            for name, flow in chart._flows.items():
-
-                if (
-                    not flow.render
-
-                    # XXX: super important to be aware of this.
-                    # or not flow.graphics.isVisible()
-                ):
-                    continue
-
-                # pass in no array which will read and render from the last
-                # passed array (normally provided by the display loop.)
-                chart.update_graphics_from_flow(
-                    name,
-                    use_vr=True,
-                )
-
-                # for each overlay on this chart auto-scale the
-                # y-range to max-min values.
-                if autoscale_overlays:
-                    overlay = chart.pi_overlay
-                    if overlay:
-                        for pi in overlay.overlays:
-                            pi.vb._set_yrange(
-                                # TODO: get the range once up front...
-                                # bars_range=br,
-                            )
-                    profiler('autoscaled linked plots')
-
-                profiler(f'<{chart_name}>.update_graphics_from_flow({name})')

piker/ui/_l1.py

@@ -199,14 +199,7 @@ class LevelLabel(YAxisLabel):
         elif self._orient_v == 'top':
             lp, rp = rect.bottomLeft(), rect.bottomRight()
 
-        p.drawLine(
+        p.drawLine(lp.x(), lp.y(), rp.x(), rp.y())
-            *map(int, [
-                lp.x(),
-                lp.y(),
-                rp.x(),
-                rp.y(),
-            ])
-        )
 
     def highlight(self, pen) -> None:
         self._pen = pen

piker/ui/_lines.py

@@ -20,7 +20,7 @@ Lines for orders, alerts, L2.
 """
 from functools import partial
 from math import floor
-from typing import Optional, Callable
+from typing import Tuple, Optional, List, Callable
 
 import pyqtgraph as pg
 from pyqtgraph import Point, functions as fn
@@ -29,8 +29,10 @@ from PyQt5.QtCore import QPointF
 
 from ._annotate import qgo_draw_markers, LevelMarker
 from ._anchors import (
+    marker_right_points,
     vbr_left,
     right_axis,
+    # pp_tight_and_right,  # wanna keep it straight in the long run
     gpath_pin,
 )
 from ..calc import humanize
@@ -102,8 +104,8 @@ class LevelLine(pg.InfiniteLine):
 
         # list of labels anchored at one of the 2 line endpoints
         # inside the viewbox
-        self._labels: list[Label] = []
+        self._labels: List[Label] = []
-        self._markers: list[(int, Label)] = []
+        self._markers: List[(int, Label)] = []
 
         # whenever this line is moved trigger label updates
         self.sigPositionChanged.connect(self.on_pos_change)
@@ -122,7 +124,7 @@ class LevelLine(pg.InfiniteLine):
         self._y_incr_mult = 1 / chart.linked.symbol.tick_size
         self._right_end_sc: float = 0
 
-    def txt_offsets(self) -> tuple[int, int]:
+    def txt_offsets(self) -> Tuple[int, int]:
         return 0, 0
 
     @property
@@ -313,6 +315,17 @@ class LevelLine(pg.InfiniteLine):
         # TODO: enter labels edit mode
         print(f'double click {ev}')
 
+    def right_point(
+        self,
+    ) -> float:
+
+        chart = self._chart
+        l1_len = chart._max_l1_line_len
+        ryaxis = chart.getAxis('right')
+        up_to_l1_sc = ryaxis.pos().x() - l1_len
+
+        return up_to_l1_sc
+
     def paint(
         self,
 
@@ -332,7 +345,7 @@ class LevelLine(pg.InfiniteLine):
         vb_left, vb_right = self._endPoints
         vb = self.getViewBox()
 
-        line_end, marker_right, r_axis_x = self._chart.marker_right_points()
+        line_end, marker_right, r_axis_x = marker_right_points(self._chart)
 
         if self.show_markers and self.markers:
 
@@ -398,7 +411,7 @@ class LevelLine(pg.InfiniteLine):
     def scene_endpoint(self) -> QPointF:
 
         if not self._right_end_sc:
-            line_end, _, _ = self._chart.marker_right_points()
+            line_end, _, _ = marker_right_points(self._chart)
             self._right_end_sc = line_end - 10
 
         return QPointF(self._right_end_sc, self.scene_y())
@@ -409,23 +422,23 @@ class LevelLine(pg.InfiniteLine):
 
     ) -> QtWidgets.QGraphicsPathItem:
 
-        self._marker = path
-        self._marker.setPen(self.currentPen)
-        self._marker.setBrush(fn.mkBrush(self.currentPen.color()))
         # add path to scene
         self.getViewBox().scene().addItem(path)
 
-        # place to just-left of L1 labels
+        self._marker = path
-        rsc = self._chart.pre_l1_xs()[0]
+
+        rsc = self.right_point()
+
+        self._marker.setPen(self.currentPen)
+        self._marker.setBrush(fn.mkBrush(self.currentPen.color()))
         path.setPos(QPointF(rsc, self.scene_y()))
 
         return path
 
     def hoverEvent(self, ev):
-        '''
+        """Mouse hover callback.
-        Mouse hover callback.
 
-        '''
+        """
         cur = self._chart.linked.cursor
 
         # hovered
@@ -601,8 +614,7 @@ def order_line(
     **line_kwargs,
 
 ) -> LevelLine:
-    '''
+    '''Convenience routine to add a line graphic representing an order
-    Convenience routine to add a line graphic representing an order
     execution submitted to the EMS via the chart's "order mode".
 
     '''
@@ -677,6 +689,7 @@ def order_line(
 
             return f'{account}: '
 
+
         label.fields = {
             'size': size,
             'size_digits': 0,

piker/ui/_ohlc.py

@@ -17,40 +17,40 @@
 Super fast OHLC sampling graphics types.
 
 """
-from __future__ import annotations
+from typing import List, Optional, Tuple
-from typing import (
-    Optional,
-    TYPE_CHECKING,
-)
 
 import numpy as np
 import pyqtgraph as pg
+from numba import njit, float64, int64  # , optional
 from PyQt5 import QtCore, QtGui, QtWidgets
 from PyQt5.QtCore import QLineF, QPointF
-from PyQt5.QtGui import QPainterPath
+# from numba import types as ntypes
+# from ..data._source import numba_ohlc_dtype
 
-from .._profile import pg_profile_enabled, ms_slower_then
+from .._profile import pg_profile_enabled
 from ._style import hcolor
-from ..log import get_logger
-
-if TYPE_CHECKING:
-    from ._chart import LinkedSplits
 
 
-log = get_logger(__name__)
+def _mk_lines_array(
+    data: List,
+    size: int,
+    elements_step: int = 6,
+) -> np.ndarray:
+    """Create an ndarray to hold lines graphics info.
+
+    """
+    return np.zeros_like(
+        data,
+        shape=(int(size), elements_step),
+        dtype=object,
+    )
 
 
-def bar_from_ohlc_row(
+def lines_from_ohlc(
     row: np.ndarray,
-    # 0.5 is no overlap between arms, 1.0 is full overlap
+    w: float
-    w: float = 0.43
+) -> Tuple[QLineF]:
 
-) -> tuple[QLineF]:
-    '''
-    Generate the minimal ``QLineF`` lines to construct a single
-    OHLC "bar" for use in the "last datum" of a series.
-
-    '''
     open, high, low, close, index = row[
         ['open', 'high', 'low', 'close', 'index']]
 
@@ -81,37 +81,280 @@ def bar_from_ohlc_row(
     return [hl, o, c]
 
 
-class BarItems(pg.GraphicsObject):
+@njit(
-    '''
+    # TODO: for now need to construct this manually for readonly arrays, see
-    "Price range" bars graphics rendered from a OHLC sampled sequence.
+    # https://github.com/numba/numba/issues/4511
+    # ntypes.Tuple((float64[:], float64[:], float64[:]))(
+    #     numba_ohlc_dtype[::1],  # contiguous
+    #     int64,
+    #     optional(float64),
+    # ),
+    nogil=True
+)
+def path_arrays_from_ohlc(
+    data: np.ndarray,
+    start: int64,
+    bar_gap: float64 = 0.43,
+) -> np.ndarray:
+    """Generate an array of lines objects from input ohlc data.
+
+    """
+    size = int(data.shape[0] * 6)
+
+    x = np.zeros(
+        # data,
+        shape=size,
+        dtype=float64,
+    )
+    y, c = x.copy(), x.copy()
+
+    # TODO: report bug for assert @
+    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
+    for i, q in enumerate(data[start:], start):
+
+        # TODO: ask numba why this doesn't work..
+        # open, high, low, close, index = q[
+        #     ['open', 'high', 'low', 'close', 'index']]
+
+        open = q['open']
+        high = q['high']
+        low = q['low']
+        close = q['close']
+        index = float64(q['index'])
+
+        istart = i * 6
+        istop = istart + 6
+
+        # x,y detail the 6 points which connect all vertexes of a ohlc bar
+        x[istart:istop] = (
+            index - bar_gap,
+            index,
+            index,
+            index,
+            index,
+            index + bar_gap,
+        )
+        y[istart:istop] = (
+            open,
+            open,
+            low,
+            high,
+            close,
+            close,
+        )
+
+        # specifies that the first edge is never connected to the
+        # prior bars last edge thus providing a small "gap"/"space"
+        # between bars determined by ``bar_gap``.
+        c[istart:istop] = (1, 1, 1, 1, 1, 0)
+
+    return x, y, c
+
+
+def gen_qpath(
+    data,
+    start,  # XXX: do we need this?
+    w,
+) -> QtGui.QPainterPath:
+
+    profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
+
+    x, y, c = path_arrays_from_ohlc(data, start, bar_gap=w)
+    profiler("generate stream with numba")
+
+    # TODO: numba the internals of this!
+    path = pg.functions.arrayToQPath(x, y, connect=c)
+    profiler("generate path with arrayToQPath")
+
+    return path
+
+
+class BarItems(pg.GraphicsObject):
+    """Price range bars graphics rendered from a OHLC sequence.
+    """
+    sigPlotChanged = QtCore.pyqtSignal(object)
+
+    # 0.5 is no overlap between arms, 1.0 is full overlap
+    w: float = 0.43
 
-    '''
     def __init__(
         self,
-        linked: LinkedSplits,
+        # scene: 'QGraphicsScene',  # noqa
         plotitem: 'pg.PlotItem',  # noqa
         pen_color: str = 'bracket',
         last_bar_color: str = 'bracket',
-
-        name: Optional[str] = None,
-
     ) -> None:
         super().__init__()
-        self.linked = linked
+
         # XXX: for the mega-lulz increasing width here increases draw
         # latency...  so probably don't do it until we figure that out.
-        self._color = pen_color
         self.bars_pen = pg.mkPen(hcolor(pen_color), width=1)
         self.last_bar_pen = pg.mkPen(hcolor(last_bar_color), width=2)
-        self._name = name
 
+        # NOTE: this prevents redraws on mouse interaction which is
+        # a huge boon for avg interaction latency.
+
+        # TODO: one question still remaining is if this makes trasform
+        # interactions slower (such as zooming) and if so maybe if/when
+        # we implement a "history" mode for the view we disable this in
+        # that mode?
         self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
-        self.path = QPainterPath()
-        self._last_bar_lines: Optional[tuple[QLineF, ...]] = None
 
-    def x_uppx(self) -> int:
+        # not sure if this is actually impoving anything but figured it
-        # we expect the downsample curve report this.
+        # was worth a shot:
-        return 0
+        # self.path.reserve(int(100e3 * 6))
+
+        self.path = QtGui.QPainterPath()
+
+        self._pi = plotitem
+
+        self._xrange: Tuple[int, int]
+        self._yrange: Tuple[float, float]
+
+        # TODO: don't render the full backing array each time
+        # self._path_data = None
+        self._last_bar_lines: Optional[Tuple[QLineF, ...]] = None
+
+        # track the current length of drawable lines within the larger array
+        self.start_index: int = 0
+        self.stop_index: int = 0
+
+    def draw_from_data(
+        self,
+        data: np.ndarray,
+        start: int = 0,
+    ) -> QtGui.QPainterPath:
+        """Draw OHLC datum graphics from a ``np.ndarray``.
+
+        This routine is usually only called to draw the initial history.
+        """
+        hist, last = data[:-1], data[-1]
+
+        self.path = gen_qpath(hist, start, self.w)
+
+        # save graphics for later reference and keep track
+        # of current internal "last index"
+        # self.start_index = len(data)
+        index = data['index']
+        self._xrange = (index[0], index[-1])
+        self._yrange = (
+            np.nanmax(data['high']),
+            np.nanmin(data['low']),
+        )
+
+        # up to last to avoid double draw of last bar
+        self._last_bar_lines = lines_from_ohlc(last, self.w)
+
+        # trigger render
+        # https://doc.qt.io/qt-5/qgraphicsitem.html#update
+        self.update()
+
+        return self.path
+
+    def update_from_array(
+        self,
+        array: np.ndarray,
+        just_history=False,
+    ) -> None:
+        """Update the last datum's bar graphic from input data array.
+
+        This routine should be interface compatible with
+        ``pg.PlotCurveItem.setData()``. Normally this method in
+        ``pyqtgraph`` seems to update all the data passed to the
+        graphics object, and then update/rerender, but here we're
+        assuming the prior graphics havent changed (OHLC history rarely
+        does) so this "should" be simpler and faster.
+
+        This routine should be made (transitively) as fast as possible.
+        """
+        # index = self.start_index
+        istart, istop = self._xrange
+
+        index = array['index']
+        first_index, last_index = index[0], index[-1]
+
+        # length = len(array)
+        prepend_length = istart - first_index
+        append_length = last_index - istop
+
+        flip_cache = False
+
+        # TODO: allow mapping only a range of lines thus
+        # only drawing as many bars as exactly specified.
+
+        if prepend_length:
+
+            # new history was added and we need to render a new path
+            new_bars = array[:prepend_length]
+            prepend_path = gen_qpath(new_bars, 0, self.w)
+
+            # XXX: SOMETHING IS MAYBE FISHY HERE what with the old_path
+            # y value not matching the first value from
+            # array[prepend_length + 1] ???
+
+            # update path
+            old_path = self.path
+            self.path = prepend_path
+            self.path.addPath(old_path)
+
+            # trigger redraw despite caching
+            self.prepareGeometryChange()
+
+        if append_length:
+            # generate new lines objects for updatable "current bar"
+            self._last_bar_lines = lines_from_ohlc(array[-1], self.w)
+
+            # generate new graphics to match provided array
+            # path appending logic:
+            # we need to get the previous "current bar(s)" for the time step
+            # and convert it to a sub-path to append to the historical set
+            # new_bars = array[istop - 1:istop + append_length - 1]
+            new_bars = array[-append_length - 1:-1]
+            append_path = gen_qpath(new_bars, 0, self.w)
+            self.path.moveTo(float(istop - self.w), float(new_bars[0]['open']))
+            self.path.addPath(append_path)
+
+            # trigger redraw despite caching
+            self.prepareGeometryChange()
+            self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
+            flip_cache = True
+
+        self._xrange = first_index, last_index
+
+        # last bar update
+        i, o, h, l, last, v = array[-1][
+            ['index', 'open', 'high', 'low', 'close', 'volume']
+        ]
+        # assert i == self.start_index - 1
+        # assert i == last_index
+        body, larm, rarm = self._last_bar_lines
+
+        # XXX: is there a faster way to modify this?
+        rarm.setLine(rarm.x1(), last, rarm.x2(), last)
+
+        # writer is responsible for changing open on "first" volume of bar
+        larm.setLine(larm.x1(), o, larm.x2(), o)
+
+        if l != h:  # noqa
+
+            if body is None:
+                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
+            else:
+                # update body
+                body.setLine(i, l, i, h)
+
+            # XXX: pretty sure this is causing an issue where the bar has
+            # a large upward move right before the next sample and the body
+            # is getting set to None since the next bar is flat but the shm
+            # array index update wasn't read by the time this code runs. Iow
+            # we're doing this removal of the body for a bar index that is
+            # now out of date / from some previous sample. It's weird
+            # though because i've seen it do this to bars i - 3 back?
+
+        self.update()
+
+        if flip_cache:
+            self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
 
     def boundingRect(self):
         # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
@@ -130,17 +373,12 @@ class BarItems(pg.GraphicsObject):
         # apparently this a lot faster says the docs?
         # https://doc.qt.io/qt-5/qpainterpath.html#controlPointRect
         hb = self.path.controlPointRect()
-        hb_tl, hb_br = (
+        hb_tl, hb_br = hb.topLeft(), hb.bottomRight()
-            hb.topLeft(),
-            hb.bottomRight(),
-        )
 
         # need to include last bar height or BR will be off
         mx_y = hb_br.y()
         mn_y = hb_tl.y()
 
-        last_lines = self._last_bar_lines
-        if last_lines:
         body_line = self._last_bar_lines[0]
         if body_line:
             mx_y = max(mx_y, max(body_line.y1(), body_line.y2()))
@@ -167,13 +405,9 @@ class BarItems(pg.GraphicsObject):
         p: QtGui.QPainter,
         opt: QtWidgets.QStyleOptionGraphicsItem,
         w: QtWidgets.QWidget
-
     ) -> None:
 
-        profiler = pg.debug.Profiler(
+        profiler = pg.debug.Profiler(disabled=not pg_profile_enabled())
-            disabled=not pg_profile_enabled(),
-            ms_threshold=ms_slower_then,
-        )
 
         # p.setCompositionMode(0)
 
@@ -184,67 +418,9 @@ class BarItems(pg.GraphicsObject):
         # lead to any perf gains other then when zoomed in to less bars
         # in view.
         p.setPen(self.last_bar_pen)
-        if self._last_bar_lines:
         p.drawLines(*tuple(filter(bool, self._last_bar_lines)))
         profiler('draw last bar')
 
         p.setPen(self.bars_pen)
         p.drawPath(self.path)
-        profiler(f'draw history path: {self.path.capacity()}')
+        profiler('draw history path')
-
-    def draw_last_datum(
-        self,
-        path: QPainterPath,
-        src_data: np.ndarray,
-        render_data: np.ndarray,
-        reset: bool,
-        array_key: str,
-
-        fields: list[str] = [
-            'index',
-            'open',
-            'high',
-            'low',
-            'close',
-        ],
-
-    ) -> None:
-
-        # relevant fields
-        ohlc = src_data[fields]
-        last_row = ohlc[-1:]
-
-        # individual values
-        last_row = i, o, h, l, last = ohlc[-1]
-
-        # generate new lines objects for updatable "current bar"
-        self._last_bar_lines = bar_from_ohlc_row(last_row)
-
-        # assert i == graphics.start_index - 1
-        # assert i == last_index
-        body, larm, rarm = self._last_bar_lines
-
-        # XXX: is there a faster way to modify this?
-        rarm.setLine(rarm.x1(), last, rarm.x2(), last)
-
-        # writer is responsible for changing open on "first" volume of bar
-        larm.setLine(larm.x1(), o, larm.x2(), o)
-
-        if l != h:  # noqa
-
-            if body is None:
-                body = self._last_bar_lines[0] = QLineF(i, l, i, h)
-            else:
-                # update body
-                body.setLine(i, l, i, h)
-
-            # XXX: pretty sure this is causing an issue where the
-            # bar has a large upward move right before the next
-            # sample and the body is getting set to None since the
-            # next bar is flat but the shm array index update wasn't
-            # read by the time this code runs. Iow we're doing this
-            # removal of the body for a bar index that is now out of
-            # date / from some previous sample. It's weird though
-            # because i've seen it do this to bars i - 3 back?
-
-        return ohlc['index'], ohlc['close']

piker/ui/_overlay.py

@@ -103,6 +103,11 @@ class ComposedGridLayout:
             dict[str, AxisItem],
         ] = {}
 
+        self._axes2pi: dict[
+            AxisItem,
+            dict[str, PlotItem],
+        ] = {}
+
         # TODO: better name?
         # construct surrounding layouts for placing outer axes and
         # their legends and title labels.
@@ -153,8 +158,8 @@ class ComposedGridLayout:
         for name, axis_info in plotitem.axes.items():
             axis = axis_info['item']
             # register this plot's (maybe re-placed) axes for lookup.
-            # print(f'inserting {name}:{axis} to index {index}')
+            self._pi2axes.setdefault(index, {})[name] = axis
-            self._pi2axes.setdefault(name, {})[index] = axis
+            self._axes2pi.setdefault(index, {})[name] = plotitem
 
         # enter plot into list for index tracking
         self.items.insert(index, plotitem)
@@ -208,12 +213,11 @@ class ComposedGridLayout:
 
             # invert insert index for layouts which are
             # not-left-to-right, top-to-bottom insert oriented
-            insert_index = index
             if name in ('top', 'left'):
-                insert_index = min(len(axes) - index, 0)
+                index = min(len(axes) - index, 0)
-                assert insert_index >= 0
+                assert index >= 0
 
-            linlayout.insertItem(insert_index, axis)
+            linlayout.insertItem(index, axis)
             axes.insert(index, axis)
 
         self._register_item(index, plotitem)
@@ -239,15 +243,13 @@ class ComposedGridLayout:
         plot: PlotItem,
         name: str,
 
-    ) -> Optional[AxisItem]:
+    ) -> AxisItem:
         '''
-        Retrieve the named axis for overlayed ``plot`` or ``None``
+        Retrieve the named axis for overlayed ``plot``.
-        if axis for that name is not shown.
 
         '''
         index = self.items.index(plot)
-        named = self._pi2axes[name]
+        return self._pi2axes[index][name]
-        return named.get(index)
 
     def pop(
         self,
@@ -339,7 +341,7 @@ def mk_relay_method(
             # halt/short circuit the graphicscene loop). Further the
             # surrounding handler for this signal must be allowed to execute
             # and get processed by **this consumer**.
-            # print(f'{vb.name} rx relayed from {relayed_from.name}')
+            print(f'{vb.name} rx relayed from {relayed_from.name}')
             ev.ignore()
 
             return slot(
@@ -349,7 +351,7 @@ def mk_relay_method(
             )
 
         if axis is not None:
-            # print(f'{vb.name} handling axis event:\n{str(ev)}')
+            print(f'{vb.name} handling axis event:\n{str(ev)}')
             ev.accept()
             return slot(
                 vb,
@@ -488,6 +490,7 @@ class PlotItemOverlay:
         vb.setZValue(1000)  # XXX: critical for scene layering/relaying
 
         self.overlays: list[PlotItem] = []
+        from piker.ui._overlay import ComposedGridLayout
         self.layout = ComposedGridLayout(
             root_plotitem,
             root_plotitem.layout,
@@ -508,7 +511,7 @@ class PlotItemOverlay:
 
     ) -> None:
 
-        index = index or len(self.overlays)
+        index = index or 0
         root = self.root_plotitem
         # layout: QGraphicsGridLayout = root.layout
         self.overlays.insert(index, plotitem)
@@ -610,26 +613,6 @@ class PlotItemOverlay:
         '''
         return self.layout.get_axis(plot, name)
 
-    def get_axes(
-        self,
-        name: str,
-
-    ) -> list[AxisItem]:
-        '''
-        Retrieve all axes for all plots with ``name: str``.
-
-        If a particular overlay doesn't have a displayed named axis
-        then it is not delivered in the returned ``list``.
-
-        '''
-        axes = []
-        for plot in self.overlays:
-            axis = self.layout.get_axis(plot, name)
-            if axis:
-                axes.append(axis)
-
-        return axes
-
     # TODO: i guess we need this if you want to detach existing plots
     # dynamically? XXX: untested as of now.
     def _disconnect_all(

piker/ui/_pathops.py

@@ -1,236 +0,0 @@
-# piker: trading gear for hackers
-# Copyright (C) 2018-present  Tyler Goodlet (in stewardship of piker0)
-
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-"""
-Super fast ``QPainterPath`` generation related operator routines.
-
-"""
-from __future__ import annotations
-from typing import (
-    # Optional,
-    TYPE_CHECKING,
-)
-
-import numpy as np
-from numpy.lib import recfunctions as rfn
-from numba import njit, float64, int64  # , optional
-# import pyqtgraph as pg
-from PyQt5 import QtGui
-# from PyQt5.QtCore import QLineF, QPointF
-
-from ..data._sharedmem import (
-    ShmArray,
-)
-# from .._profile import pg_profile_enabled, ms_slower_then
-from ._compression import (
-    ds_m4,
-)
-
-if TYPE_CHECKING:
-    from ._flows import Renderer
-
-
-def xy_downsample(
-    x,
-    y,
-    uppx,
-
-    x_spacer: float = 0.5,
-
-) -> tuple[np.ndarray, np.ndarray]:
-
-    # downsample whenever more then 1 pixels per datum can be shown.
-    # always refresh data bounds until we get diffing
-    # working properly, see above..
-    bins, x, y = ds_m4(
-        x,
-        y,
-        uppx,
-    )
-
-    # flatten output to 1d arrays suitable for path-graphics generation.
-    x = np.broadcast_to(x[:, None], y.shape)
-    x = (x + np.array(
-        [-x_spacer, 0, 0, x_spacer]
-    )).flatten()
-    y = y.flatten()
-
-    return x, y
-
-
-@njit(
-    # TODO: for now need to construct this manually for readonly arrays, see
-    # https://github.com/numba/numba/issues/4511
-    # ntypes.tuple((float64[:], float64[:], float64[:]))(
-    #     numba_ohlc_dtype[::1],  # contiguous
-    #     int64,
-    #     optional(float64),
-    # ),
-    nogil=True
-)
-def path_arrays_from_ohlc(
-    data: np.ndarray,
-    start: int64,
-    bar_gap: float64 = 0.43,
-
-) -> np.ndarray:
-    '''
-    Generate an array of lines objects from input ohlc data.
-
-    '''
-    size = int(data.shape[0] * 6)
-
-    x = np.zeros(
-        # data,
-        shape=size,
-        dtype=float64,
-    )
-    y, c = x.copy(), x.copy()
-
-    # TODO: report bug for assert @
-    # /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
-    for i, q in enumerate(data[start:], start):
-
-        # TODO: ask numba why this doesn't work..
-        # open, high, low, close, index = q[
-        #     ['open', 'high', 'low', 'close', 'index']]
-
-        open = q['open']
-        high = q['high']
-        low = q['low']
-        close = q['close']
-        index = float64(q['index'])
-
-        istart = i * 6
-        istop = istart + 6
-
-        # x,y detail the 6 points which connect all vertexes of a ohlc bar
-        x[istart:istop] = (
-            index - bar_gap,
-            index,
-            index,
-            index,
-            index,
-            index + bar_gap,
-        )
-        y[istart:istop] = (
-            open,
-            open,
-            low,
-            high,
-            close,
-            close,
-        )
-
-        # specifies that the first edge is never connected to the
-        # prior bars last edge thus providing a small "gap"/"space"
-        # between bars determined by ``bar_gap``.
-        c[istart:istop] = (1, 1, 1, 1, 1, 0)
-
-    return x, y, c
-
-
-def gen_ohlc_qpath(
-    r: Renderer,
-    data: np.ndarray,
-    array_key: str,  # we ignore this
-    vr: tuple[int, int],
-
-    start: int = 0,  # XXX: do we need this?
-    # 0.5 is no overlap between arms, 1.0 is full overlap
-    w: float = 0.43,
-
-) -> QtGui.QPainterPath:
-    '''
-    More or less direct proxy to ``path_arrays_from_ohlc()``
-    but with closed in kwargs for line spacing.
-
-    '''
-    x, y, c = path_arrays_from_ohlc(
-        data,
-        start,
-        bar_gap=w,
-    )
-    return x, y, c
-
-
-def ohlc_to_line(
-    ohlc_shm: ShmArray,
-    data_field: str,
-    fields: list[str] = ['open', 'high', 'low', 'close']
-
-) -> tuple[
-    np.ndarray,
-    np.ndarray,
-]:
-    '''
-    Convert an input struct-array holding OHLC samples into a pair of
-    flattened x, y arrays with the same size (datums wise) as the source
-    data.
-
-    '''
-    y_out = ohlc_shm.ustruct(fields)
-    first = ohlc_shm._first.value
-    last = ohlc_shm._last.value
-
-    # write pushed data to flattened copy
-    y_out[first:last] = rfn.structured_to_unstructured(
-        ohlc_shm.array[fields]
-    )
-
-    # generate an flat-interpolated x-domain
-    x_out = (
-        np.broadcast_to(
-            ohlc_shm._array['index'][:, None],
-            (
-                ohlc_shm._array.size,
-                # 4,  # only ohlc
-                y_out.shape[1],
-            ),
-        ) + np.array([-0.5, 0, 0, 0.5])
-    )
-    assert y_out.any()
-
-    return (
-        x_out,
-        y_out,
-    )
-
-
-def to_step_format(
-    shm: ShmArray,
-    data_field: str,
-    index_field: str = 'index',
-
-) -> tuple[int, np.ndarray, np.ndarray]:
-    '''
-    Convert an input 1d shm array to a "step array" format
-    for use by path graphics generation.
-
-    '''
-    i = shm._array['index'].copy()
-    out = shm._array[data_field].copy()
-
-    x_out = np.broadcast_to(
-        i[:, None],
-        (i.size, 2),
-    ) + np.array([-0.5, 0.5])
-
-    y_out = np.empty((len(out), 2), dtype=out.dtype)
-    y_out[:] = out[:, np.newaxis]
-
-    # start y at origin level
-    y_out[0, 0] = 0
-    return x_out, y_out

piker/ui/_position.py

@@ -71,10 +71,10 @@ async def update_pnl_from_feed(
     log.info(f'Starting pnl display for {pp.alloc.account}')
 
     if live.size < 0:
-        types = ('ask', 'last', 'last', 'dark_trade')
+        types = ('ask', 'last', 'last', 'utrade')
 
     elif live.size > 0:
-        types = ('bid', 'last', 'last', 'dark_trade')
+        types = ('bid', 'last', 'last', 'utrade')
 
     else:
         log.info(f'No position (yet) for {tracker.alloc.account}@{key}')
@@ -119,8 +119,7 @@ async def update_pnl_from_feed(
 
 @dataclass
 class SettingsPane:
-    '''
+    '''Composite set of widgets plus an allocator model for configuring
-    Composite set of widgets plus an allocator model for configuring
     order entry sizes and position limits per tradable instrument.
 
     '''
@@ -152,8 +151,7 @@ class SettingsPane:
         key: str,
 
     ) -> None:
-        '''
+        '''Called on any order pane drop down selection change.
-        Called on any order pane drop down selection change.
 
         '''
         log.info(f'selection input {key}:{text}')
@@ -166,8 +164,7 @@ class SettingsPane:
         value: str,
 
     ) -> bool:
-        '''
+        '''Called on any order pane edit field value change.
-        Called on any order pane edit field value change.
 
         '''
         mode = self.order_mode
@@ -222,36 +219,16 @@ class SettingsPane:
             else:
                 value = puterize(value)
                 if key == 'limit':
-                    pp = mode.current_pp.live_pp
-
                     if size_unit == 'currency':
-                        dsize = pp.dsize
-                        if dsize > value:
-                            log.error(
-                                f'limit must > then current pp: {dsize}'
-                            )
-                            raise ValueError
-
                         alloc.currency_limit = value
-
                     else:
-                        size = pp.size
-                        if size > value:
-                            log.error(
-                                f'limit must > then current pp: {size}'
-                            )
-                            raise ValueError
-
                         alloc.units_limit = value
 
                 elif key == 'slots':
-                    if value <= 0:
-                        raise ValueError('slots must be > 0')
                     alloc.slots = int(value)
 
                 else:
-                    log.error(f'Unknown setting {key}')
+                    raise ValueError(f'Unknown setting {key}')
-                    raise ValueError
 
             log.info(f'settings change: {key}: {value}')
 
@@ -386,8 +363,7 @@ def position_line(
     marker: Optional[LevelMarker] = None,
 
 ) -> LevelLine:
-    '''
+    '''Convenience routine to create a line graphic representing a "pp"
-    Convenience routine to create a line graphic representing a "pp"
     aka the acro for a,
     "{piker, private, personal, puny, <place your p-word here>} position".
 
@@ -441,8 +417,7 @@ def position_line(
 
 
 class PositionTracker:
-    '''
+    '''Track and display real-time positions for a single symbol
-    Track and display real-time positions for a single symbol
     over multiple accounts on a single chart.
 
     Graphically composed of a level line and marker as well as labels
@@ -522,8 +497,7 @@ class PositionTracker:
 
     @property
     def pane(self) -> FieldsForm:
-        '''
+        '''Return handle to pp side pane form.
-        Return handle to pp side pane form.
 
         '''
         return self.chart.linked.godwidget.pp_pane
@@ -533,8 +507,7 @@ class PositionTracker:
         marker: LevelMarker
 
     ) -> None:
-        '''
+        '''Update all labels.
-        Update all labels.
 
         Meant to be called from the maker ``.paint()``
         for immediate, lag free label draws.

piker/ui/_search.py

@@ -49,6 +49,7 @@ from PyQt5 import QtCore
 from PyQt5 import QtWidgets
 from PyQt5.QtCore import (
     Qt,
+    # QSize,
     QModelIndex,
     QItemSelectionModel,
 )
@@ -125,10 +126,6 @@ class CompleterView(QTreeView):
         # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored)
 
         # ux settings
-        self.setSizePolicy(
-            QtWidgets.QSizePolicy.Expanding,
-            QtWidgets.QSizePolicy.Expanding,
-        )
         self.setItemsExpandable(True)
         self.setExpandsOnDoubleClick(False)
         self.setAnimated(False)
@@ -156,58 +153,23 @@ class CompleterView(QTreeView):
 
         self.setStyleSheet(f"font: {size}px")
 
-    # def resizeEvent(self, event: 'QEvent') -> None:
+    def resize(self):
-    #     event.accept()
-    #     super().resizeEvent(event)
-
-    def on_resize(self) -> None:
-        '''
-        Resize relay event from god.
-
-        '''
-        self.resize_to_results()
-
-    def resize_to_results(self):
         model = self.model()
         cols = model.columnCount()
-        # rows = model.rowCount()
 
-        col_w_tot = 0
         for i in range(cols):
             self.resizeColumnToContents(i)
-            col_w_tot += self.columnWidth(i)
 
-        win = self.window()
+        # inclusive of search bar and header "rows" in pixel terms
-        win_h = win.height()
+        rows = 100
-        edit_h = self.parent().bar.height()
+        # max_rows = 8  # 6 + search and headers
-        sb_h = win.statusBar().height()
+        row_px = self.rowHeight(self.currentIndex())
+        # print(f'font_h: {font_h}\n px_height: {px_height}')
 
         # TODO: probably make this more general / less hacky
-        # we should figure out the exact number of rows to allow
+        self.setMinimumSize(self.width(), rows * row_px)
-        # inclusive of search bar and header "rows", in pixel terms.
+        self.setMaximumSize(self.width() + 10, rows * row_px)
-        # Eventually when we have an "info" widget below the results we
+        self.setFixedWidth(333)
-        # will want space for it and likely terminating the results-view
-        # space **exactly on a row** would be ideal.
-        # if row_px > 0:
-        #     rows = ceil(window_h / row_px) - 4
-        # else:
-        #     rows = 16
-        # self.setFixedHeight(rows * row_px)
-        # self.resize(self.width(), rows * row_px)
-
-        # NOTE: if the heigh set here is **too large** then the resize
-        # event will perpetually trigger as the window causes some kind
-        # of recompute of callbacks.. so we have to ensure it's limited.
-        h = win_h - (edit_h + 1.666*sb_h)
-        assert h > 0
-        self.setFixedHeight(round(h))
-
-        # size to width of longest result seen thus far
-        # TODO: should we always dynamically scale to longest result?
-        if self.width() < col_w_tot:
-            self.setFixedWidth(col_w_tot)
-
-        self.update()
 
     def is_selecting_d1(self) -> bool:
         cidx = self.selectionModel().currentIndex()
@@ -256,8 +218,7 @@ class CompleterView(QTreeView):
         idx: QModelIndex,
 
     ) -> QStandardItem:
-        '''
+        '''Select and return the item at index ``idx``.
-        Select and return the item at index ``idx``.
 
         '''
         sel = self.selectionModel()
@@ -272,8 +233,7 @@ class CompleterView(QTreeView):
         return model.itemFromIndex(idx)
 
     def select_first(self) -> QStandardItem:
-        '''
+        '''Select the first depth >= 2 entry from the completer tree and
-        Select the first depth >= 2 entry from the completer tree and
         return it's item.
 
         '''
@@ -336,8 +296,7 @@ class CompleterView(QTreeView):
         section: str,
 
     ) -> Optional[QModelIndex]:
-        '''
+        '''Find the *first* depth = 1 section matching ``section`` in
-        Find the *first* depth = 1 section matching ``section`` in
         the tree and return its index.
 
         '''
@@ -375,7 +334,7 @@ class CompleterView(QTreeView):
             else:
                 model.setItem(idx.row(), 1, QStandardItem())
 
-            self.resize_to_results()
+            self.resize()
 
             return idx
         else:
@@ -446,7 +405,7 @@ class CompleterView(QTreeView):
 
     def show_matches(self) -> None:
         self.show()
-        self.resize_to_results()
+        self.resize()
 
 
 class SearchBar(Edit):
@@ -466,7 +425,6 @@ class SearchBar(Edit):
         self.godwidget = godwidget
         super().__init__(parent, **kwargs)
         self.view: CompleterView = view
-        godwidget._widgets[view.mode_name] = view
 
     def show(self) -> None:
         super().show()
@@ -501,7 +459,7 @@ class SearchWidget(QtWidgets.QWidget):
         # size it as we specify
         self.setSizePolicy(
             QtWidgets.QSizePolicy.Fixed,
-            QtWidgets.QSizePolicy.Expanding,
+            QtWidgets.QSizePolicy.Fixed,
         )
 
         self.godwidget = godwidget

piker/ui/_style.py

@@ -14,16 +14,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-'''
+"""
 Qt UI styling.
-
+"""
-'''
 from typing import Optional, Dict
 import math
 
 import pyqtgraph as pg
 from PyQt5 import QtCore, QtGui
-from PyQt5.QtCore import Qt, QCoreApplication
 from qdarkstyle import DarkPalette
 
 from ..log import get_logger
@@ -122,10 +120,6 @@ class DpiAwareFont:
 
         dpi = mn_dpi
 
-        mult = 1.0
-
-        # No implicit DPI scaling was done by the DE so let's engage
-        # some hackery ad-hoc scaling shiat.
         # dpi is likely somewhat scaled down so use slightly larger font size
         if scale >= 1.1 and self._font_size:
 
@@ -140,16 +134,6 @@ class DpiAwareFont:
             # relative aspect ratios or something?
             inches *= mult
 
-        # XXX: if additionally we detect a known DE scaling factor we
-        # also scale *up* our font size on top of the existing
-        # heuristical (aka no clue why it works) scaling from the block
-        # above XD
-        if (
-            hasattr(Qt, 'AA_EnableHighDpiScaling')
-            and QCoreApplication.testAttribute(Qt.AA_EnableHighDpiScaling)
-        ):
-            inches *= round(scale)
-
         # TODO: we might want to fiddle with incrementing font size by
         # +1 for the edge cases above. it seems doing it via scaling is
         # always going to hit that error in range mapping from inches:
@@ -157,8 +141,8 @@ class DpiAwareFont:
         self._font_inches = inches
         font_size = math.floor(inches * dpi)
 
-        log.debug(
+        log.info(
-            f"screen:{screen.name()}\n"
+            f"screen:{screen.name()}]\n"
             f"pDPI: {pdpi}, lDPI: {ldpi}, scale: {scale}\n"
             f"\nOur best guess font size is {font_size}\n"
         )
@@ -203,6 +187,8 @@ _xaxis_at = 'bottom'
 # charting config
 CHART_MARGINS = (0, 0, 2, 2)
 _min_points_to_show = 6
+_bars_to_left_in_follow_mode = int(61*6)
+_bars_from_right_in_follow_mode = round(0.16 * _bars_to_left_in_follow_mode)
 _tina_mode = False
 
 

piker/ui/_window.py

@@ -153,7 +153,8 @@ class MainWindow(QtGui.QMainWindow):
     # XXX: for tiling wms this should scale
     # with the alloted window size.
     # TODO: detect for tiling and if untrue set some size?
-    size = (300, 500)
+    # size = (300, 500)
+    size = (0, 0)
 
     title = 'piker chart (ur symbol is loading bby)'
 
@@ -164,7 +165,6 @@ class MainWindow(QtGui.QMainWindow):
 
         self._status_bar: QStatusBar = None
         self._status_label: QLabel = None
-        self._size: Optional[tuple[int, int]] = None
 
     @property
     def mode_label(self) -> QtGui.QLabel:
@@ -269,29 +269,6 @@ class MainWindow(QtGui.QMainWindow):
         assert screen, "Wow Qt is dumb as shit and has no screen..."
         return screen
 
-    def configure_to_desktop(
-        self,
-        size: Optional[tuple[int, int]] = None,
-
-    ) -> None:
-        '''
-        Explicitly size the window dimensions (for stacked window
-        managers).
-
-        For tina systems (like windoze) try to do a sane window size on
-        startup.
-
-        '''
-        # https://stackoverflow.com/a/18975846
-        if not size and not self._size:
-            app = QtGui.QApplication.instance()
-            geo = self.current_screen().geometry()
-            h, w = geo.height(), geo.width()
-            # use approx 1/3 of the area of the screen by default
-            self._size = round(w * .666), round(h * .666)
-
-        self.resize(*size or self._size)
-
 
 # singleton app per actor
 _qt_win: QtGui.QMainWindow = None

piker/ui/cli.py

@@ -122,8 +122,7 @@ def optschain(config, symbol, date, rate, test):
 @cli.command()
 @click.option(
     '--profile',
-    '-p',
+    is_flag=True,
-    default=None,
     help='Enable pyqtgraph profiling'
 )
 @click.option(
@@ -134,16 +133,9 @@ def optschain(config, symbol, date, rate, test):
 @click.argument('symbol', required=True)
 @click.pass_obj
 def chart(config, symbol, profile, pdb):
-    '''
+    """Start a real-time chartng UI
-    Start a real-time chartng UI
+    """
-
-    '''
-    # eg. ``--profile 3`` reports profiling for anything slower then 3 ms.
-    if profile is not None:
     from .. import _profile
-        _profile._pg_profile = True
-        _profile.ms_slower_then = float(profile)
-
     from ._app import _main
 
     if '.' not in symbol:
@@ -153,6 +145,8 @@ def chart(config, symbol, profile, pdb):
         ))
         return
 
+    # toggle to enable profiling
+    _profile._pg_profile = profile
 
     # global opts
     brokernames = config['brokers']

piker/ui/order_mode.py

@@ -22,7 +22,6 @@ from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
 from functools import partial
 from pprint import pformat
-import platform
 import time
 from typing import Optional, Dict, Callable, Any
 import uuid
@@ -30,7 +29,6 @@ import uuid
 from pydantic import BaseModel
 import tractor
 import trio
-from PyQt5.QtCore import Qt
 
 from .. import config
 from ..clearing._client import open_ems, OrderBook
@@ -38,7 +36,6 @@ from ..clearing._allocate import (
     mk_allocator,
     Position,
 )
-from ._style import _font
 from ..data._source import Symbol
 from ..data.feed import Feed
 from ..log import get_logger
@@ -48,8 +45,7 @@ from ._position import (
     PositionTracker,
     SettingsPane,
 )
-from ._forms import FieldsForm
+from ._label import FormatLabel
-# from ._label import FormatLabel
 from ._window import MultiStatus
 from ..clearing._messages import Order, BrokerdPosition
 from ._forms import open_form_input_handling
@@ -110,8 +106,7 @@ def on_level_change_update_next_order_info(
 
 @dataclass
 class OrderMode:
-    '''
+    '''Major UX mode for placing orders on a chart view providing so
-    Major UX mode for placing orders on a chart view providing so
     called, "chart trading".
 
     This is the other "main" mode that pairs with "view mode" (when
@@ -271,14 +266,13 @@ class OrderMode:
 
         '''
         staged = self._staged_order
-        symbol: Symbol = staged.symbol
+        symbol = staged.symbol
         oid = str(uuid.uuid4())
 
         # format order data for ems
-        fqsn = symbol.front_fqsn()
         order = staged.copy(
             update={
-                'symbol': fqsn,
+                'symbol': symbol.key,
                 'oid': oid,
             }
         )
@@ -435,19 +429,13 @@ class OrderMode:
 
         # TODO: make this not trash.
         # XXX: linux only for now
-        if platform.system() == "Windows":
-            return
-
         result = await trio.run_process(
             [
                 'notify-send',
                 '-u', 'normal',
-                '-t', '1616',
+                '-t', '10000',
                 'piker',
-
+                f'alert: {msg}',
-                # TODO: add in standard fill/exec info that maybe we
-                # pack in a broker independent way?
-                f'{msg["resp"]}: {msg["trigger_price"]}',
             ],
         )
         log.runtime(result)
@@ -523,7 +511,8 @@ async def open_order_mode(
 
     feed: Feed,
     chart: 'ChartPlotWidget',  # noqa
-    fqsn: str,
+    symbol: Symbol,
+    brokername: str,
     started: trio.Event,
 
 ) -> None:
@@ -549,7 +538,8 @@ async def open_order_mode(
 
     # spawn EMS actor-service
     async with (
-        open_ems(fqsn) as (
+
+        open_ems(brokername, symbol) as (
             book,
             trades_stream,
             position_msgs,
@@ -558,7 +548,8 @@ async def open_order_mode(
         trio.open_nursery() as tn,
 
     ):
-        log.info(f'Opening order mode for {fqsn}')
+        log.info(f'Opening order mode for {brokername}.{symbol.key}')
+
         view = chart.view
 
         # annotations editors
@@ -567,7 +558,7 @@ async def open_order_mode(
 
         # symbol id
         symbol = chart.linked.symbol
-        symkey = symbol.front_fqsn()
+        symkey = symbol.key
 
         # map of per-provider account keys to position tracker instances
         trackers: dict[str, PositionTracker] = {}
@@ -611,7 +602,7 @@ async def open_order_mode(
                 log.info(f'Loading pp for {symkey}:\n{pformat(msg)}')
                 startup_pp.update_from_msg(msg)
 
-            # allocator config
+            # allocator
             alloc = mk_allocator(
                 symbol=symbol,
                 account=account_name,
@@ -642,21 +633,61 @@ async def open_order_mode(
                 pp_tracker.hide_info()
 
         # setup order mode sidepane widgets
-        form: FieldsForm = chart.sidepane
+        form = chart.sidepane
-        form.vbox.setSpacing(
+        vbox = form.vbox
-            int((1 + 5/8)*_font.px_size)
+
+        from textwrap import dedent
+
+        from PyQt5.QtCore import Qt
+
+        from ._style import _font, _font_small
+        from ..calc import humanize
+
+        feed_label = FormatLabel(
+            fmt_str=dedent("""
+            actor: **{actor_name}**\n
+            |_ @**{host}:{port}**\n
+            |_ throttle_hz: **{throttle_rate}**\n
+            |_ streams: **{symbols}**\n
+            |_ shm: **{shm}**\n
+            """),
+            font=_font.font,
+            font_size=_font_small.px_size,
+            font_color='default_lightest',
         )
 
-        from ._feedstatus import mk_feed_label
-
-        feed_label = mk_feed_label(
-            form,
-            feed,
-            chart,
-        )
-
-        # XXX: we set this because?
         form.feed_label = feed_label
+
+        # add feed info label to top
+        vbox.insertWidget(
+            0,
+            feed_label,
+            alignment=Qt.AlignBottom,
+        )
+        # vbox.setAlignment(feed_label, Qt.AlignBottom)
+        # vbox.setAlignment(Qt.AlignBottom)
+        blank_h = chart.height() - (
+            form.height() +
+            form.fill_bar.height()
+            # feed_label.height()
+        )
+        vbox.setSpacing((1 + 5/8)*_font.px_size)
+
+        # fill in brokerd feed info
+        host, port = feed.portal.channel.raddr
+        if host == '127.0.0.1':
+            host = 'localhost'
+        mpshm = feed.shm._shm
+        shmstr = f'{humanize(mpshm.size)}'
+        form.feed_label.format(
+            actor_name=feed.portal.channel.uid[0],
+            host=host,
+            port=port,
+            symbols=len(feed.symbols),
+            shm=shmstr,
+            throttle_rate=feed.throttle_rate,
+        )
+
         order_pane = SettingsPane(
             form=form,
             # XXX: ugh, so hideous...
@@ -667,11 +698,6 @@ async def open_order_mode(
         )
         order_pane.set_accounts(list(trackers.keys()))
 
-        form.vbox.addWidget(
-            feed_label,
-            alignment=Qt.AlignBottom,
-        )
-
         # update pp icons
         for name, tracker in trackers.items():
             order_pane.update_account_icons({name: tracker.live_pp})
@@ -782,18 +808,8 @@ async def process_trades_and_update_ui(
             'position',
         ):
             sym = mode.chart.linked.symbol
-            pp_msg_symbol = msg['symbol'].lower()
+            if msg['symbol'].lower() in sym.key:
-            fqsn = sym.front_fqsn()
-            broker, key = sym.front_feed()
-            # print(
-            #     f'pp msg symbol: {pp_msg_symbol}\n',
-            #     f'fqsn: {fqsn}\n',
-            #     f'front key: {key}\n',
-            # )
 
-            if (
-                pp_msg_symbol == fqsn.replace(f'.{broker}', '')
-            ):
                 tracker = mode.trackers[msg['account']]
                 tracker.live_pp.update_from_msg(msg)
                 # update order pane widgets
@@ -873,9 +889,7 @@ async def process_trades_and_update_ui(
                 mode.lines.remove_line(uuid=oid)
 
         # each clearing tick is responded individually
-        elif resp in (
+        elif resp in ('broker_filled',):
-            'broker_filled',
-        ):
 
             known_order = book._sent_orders.get(oid)
             if not known_order:

requirements.txt

@@ -1,21 +1,9 @@
 # we require a pinned dev branch to get some edge features that
 # are often untested in tractor's CI and/or being tested by us
 # first before committing as core features in tractor's base.
--e git+https://github.com/goodboy/tractor.git@master#egg=tractor
+-e git+git://github.com/goodboy/tractor.git@master#egg=tractor
 
 # `pyqtgraph` peeps keep breaking, fixing, improving so might as well
 # pin this to a dev branch that we have more control over especially
 # as more graphics stuff gets hashed out.
--e git+https://github.com/pikers/pyqtgraph.git@piker_pin#egg=pyqtgraph
+-e git+git://github.com/pikers/pyqtgraph.git@piker_pin#egg=pyqtgraph
-
-
-# our async client for ``marketstore`` (the tsdb)
--e git+https://github.com/pikers/anyio-marketstore.git@master#egg=anyio-marketstore
-
-
-# ``trimeter`` for asysnc history fetching
--e git+https://github.com/python-trio/trimeter.git@master#egg=trimeter
-
-
-# ``asyncvnc`` for sending interactions to ib-gw inside docker
--e git+https://github.com/pikers/asyncvnc.git@vid_passthrough#egg=asyncvnc

setup.py

@@ -51,56 +51,42 @@ setup(
         # async
         'trio',
         'trio-websocket',
-        'msgspec',  # performant IPC messaging
+        # 'tractor',  # from github currently
         'async_generator',
 
-        # from github currently (see requirements.txt)
-        # 'trimeter',  # not released yet..
-        # 'tractor',
-        # asyncvnc,
-
         # brokers
         'asks==2.4.8',
         'ib_insync',
 
         # numerics
-        'pendulum', # easier datetimes
+        'arrow',  # better datetimes
         'bidict',  # 2 way map
         'cython',
         'numpy',
         'numba',
+        'pandas',
+        'msgpack-numpy',
 
         # UI
         'PyQt5',
-        # 'pyqtgraph',  from our fork see reqs.txt
+        'pyqtgraph',
-        'qdarkstyle >= 3.0.2',  # themeing
+        'qdarkstyle >= 3.0.2',
-        'fuzzywuzzy[speedup]',  # fuzzy search
+        # fuzzy search
+        'fuzzywuzzy[speedup]',
 
         # tsdbs
-        # anyio-marketstore  # from gh see reqs.txt
+        'pymarketstore',
     ],
-    extras_require={
-        'tsdb': [
-            'docker',
-        ],
-
-    },
     tests_require=['pytest'],
-    python_requires=">=3.10",
+    python_requires=">=3.9",  # literally for ``datetime.datetime.fromisoformat``...
-    keywords=[
+    keywords=["async", "trading", "finance", "quant", "charting"],
-        "async",
-        "trading",
-        "finance",
-        "quant",
-        "charting",
-    ],
     classifiers=[
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: ',
         'Operating System :: POSIX :: Linux',
         "Programming Language :: Python :: Implementation :: CPython",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.9",
         'Intended Audience :: Financial and Insurance Industry',
         'Intended Audience :: Science/Research',
         'Intended Audience :: Developers',

snippets/ib_data_reset.py

@@ -1,5 +1,5 @@
 # piker: trading gear for hackers
-# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+# Copyright (C) Tyler Goodlet (in stewardship for piker0)
 
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -30,13 +30,11 @@ orig_win_id = t.find_focused().window
 # for tws
 win_names: list[str] = [
     'Interactive Brokers',  # tws running in i3
-    'IB Gateway',  # gw running in i3
+    'IB Gateway.',  # gw running in i3
-    # 'IB',  # gw running in i3 (newer version?)
 ]
 
 for name in win_names:
-    results = t.find_titled(name)
+    results = t.find_named(name)
-    print(f'results for {name}: {results}')
     if results:
         con = results[0]
         print(f'Resetting data feed for {name}')
@@ -49,15 +47,6 @@ for name in win_names:
         # https://github.com/rr-/pyxdotool
         # https://github.com/ShaneHutter/pyxdotool
         # https://github.com/cphyc/pyxdotool
-
-        # TODO: only run the reconnect (2nd) kc on a detected
-        # disconnect?
-        for key_combo, timeout in [
-            # only required if we need a connection reset.
-            # ('ctrl+alt+r', 12),
-            # data feed reset.
-            ('ctrl+alt+f', 6)
-        ]:
         subprocess.call([
             'xdotool',
             'windowactivate', '--sync', win_id,
@@ -67,13 +56,12 @@ for name in win_names:
             'mousemove_relative', '--sync', str(w-4), str(h-4),
 
             # NOTE: we may need to stick a `--retry 3` in here..
-                'click', '--window', win_id,
+            'click', '--window', win_id, '--repeat', '3', '1',
-                '--repeat', '3', '1',
 
             # hackzorzes
-                'key', key_combo,
+            'key', 'ctrl+alt+f',
             ],
-                timeout=timeout,
+            timeout=1,
         )
 
 # re-activate and focus original window