Compare commits
No commits in common. "basic_buy_bot" and "310_plus" have entirely different histories.
basic_buy_
...
310_plus
|
|
@ -3,9 +3,10 @@ name: CI
|
|||
|
||||
on:
|
||||
# Triggers the workflow on push or pull request events but only for the master branch
|
||||
pull_request:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
|
@ -13,49 +14,19 @@ on:
|
|||
|
||||
jobs:
|
||||
|
||||
# test that we can generate a software distribution and install it
|
||||
# thus avoid missing file issues after packaging.
|
||||
sdist-linux:
|
||||
name: 'sdist'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Build sdist
|
||||
run: python setup.py sdist --formats=zip
|
||||
|
||||
- name: Install sdist from .zips
|
||||
run: python -m pip install dist/*.zip
|
||||
|
||||
testing:
|
||||
name: 'install + test-suite'
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
# elastic only
|
||||
# - name: Build DB container
|
||||
# run: docker build -t piker:elastic dockering/elastic
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
# elastic only
|
||||
# - name: Install dependencies
|
||||
# run: pip install -U .[es] -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -U . -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
|
||||
|
|
|
|||
128
README.rst
128
README.rst
|
|
@ -13,14 +13,13 @@ computational trading targeted at `hardcore Linux users <comp_trader>`_ .
|
|||
we use as much bleeding edge tech as possible including (but not limited to):
|
||||
|
||||
- latest python for glue_
|
||||
- trio_ & tractor_ for our distributed, multi-core, real-time streaming
|
||||
`structured concurrency`_ runtime B)
|
||||
- trio_ for `structured concurrency`_
|
||||
- tractor_ for distributed, multi-core, real-time streaming
|
||||
- marketstore_ for historical and real-time tick data persistence and sharing
|
||||
- techtonicdb_ for L2 book storage
|
||||
- Qt_ for pristine high performance UIs
|
||||
- pyqtgraph_ for real-time charting
|
||||
- ``polars`` ``numpy`` and ``numba`` for `fast numerics`_
|
||||
- `apache arrow and parquet`_ for time series history management
|
||||
persistence and sharing
|
||||
- (prototyped) techtonicdb_ for L2 book storage
|
||||
- ``numpy`` and ``numba`` for `fast numerics`_
|
||||
|
||||
.. |travis| image:: https://img.shields.io/travis/pikers/piker/master.svg
|
||||
:target: https://travis-ci.org/pikers/piker
|
||||
|
|
@ -32,7 +31,6 @@ we use as much bleeding edge tech as possible including (but not limited to):
|
|||
.. _Qt: https://www.qt.io/
|
||||
.. _pyqtgraph: https://github.com/pyqtgraph/pyqtgraph
|
||||
.. _glue: https://numpy.org/doc/stable/user/c-info.python-as-glue.html#using-python-as-glue
|
||||
.. _apache arrow and parquet: https://arrow.apache.org/faq/
|
||||
.. _fast numerics: https://zerowithdot.com/python-numpy-and-pandas-performance/
|
||||
.. _comp_trader: https://jfaleiro.wordpress.com/2019/10/09/computational-trader/
|
||||
|
||||
|
|
@ -60,33 +58,8 @@ building the best looking, most reliable, keyboard friendly trading
|
|||
platform is the dream; join the cause.
|
||||
|
||||
|
||||
sane install with `poetry`
|
||||
**************************
|
||||
TODO!
|
||||
|
||||
|
||||
rigorous install on ``nixos`` using ``poetry2nix``
|
||||
**************************************************
|
||||
TODO!
|
||||
|
||||
|
||||
hacky install on nixos
|
||||
**********************
|
||||
`NixOS` is our core devs' distro of choice for which we offer
|
||||
a stringently defined development shell envoirment that can be loaded with::
|
||||
|
||||
nix-shell develop.nix
|
||||
|
||||
this will setup the required python environment to run piker, make sure to
|
||||
run::
|
||||
|
||||
pip install -r requirements.txt -e .
|
||||
|
||||
once after loading the shell
|
||||
|
||||
|
||||
install wild-west style via `pip`
|
||||
*********************************
|
||||
install
|
||||
*******
|
||||
``piker`` is currently under heavy pre-alpha development and as such
|
||||
should be cloned from this repo and hacked on directly.
|
||||
|
||||
|
|
@ -99,6 +72,93 @@ for a development install::
|
|||
pip install -r requirements.txt -e .
|
||||
|
||||
|
||||
install for tinas
|
||||
*****************
|
||||
for windows peeps you can start by installing all the prerequisite software:
|
||||
|
||||
- install git with all default settings - https://git-scm.com/download/win
|
||||
- install anaconda all default settings - https://www.anaconda.com/products/individual
|
||||
- install microsoft build tools (check the box for Desktop development for C++, you might be able to uncheck some optional downloads) - https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
||||
- install visual studio code default settings - https://code.visualstudio.com/download
|
||||
|
||||
|
||||
then, `crack a conda shell`_ and run the following commands::
|
||||
|
||||
mkdir code # create code directory
|
||||
cd code # change directory to code
|
||||
git clone https://github.com/pikers/piker.git # downloads piker installation package from github
|
||||
cd piker # change directory to piker
|
||||
|
||||
conda create -n pikonda # creates conda environment named pikonda
|
||||
conda activate pikonda # activates pikonda
|
||||
|
||||
conda install -c conda-forge python-levenshtein # in case it is not already installed
|
||||
conda install pip # may already be installed
|
||||
pip # will show if pip is installed
|
||||
|
||||
pip install -e . -r requirements.txt # install piker in editable mode
|
||||
|
||||
test Piker to see if it is working::
|
||||
|
||||
piker -b binance chart btcusdt.binance # formatting for loading a chart
|
||||
piker -b kraken -b binance chart xbtusdt.kraken
|
||||
piker -b kraken -b binance -b ib chart qqq.nasdaq.ib
|
||||
piker -b ib chart tsla.nasdaq.ib
|
||||
|
||||
potential error::
|
||||
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\user\\AppData\\Roaming\\piker\\brokers.toml'
|
||||
|
||||
solution:
|
||||
|
||||
- navigate to file directory above (may be different on your machine, location should be listed in the error code)
|
||||
- copy and paste file from 'C:\\Users\\user\\code\\data/brokers.toml' or create a blank file using notepad at the location above
|
||||
|
||||
Visual Studio Code setup:
|
||||
|
||||
- now that piker is installed we can set up vscode as the default terminal for running piker and editing the code
|
||||
- open Visual Studio Code
|
||||
- file --> Add Folder to Workspace --> C:\Users\user\code\piker (adds piker directory where all piker files are located)
|
||||
- file --> Save Workspace As --> save it wherever you want and call it whatever you want, this is going to be your default workspace for running and editing piker code
|
||||
- ctrl + shift + p --> start typing Python: Select Interpetter --> when the option comes up select it --> Select at the workspace level --> select the one that shows ('pikonda')
|
||||
- change the default terminal to cmd.exe instead of powershell (default)
|
||||
- now when you create a new terminal VScode should automatically activate you conda env so that piker can be run as the first command after a new terminal is created
|
||||
|
||||
also, try out fancyzones as part of powertoyz for a decent tiling windows manager to manage all the cool new software you are going to be running.
|
||||
|
||||
.. _conda installed: https://
|
||||
.. _C++ build toolz: https://
|
||||
.. _crack a conda shell: https://
|
||||
.. _vscode: https://
|
||||
|
||||
.. link to the tina guide
|
||||
.. _setup a coolio tiled wm console: https://
|
||||
|
||||
provider support
|
||||
****************
|
||||
for live data feeds the in-progress set of supported brokers is:
|
||||
|
||||
- IB_ via ``ib_insync``, also see our `container docs`_
|
||||
- binance_ and kraken_ for crypto over their public websocket API
|
||||
- questrade_ (ish) which comes with effectively free L1
|
||||
|
||||
coming soon...
|
||||
|
||||
- webull_ via the reverse engineered public API
|
||||
- yahoo via yliveticker_
|
||||
|
||||
if you want your broker supported and they have an API let us know.
|
||||
|
||||
.. _IB: https://interactivebrokers.github.io/tws-api/index.html
|
||||
.. _container docs: https://github.com/pikers/piker/tree/master/dockering/ib
|
||||
.. _questrade: https://www.questrade.com/api/documentation
|
||||
.. _kraken: https://www.kraken.com/features/api#public-market-data
|
||||
.. _binance: https://github.com/pikers/piker/pull/182
|
||||
.. _webull: https://github.com/tedchou12/webull
|
||||
.. _yliveticker: https://github.com/yahoofinancelive/yliveticker
|
||||
.. _coinbase: https://docs.pro.coinbase.com/#websocket-feed
|
||||
|
||||
|
||||
check out our charts
|
||||
********************
|
||||
bet you weren't expecting this from the foss::
|
||||
|
|
|
|||
|
|
@ -1,50 +1,19 @@
|
|||
################
|
||||
# ---- CEXY ----
|
||||
################
|
||||
[binance]
|
||||
accounts.usdtm = 'futes'
|
||||
futes.use_testnet = true
|
||||
futes.api_key = ''
|
||||
futes.api_secret = ''
|
||||
|
||||
accounts.spot = 'spot'
|
||||
spot.use_testnet = true
|
||||
spot.api_key = ''
|
||||
spot.api_secret = ''
|
||||
|
||||
|
||||
[deribit]
|
||||
key_id = ''
|
||||
key_secret = ''
|
||||
|
||||
|
||||
[kraken]
|
||||
key_descr = ''
|
||||
api_key = ''
|
||||
secret = ''
|
||||
|
||||
|
||||
[kucoin]
|
||||
key_id = ''
|
||||
key_secret = ''
|
||||
key_passphrase = ''
|
||||
|
||||
|
||||
################
|
||||
# -- BROKERZ ---
|
||||
################
|
||||
[questrade]
|
||||
refresh_token = ''
|
||||
access_token = ''
|
||||
api_server = 'https://api06.iq.questrade.com/'
|
||||
refresh_token = ""
|
||||
access_token = ""
|
||||
api_server = "https://api06.iq.questrade.com/"
|
||||
expires_in = 1800
|
||||
token_type = 'Bearer'
|
||||
token_type = "Bearer"
|
||||
expires_at = 1616095326.355846
|
||||
|
||||
[kraken]
|
||||
key_descr = "api_0"
|
||||
api_key = ""
|
||||
secret = ""
|
||||
|
||||
[ib]
|
||||
hosts = [
|
||||
'127.0.0.1',
|
||||
"127.0.0.1",
|
||||
]
|
||||
# XXX: the order in which ports will be scanned
|
||||
# (by the `brokerd` daemon-actor)
|
||||
|
|
@ -61,8 +30,8 @@ ports = [
|
|||
# is not supported so you have to manually download
|
||||
# and XML report and put it in a location that can be
|
||||
# accessed by the ``brokerd.ib`` backend code for parsing.
|
||||
flex_token = ''
|
||||
flex_trades_query_id = '' # live account
|
||||
flex_token = '666666666666666666666666'
|
||||
flex_trades_query_id = '666666' # live account
|
||||
|
||||
# when clients are being scanned this determines
|
||||
# which clients are preferred to be used for data
|
||||
|
|
@ -78,6 +47,6 @@ prefer_data_account = [
|
|||
# the order in which accounts will be selectable
|
||||
# in the order mode UI (if found via clients during
|
||||
# API-app scanning)when a new symbol is loaded.
|
||||
paper = 'XX0000000'
|
||||
margin = 'X0000000'
|
||||
ira = 'X0000000'
|
||||
paper = "XX0000000"
|
||||
margin = "X0000000"
|
||||
ira = "X0000000"
|
||||
|
|
|
|||
|
|
@ -1,12 +0,0 @@
|
|||
[network]
|
||||
tsdb.backend = 'marketstore'
|
||||
tsdb.host = 'localhost'
|
||||
tsdb.grpc_port = 5995
|
||||
|
||||
[ui]
|
||||
# set custom font + size which will scale entire UI
|
||||
# font_size = 16
|
||||
# font_name = 'Monospaced'
|
||||
|
||||
# colorscheme = 'default' # UNUSED
|
||||
# graphics.update_throttle = 60 # Hz # TODO
|
||||
42
develop.nix
42
develop.nix
|
|
@ -1,42 +0,0 @@
|
|||
with (import <nixpkgs> {});
|
||||
with python310Packages;
|
||||
stdenv.mkDerivation {
|
||||
name = "pip-env";
|
||||
buildInputs = [
|
||||
# System requirements.
|
||||
readline
|
||||
|
||||
# TODO: hacky non-poetry install stuff we need to get rid of!!
|
||||
virtualenv
|
||||
setuptools
|
||||
pip
|
||||
|
||||
# obviously, and see below for hacked linking
|
||||
pyqt5
|
||||
|
||||
# Python requirements (enough to get a virtualenv going).
|
||||
python310Full
|
||||
|
||||
# numerics deps
|
||||
python310Packages.python-Levenshtein
|
||||
python310Packages.fastparquet
|
||||
python310Packages.polars
|
||||
|
||||
];
|
||||
src = null;
|
||||
shellHook = ''
|
||||
# Allow the use of wheels.
|
||||
SOURCE_DATE_EPOCH=$(date +%s)
|
||||
|
||||
# Augment the dynamic linker path
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${R}/lib/R/lib:${readline}/lib
|
||||
|
||||
export QT_QPA_PLATFORM_PLUGIN_PATH="${qt5.qtbase.bin}/lib/qt-${qt5.qtbase.version}/plugins";
|
||||
|
||||
if [ ! -d "venv" ]; then
|
||||
virtualenv venv
|
||||
fi
|
||||
|
||||
source venv/bin/activate
|
||||
'';
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
FROM elasticsearch:7.17.4
|
||||
|
||||
ENV ES_JAVA_OPTS "-Xms2g -Xmx2g"
|
||||
ENV ELASTIC_USERNAME "elastic"
|
||||
ENV ELASTIC_PASSWORD "password"
|
||||
|
||||
COPY elasticsearch.yml /usr/share/elasticsearch/config/
|
||||
|
||||
RUN printf "password" | ./bin/elasticsearch-keystore add -f -x "bootstrap.password"
|
||||
|
||||
EXPOSE 19200
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
network.host: 0.0.0.0
|
||||
|
||||
http.port: 19200
|
||||
|
||||
discovery.type: single-node
|
||||
|
|
@ -2,26 +2,12 @@
|
|||
# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
|
||||
version: "3.5"
|
||||
|
||||
|
||||
services:
|
||||
|
||||
ib_gw_paper:
|
||||
|
||||
# apparently java is a mega cukc:
|
||||
# https://stackoverflow.com/a/56895801
|
||||
# https://bugs.openjdk.org/browse/JDK-8150460
|
||||
ulimits:
|
||||
# nproc: 65535
|
||||
nproc: 6000
|
||||
nofile:
|
||||
soft: 2000
|
||||
hard: 3000
|
||||
|
||||
ib-gateway:
|
||||
# other image tags available:
|
||||
# https://github.com/waytrade/ib-gateway-docker#supported-tags
|
||||
# image: waytrade/ib-gateway:981.3j
|
||||
image: waytrade/ib-gateway:1012.2i
|
||||
restart: 'no' # restart on boot whenev there's a crash or user clicsk
|
||||
image: waytrade/ib-gateway:981.3j
|
||||
restart: always
|
||||
network_mode: 'host'
|
||||
|
||||
volumes:
|
||||
|
|
@ -53,12 +39,14 @@ services:
|
|||
# this compose file which looks something like:
|
||||
# TWS_USERID='myuser'
|
||||
# TWS_PASSWORD='guest'
|
||||
# TRADING_MODE=paper (or live)
|
||||
# VNC_SERVER_PASSWORD='diggity'
|
||||
|
||||
environment:
|
||||
TWS_USERID: ${TWS_USERID}
|
||||
TWS_PASSWORD: ${TWS_PASSWORD}
|
||||
TRADING_MODE: 'paper'
|
||||
VNC_SERVER_PASSWORD: 'doggy'
|
||||
VNC_SERVER_PORT: '3003'
|
||||
TRADING_MODE: ${TRADING_MODE:-paper}
|
||||
VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
|
||||
|
||||
# ports:
|
||||
# - target: 4002
|
||||
|
|
@ -74,40 +62,3 @@ services:
|
|||
# - "127.0.0.1:4001:4001"
|
||||
# - "127.0.0.1:4002:4002"
|
||||
# - "127.0.0.1:5900:5900"
|
||||
|
||||
# ib_gw_live:
|
||||
# image: waytrade/ib-gateway:1012.2i
|
||||
# restart: no
|
||||
# network_mode: 'host'
|
||||
|
||||
# volumes:
|
||||
# - type: bind
|
||||
# source: ./jts_live.ini
|
||||
# target: /root/jts/jts.ini
|
||||
# # don't let ibc clobber this file for
|
||||
# # the main reason of not having a stupid
|
||||
# # timezone set..
|
||||
# read_only: true
|
||||
|
||||
# # force our own ibc config
|
||||
# - type: bind
|
||||
# source: ./ibc.ini
|
||||
# target: /root/ibc/config.ini
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./fork_ports_delayed.sh
|
||||
# target: /root/scripts/fork_ports_delayed.sh
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./run_x11_vnc.sh
|
||||
# target: /root/scripts/run_x11_vnc.sh
|
||||
# read_only: true
|
||||
|
||||
# # NOTE: to fill these out, define an `.env` file in the same dir as
|
||||
# # this compose file which looks something like:
|
||||
# environment:
|
||||
# TRADING_MODE: 'live'
|
||||
# VNC_SERVER_PASSWORD: 'doggy'
|
||||
# VNC_SERVER_PORT: '3004'
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes
|
|||
#
|
||||
# The default value is 60.
|
||||
|
||||
LoginDialogDisplayTimeout=20
|
||||
LoginDialogDisplayTimeout = 60
|
||||
|
||||
|
||||
|
||||
|
|
@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary
|
|||
# be set dynamically at run-time: most users will never need it,
|
||||
# so don't use it unless you know you need it.
|
||||
|
||||
; OverrideTwsApiPort=4002
|
||||
OverrideTwsApiPort=4002
|
||||
|
||||
|
||||
# Read-only Login
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
[IBGateway]
|
||||
ApiOnly=true
|
||||
LocalServerPort=4001
|
||||
# NOTE: must be set if using IBC's "reject" mode
|
||||
TrustedIPs=127.0.0.1
|
||||
; RemoteHostOrderRouting=ndc1.ibllc.com
|
||||
; WriteDebug=true
|
||||
; RemotePortOrderRouting=4001
|
||||
; useRemoteSettings=false
|
||||
; tradingMode=p
|
||||
; Steps=8
|
||||
; colorPalletName=dark
|
||||
|
||||
# window geo, this may be useful for sending `xdotool` commands?
|
||||
; MainWindow.Width=1986
|
||||
; screenHeight=3960
|
||||
|
||||
|
||||
[Logon]
|
||||
Locale=en
|
||||
# most markets are oriented around this zone
|
||||
# so might as well hard code it.
|
||||
TimeZone=America/New_York
|
||||
UseSSL=true
|
||||
displayedproxymsg=1
|
||||
os_titlebar=true
|
||||
s3store=true
|
||||
useRemoteSettings=false
|
||||
|
||||
[Communication]
|
||||
ctciAutoEncrypt=true
|
||||
Region=usr
|
||||
; Peer=cdc1.ibllc.com:4001
|
||||
|
|
@ -1,35 +1,16 @@
|
|||
#!/bin/sh
|
||||
# start vnc server and listen for connections
|
||||
# on port specced in `$VNC_SERVER_PORT`
|
||||
|
||||
# start VNC server
|
||||
x11vnc \
|
||||
-listen 127.0.0.1 \
|
||||
-allow 127.0.0.1 \
|
||||
-rfbport "${VNC_SERVER_PORT}" \
|
||||
-ncache_cr \
|
||||
-listen localhost \
|
||||
-display :1 \
|
||||
-forever \
|
||||
-shared \
|
||||
-logappend /var/log/x11vnc.log \
|
||||
-bg \
|
||||
-nowf \
|
||||
-noxdamage \
|
||||
-noxfixes \
|
||||
-no6 \
|
||||
-noipv6 \
|
||||
|
||||
|
||||
# -nowcr \
|
||||
# TODO: can't use this because of ``asyncvnc`` issue:
|
||||
-autoport 3003 \
|
||||
# can't use this because of ``asyncvnc`` issue:
|
||||
# https://github.com/barneygale/asyncvnc/issues/1
|
||||
# -passwd 'ibcansmbz'
|
||||
|
||||
# XXX: optional graphics caching flags that seem to rekt the overlay
|
||||
# of the 2 gw windows? When running a single gateway
|
||||
# this seems to maybe optimize some memory usage?
|
||||
# -ncache_cr \
|
||||
# -ncache \
|
||||
|
||||
# NOTE: this will prevent logs from going to the console.
|
||||
# -logappend /var/log/x11vnc.log \
|
||||
|
||||
# where to start allocating ports
|
||||
# -autoport "${VNC_SERVER_PORT}" \
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
### NOTE this is likely out of date given it was written some
|
||||
(years) time ago by a user that has since not really partaken in
|
||||
contributing since.
|
||||
|
||||
install for tinas
|
||||
*****************
|
||||
for windows peeps you can start by installing all the prerequisite software:
|
||||
|
||||
- install git with all default settings - https://git-scm.com/download/win
|
||||
- install anaconda all default settings - https://www.anaconda.com/products/individual
|
||||
- install microsoft build tools (check the box for Desktop development for C++, you might be able to uncheck some optional downloads) - https://visualstudio.microsoft.com/visual-cpp-build-tools/
|
||||
- install visual studio code default settings - https://code.visualstudio.com/download
|
||||
|
||||
|
||||
then, `crack a conda shell`_ and run the following commands::
|
||||
|
||||
mkdir code # create code directory
|
||||
cd code # change directory to code
|
||||
git clone https://github.com/pikers/piker.git # downloads piker installation package from github
|
||||
cd piker # change directory to piker
|
||||
|
||||
conda create -n pikonda # creates conda environment named pikonda
|
||||
conda activate pikonda # activates pikonda
|
||||
|
||||
conda install -c conda-forge python-levenshtein # in case it is not already installed
|
||||
conda install pip # may already be installed
|
||||
pip # will show if pip is installed
|
||||
|
||||
pip install -e . -r requirements.txt # install piker in editable mode
|
||||
|
||||
test Piker to see if it is working::
|
||||
|
||||
piker -b binance chart btcusdt.binance # formatting for loading a chart
|
||||
piker -b kraken -b binance chart xbtusdt.kraken
|
||||
piker -b kraken -b binance -b ib chart qqq.nasdaq.ib
|
||||
piker -b ib chart tsla.nasdaq.ib
|
||||
|
||||
potential error::
|
||||
|
||||
FileNotFoundError: [Errno 2] No such file or directory: 'C:\\Users\\user\\AppData\\Roaming\\piker\\brokers.toml'
|
||||
|
||||
solution:
|
||||
|
||||
- navigate to file directory above (may be different on your machine, location should be listed in the error code)
|
||||
- copy and paste file from 'C:\\Users\\user\\code\\data/brokers.toml' or create a blank file using notepad at the location above
|
||||
|
||||
Visual Studio Code setup:
|
||||
|
||||
- now that piker is installed we can set up vscode as the default terminal for running piker and editing the code
|
||||
- open Visual Studio Code
|
||||
- file --> Add Folder to Workspace --> C:\Users\user\code\piker (adds piker directory where all piker files are located)
|
||||
- file --> Save Workspace As --> save it wherever you want and call it whatever you want, this is going to be your default workspace for running and editing piker code
|
||||
- ctrl + shift + p --> start typing Python: Select Interpetter --> when the option comes up select it --> Select at the workspace level --> select the one that shows ('pikonda')
|
||||
- change the default terminal to cmd.exe instead of powershell (default)
|
||||
- now when you create a new terminal VScode should automatically activate you conda env so that piker can be run as the first command after a new terminal is created
|
||||
|
||||
also, try out fancyzones as part of powertoyz for a decent tiling windows manager to manage all the cool new software you are going to be running.
|
||||
|
||||
.. _conda installed: https://
|
||||
.. _C++ build toolz: https://
|
||||
.. _crack a conda shell: https://
|
||||
.. _vscode: https://
|
||||
|
||||
.. link to the tina guide
|
||||
.. _setup a coolio tiled wm console: https://
|
||||
|
||||
provider support
|
||||
****************
|
||||
for live data feeds the in-progress set of supported brokers is:
|
||||
|
||||
- IB_ via ``ib_insync``, also see our `container docs`_
|
||||
- binance_ and kraken_ for crypto over their public websocket API
|
||||
- questrade_ (ish) which comes with effectively free L1
|
||||
|
||||
coming soon...
|
||||
|
||||
- webull_ via the reverse engineered public API
|
||||
- yahoo via yliveticker_
|
||||
|
||||
if you want your broker supported and they have an API let us know.
|
||||
|
||||
.. _IB: https://interactivebrokers.github.io/tws-api/index.html
|
||||
.. _container docs: https://github.com/pikers/piker/tree/master/dockering/ib
|
||||
.. _questrade: https://www.questrade.com/api/documentation
|
||||
.. _kraken: https://www.kraken.com/features/api#public-market-data
|
||||
.. _binance: https://github.com/pikers/piker/pull/182
|
||||
.. _webull: https://github.com/tedchou12/webull
|
||||
.. _yliveticker: https://github.com/yahoofinancelive/yliveticker
|
||||
.. _coinbase: https://docs.pro.coinbase.com/#websocket-feed
|
||||
|
||||
|
||||
|
|
@ -1,263 +0,0 @@
|
|||
# from pprint import pformat
|
||||
from functools import partial
|
||||
from decimal import Decimal
|
||||
from typing import Callable
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
from uuid import uuid4
|
||||
|
||||
from piker.service import maybe_open_pikerd
|
||||
from piker.accounting import dec_digits
|
||||
from piker.clearing import (
|
||||
open_ems,
|
||||
OrderClient,
|
||||
)
|
||||
# TODO: we should probably expose these top level in this subsys?
|
||||
from piker.clearing._messages import (
|
||||
Order,
|
||||
Status,
|
||||
BrokerdPosition,
|
||||
)
|
||||
from piker.data import (
|
||||
iterticks,
|
||||
Flume,
|
||||
open_feed,
|
||||
Feed,
|
||||
# ShmArray,
|
||||
)
|
||||
|
||||
|
||||
# TODO: handle other statuses:
|
||||
# - fills, errors, and position tracking
|
||||
async def wait_for_order_status(
|
||||
trades_stream: tractor.MsgStream,
|
||||
oid: str,
|
||||
expect_status: str,
|
||||
|
||||
) -> tuple[
|
||||
list[Status],
|
||||
list[BrokerdPosition],
|
||||
]:
|
||||
'''
|
||||
Wait for a specific order status for a given dialog, return msg flow
|
||||
up to that msg and any position update msgs in a tuple.
|
||||
|
||||
'''
|
||||
# Wait for position message before moving on to verify flow(s)
|
||||
# for the multi-order position entry/exit.
|
||||
status_msgs: list[Status] = []
|
||||
pp_msgs: list[BrokerdPosition] = []
|
||||
|
||||
async for msg in trades_stream:
|
||||
match msg:
|
||||
case {'name': 'position'}:
|
||||
ppmsg = BrokerdPosition(**msg)
|
||||
pp_msgs.append(ppmsg)
|
||||
|
||||
case {
|
||||
'name': 'status',
|
||||
}:
|
||||
msg = Status(**msg)
|
||||
status_msgs.append(msg)
|
||||
|
||||
# if we get the status we expect then return all
|
||||
# collected msgs from the brokerd dialog up to the
|
||||
# exected msg B)
|
||||
if (
|
||||
msg.resp == expect_status
|
||||
and msg.oid == oid
|
||||
):
|
||||
return status_msgs, pp_msgs
|
||||
|
||||
|
||||
async def bot_main():
|
||||
'''
|
||||
Boot the piker runtime, open an ems connection, submit
|
||||
and process orders statuses in real-time.
|
||||
|
||||
'''
|
||||
ll: str = 'info'
|
||||
|
||||
# open an order ctl client, live data feed, trio nursery for
|
||||
# spawning an order trailer task
|
||||
client: OrderClient
|
||||
trades_stream: tractor.MsgStream
|
||||
feed: Feed
|
||||
accounts: list[str]
|
||||
|
||||
fqme: str = 'btcusdt.usdtm.perp.binance'
|
||||
|
||||
async with (
|
||||
|
||||
# TODO: do this implicitly inside `open_ems()` ep below?
|
||||
# init and sync actor-service runtime
|
||||
maybe_open_pikerd(
|
||||
loglevel=ll,
|
||||
debug_mode=True,
|
||||
|
||||
),
|
||||
open_ems(
|
||||
fqme,
|
||||
mode='paper', # {'live', 'paper'}
|
||||
# mode='live', # for real-brokerd submissions
|
||||
loglevel=ll,
|
||||
|
||||
) as (
|
||||
client, # OrderClient
|
||||
trades_stream, # tractor.MsgStream startup_pps,
|
||||
_, # positions
|
||||
accounts,
|
||||
_, # dialogs
|
||||
),
|
||||
|
||||
open_feed(
|
||||
fqmes=[fqme],
|
||||
loglevel=ll,
|
||||
|
||||
# TODO: if you want to throttle via downsampling
|
||||
# how many tick updates your feed received on
|
||||
# quote streams B)
|
||||
# tick_throttle=10,
|
||||
) as feed,
|
||||
|
||||
trio.open_nursery() as tn,
|
||||
):
|
||||
assert accounts
|
||||
print(f'Loaded binance accounts: {accounts}')
|
||||
|
||||
flume: Flume = feed.flumes[fqme]
|
||||
min_tick = Decimal(flume.mkt.price_tick)
|
||||
min_tick_digits: int = dec_digits(min_tick)
|
||||
price_round: Callable = partial(
|
||||
round,
|
||||
ndigits=min_tick_digits,
|
||||
)
|
||||
|
||||
quote_stream: trio.abc.ReceiveChannel = feed.streams['binance']
|
||||
|
||||
|
||||
# always keep live limit 0.003% below last
|
||||
# clearing price
|
||||
clear_margin: float = 0.9997
|
||||
|
||||
async def trailer(
|
||||
order: Order,
|
||||
):
|
||||
# ref shm OHLCV array history, if you want
|
||||
# s_shm: ShmArray = flume.rt_shm
|
||||
# m_shm: ShmArray = flume.hist_shm
|
||||
|
||||
# NOTE: if you wanted to frame ticks by type like the
|
||||
# the quote throttler does.. and this is probably
|
||||
# faster in terms of getting the latest tick type
|
||||
# embedded value of interest?
|
||||
# from piker.data._sampling import frame_ticks
|
||||
|
||||
async for quotes in quote_stream:
|
||||
for fqme, quote in quotes.items():
|
||||
# print(
|
||||
# f'{quote["symbol"]} -> {quote["ticks"]}\n'
|
||||
# f'last 1s OHLC:\n{s_shm.array[-1]}\n'
|
||||
# f'last 1m OHLC:\n{m_shm.array[-1]}\n'
|
||||
# )
|
||||
|
||||
for tick in iterticks(
|
||||
quote,
|
||||
reverse=True,
|
||||
# types=('trade', 'dark_trade'), # defaults
|
||||
):
|
||||
|
||||
await client.update(
|
||||
uuid=order.oid,
|
||||
price=price_round(
|
||||
clear_margin
|
||||
*
|
||||
tick['price']
|
||||
),
|
||||
)
|
||||
msgs, pps = await wait_for_order_status(
|
||||
trades_stream,
|
||||
order.oid,
|
||||
'open'
|
||||
)
|
||||
# if multiple clears per quote just
|
||||
# skip to the next quote?
|
||||
break
|
||||
|
||||
|
||||
# get first live quote to be sure we submit the initial
|
||||
# live buy limit low enough that it doesn't clear due to
|
||||
# a stale initial price from the data feed layer!
|
||||
first_ask_price: float | None = None
|
||||
async for quotes in quote_stream:
|
||||
for fqme, quote in quotes.items():
|
||||
# print(quote['symbol'])
|
||||
for tick in iterticks(quote, types=('ask')):
|
||||
first_ask_price: float = tick['price']
|
||||
break
|
||||
|
||||
if first_ask_price:
|
||||
break
|
||||
|
||||
# setup order dialog via first msg
|
||||
price: float = price_round(
|
||||
clear_margin
|
||||
*
|
||||
first_ask_price,
|
||||
)
|
||||
|
||||
# compute a 1k USD sized pos
|
||||
size: float = round(1e3/price, ndigits=3)
|
||||
|
||||
order = Order(
|
||||
|
||||
# docs on how this all works, bc even i'm not entirely
|
||||
# clear XD. also we probably want to figure out how to
|
||||
# offer both the paper engine running and the brokerd
|
||||
# order ctl tasks with the ems choosing which stream to
|
||||
# route msgs on given the account value!
|
||||
account='paper', # use built-in paper clearing engine and .accounting
|
||||
# account='binance.usdtm', # for live binance futes
|
||||
|
||||
oid=str(uuid4()),
|
||||
exec_mode='live', # {'dark', 'live', 'alert'}
|
||||
|
||||
action='buy', # TODO: remove this from our schema?
|
||||
|
||||
size=size,
|
||||
symbol=fqme,
|
||||
price=price,
|
||||
brokers=['binance'],
|
||||
)
|
||||
await client.send(order)
|
||||
|
||||
msgs, pps = await wait_for_order_status(
|
||||
trades_stream,
|
||||
order.oid,
|
||||
'open',
|
||||
)
|
||||
|
||||
assert not pps
|
||||
assert msgs[-1].oid == order.oid
|
||||
|
||||
# start "trailer task" which tracks rt quote stream
|
||||
tn.start_soon(trailer, order)
|
||||
|
||||
try:
|
||||
# wait for ctl-c from user..
|
||||
await trio.sleep_forever()
|
||||
except KeyboardInterrupt:
|
||||
# cancel the open order
|
||||
await client.cancel(order.oid)
|
||||
|
||||
msgs, pps = await wait_for_order_status(
|
||||
trades_stream,
|
||||
order.oid,
|
||||
'canceled'
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
trio.run(bot_main)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers.
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,14 +14,7 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"""
|
||||
piker: trading gear for hackers.
|
||||
|
||||
'''
|
||||
from .service import open_piker_runtime
|
||||
from .data.feed import open_feed
|
||||
|
||||
__all__ = [
|
||||
'open_piker_runtime',
|
||||
'open_feed',
|
||||
]
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,71 +14,37 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"""
|
||||
Cacheing apis and toolz.
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import (
|
||||
Awaitable,
|
||||
Callable,
|
||||
ParamSpec,
|
||||
TypeVar,
|
||||
from contextlib import (
|
||||
asynccontextmanager,
|
||||
)
|
||||
|
||||
from tractor.trionics import maybe_open_context
|
||||
|
||||
from .brokers import get_brokermod
|
||||
from .log import get_logger
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
P = ParamSpec("P")
|
||||
|
||||
|
||||
# TODO: move this to `tractor.trionics`..
|
||||
# - egs. to replicate for tests: https://github.com/aio-libs/async-lru#usage
|
||||
# - their suite as well:
|
||||
# https://github.com/aio-libs/async-lru/tree/master/tests
|
||||
# - asked trio_util about it too:
|
||||
# https://github.com/groove-x/trio-util/issues/21
|
||||
def async_lifo_cache(
|
||||
maxsize=128,
|
||||
|
||||
# NOTE: typing style was learned from:
|
||||
# https://stackoverflow.com/a/71132186
|
||||
) -> Callable[
|
||||
Callable[P, Awaitable[T]],
|
||||
Callable[
|
||||
Callable[P, Awaitable[T]],
|
||||
Callable[P, Awaitable[T]],
|
||||
],
|
||||
]:
|
||||
'''
|
||||
Async ``cache`` with a LIFO policy.
|
||||
def async_lifo_cache(maxsize=128):
|
||||
"""Async ``cache`` with a LIFO policy.
|
||||
|
||||
Implemented my own since no one else seems to have
|
||||
a standard. I'll wait for the smarter people to come
|
||||
up with one, but until then...
|
||||
|
||||
NOTE: when decorating, due to this simple/naive implementation, you
|
||||
MUST call the decorator like,
|
||||
|
||||
.. code:: python
|
||||
|
||||
@async_lifo_cache()
|
||||
async def cache_target():
|
||||
|
||||
'''
|
||||
"""
|
||||
cache = OrderedDict()
|
||||
|
||||
def decorator(
|
||||
fn: Callable[P, Awaitable[T]],
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
def decorator(fn):
|
||||
|
||||
async def decorated(
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> T:
|
||||
async def wrapper(*args):
|
||||
key = args
|
||||
try:
|
||||
return cache[key]
|
||||
|
|
@ -87,13 +53,27 @@ def async_lifo_cache(
|
|||
# discard last added new entry
|
||||
cache.popitem()
|
||||
|
||||
# call underlying
|
||||
cache[key] = await fn(
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
# do it
|
||||
cache[key] = await fn(*args)
|
||||
return cache[key]
|
||||
|
||||
return decorated
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def open_cached_client(
|
||||
brokername: str,
|
||||
) -> 'Client': # noqa
|
||||
'''
|
||||
Get a cached broker client from the current actor's local vars.
|
||||
|
||||
If one has not been setup do it and cache it.
|
||||
|
||||
'''
|
||||
brokermod = get_brokermod(brokername)
|
||||
async with maybe_open_context(
|
||||
acm_func=brokermod.get_client,
|
||||
) as (cache_hit, client):
|
||||
yield client
|
||||
|
|
|
|||
|
|
@ -0,0 +1,561 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Structured, daemon tree service management.
|
||||
|
||||
"""
|
||||
from typing import Optional, Union, Callable, Any
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from collections import defaultdict
|
||||
|
||||
from pydantic import BaseModel
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from .log import get_logger, get_console_log
|
||||
from .brokers import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
|
||||
_registry_addr = ('127.0.0.1', 6116)
|
||||
_tractor_kwargs: dict[str, Any] = {
|
||||
# use a different registry addr then tractor's default
|
||||
'arbiter_addr': _registry_addr
|
||||
}
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
]
|
||||
|
||||
|
||||
class Services(BaseModel):
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[str, tuple[trio.CancelScope, tractor.Portal]] = {}
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
**kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
async with portal.open_context(
|
||||
target,
|
||||
**kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
task_status.started((cs, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
ctx_res = await ctx.result()
|
||||
except tractor.ContextCancelled:
|
||||
return await self.cancel_service(name)
|
||||
else:
|
||||
# wait on any error from the sub-actor
|
||||
# NOTE: this will block indefinitely until
|
||||
# cancelled either by error from the target
|
||||
# context function or by being cancelled here by
|
||||
# the surrounding cancel scope
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
cs, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal)
|
||||
|
||||
return cs, first
|
||||
|
||||
# TODO: per service cancellation by scope, we aren't using this
|
||||
# anywhere right?
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
) -> Any:
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal = self.service_tasks[name]
|
||||
# XXX: not entirely sure why this is required,
|
||||
# and should probably be better fine tuned in
|
||||
# ``tractor``?
|
||||
cs.cancel()
|
||||
return await portal.cancel_actor()
|
||||
|
||||
|
||||
_services: Optional[Services] = None
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a root piker daemon who's lifetime extends indefinitely
|
||||
until cancelled.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=_root_dname,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
):
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
|
||||
# # setup service mngr singleton instance
|
||||
# async with AsyncExitStack() as stack:
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
_services = Services(
|
||||
actor_n=actor_nursery,
|
||||
service_n=service_nursery,
|
||||
debug_mode=debug_mode,
|
||||
)
|
||||
|
||||
yield _services
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically
|
||||
sync with existing piker actors in local network
|
||||
based on configuration.
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
):
|
||||
yield tractor.current_actor()
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_runtime(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
"""
|
||||
Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
"""
|
||||
settings = _tractor_kwargs
|
||||
settings.update(kwargs)
|
||||
|
||||
if not tractor.current_actor(err_on_no_runtime=False):
|
||||
async with tractor.open_root_actor(
|
||||
loglevel=loglevel,
|
||||
**settings,
|
||||
):
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> Union[tractor._portal.Portal, Services]:
|
||||
"""If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
"""
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
async with maybe_open_runtime(loglevel, **kwargs):
|
||||
|
||||
async with tractor.find_actor(_root_dname) as portal:
|
||||
# assert portal is not None
|
||||
if portal is not None:
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
debug_mode=kwargs.get('debug_mode', False),
|
||||
|
||||
) as _:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
yield None
|
||||
|
||||
|
||||
# brokerd enabled modules
|
||||
_data_mods = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
class Brokerd:
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> Optional[tractor.Portal]:
|
||||
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
spawn_args: dict[str, Any],
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Brokerd.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(f"Couldn't find any existing {service_name}")
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
if pikerd_portal is None:
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
await service_task_target(**spawn_args)
|
||||
|
||||
else:
|
||||
# tell the remote `pikerd` to start the target,
|
||||
# the target can't return a non-serializable value
|
||||
# since it is expected that service startingn is
|
||||
# non-blocking and the target task will persist running
|
||||
# on `pikerd` after the client requesting it's start
|
||||
# disconnects.
|
||||
await pikerd_portal.run(
|
||||
service_task_target,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
dname = f'brokerd.{brokername}'
|
||||
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
modpath = brokermod.__name__
|
||||
broker_enable = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
broker_enable.append(subpath)
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + broker_enable,
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from .data import _setup_persistent_brokerd
|
||||
|
||||
await _services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
_setup_persistent_brokerd,
|
||||
brokername=brokername,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client
|
||||
who wishes to use the sub-actor-daemon.
|
||||
|
||||
'''
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={'brokername': brokername, 'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: Optional[str] = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from .clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await _services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
_setup_persistent_emsd,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
# TODO: ideally we can start the tsdb "on demand" but it's
|
||||
# probably going to require "rootless" docker, at least if we don't
|
||||
# want to expect the user to start ``pikerd`` with root perms all the
|
||||
# time.
|
||||
# async def maybe_open_marketstored(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> tractor._portal.Portal: # noqa
|
||||
|
||||
# async with maybe_spawn_daemon(
|
||||
|
||||
# 'marketstored',
|
||||
# service_task_target=spawn_emsd,
|
||||
# spawn_args={'loglevel': loglevel},
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
|
||||
# ) as portal:
|
||||
# yield portal
|
||||
|
|
@ -18,10 +18,7 @@
|
|||
Profiling wrappers for internal libs.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from time import perf_counter
|
||||
from functools import wraps
|
||||
|
||||
# NOTE: you can pass a flag to enable this:
|
||||
|
|
@ -47,193 +44,3 @@ def timeit(fn):
|
|||
return res
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# Modified version of ``pyqtgraph.debug.Profiler`` that
|
||||
# core seems hesitant to land in:
|
||||
# https://github.com/pyqtgraph/pyqtgraph/pull/2281
|
||||
class Profiler(object):
|
||||
'''
|
||||
Simple profiler allowing measurement of multiple time intervals.
|
||||
|
||||
By default, profilers are disabled. To enable profiling, set the
|
||||
environment variable `PYQTGRAPHPROFILE` to a comma-separated list of
|
||||
fully-qualified names of profiled functions.
|
||||
|
||||
Calling a profiler registers a message (defaulting to an increasing
|
||||
counter) that contains the time elapsed since the last call. When the
|
||||
profiler is about to be garbage-collected, the messages are passed to the
|
||||
outer profiler if one is running, or printed to stdout otherwise.
|
||||
|
||||
If `delayed` is set to False, messages are immediately printed instead.
|
||||
|
||||
Example:
|
||||
def function(...):
|
||||
profiler = Profiler()
|
||||
... do stuff ...
|
||||
profiler('did stuff')
|
||||
... do other stuff ...
|
||||
profiler('did other stuff')
|
||||
# profiler is garbage-collected and flushed at function end
|
||||
|
||||
If this function is a method of class C, setting `PYQTGRAPHPROFILE` to
|
||||
"C.function" (without the module name) will enable this profiler.
|
||||
|
||||
For regular functions, use the qualified name of the function, stripping
|
||||
only the initial "pyqtgraph." prefix from the module.
|
||||
'''
|
||||
|
||||
_profilers = os.environ.get("PYQTGRAPHPROFILE", None)
|
||||
_profilers = _profilers.split(",") if _profilers is not None else []
|
||||
|
||||
_depth = 0
|
||||
|
||||
# NOTE: without this defined at the class level
|
||||
# you won't see apprpriately "nested" sub-profiler
|
||||
# instance calls.
|
||||
_msgs = []
|
||||
|
||||
# set this flag to disable all or individual profilers at runtime
|
||||
disable = False
|
||||
|
||||
class DisabledProfiler(object):
|
||||
def __init__(self, *args, **kwds):
|
||||
pass
|
||||
|
||||
def __call__(self, *args):
|
||||
pass
|
||||
|
||||
def finish(self):
|
||||
pass
|
||||
|
||||
def mark(self, msg=None):
|
||||
pass
|
||||
|
||||
_disabledProfiler = DisabledProfiler()
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
msg=None,
|
||||
disabled='env',
|
||||
delayed=True,
|
||||
ms_threshold: float = 0.0,
|
||||
):
|
||||
"""Optionally create a new profiler based on caller's qualname.
|
||||
|
||||
``ms_threshold`` can be set to value in ms for which, if the
|
||||
total measured time of the lifetime of this profiler is **less
|
||||
than** this value, then no profiling messages will be printed.
|
||||
Setting ``delayed=False`` disables this feature since messages
|
||||
are emitted immediately.
|
||||
|
||||
"""
|
||||
if (
|
||||
disabled is True
|
||||
or (
|
||||
disabled == 'env'
|
||||
and len(cls._profilers) == 0
|
||||
)
|
||||
):
|
||||
return cls._disabledProfiler
|
||||
|
||||
# determine the qualified name of the caller function
|
||||
caller_frame = sys._getframe(1)
|
||||
try:
|
||||
caller_object_type = type(caller_frame.f_locals["self"])
|
||||
|
||||
except KeyError: # we are in a regular function
|
||||
qualifier = caller_frame.f_globals["__name__"].split(".", 1)[-1]
|
||||
|
||||
else: # we are in a method
|
||||
qualifier = caller_object_type.__name__
|
||||
func_qualname = qualifier + "." + caller_frame.f_code.co_name
|
||||
|
||||
if disabled == 'env' and func_qualname not in cls._profilers:
|
||||
# don't do anything
|
||||
return cls._disabledProfiler
|
||||
|
||||
cls._depth += 1
|
||||
obj = super(Profiler, cls).__new__(cls)
|
||||
obj._msgs = []
|
||||
|
||||
# create an actual profiling object
|
||||
if cls._depth < 1:
|
||||
cls._msgs = []
|
||||
|
||||
obj._name = msg or func_qualname
|
||||
obj._delayed = delayed
|
||||
obj._markCount = 0
|
||||
obj._finished = False
|
||||
obj._firstTime = obj._lastTime = perf_counter()
|
||||
obj._mt = ms_threshold
|
||||
obj._newMsg("> Entering " + obj._name)
|
||||
return obj
|
||||
|
||||
def __call__(self, msg=None):
|
||||
"""Register or print a new message with timing information.
|
||||
"""
|
||||
if self.disable:
|
||||
return
|
||||
if msg is None:
|
||||
msg = str(self._markCount)
|
||||
|
||||
self._markCount += 1
|
||||
newTime = perf_counter()
|
||||
tot_ms = (newTime - self._firstTime) * 1000
|
||||
ms = (newTime - self._lastTime) * 1000
|
||||
self._newMsg(
|
||||
f" {msg}: {ms:0.4f}, tot:{tot_ms:0.4f}"
|
||||
)
|
||||
|
||||
self._lastTime = newTime
|
||||
|
||||
def mark(self, msg=None):
|
||||
self(msg)
|
||||
|
||||
def _newMsg(self, msg, *args):
|
||||
msg = " " * (self._depth - 1) + msg
|
||||
if self._delayed:
|
||||
self._msgs.append((msg, args))
|
||||
else:
|
||||
print(msg % args)
|
||||
|
||||
def __del__(self):
|
||||
self.finish()
|
||||
|
||||
def finish(self, msg=None):
|
||||
"""Add a final message; flush the message list if no parent profiler.
|
||||
"""
|
||||
if self._finished or self.disable:
|
||||
return
|
||||
|
||||
self._finished = True
|
||||
if msg is not None:
|
||||
self(msg)
|
||||
|
||||
tot_ms = (perf_counter() - self._firstTime) * 1000
|
||||
self._newMsg(
|
||||
"< Exiting %s, total time: %0.4f ms",
|
||||
self._name,
|
||||
tot_ms,
|
||||
)
|
||||
|
||||
if tot_ms < self._mt:
|
||||
# print(f'{tot_ms} < {self._mt}, clearing')
|
||||
# NOTE: this list **must** be an instance var to avoid
|
||||
# deleting common messages during GC I think?
|
||||
self._msgs.clear()
|
||||
# else:
|
||||
# print(f'{tot_ms} > {self._mt}, not clearing')
|
||||
|
||||
# XXX: why is this needed?
|
||||
# don't we **want to show** nested profiler messages?
|
||||
if self._msgs: # and self._depth < 1:
|
||||
|
||||
# if self._msgs:
|
||||
print("\n".join([m[0] % m[1] for m in self._msgs]))
|
||||
|
||||
# clear all entries
|
||||
self._msgs.clear()
|
||||
# type(self)._msgs = []
|
||||
|
||||
type(self)._depth -= 1
|
||||
|
|
|
|||
|
|
@ -1,117 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"Accounting for degens": count dem numberz that tracks how much you got
|
||||
for tendiez.
|
||||
|
||||
'''
|
||||
from ..log import get_logger
|
||||
|
||||
from ._ledger import (
|
||||
iter_by_dt,
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
)
|
||||
from ._pos import (
|
||||
load_pps_from_ledger,
|
||||
open_pps,
|
||||
Position,
|
||||
PpTable,
|
||||
)
|
||||
from ._mktinfo import (
|
||||
Asset,
|
||||
dec_digits,
|
||||
digits_to_dec,
|
||||
MktPair,
|
||||
Symbol,
|
||||
unpack_fqme,
|
||||
_derivs as DerivTypes,
|
||||
)
|
||||
from ._allocate import (
|
||||
mk_allocator,
|
||||
Allocator,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'Allocator',
|
||||
'Asset',
|
||||
'MktPair',
|
||||
'Position',
|
||||
'PpTable',
|
||||
'Symbol',
|
||||
'Transaction',
|
||||
'TransactionLedger',
|
||||
'dec_digits',
|
||||
'digits_to_dec',
|
||||
'iter_by_dt',
|
||||
'load_pps_from_ledger',
|
||||
'mk_allocator',
|
||||
'open_pps',
|
||||
'open_trade_ledger',
|
||||
'unpack_fqme',
|
||||
'DerivTypes',
|
||||
]
|
||||
|
||||
|
||||
def get_likely_pair(
|
||||
src: str,
|
||||
dst: str,
|
||||
bs_mktid: str,
|
||||
|
||||
) -> str | None:
|
||||
'''
|
||||
Attempt to get the likely trading pair matching a given destination
|
||||
asset `dst: str`.
|
||||
|
||||
'''
|
||||
try:
|
||||
src_name_start = bs_mktid.rindex(src)
|
||||
except (
|
||||
ValueError, # substr not found
|
||||
):
|
||||
# TODO: handle nested positions..(i.e.
|
||||
# positions where the src fiat was used to
|
||||
# buy some other dst which was furhter used
|
||||
# to buy another dst..)
|
||||
# log.warning(
|
||||
# f'No src fiat {src} found in {bs_mktid}?'
|
||||
# )
|
||||
return
|
||||
|
||||
likely_dst = bs_mktid[:src_name_start]
|
||||
if likely_dst == dst:
|
||||
return bs_mktid
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
from pprint import pformat
|
||||
|
||||
args = sys.argv
|
||||
assert len(args) > 1, 'Specifiy account(s) from `brokers.toml`'
|
||||
args = args[1:]
|
||||
for acctid in args:
|
||||
broker, name = acctid.split('.')
|
||||
trans, updated_pps = load_pps_from_ledger(broker, name)
|
||||
print(
|
||||
f'Processing transactions into pps for {broker}:{acctid}\n'
|
||||
f'{pformat(trans)}\n\n'
|
||||
f'{pformat(updated_pps)}'
|
||||
)
|
||||
|
|
@ -1,339 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Trade and transaction ledger processing.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from collections import UserDict
|
||||
from contextlib import contextmanager as cm
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Iterator,
|
||||
Union,
|
||||
Generator
|
||||
)
|
||||
|
||||
from pendulum import (
|
||||
datetime,
|
||||
DateTime,
|
||||
from_timestamp,
|
||||
parse,
|
||||
)
|
||||
import tomli_w # for fast ledger writing
|
||||
|
||||
from .. import config
|
||||
from ..data.types import Struct
|
||||
from ..log import get_logger
|
||||
from ._mktinfo import (
|
||||
Symbol, # legacy
|
||||
MktPair,
|
||||
Asset,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Transaction(Struct, frozen=True):
|
||||
|
||||
# TODO: unify this with the `MktPair`,
|
||||
# once we have that as a required field,
|
||||
# we don't really need the fqme any more..
|
||||
fqme: str
|
||||
|
||||
tid: Union[str, int] # unique transaction id
|
||||
size: float
|
||||
price: float
|
||||
cost: float # commisions or other additional costs
|
||||
dt: datetime
|
||||
|
||||
# TODO: we can drop this right since we
|
||||
# can instead expect the backend to provide this
|
||||
# via the `MktPair`?
|
||||
expiry: datetime | None = None
|
||||
|
||||
# TODO: drop the Symbol type, construct using
|
||||
# t.sys (the transaction system)
|
||||
|
||||
# the underlying "transaction system", normally one of a ``MktPair``
|
||||
# (a description of a tradable double auction) or a ledger-recorded
|
||||
# ("ledger" in any sense as long as you can record transfers) of any
|
||||
# sort) ``Asset``.
|
||||
sym: MktPair | Asset | Symbol | None = None
|
||||
|
||||
@property
|
||||
def sys(self) -> Symbol:
|
||||
return self.sym
|
||||
|
||||
# (optional) key-id defined by the broker-service backend which
|
||||
# ensures the instrument-symbol market key for this record is unique
|
||||
# in the "their backend/system" sense; i.e. this uid for the market
|
||||
# as defined (internally) in some namespace defined by the broker
|
||||
# service.
|
||||
bs_mktid: str | int | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
dct = super().to_dict()
|
||||
|
||||
# TODO: switch to sys!
|
||||
dct.pop('sym')
|
||||
|
||||
# ensure we use a pendulum formatted
|
||||
# ISO style str here!@
|
||||
dct['dt'] = str(self.dt)
|
||||
return dct
|
||||
|
||||
|
||||
class TransactionLedger(UserDict):
|
||||
'''
|
||||
Very simple ``dict`` wrapper + ``pathlib.Path`` handle to
|
||||
a TOML formatted transaction file for enabling file writes
|
||||
dynamically whilst still looking exactly like a ``dict`` from the
|
||||
outside.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
ledger_dict: dict,
|
||||
file_path: Path,
|
||||
tx_sort: Callable,
|
||||
|
||||
) -> None:
|
||||
self.file_path = file_path
|
||||
self.tx_sort = tx_sort
|
||||
super().__init__(ledger_dict)
|
||||
|
||||
def update_from_t(
|
||||
self,
|
||||
t: Transaction,
|
||||
) -> None:
|
||||
'''
|
||||
Given an input `Transaction`, cast to `dict` and update
|
||||
from it's transaction id.
|
||||
|
||||
'''
|
||||
self.data[t.tid] = t.to_dict()
|
||||
|
||||
def iter_trans(
|
||||
self,
|
||||
mkt_by_fqme: dict[str, MktPair],
|
||||
broker: str = 'paper',
|
||||
|
||||
) -> Generator[
|
||||
tuple[str, Transaction],
|
||||
None,
|
||||
None,
|
||||
]:
|
||||
'''
|
||||
Deliver trades records in ``(key: str, t: Transaction)``
|
||||
form via generator.
|
||||
|
||||
'''
|
||||
if broker != 'paper':
|
||||
raise NotImplementedError('Per broker support not dun yet!')
|
||||
|
||||
# TODO: lookup some standard normalizer
|
||||
# func in the backend?
|
||||
# from ..brokers import get_brokermod
|
||||
# mod = get_brokermod(broker)
|
||||
# trans_dict = mod.norm_trade_records(self.data)
|
||||
|
||||
# NOTE: instead i propose the normalizer is
|
||||
# a one shot routine (that can be lru cached)
|
||||
# and instead call it for each entry incrementally:
|
||||
# normer = mod.norm_trade_record(txdict)
|
||||
|
||||
# TODO: use tx_sort here yah?
|
||||
for txdict in self.tx_sort(self.data.values()):
|
||||
# for tid, txdict in self.data.items():
|
||||
# special field handling for datetimes
|
||||
# to ensure pendulum is used!
|
||||
tid: str = txdict['tid']
|
||||
fqme: str = txdict.get('fqme') or txdict['fqsn']
|
||||
dt: DateTime = parse(txdict['dt'])
|
||||
expiry: str | None = txdict.get('expiry')
|
||||
|
||||
if not (mkt := mkt_by_fqme.get(fqme)):
|
||||
# we can't build a trans if we don't have
|
||||
# the ``.sys: MktPair`` info, so skip.
|
||||
continue
|
||||
|
||||
tx = Transaction(
|
||||
fqme=fqme,
|
||||
tid=txdict['tid'],
|
||||
dt=dt,
|
||||
price=txdict['price'],
|
||||
size=txdict['size'],
|
||||
cost=txdict.get('cost', 0),
|
||||
bs_mktid=txdict['bs_mktid'],
|
||||
|
||||
# TODO: change to .sys!
|
||||
sym=mkt,
|
||||
expiry=parse(expiry) if expiry else None,
|
||||
)
|
||||
yield tid, tx
|
||||
|
||||
def to_trans(
|
||||
self,
|
||||
**kwargs,
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Return entire output from ``.iter_trans()`` in a ``dict``.
|
||||
|
||||
'''
|
||||
return dict(self.iter_trans(**kwargs))
|
||||
|
||||
def write_config(
|
||||
self,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Render the self.data ledger dict to it's TOML file form.
|
||||
|
||||
'''
|
||||
cpy = self.data.copy()
|
||||
towrite: dict[str, Any] = {}
|
||||
for tid, trans in cpy.items():
|
||||
|
||||
# drop key for non-expiring assets
|
||||
txdict = towrite[tid] = self.data[tid]
|
||||
if (
|
||||
'expiry' in txdict
|
||||
and txdict['expiry'] is None
|
||||
):
|
||||
txdict.pop('expiry')
|
||||
|
||||
# re-write old acro-key
|
||||
fqme = txdict.get('fqsn')
|
||||
if fqme:
|
||||
txdict['fqme'] = fqme
|
||||
|
||||
with self.file_path.open(mode='wb') as fp:
|
||||
tomli_w.dump(towrite, fp)
|
||||
|
||||
|
||||
def iter_by_dt(
|
||||
records: dict[str, dict[str, Any]] | list[dict],
|
||||
|
||||
# NOTE: parsers are looked up in the insert order
|
||||
# so if you know that the record stats show some field
|
||||
# is more common then others, stick it at the top B)
|
||||
parsers: dict[tuple[str], Callable] = {
|
||||
'dt': None, # parity case
|
||||
'datetime': parse, # datetime-str
|
||||
'time': from_timestamp, # float epoch
|
||||
},
|
||||
key: Callable | None = None,
|
||||
|
||||
) -> Iterator[tuple[str, dict]]:
|
||||
'''
|
||||
Iterate entries of a ``records: dict`` table sorted by entry recorded
|
||||
datetime presumably set at the ``'dt'`` field in each entry.
|
||||
|
||||
'''
|
||||
def dyn_parse_to_dt(txdict: dict[str, Any]) -> DateTime:
|
||||
k, v, parser = next(
|
||||
(k, txdict[k], parsers[k]) for k in parsers if k in txdict
|
||||
)
|
||||
return parser(v) if parser else v
|
||||
|
||||
if isinstance(records, dict):
|
||||
records = records.values()
|
||||
|
||||
for entry in sorted(
|
||||
records,
|
||||
key=key or dyn_parse_to_dt,
|
||||
):
|
||||
yield entry
|
||||
|
||||
|
||||
def load_ledger(
|
||||
brokername: str,
|
||||
acctid: str,
|
||||
|
||||
) -> tuple[dict, Path]:
|
||||
'''
|
||||
Load a ledger (TOML) file from user's config directory:
|
||||
$CONFIG_DIR/accounting/ledgers/trades_<brokername>_<acctid>.toml
|
||||
|
||||
Return its `dict`-content and file path.
|
||||
|
||||
'''
|
||||
import time
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
ldir: Path = config._config_dir / 'accounting' / 'ledgers'
|
||||
if not ldir.is_dir():
|
||||
ldir.mkdir()
|
||||
|
||||
fname = f'trades_{brokername}_{acctid}.toml'
|
||||
fpath: Path = ldir / fname
|
||||
|
||||
if not fpath.is_file():
|
||||
log.info(
|
||||
f'Creating new local trades ledger: {fpath}'
|
||||
)
|
||||
fpath.touch()
|
||||
|
||||
with fpath.open(mode='rb') as cf:
|
||||
start = time.time()
|
||||
ledger_dict = tomllib.load(cf)
|
||||
log.debug(f'Ledger load took {time.time() - start}s')
|
||||
|
||||
return ledger_dict, fpath
|
||||
|
||||
|
||||
@cm
|
||||
def open_trade_ledger(
|
||||
broker: str,
|
||||
account: str,
|
||||
|
||||
# default is to sort by detected datetime-ish field
|
||||
tx_sort: Callable = iter_by_dt,
|
||||
|
||||
) -> Generator[TransactionLedger, None, None]:
|
||||
'''
|
||||
Indempotently create and read in a trade log file from the
|
||||
``<configuration_dir>/ledgers/`` directory.
|
||||
|
||||
Files are named per broker account of the form
|
||||
``<brokername>_<accountname>.toml``. The ``accountname`` here is the
|
||||
name as defined in the user's ``brokers.toml`` config.
|
||||
|
||||
'''
|
||||
ledger_dict, fpath = load_ledger(broker, account)
|
||||
cpy = ledger_dict.copy()
|
||||
ledger = TransactionLedger(
|
||||
ledger_dict=cpy,
|
||||
file_path=fpath,
|
||||
tx_sort=tx_sort,
|
||||
)
|
||||
try:
|
||||
yield ledger
|
||||
finally:
|
||||
if ledger.data != ledger_dict:
|
||||
|
||||
# TODO: show diff output?
|
||||
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
|
||||
log.info(f'Updating ledger for {fpath}:\n')
|
||||
ledger.write_config()
|
||||
|
|
@ -1,707 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Market (pair) meta-info layer: sane addressing semantics and meta-data
|
||||
for cross-provider marketplaces.
|
||||
|
||||
We intoduce the concept of,
|
||||
|
||||
- a FQMA: fully qualified market address,
|
||||
- a sane schema for FQMAs including derivatives,
|
||||
- a msg-serializeable description of markets for
|
||||
easy sharing with other pikers B)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from decimal import (
|
||||
Decimal,
|
||||
ROUND_HALF_EVEN,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Literal,
|
||||
)
|
||||
|
||||
from ..data.types import Struct
|
||||
|
||||
|
||||
# TODO: make these literals..
|
||||
_underlyings: list[str] = [
|
||||
'stock',
|
||||
'bond',
|
||||
'crypto',
|
||||
'fiat',
|
||||
'commodity',
|
||||
]
|
||||
|
||||
_crypto_derivs: list[str] = [
|
||||
'perpetual_future',
|
||||
'crypto_future',
|
||||
]
|
||||
|
||||
_derivs: list[str] = [
|
||||
'swap',
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
|
||||
# if we can't figure it out, presume the worst XD
|
||||
'unknown',
|
||||
]
|
||||
|
||||
# NOTE: a tag for other subsystems to try
|
||||
# and do default settings for certain things:
|
||||
# - allocator does unit vs. dolla size limiting.
|
||||
AssetTypeName: Literal[
|
||||
_underlyings
|
||||
+
|
||||
_derivs
|
||||
+
|
||||
_crypto_derivs
|
||||
]
|
||||
|
||||
# egs. stock, futer, option, bond etc.
|
||||
|
||||
|
||||
def dec_digits(
|
||||
value: float | str | Decimal,
|
||||
|
||||
) -> int:
|
||||
'''
|
||||
Return the number of precision digits read from a decimal or float
|
||||
value.
|
||||
|
||||
'''
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(
|
||||
-Decimal(str(value)).as_tuple().exponent
|
||||
)
|
||||
|
||||
|
||||
float_digits = dec_digits
|
||||
|
||||
|
||||
def digits_to_dec(
|
||||
ndigits: int,
|
||||
) -> Decimal:
|
||||
'''
|
||||
Return the minimum float value for an input integer value.
|
||||
|
||||
eg. 3 -> 0.001
|
||||
|
||||
'''
|
||||
if ndigits == 0:
|
||||
return Decimal('0')
|
||||
|
||||
return Decimal('0.' + '0'*(ndigits-1) + '1')
|
||||
|
||||
|
||||
class Asset(Struct, frozen=True):
|
||||
'''
|
||||
Container type describing any transactable asset and its
|
||||
contract-like and/or underlying technology meta-info.
|
||||
|
||||
'''
|
||||
name: str
|
||||
atype: str # AssetTypeName
|
||||
|
||||
# minimum transaction size / precision.
|
||||
# eg. for buttcoin this is a "satoshi".
|
||||
tx_tick: Decimal
|
||||
|
||||
# NOTE: additional info optionally packed in by the backend, but
|
||||
# should not be explicitly required in our generic API.
|
||||
info: dict | None = None
|
||||
|
||||
# TODO?
|
||||
# _to_dict_skip = {'info'}
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
quantized form of the digit precision defined
|
||||
by ``self.lot_tick_size``.
|
||||
|
||||
'''
|
||||
digits = float_digits(self.tx_tick)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def guess_from_mkt_ep_key(
|
||||
cls,
|
||||
mkt_ep_key: str,
|
||||
atype: str | None = None,
|
||||
|
||||
) -> Asset:
|
||||
'''
|
||||
A hacky guess method for presuming a (target) asset's properties
|
||||
based on either the actualy market endpoint key, or config settings
|
||||
from the user.
|
||||
|
||||
'''
|
||||
atype = atype or 'unknown'
|
||||
|
||||
# attempt to strip off any source asset
|
||||
# via presumed syntax of:
|
||||
# - <dst>/<src>
|
||||
# - <dst>.<src>
|
||||
# - etc.
|
||||
for char in ['/', '.']:
|
||||
dst, _, src = mkt_ep_key.partition(char)
|
||||
if src:
|
||||
if not atype:
|
||||
atype = 'fiat'
|
||||
break
|
||||
|
||||
return Asset(
|
||||
name=dst,
|
||||
atype=atype,
|
||||
tx_tick=Decimal('0.01'),
|
||||
)
|
||||
|
||||
|
||||
def maybe_cons_tokens(
|
||||
tokens: list[Any],
|
||||
delim_char: str = '.',
|
||||
) -> str:
|
||||
'''
|
||||
Construct `str` output from a maybe-concatenation of input
|
||||
sequence of elements in ``tokens``.
|
||||
|
||||
'''
|
||||
return delim_char.join(filter(bool, tokens)).lower()
|
||||
|
||||
|
||||
class MktPair(Struct, frozen=True):
|
||||
'''
|
||||
Market description for a pair of assets which are tradeable:
|
||||
a market which enables transactions of the form,
|
||||
buy: source asset -> destination asset
|
||||
sell: destination asset -> source asset
|
||||
|
||||
The main intention of this type is for a **simple** cross-asset
|
||||
venue/broker normalized descrption type from which all
|
||||
market-auctions can be mapped from FQME identifiers.
|
||||
|
||||
TODO: our eventual target fqme format/schema is:
|
||||
<dst>/<src>.<expiry>.<con_info_1>.<con_info_2>. -> .<venue>.<broker>
|
||||
^ -- optional tokens ------------------------------- ^
|
||||
|
||||
|
||||
Notes:
|
||||
------
|
||||
|
||||
Some venues provide a different semantic (which we frankly find
|
||||
confusing and non-general) such as "base" and "quote" asset.
|
||||
For example this is how `binance` defines the terms:
|
||||
|
||||
https://binance-docs.github.io/apidocs/websocket_api/en/#public-api-definitions
|
||||
https://binance-docs.github.io/apidocs/futures/en/#public-endpoints-info
|
||||
|
||||
- *base* asset refers to the asset that is the *quantity* of a symbol.
|
||||
- *quote* asset refers to the asset that is the *price* of a symbol.
|
||||
|
||||
In other words the "quote" asset is the asset that the market
|
||||
is pricing "buys" *in*, and the *base* asset it the one that the market
|
||||
allows you to "buy" an *amount of*. Put more simply the *quote*
|
||||
asset is our "source" asset and the *base* asset is our "destination"
|
||||
asset.
|
||||
|
||||
This defintion can be further understood reading our
|
||||
`.brokers.binance.api.Pair` type wherein the
|
||||
`Pair.[quote/base]AssetPrecision` field determines the (transfer)
|
||||
transaction precision available per asset; i.e. the satoshis
|
||||
unit in bitcoin for representing the minimum size of a
|
||||
transaction that can take place on the blockchain.
|
||||
|
||||
'''
|
||||
dst: str | Asset
|
||||
# "destination asset" (name) used to buy *to*
|
||||
# (or used to sell *from*)
|
||||
|
||||
price_tick: Decimal # minimum price increment
|
||||
size_tick: Decimal # minimum size (aka vlm) increment
|
||||
# the tick size is the number describing the smallest step in value
|
||||
# available in this market between the source and destination
|
||||
# assets.
|
||||
# https://en.wikipedia.org/wiki/Tick_size
|
||||
# https://en.wikipedia.org/wiki/Commodity_tick
|
||||
# https://en.wikipedia.org/wiki/Percentage_in_point
|
||||
|
||||
# unique "broker id" since every market endpoint provider
|
||||
# has their own nomenclature and schema for market maps.
|
||||
bs_mktid: str
|
||||
broker: str # the middle man giving access
|
||||
|
||||
# NOTE: to start this field is optional but should eventually be
|
||||
# required; the reason is for backward compat since more positioning
|
||||
# calculations were not originally stored with a src asset..
|
||||
|
||||
src: str | Asset = ''
|
||||
# "source asset" (name) used to buy *from*
|
||||
# (or used to sell *to*).
|
||||
|
||||
venue: str = '' # market venue provider name
|
||||
expiry: str = '' # for derivs, expiry datetime parseable str
|
||||
|
||||
# destination asset's financial type/classification name
|
||||
# NOTE: this is required for the order size allocator system,
|
||||
# since we use different default settings based on the type
|
||||
# of the destination asset, eg. futes use a units limits vs.
|
||||
# equities a $limit.
|
||||
# dst_type: AssetTypeName | None = None
|
||||
|
||||
# source asset's financial type/classification name
|
||||
# TODO: is a src type required for trading?
|
||||
# there's no reason to need any more then the one-way alloc-limiter
|
||||
# config right?
|
||||
# src_type: AssetTypeName
|
||||
|
||||
# for derivs, info describing contract, egs.
|
||||
# strike price, call or put, swap type, exercise model, etc.
|
||||
contract_info: list[str] | None = None
|
||||
|
||||
_atype: str = ''
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict[str, Any],
|
||||
|
||||
) -> MktPair:
|
||||
'''
|
||||
Constructor for a received msg-dict normally received over IPC.
|
||||
|
||||
'''
|
||||
dst_asset_msg = msg.pop('dst')
|
||||
src_asset_msg = msg.pop('src')
|
||||
|
||||
if isinstance(dst_asset_msg, str):
|
||||
src: str = str(src_asset_msg)
|
||||
assert isinstance(src, str)
|
||||
return cls.from_fqme(
|
||||
dst_asset_msg,
|
||||
src=src,
|
||||
**msg,
|
||||
)
|
||||
|
||||
else:
|
||||
# NOTE: we call `.copy()` here to ensure
|
||||
# type casting!
|
||||
dst = Asset(**dst_asset_msg).copy()
|
||||
if not isinstance(src_asset_msg, str):
|
||||
src = Asset(**src_asset_msg).copy()
|
||||
else:
|
||||
src = str(src_asset_msg)
|
||||
|
||||
return cls(
|
||||
dst=dst,
|
||||
src=src,
|
||||
**msg,
|
||||
# XXX NOTE: ``msgspec`` can encode `Decimal`
|
||||
# but it doesn't decide to it by default since
|
||||
# we aren't spec-cing these msgs as structs, SO
|
||||
# we have to ensure we do a struct type case (which `.copy()`
|
||||
# does) to ensure we get the right type!
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return isinstance(self.dst, Asset)
|
||||
|
||||
@classmethod
|
||||
def from_fqme(
|
||||
cls,
|
||||
fqme: str,
|
||||
|
||||
price_tick: float | str,
|
||||
size_tick: float | str,
|
||||
bs_mktid: str,
|
||||
|
||||
broker: str | None = None,
|
||||
**kwargs,
|
||||
|
||||
) -> MktPair:
|
||||
|
||||
_fqme: str = fqme
|
||||
if (
|
||||
broker
|
||||
and broker not in fqme
|
||||
):
|
||||
_fqme = f'{fqme}.{broker}'
|
||||
|
||||
broker, mkt_ep_key, venue, suffix = unpack_fqme(_fqme)
|
||||
dst: Asset = Asset.guess_from_mkt_ep_key(
|
||||
mkt_ep_key,
|
||||
atype=kwargs.get('_atype'),
|
||||
)
|
||||
|
||||
# XXX: loading from a fqme string will
|
||||
# leave this pair as "un resolved" meaning
|
||||
# we don't yet have `.dst` set as an `Asset`
|
||||
# which we expect to be filled in by some
|
||||
# backend client with access to that data-info.
|
||||
return cls(
|
||||
# XXX: not resolved to ``Asset`` :(
|
||||
dst=dst,
|
||||
|
||||
broker=broker,
|
||||
venue=venue,
|
||||
# XXX NOTE: we presume this token
|
||||
# if the expiry for now!
|
||||
expiry=suffix,
|
||||
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
**kwargs,
|
||||
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
'''
|
||||
The "endpoint key" for this market.
|
||||
|
||||
'''
|
||||
return self.pair
|
||||
|
||||
def pair(
|
||||
self,
|
||||
delim_char: str | None = None,
|
||||
) -> str:
|
||||
'''
|
||||
The "endpoint asset pair key" for this market.
|
||||
Eg. mnq/usd or btc/usdt or xmr/btc
|
||||
|
||||
In most other tina platforms this is referred to as the
|
||||
"symbol".
|
||||
|
||||
'''
|
||||
return maybe_cons_tokens(
|
||||
[str(self.dst),
|
||||
str(self.src)],
|
||||
# TODO: make the default '/'
|
||||
delim_char=delim_char or '',
|
||||
)
|
||||
|
||||
@property
|
||||
def suffix(self) -> str:
|
||||
'''
|
||||
The "contract suffix" for this market.
|
||||
|
||||
Eg. mnq/usd.20230616.cme.ib
|
||||
^ ----- ^
|
||||
or tsla/usd.20230324.200c.cboe.ib
|
||||
^ ---------- ^
|
||||
|
||||
In most other tina platforms they only show you these details in
|
||||
some kinda "meta data" format, we have FQMEs so we do this up
|
||||
front and explicit.
|
||||
|
||||
'''
|
||||
field_strs = [self.expiry]
|
||||
con_info = self.contract_info
|
||||
if con_info is not None:
|
||||
field_strs.extend(con_info)
|
||||
|
||||
return maybe_cons_tokens(field_strs)
|
||||
|
||||
def get_fqme(
|
||||
self,
|
||||
|
||||
# NOTE: allow dropping the source asset from the
|
||||
# market endpoint's pair key. Eg. to change
|
||||
# mnq/usd.<> -> mnq.<> which is useful when
|
||||
# searching (legacy) stock exchanges.
|
||||
without_src: bool = False,
|
||||
delim_char: str | None = None,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Return the fully qualified market endpoint-address for the
|
||||
pair of transacting assets.
|
||||
|
||||
fqme = "fully qualified market endpoint"
|
||||
|
||||
And yes, you pronounce it colloquially as read..
|
||||
|
||||
Basically the idea here is for all client code (consumers of piker's
|
||||
APIs which query the data/broker-provider agnostic layer(s)) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string-key of the current form:
|
||||
|
||||
<market-instrument-name>
|
||||
.<venue>
|
||||
.<expiry>
|
||||
.<derivative-suffix-info>
|
||||
.<brokerbackendname>
|
||||
|
||||
eg. for an explicit daq mini futes contract: mnq.cme.20230317.ib
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.)
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
TODO:
|
||||
See community discussion on naming and nomenclature, order
|
||||
of addressing hierarchy, general schema, internal representation:
|
||||
|
||||
https://github.com/pikers/piker/issues/467
|
||||
|
||||
'''
|
||||
key: str = (
|
||||
self.pair(delim_char=delim_char)
|
||||
if not without_src
|
||||
else str(self.dst)
|
||||
)
|
||||
|
||||
return maybe_cons_tokens([
|
||||
key, # final "pair name" (eg. qqq[/usd], btcusdt)
|
||||
self.venue,
|
||||
self.suffix, # includes expiry and other con info
|
||||
self.broker,
|
||||
])
|
||||
|
||||
# NOTE: the main idea behind an fqme is to map a "market address"
|
||||
# to some endpoint from a transaction provider (eg. a broker) such
|
||||
# that we build a table of `fqme: str -> bs_mktid: Any` where any "piker
|
||||
# market address" maps 1-to-1 to some broker trading endpoint.
|
||||
# @cached_property
|
||||
fqme = property(get_fqme)
|
||||
|
||||
def get_bs_fqme(
|
||||
self,
|
||||
**kwargs,
|
||||
) -> str:
|
||||
'''
|
||||
FQME sin broker part XD
|
||||
|
||||
'''
|
||||
sin_broker, *_ = self.get_fqme(**kwargs).rpartition('.')
|
||||
return sin_broker
|
||||
|
||||
bs_fqme = property(get_bs_fqme)
|
||||
|
||||
@property
|
||||
def fqsn(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
quantity_type: Literal['price', 'size'] = 'size',
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
and ``.size_tick``'s # of digits.
|
||||
|
||||
'''
|
||||
match quantity_type:
|
||||
case 'price':
|
||||
digits = float_digits(self.price_tick)
|
||||
case 'size':
|
||||
digits = float_digits(self.size_tick)
|
||||
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# TODO: BACKWARD COMPAT, TO REMOVE?
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
|
||||
# if set explicitly then use it!
|
||||
if self._atype:
|
||||
return self._atype
|
||||
|
||||
if isinstance(self.dst, Asset):
|
||||
return str(self.dst.atype)
|
||||
|
||||
return 'unknown'
|
||||
|
||||
@property
|
||||
def price_tick_digits(self) -> int:
|
||||
return float_digits(self.price_tick)
|
||||
|
||||
@property
|
||||
def size_tick_digits(self) -> int:
|
||||
return float_digits(self.size_tick)
|
||||
|
||||
|
||||
def unpack_fqme(
|
||||
fqme: str,
|
||||
|
||||
broker: str | None = None
|
||||
|
||||
) -> tuple[str, ...]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqme.split('.')
|
||||
|
||||
match tokens:
|
||||
case [mkt_ep, broker]:
|
||||
# probably crypto
|
||||
return (
|
||||
broker,
|
||||
mkt_ep,
|
||||
'',
|
||||
'',
|
||||
)
|
||||
|
||||
# TODO: swap venue and suffix/deriv-info here?
|
||||
case [mkt_ep, venue, suffix, broker]:
|
||||
pass
|
||||
|
||||
# handle `bs_mktid` + `broker` input case
|
||||
case [
|
||||
mkt_ep, venue, suffix
|
||||
] if (
|
||||
broker
|
||||
and suffix != broker
|
||||
):
|
||||
pass
|
||||
|
||||
case [mkt_ep, venue, broker]:
|
||||
suffix = ''
|
||||
|
||||
case _:
|
||||
raise ValueError(f'Invalid fqme: {fqme}')
|
||||
|
||||
return (
|
||||
broker,
|
||||
mkt_ep,
|
||||
venue,
|
||||
# '.'.join([mkt_ep, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
class Symbol(Struct):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
|
||||
broker: str = ''
|
||||
venue: str = ''
|
||||
|
||||
# precision descriptors for price and vlm
|
||||
tick_size: Decimal = Decimal('0.01')
|
||||
lot_tick_size: Decimal = Decimal('0.0')
|
||||
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@classmethod
|
||||
def from_fqme(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
) -> Symbol:
|
||||
broker, mktep, venue, suffix = unpack_fqme(fqsn)
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
broker=broker,
|
||||
key=mktep,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_size,
|
||||
venue=venue,
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def tick_size_digits(self) -> int:
|
||||
return float_digits(self.tick_size)
|
||||
|
||||
@property
|
||||
def lot_size_digits(self) -> int:
|
||||
return float_digits(self.lot_tick_size)
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return Decimal(str(self.tick_size))
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return Decimal(str(self.lot_tick_size))
|
||||
|
||||
@property
|
||||
def broker(self) -> str:
|
||||
return list(self.broker_info.keys())[0]
|
||||
|
||||
@property
|
||||
def fqme(self) -> str:
|
||||
return maybe_cons_tokens([
|
||||
self.key, # final "pair name" (eg. qqq[/usd], btcusdt)
|
||||
self.venue,
|
||||
self.suffix, # includes expiry and other con info
|
||||
self.broker,
|
||||
])
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
) -> Decimal:
|
||||
digits = float_digits(self.lot_tick_size)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,293 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
CLI front end for trades ledger and position tracking management.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from rich.console import Console
|
||||
from rich.markdown import Markdown
|
||||
import polars as pl
|
||||
import tractor
|
||||
import trio
|
||||
import typer
|
||||
|
||||
from ..log import get_logger
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
)
|
||||
from ..clearing._messages import BrokerdPosition
|
||||
from ..calc import humanize
|
||||
from ..brokers._daemon import broker_init
|
||||
from ._ledger import (
|
||||
load_ledger,
|
||||
# open_trade_ledger,
|
||||
# TransactionLedger,
|
||||
)
|
||||
from ._pos import (
|
||||
PpTable,
|
||||
load_pps_from_ledger,
|
||||
# load_account,
|
||||
)
|
||||
|
||||
|
||||
ledger = typer.Typer()
|
||||
|
||||
|
||||
def unpack_fqan(
|
||||
fully_qualified_account_name: str,
|
||||
console: Console | None = None,
|
||||
) -> tuple | bool:
|
||||
try:
|
||||
brokername, account = fully_qualified_account_name.split('.')
|
||||
return brokername, account
|
||||
except ValueError:
|
||||
if console is not None:
|
||||
md = Markdown(
|
||||
f'=> `{fully_qualified_account_name}` <=\n\n'
|
||||
'is not a valid '
|
||||
'__fully qualified account name?__\n\n'
|
||||
'Your account name needs to be of the form '
|
||||
'`<brokername>.<account_name>`\n'
|
||||
)
|
||||
console.print(md)
|
||||
return False
|
||||
|
||||
|
||||
@ledger.command()
|
||||
def sync(
|
||||
fully_qualified_account_name: str,
|
||||
pdb: bool = False,
|
||||
|
||||
loglevel: str = typer.Option(
|
||||
'error',
|
||||
"-l",
|
||||
),
|
||||
):
|
||||
log = get_logger(loglevel)
|
||||
console = Console()
|
||||
|
||||
pair: tuple[str, str]
|
||||
if not (pair := unpack_fqan(
|
||||
fully_qualified_account_name,
|
||||
console,
|
||||
)):
|
||||
return
|
||||
|
||||
brokername, account = pair
|
||||
|
||||
brokermod, start_kwargs, deamon_ep = broker_init(
|
||||
brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
brokername: str = brokermod.name
|
||||
|
||||
async def main():
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='ledger_cli',
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
|
||||
) as (actor, sockaddr),
|
||||
|
||||
tractor.open_nursery() as an,
|
||||
):
|
||||
try:
|
||||
log.info(
|
||||
f'Piker runtime up as {actor.uid}@{sockaddr}'
|
||||
)
|
||||
|
||||
portal = await an.start_actor(
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
**start_kwargs,
|
||||
)
|
||||
|
||||
from ..clearing import (
|
||||
open_brokerd_dialog,
|
||||
)
|
||||
brokerd_stream: tractor.MsgStream
|
||||
|
||||
async with (
|
||||
# engage the brokerd daemon context
|
||||
portal.open_context(
|
||||
deamon_ep,
|
||||
brokername=brokername,
|
||||
loglevel=loglevel,
|
||||
),
|
||||
|
||||
# manually open the brokerd trade dialog EP
|
||||
# (what the EMS normally does internall) B)
|
||||
open_brokerd_dialog(
|
||||
brokermod,
|
||||
portal,
|
||||
exec_mode=(
|
||||
'paper'
|
||||
if account == 'paper'
|
||||
else 'live'
|
||||
),
|
||||
loglevel=loglevel,
|
||||
) as (
|
||||
brokerd_stream,
|
||||
pp_msg_table,
|
||||
accounts,
|
||||
),
|
||||
):
|
||||
try:
|
||||
assert len(accounts) == 1
|
||||
if not pp_msg_table:
|
||||
ld, fpath = load_ledger(brokername, account)
|
||||
assert not ld, f'WTF did we fail to parse ledger:\n{ld}'
|
||||
|
||||
console.print(
|
||||
'[yellow]'
|
||||
'No pps found for '
|
||||
f'`{brokername}.{account}` '
|
||||
'account!\n\n'
|
||||
'[/][underline]'
|
||||
'None of the following ledger files exist:\n\n[/]'
|
||||
f'{fpath.as_uri()}\n'
|
||||
)
|
||||
return
|
||||
|
||||
pps_by_symbol: dict[str, BrokerdPosition] = pp_msg_table[
|
||||
brokername,
|
||||
account,
|
||||
]
|
||||
|
||||
summary: str = (
|
||||
'[dim underline]Piker Position Summary[/] '
|
||||
f'[dim blue underline]{brokername}[/]'
|
||||
'[dim].[/]'
|
||||
f'[blue underline]{account}[/]'
|
||||
f'[dim underline] -> total pps: [/]'
|
||||
f'[green]{len(pps_by_symbol)}[/]\n'
|
||||
)
|
||||
# for ppdict in positions:
|
||||
for fqme, ppmsg in pps_by_symbol.items():
|
||||
# ppmsg = BrokerdPosition(**ppdict)
|
||||
size = ppmsg.size
|
||||
if size:
|
||||
ppu: float = round(
|
||||
ppmsg.avg_price,
|
||||
ndigits=2,
|
||||
)
|
||||
cost_basis: str = humanize(size * ppu)
|
||||
h_size: str = humanize(size)
|
||||
|
||||
if size < 0:
|
||||
pcolor = 'red'
|
||||
else:
|
||||
pcolor = 'green'
|
||||
|
||||
# sematic-highlight of fqme
|
||||
fqme = ppmsg.symbol
|
||||
tokens = fqme.split('.')
|
||||
styled_fqme = f'[blue underline]{tokens[0]}[/]'
|
||||
for tok in tokens[1:]:
|
||||
styled_fqme += '[dim].[/]'
|
||||
styled_fqme += f'[dim blue underline]{tok}[/]'
|
||||
|
||||
# TODO: instead display in a ``rich.Table``?
|
||||
summary += (
|
||||
styled_fqme +
|
||||
'[dim]: [/]'
|
||||
f'[{pcolor}]{h_size}[/]'
|
||||
'[dim blue]u @[/]'
|
||||
f'[{pcolor}]{ppu}[/]'
|
||||
'[dim blue] = [/]'
|
||||
f'[{pcolor}]$ {cost_basis}\n[/]'
|
||||
)
|
||||
|
||||
console.print(summary)
|
||||
|
||||
finally:
|
||||
# exit via ctx cancellation.
|
||||
brokerd_ctx: tractor.Context = brokerd_stream._ctx
|
||||
await brokerd_ctx.cancel(timeout=1)
|
||||
|
||||
# TODO: once ported to newer tractor branch we should
|
||||
# be able to do a loop like this:
|
||||
# while brokerd_ctx.cancel_called_remote is None:
|
||||
# await trio.sleep(0.01)
|
||||
# await brokerd_ctx.cancel()
|
||||
|
||||
finally:
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@ledger.command()
|
||||
def disect(
|
||||
# "fully_qualified_account_name"
|
||||
fqan: str,
|
||||
bs_mktid: str, # for ib
|
||||
pdb: bool = False,
|
||||
|
||||
loglevel: str = typer.Option(
|
||||
'error',
|
||||
"-l",
|
||||
),
|
||||
):
|
||||
pair: tuple[str, str]
|
||||
if not (pair := unpack_fqan(fqan)):
|
||||
raise ValueError('{fqan} malformed!?')
|
||||
|
||||
brokername, account = pair
|
||||
|
||||
# ledger: TransactionLedger
|
||||
records: dict[str, dict]
|
||||
table: PpTable
|
||||
records, table = load_pps_from_ledger(
|
||||
brokername,
|
||||
account,
|
||||
filter_by_ids={bs_mktid},
|
||||
)
|
||||
df = pl.DataFrame(
|
||||
list(records.values()),
|
||||
# schema=[
|
||||
# ('tid', str),
|
||||
# ('fqme', str),
|
||||
# ('dt', str),
|
||||
# ('size', pl.Float64),
|
||||
# ('price', pl.Float64),
|
||||
# ('cost', pl.Float64),
|
||||
# ('expiry', str),
|
||||
# ('bs_mktid', str),
|
||||
# ],
|
||||
).select([
|
||||
pl.col('fqme'),
|
||||
pl.col('dt').str.to_datetime(),
|
||||
# pl.col('expiry').dt.datetime(),
|
||||
pl.col('size'),
|
||||
pl.col('price'),
|
||||
])
|
||||
|
||||
assert not df.is_empty()
|
||||
breakpoint()
|
||||
# tractor.pause_from_sync()
|
||||
# with open_trade_ledger(
|
||||
# brokername,
|
||||
# account,
|
||||
# ) as ledger:
|
||||
# for tid, rec in ledger.items():
|
||||
# bs_mktid: str = rec['bs_mktid']
|
||||
|
|
@ -17,95 +17,33 @@
|
|||
"""
|
||||
Broker clients, daemons and general back end machinery.
|
||||
"""
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from importlib import import_module
|
||||
from types import ModuleType
|
||||
|
||||
from tractor.trionics import maybe_open_context
|
||||
# TODO: move to urllib3/requests once supported
|
||||
import asks
|
||||
asks.init('trio')
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
BrokerError,
|
||||
SymbolNotFound,
|
||||
NoData,
|
||||
DataUnavailable,
|
||||
DataThrottle,
|
||||
resproc,
|
||||
get_logger,
|
||||
)
|
||||
|
||||
__all__: list[str] = [
|
||||
'BrokerError',
|
||||
'SymbolNotFound',
|
||||
'NoData',
|
||||
'DataUnavailable',
|
||||
'DataThrottle',
|
||||
'resproc',
|
||||
'get_logger',
|
||||
]
|
||||
|
||||
__brokers__: list[str] = [
|
||||
__brokers__ = [
|
||||
'binance',
|
||||
'questrade',
|
||||
'robinhood',
|
||||
'ib',
|
||||
'kraken',
|
||||
'kucoin'
|
||||
|
||||
# broken but used to work
|
||||
# 'questrade',
|
||||
# 'robinhood',
|
||||
|
||||
# TODO: we should get on these stat!
|
||||
# alpaca
|
||||
# wstrade
|
||||
# iex
|
||||
|
||||
# deribit
|
||||
# bitso
|
||||
]
|
||||
|
||||
|
||||
def get_brokermod(brokername: str) -> ModuleType:
|
||||
'''
|
||||
Return the imported broker module by name.
|
||||
|
||||
'''
|
||||
"""Return the imported broker module by name.
|
||||
"""
|
||||
module = import_module('.' + brokername, 'piker.brokers')
|
||||
# we only allow monkeying because it's for internal keying
|
||||
module.name = module.__name__.split('.')[-1]
|
||||
module.name = module.__name__.split('.')[-1]
|
||||
return module
|
||||
|
||||
|
||||
def iter_brokermods():
|
||||
'''
|
||||
Iterate all built-in broker modules.
|
||||
|
||||
'''
|
||||
"""Iterate all built-in broker modules.
|
||||
"""
|
||||
for name in __brokers__:
|
||||
yield get_brokermod(name)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_cached_client(
|
||||
brokername: str,
|
||||
**kwargs,
|
||||
|
||||
) -> 'Client': # noqa
|
||||
'''
|
||||
Get a cached broker client from the current actor's local vars.
|
||||
|
||||
If one has not been setup do it and cache it.
|
||||
|
||||
'''
|
||||
brokermod = get_brokermod(brokername)
|
||||
async with maybe_open_context(
|
||||
acm_func=brokermod.get_client,
|
||||
kwargs=kwargs,
|
||||
|
||||
) as (cache_hit, client):
|
||||
|
||||
if cache_hit:
|
||||
log.info(f'Reusing existing {client}')
|
||||
|
||||
yield client
|
||||
|
|
|
|||
|
|
@ -1,279 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Broker-daemon-actor "endpoint-hooks": the service task entry points for
|
||||
``brokerd``.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
AsyncContextManager,
|
||||
)
|
||||
import exceptiongroup as eg
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from . import _util
|
||||
from . import get_brokermod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..data import _FeedsBus
|
||||
|
||||
# `brokerd` enabled modules
|
||||
# TODO: move this def to the `.data` subpkg..
|
||||
# NOTE: keeping this list as small as possible is part of our caps-sec
|
||||
# model and should be treated with utmost care!
|
||||
_data_mods: str = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.brokers._daemon',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
# TODO: we should rename the daemon to datad prolly once we split up
|
||||
# broker vs. data tasks into separate actors?
|
||||
@tractor.context
|
||||
async def _setup_persistent_brokerd(
|
||||
ctx: tractor.Context,
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Allocate a actor-wide service nursery in ``brokerd``
|
||||
such that feeds can be run in the background persistently by
|
||||
the broker backend as needed.
|
||||
|
||||
'''
|
||||
# NOTE: we only need to setup logging once (and only) here
|
||||
# since all hosted daemon tasks will reference this same
|
||||
# log instance's (actor local) state and thus don't require
|
||||
# any further (level) configuration on their own B)
|
||||
log = _util.get_console_log(
|
||||
loglevel or tractor.current_actor().loglevel,
|
||||
name=f'{_util.subsys}.{brokername}',
|
||||
)
|
||||
|
||||
# set global for this actor to this new process-wide instance B)
|
||||
_util.log = log
|
||||
|
||||
# further, set the log level on any broker broker specific
|
||||
# logger instance.
|
||||
|
||||
from piker.data import feed
|
||||
assert not feed._bus
|
||||
|
||||
# allocate a nursery to the bus for spawning background
|
||||
# tasks to service client IPC requests, normally
|
||||
# `tractor.Context` connections to explicitly required
|
||||
# `brokerd` endpoints such as:
|
||||
# - `stream_quotes()`,
|
||||
# - `manage_history()`,
|
||||
# - `allocate_persistent_feed()`,
|
||||
# - `open_symbol_search()`
|
||||
# NOTE: see ep invocation details inside `.data.feed`.
|
||||
try:
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
bus: _FeedsBus = feed.get_feed_bus(
|
||||
brokername,
|
||||
service_nursery,
|
||||
)
|
||||
assert bus is feed._bus
|
||||
|
||||
# unblock caller
|
||||
await ctx.started()
|
||||
|
||||
# we pin this task to keep the feeds manager active until the
|
||||
# parent actor decides to tear it down
|
||||
await trio.sleep_forever()
|
||||
|
||||
except eg.ExceptionGroup:
|
||||
# TODO: likely some underlying `brokerd` IPC connection
|
||||
# broke so here we handle a respawn and re-connect attempt!
|
||||
# This likely should pair with development of the OCO task
|
||||
# nusery in dev over @ `tractor` B)
|
||||
# https://github.com/goodboy/tractor/pull/363
|
||||
raise
|
||||
|
||||
|
||||
def broker_init(
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**start_actor_kwargs,
|
||||
|
||||
) -> tuple[
|
||||
ModuleType,
|
||||
dict,
|
||||
AsyncContextManager,
|
||||
]:
|
||||
'''
|
||||
Given an input broker name, load all named arguments
|
||||
which can be passed for daemon endpoint + context spawn
|
||||
as required in every `brokerd` (actor) service.
|
||||
|
||||
This includes:
|
||||
- load the appropriate <brokername>.py pkg module,
|
||||
- reads any declared `__enable_modules__: listr[str]` which will be
|
||||
passed to `tractor.ActorNursery.start_actor(enabled_modules=<this>)`
|
||||
at actor start time,
|
||||
- deliver a references to the daemon lifetime fixture, which
|
||||
for now is always the `_setup_persistent_brokerd()` context defined
|
||||
above.
|
||||
|
||||
'''
|
||||
from ..brokers import get_brokermod
|
||||
brokermod = get_brokermod(brokername)
|
||||
modpath: str = brokermod.__name__
|
||||
|
||||
start_actor_kwargs['name'] = f'brokerd.{brokername}'
|
||||
start_actor_kwargs.update(
|
||||
getattr(
|
||||
brokermod,
|
||||
'_spawn_kwargs',
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
# XXX TODO: make this not so hacky/monkeypatched..
|
||||
# -> we need a sane way to configure the logging level for all
|
||||
# code running in brokerd.
|
||||
# if utilmod := getattr(brokermod, '_util', False):
|
||||
# utilmod.log.setLevel(loglevel.upper())
|
||||
|
||||
# lookup actor-enabled modules declared by the backend offering the
|
||||
# `brokerd` endpoint(s).
|
||||
enabled: list[str]
|
||||
enabled = start_actor_kwargs['enable_modules'] = [
|
||||
__name__, # so that eps from THIS mod can be invoked
|
||||
modpath,
|
||||
]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath: str = f'{modpath}.{submodname}'
|
||||
enabled.append(subpath)
|
||||
|
||||
# TODO XXX: DO WE NEED THIS?
|
||||
# enabled.append('piker.data.feed')
|
||||
|
||||
return (
|
||||
brokermod,
|
||||
start_actor_kwargs, # to `ActorNursery.start_actor()`
|
||||
|
||||
# XXX see impl above; contains all (actor global)
|
||||
# setup/teardown expected in all `brokerd` actor instances.
|
||||
_setup_persistent_brokerd,
|
||||
)
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
from piker.service._util import log # use service mngr log
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
(
|
||||
brokermode,
|
||||
tractor_kwargs,
|
||||
daemon_fixture_ep,
|
||||
) = broker_init(
|
||||
brokername,
|
||||
loglevel,
|
||||
**tractor_kwargs,
|
||||
)
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
from piker.service import Services
|
||||
|
||||
dname: str = tractor_kwargs.pop('name') # f'brokerd.{brokername}'
|
||||
portal = await Services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + tractor_kwargs.pop('enable_modules'),
|
||||
debug_mode=Services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# NOTE: the service mngr expects an already spawned actor + its
|
||||
# portal ref in order to do non-blocking setup of brokerd
|
||||
# service nursery.
|
||||
await Services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
daemon_fixture_ep,
|
||||
brokername=brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client who wishes to
|
||||
use the sub-actor-daemon but is fine with re-using any existing
|
||||
and contactable `brokerd`.
|
||||
|
||||
Mas o menos, acts as a cached-actor-getter factory.
|
||||
|
||||
'''
|
||||
from piker.service import maybe_spawn_daemon
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={
|
||||
'brokername': brokername,
|
||||
},
|
||||
loglevel=loglevel,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -15,31 +15,13 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Handy cross-broker utils.
|
||||
|
||||
Handy utils.
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
import json
|
||||
import asks
|
||||
import logging
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
colorize_json,
|
||||
)
|
||||
subsys: str = 'piker.brokers'
|
||||
|
||||
# NOTE: level should be reset by any actor that is spawned
|
||||
# as well as given a (more) explicit name/key such
|
||||
# as `piker.brokers.binance` matching the subpkg.
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
from ..log import colorize_json
|
||||
|
||||
|
||||
class BrokerError(Exception):
|
||||
|
|
@ -87,6 +69,7 @@ class DataThrottle(BrokerError):
|
|||
# TODO: add in throttle metrics/feedback
|
||||
|
||||
|
||||
|
||||
def resproc(
|
||||
resp: asks.response_objects.Response,
|
||||
log: logging.Logger,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,566 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Binance backend
|
||||
|
||||
"""
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
Any, Union, Optional,
|
||||
AsyncGenerator, Callable,
|
||||
)
|
||||
import time
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import pendulum
|
||||
import asks
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
from pydantic.dataclasses import dataclass
|
||||
from pydantic import BaseModel
|
||||
import wsproto
|
||||
|
||||
from .._cacheables import open_cached_client
|
||||
from ._util import resproc, SymbolNotFound
|
||||
from ..log import get_logger, get_console_log
|
||||
from ..data import ShmArray
|
||||
from ..data._web_bs import open_autorecon_ws, NoBsWs
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_url = 'https://api.binance.com'
|
||||
|
||||
|
||||
# Broker specific ohlc schema (rest)
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('bar_wap', float), # will be zeroed by sampler if not filled
|
||||
|
||||
# XXX: some additional fields are defined in the docs:
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
|
||||
# ('close_time', int),
|
||||
# ('quote_vol', float),
|
||||
# ('num_trades', int),
|
||||
# ('buy_base_vol', float),
|
||||
# ('buy_quote_vol', float),
|
||||
# ('ignore', float),
|
||||
]
|
||||
|
||||
# UI components allow this to be declared such that additional
|
||||
# (historical) fields can be exposed.
|
||||
ohlc_dtype = np.dtype(_ohlc_dtype)
|
||||
|
||||
_show_wap_in_history = False
|
||||
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
class Pair(BaseModel):
|
||||
symbol: str
|
||||
status: str
|
||||
|
||||
baseAsset: str
|
||||
baseAssetPrecision: int
|
||||
quoteAsset: str
|
||||
quotePrecision: int
|
||||
quoteAssetPrecision: int
|
||||
|
||||
baseCommissionPrecision: int
|
||||
quoteCommissionPrecision: int
|
||||
|
||||
orderTypes: list[str]
|
||||
|
||||
icebergAllowed: bool
|
||||
ocoAllowed: bool
|
||||
quoteOrderQtyMarketAllowed: bool
|
||||
isSpotTradingAllowed: bool
|
||||
isMarginTradingAllowed: bool
|
||||
|
||||
filters: list[dict[str, Union[str, int, float]]]
|
||||
permissions: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class OHLC:
|
||||
"""Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-streams
|
||||
|
||||
"""
|
||||
time: int
|
||||
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float
|
||||
|
||||
close_time: int
|
||||
|
||||
quote_vol: float
|
||||
num_trades: int
|
||||
buy_base_vol: float
|
||||
buy_quote_vol: float
|
||||
ignore: int
|
||||
|
||||
# null the place holder for `bar_wap` until we
|
||||
# figure out what to extract for this.
|
||||
bar_wap: float = 0.0
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def binance_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._pairs: dict[str, Any] = {}
|
||||
|
||||
async def _api(
|
||||
self,
|
||||
method: str,
|
||||
params: dict,
|
||||
) -> dict[str, Any]:
|
||||
resp = await self._sesh.get(
|
||||
path=f'/api/v3/{method}',
|
||||
params=params,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def symbol_info(
|
||||
|
||||
self,
|
||||
sym: Optional[str] = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''Get symbol info for the exchange.
|
||||
|
||||
'''
|
||||
# TODO: we can load from our self._pairs cache
|
||||
# on repeat calls...
|
||||
|
||||
# will retrieve all symbols by default
|
||||
params = {}
|
||||
|
||||
if sym is not None:
|
||||
sym = sym.upper()
|
||||
params = {'symbol': sym}
|
||||
|
||||
resp = await self._api(
|
||||
'exchangeInfo',
|
||||
params=params,
|
||||
)
|
||||
|
||||
entries = resp['symbols']
|
||||
if not entries:
|
||||
raise SymbolNotFound(f'{sym} not found')
|
||||
|
||||
syms = {item['symbol']: item for item in entries}
|
||||
|
||||
if sym is not None:
|
||||
return syms[sym]
|
||||
else:
|
||||
return syms
|
||||
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = None,
|
||||
) -> dict[str, Any]:
|
||||
if self._pairs is not None:
|
||||
data = self._pairs
|
||||
else:
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
data,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0]['symbol']: item[0]
|
||||
for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str,
|
||||
start_dt: Optional[datetime] = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
limit: int = 1000, # <- max allowed per query
|
||||
as_np: bool = True,
|
||||
|
||||
) -> dict:
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_time = binance_timestamp(start_dt)
|
||||
end_time = binance_timestamp(end_dt)
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
bars = await self._api(
|
||||
'klines',
|
||||
params={
|
||||
'symbol': symbol.upper(),
|
||||
'interval': '1m',
|
||||
'startTime': start_time,
|
||||
'endTime': end_time,
|
||||
'limit': limit
|
||||
}
|
||||
)
|
||||
|
||||
# TODO: pack this bars scheme into a ``pydantic`` validator type:
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
|
||||
# TODO: we should port this to ``pydantic`` to avoid doing
|
||||
# manual validation ourselves..
|
||||
new_bars = []
|
||||
for i, bar in enumerate(bars):
|
||||
|
||||
bar = OHLC(*bar)
|
||||
|
||||
row = []
|
||||
for j, (name, ftype) in enumerate(_ohlc_dtype[1:]):
|
||||
|
||||
# TODO: maybe we should go nanoseconds on all
|
||||
# history time stamps?
|
||||
if name == 'time':
|
||||
# convert to epoch seconds: float
|
||||
row.append(bar.time / 1000.0)
|
||||
|
||||
else:
|
||||
row.append(getattr(bar, name))
|
||||
|
||||
new_bars.append((i,) + tuple(row))
|
||||
|
||||
array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars
|
||||
return array
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> Client:
|
||||
client = Client()
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
|
||||
|
||||
# validation type
|
||||
class AggTrade(BaseModel):
|
||||
e: str # Event type
|
||||
E: int # Event time
|
||||
s: str # Symbol
|
||||
a: int # Aggregate trade ID
|
||||
p: float # Price
|
||||
q: float # Quantity
|
||||
f: int # First trade ID
|
||||
l: int # Last trade ID
|
||||
T: int # Trade time
|
||||
m: bool # Is the buyer the market maker?
|
||||
M: bool # Ignore
|
||||
|
||||
|
||||
async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]:
|
||||
|
||||
timeouts = 0
|
||||
while True:
|
||||
|
||||
with trio.move_on_after(3) as cs:
|
||||
msg = await ws.recv_msg()
|
||||
|
||||
if cs.cancelled_caught:
|
||||
|
||||
timeouts += 1
|
||||
if timeouts > 2:
|
||||
log.error("binance feed seems down and slow af? rebooting...")
|
||||
await ws._connect()
|
||||
|
||||
continue
|
||||
|
||||
# for l1 streams binance doesn't add an event type field so
|
||||
# identify those messages by matching keys
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams
|
||||
|
||||
if msg.get('u'):
|
||||
sym = msg['s']
|
||||
bid = float(msg['b'])
|
||||
bsize = float(msg['B'])
|
||||
ask = float(msg['a'])
|
||||
asize = float(msg['A'])
|
||||
|
||||
yield 'l1', {
|
||||
'symbol': sym,
|
||||
'ticks': [
|
||||
{'type': 'bid', 'price': bid, 'size': bsize},
|
||||
{'type': 'bsize', 'price': bid, 'size': bsize},
|
||||
{'type': 'ask', 'price': ask, 'size': asize},
|
||||
{'type': 'asize', 'price': ask, 'size': asize}
|
||||
]
|
||||
}
|
||||
|
||||
elif msg.get('e') == 'aggTrade':
|
||||
|
||||
# validate
|
||||
msg = AggTrade(**msg)
|
||||
|
||||
# TODO: type out and require this quote format
|
||||
# from all backends!
|
||||
yield 'trade', {
|
||||
'symbol': msg.s,
|
||||
'last': msg.p,
|
||||
'brokerd_ts': time.time(),
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': msg.p,
|
||||
'size': msg.q,
|
||||
'broker_ts': msg.T,
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def make_sub(pairs: list[str], sub_name: str, uid: int) -> dict[str, str]:
|
||||
"""Create a request subscription packet dict.
|
||||
|
||||
https://binance-docs.github.io/apidocs/spot/en/#live-subscribing-unsubscribing-to-streams
|
||||
"""
|
||||
return {
|
||||
'method': 'SUBSCRIBE',
|
||||
'params': [
|
||||
f'{pair.lower()}@{sub_name}'
|
||||
for pair in pairs
|
||||
],
|
||||
'id': uid
|
||||
}
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
symbol: str,
|
||||
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
async def backfill_bars(
|
||||
sym: str,
|
||||
shm: ShmArray, # type: ignore # noqa
|
||||
task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
"""Fill historical bars into shared mem / storage afap.
|
||||
"""
|
||||
with trio.CancelScope() as cs:
|
||||
async with open_cached_client('binance') as client:
|
||||
bars = await client.bars(symbol=sym)
|
||||
shm.push(bars)
|
||||
task_status.started(cs)
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym_infos = {}
|
||||
uid = 0
|
||||
|
||||
async with (
|
||||
open_cached_client('binance') as client,
|
||||
send_chan as send_chan,
|
||||
):
|
||||
|
||||
# keep client cached for real-time section
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
for sym in symbols:
|
||||
d = cache[sym.upper()]
|
||||
syminfo = Pair(**d) # validation
|
||||
|
||||
si = sym_infos[sym] = syminfo.dict()
|
||||
|
||||
# XXX: after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
|
||||
si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
|
||||
si['asset_type'] = 'crypto'
|
||||
|
||||
symbol = symbols[0]
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
symbol: {
|
||||
'symbol_info': sym_infos[sym],
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: wsproto.WSConnection):
|
||||
# setup subs
|
||||
|
||||
# trade data (aka L1)
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#symbol-order-book-ticker
|
||||
l1_sub = make_sub(symbols, 'bookTicker', uid)
|
||||
await ws.send_msg(l1_sub)
|
||||
|
||||
# aggregate (each order clear by taker **not** by maker)
|
||||
# trades data:
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#aggregate-trade-streams
|
||||
agg_trades_sub = make_sub(symbols, 'aggTrade', uid)
|
||||
await ws.send_msg(agg_trades_sub)
|
||||
|
||||
# ack from ws server
|
||||
res = await ws.recv_msg()
|
||||
assert res['id'] == uid
|
||||
|
||||
yield
|
||||
|
||||
subs = []
|
||||
for sym in symbols:
|
||||
subs.append("{sym}@aggTrade")
|
||||
subs.append("{sym}@bookTicker")
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
await ws.send_msg({
|
||||
"method": "UNSUBSCRIBE",
|
||||
"params": subs,
|
||||
"id": uid,
|
||||
})
|
||||
|
||||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
async with open_autorecon_ws(
|
||||
'wss://stream.binance.com/ws',
|
||||
fixture=subscribe,
|
||||
) as ws:
|
||||
|
||||
# pull a first quote and deliver
|
||||
msg_gen = stream_messages(ws)
|
||||
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
while typ != 'trade':
|
||||
# TODO: use ``anext()`` when it lands in 3.10!
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
# signal to caller feed is ready for consumption
|
||||
feed_is_live.set()
|
||||
|
||||
# import time
|
||||
# last = time.time()
|
||||
|
||||
# start streaming
|
||||
async for typ, msg in msg_gen:
|
||||
|
||||
# period = time.time() - last
|
||||
# hz = 1/period if period else float('inf')
|
||||
# if hz > 60:
|
||||
# log.info(f'Binance quotez : {hz}')
|
||||
|
||||
topic = msg['symbol'].lower()
|
||||
await send_chan.send({topic: msg})
|
||||
# last = time.time()
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> Client:
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# results = await client.symbol_info(sym=pattern.upper())
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
cache,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send(
|
||||
{item[0]['symbol']: item[0]
|
||||
for item in matches}
|
||||
)
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez (aka ze jefe)
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
binancial secs on the floor, in the office, behind the dumpster.
|
||||
|
||||
"""
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
get_mkt_info,
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
open_trade_dialog,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
'get_mkt_info',
|
||||
'open_trade_dialog',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
]
|
||||
|
||||
|
||||
# `brokerd` modules
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
|
@ -1,895 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez (aka ze jefe)
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Binance clients for http and ws APIs.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import ChainMap
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from datetime import datetime
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Type,
|
||||
)
|
||||
import hmac
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import trio
|
||||
from pendulum import (
|
||||
now,
|
||||
)
|
||||
import asks
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
|
||||
from piker import config
|
||||
from piker.clearing._messages import (
|
||||
Order,
|
||||
)
|
||||
from piker.accounting import (
|
||||
Asset,
|
||||
digits_to_dec,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
from piker.data import def_iohlcv_fields
|
||||
from piker.brokers import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
get_logger,
|
||||
)
|
||||
from .venues import (
|
||||
PAIRTYPES,
|
||||
Pair,
|
||||
MarketType,
|
||||
|
||||
_spot_url,
|
||||
_futes_url,
|
||||
_testnet_futes_url,
|
||||
_testnet_spot_url,
|
||||
)
|
||||
|
||||
log = get_logger('piker.brokers.binance')
|
||||
|
||||
|
||||
def get_config() -> dict:
|
||||
|
||||
conf: dict
|
||||
path: Path
|
||||
conf, path = config.load()
|
||||
|
||||
section = conf.get('binance')
|
||||
|
||||
if not section:
|
||||
log.warning(f'No config section found for binance in {path}')
|
||||
return {}
|
||||
|
||||
return section
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# Broker specific ohlc schema (rest)
|
||||
# XXX TODO? some additional fields are defined in the docs:
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
|
||||
# _ohlc_dtype = [
|
||||
# ('close_time', int),
|
||||
# ('quote_vol', float),
|
||||
# ('num_trades', int),
|
||||
# ('buy_base_vol', float),
|
||||
# ('buy_quote_vol', float),
|
||||
# ('ignore', float),
|
||||
# ]
|
||||
|
||||
|
||||
class OHLC(Struct):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-streams
|
||||
|
||||
'''
|
||||
time: int # epoch in ms
|
||||
|
||||
open: float
|
||||
high: float
|
||||
low: float
|
||||
close: float
|
||||
volume: float
|
||||
|
||||
close_time: int
|
||||
|
||||
quote_vol: float
|
||||
num_trades: int
|
||||
buy_base_vol: float
|
||||
buy_quote_vol: float
|
||||
ignore: int
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def binance_timestamp(
|
||||
when: datetime
|
||||
) -> int:
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
class Client:
|
||||
'''
|
||||
Async ReST API client using ``trio`` + ``asks`` B)
|
||||
|
||||
Supports all of the spot, margin and futures endpoints depending
|
||||
on method.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
# TODO: change this to `Client.[mkt_]venue: MarketType`?
|
||||
mkt_mode: MarketType = 'spot',
|
||||
|
||||
) -> None:
|
||||
# build out pair info tables for each market type
|
||||
# and wrap in a chain-map view for search / query.
|
||||
self._spot_pairs: dict[str, Pair] = {} # spot info table
|
||||
self._ufutes_pairs: dict[str, Pair] = {} # usd-futures table
|
||||
self._venue2pairs: dict[str, dict] = {
|
||||
'spot': self._spot_pairs,
|
||||
'usdtm_futes': self._ufutes_pairs,
|
||||
}
|
||||
|
||||
self._venue2assets: dict[
|
||||
str,
|
||||
dict[str, dict] | None,
|
||||
] = {
|
||||
# NOTE: only the spot table contains a dict[str, Asset]
|
||||
# since others (like futes, opts) can just do lookups
|
||||
# from a list of names to the spot equivalent.
|
||||
'spot': {},
|
||||
'usdtm_futes': {},
|
||||
# 'coinm_futes': {},
|
||||
}
|
||||
|
||||
# NOTE: only stick in the spot table for now until exchange info
|
||||
# is loaded, since at that point we'll suffix all the futes
|
||||
# market symbols for use by search. See `.exch_info()`.
|
||||
self._pairs: ChainMap[str, Pair] = ChainMap()
|
||||
|
||||
# spot EPs sesh
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location: str = _spot_url
|
||||
# spot testnet
|
||||
self._test_sesh: asks.Session = asks.Session(connections=4)
|
||||
self._test_sesh.base_location: str = _testnet_spot_url
|
||||
|
||||
# margin and extended spot endpoints session.
|
||||
self._sapi_sesh = asks.Session(connections=4)
|
||||
self._sapi_sesh.base_location: str = _spot_url
|
||||
|
||||
# futes EPs sesh
|
||||
self._fapi_sesh = asks.Session(connections=4)
|
||||
self._fapi_sesh.base_location: str = _futes_url
|
||||
# futes testnet
|
||||
self._test_fapi_sesh: asks.Session = asks.Session(connections=4)
|
||||
self._test_fapi_sesh.base_location: str = _testnet_futes_url
|
||||
|
||||
# global client "venue selection" mode.
|
||||
# set this when you want to switch venues and not have to
|
||||
# specify the venue for the next request.
|
||||
self.mkt_mode: MarketType = mkt_mode
|
||||
|
||||
# per 8
|
||||
self.venue_sesh: dict[
|
||||
str, # venue key
|
||||
tuple[asks.Session, str] # session, eps path
|
||||
] = {
|
||||
'spot': (self._sesh, '/api/v3/'),
|
||||
'spot_testnet': (self._test_sesh, '/fapi/v1/'),
|
||||
|
||||
'margin': (self._sapi_sesh, '/sapi/v1/'),
|
||||
|
||||
'usdtm_futes': (self._fapi_sesh, '/fapi/v1/'),
|
||||
'usdtm_futes_testnet': (self._test_fapi_sesh, '/fapi/v1/'),
|
||||
|
||||
# 'futes_coin': self._dapi, # TODO
|
||||
}
|
||||
|
||||
# lookup for going from `.mkt_mode: str` to the config
|
||||
# subsection `key: str`
|
||||
self.venue2configkey: dict[str, str] = {
|
||||
'spot': 'spot',
|
||||
'margin': 'spot',
|
||||
'usdtm_futes': 'futes',
|
||||
# 'coinm_futes': 'futes',
|
||||
}
|
||||
self.confkey2venuekeys: dict[str, list[str]] = {
|
||||
'spot': ['spot'], # 'margin'],
|
||||
'futes': ['usdtm_futes'],
|
||||
}
|
||||
|
||||
# for creating API keys see,
|
||||
# https://www.binance.com/en/support/faq/how-to-create-api-keys-on-binance-360002502072
|
||||
self.conf: dict = get_config()
|
||||
|
||||
for key, subconf in self.conf.items():
|
||||
if api_key := subconf.get('api_key', ''):
|
||||
venue_keys: list[str] = self.confkey2venuekeys[key]
|
||||
|
||||
venue_key: str
|
||||
sesh: asks.Session
|
||||
for venue_key in venue_keys:
|
||||
sesh, _ = self.venue_sesh[venue_key]
|
||||
|
||||
api_key_header: dict = {
|
||||
# taken from official:
|
||||
# https://github.com/binance/binance-futures-connector-python/blob/main/binance/api.py#L47
|
||||
"Content-Type": "application/json;charset=utf-8",
|
||||
|
||||
# TODO: prolly should just always query and copy
|
||||
# in the real latest ver?
|
||||
"User-Agent": "binance-connector/6.1.6smbz6",
|
||||
"X-MBX-APIKEY": api_key,
|
||||
}
|
||||
sesh.headers.update(api_key_header)
|
||||
|
||||
# if `.use_tesnet = true` in the config then
|
||||
# also add headers for the testnet session which
|
||||
# will be used for all order control
|
||||
if subconf.get('use_testnet', False):
|
||||
testnet_sesh, _ = self.venue_sesh[
|
||||
venue_key + '_testnet'
|
||||
]
|
||||
testnet_sesh.headers.update(api_key_header)
|
||||
|
||||
def _mk_sig(
|
||||
self,
|
||||
data: dict,
|
||||
venue: str,
|
||||
|
||||
) -> str:
|
||||
|
||||
# look up subconfig (spot or futes) section using
|
||||
# venue specific key lookup to figure out which mkt
|
||||
# we need a key for.
|
||||
section_name: str = self.venue2configkey[venue]
|
||||
subconf: dict | None = self.conf.get(section_name)
|
||||
if subconf is None:
|
||||
raise config.ConfigurationError(
|
||||
f'binance configuration is missing a `{section_name}` section '
|
||||
'to define the creds for auth-ed endpoints!?'
|
||||
)
|
||||
|
||||
|
||||
# XXX: Info on security and authentification
|
||||
# https://binance-docs.github.io/apidocs/#endpoint-security-type
|
||||
if not (api_secret := subconf.get('api_secret')):
|
||||
raise config.NoSignature(
|
||||
"Can't generate a signature without setting up credentials"
|
||||
)
|
||||
|
||||
query_str: str = '&'.join([
|
||||
f'{key}={value}'
|
||||
for key, value in data.items()
|
||||
])
|
||||
|
||||
msg_auth = hmac.new(
|
||||
api_secret.encode('utf-8'),
|
||||
query_str.encode('utf-8'),
|
||||
hashlib.sha256
|
||||
)
|
||||
return msg_auth.hexdigest()
|
||||
|
||||
# TODO: factor all these _api methods into a single impl
|
||||
# which looks up the parent path for eps depending on a
|
||||
# mkt_mode: MarketType input!
|
||||
async def _api(
|
||||
self,
|
||||
endpoint: str, # ReST endpoint key
|
||||
params: dict,
|
||||
|
||||
method: str = 'get',
|
||||
venue: str | None = None, # if None use `.mkt_mode` state
|
||||
signed: bool = False,
|
||||
allow_testnet: bool = False,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Make a ReST API request via
|
||||
- a /api/v3/ SPOT, or
|
||||
- /fapi/v3/ USD-M FUTURES, or
|
||||
- /api/v3/ SPOT/MARGIN
|
||||
|
||||
account/market endpoint request depending on either passed in `venue: str`
|
||||
or the current setting `.mkt_mode: str` setting, default `'spot'`.
|
||||
|
||||
|
||||
Docs per venue API:
|
||||
|
||||
SPOT: market-data and spot-account-trade eps use this
|
||||
---- endpoing parent path:
|
||||
- https://binance-docs.github.io/apidocs/spot/en/#market-data-endpoints
|
||||
- https://binance-docs.github.io/apidocs/spot/en/#spot-account-trade
|
||||
|
||||
MARGIN: and advancecd spot account eps:
|
||||
------
|
||||
- https://binance-docs.github.io/apidocs/spot/en/#margin-account-trade
|
||||
- https://binance-docs.github.io/apidocs/spot/en/#listen-key-spot
|
||||
- https://binance-docs.github.io/apidocs/spot/en/#spot-algo-endpoints
|
||||
|
||||
USD-M FUTES:
|
||||
-----------
|
||||
- https://binance-docs.github.io/apidocs/futures/en/#market-data-endpoints
|
||||
|
||||
'''
|
||||
venue_key: str = venue or self.mkt_mode
|
||||
|
||||
if signed:
|
||||
params['signature'] = self._mk_sig(
|
||||
params,
|
||||
venue=venue_key,
|
||||
)
|
||||
|
||||
sesh: asks.Session
|
||||
path: str
|
||||
|
||||
# Check if we're configured to route order requests to the
|
||||
# venue equivalent's testnet.
|
||||
use_testnet: bool = False
|
||||
section_name: str = self.venue2configkey[venue_key]
|
||||
if subconf := self.conf.get(section_name):
|
||||
use_testnet = (
|
||||
subconf.get('use_testnet', False)
|
||||
and allow_testnet
|
||||
)
|
||||
|
||||
if (
|
||||
use_testnet
|
||||
and method not in {
|
||||
'klines',
|
||||
'exchangeInfo',
|
||||
}
|
||||
):
|
||||
# NOTE: only use testnet if user set brokers.toml config
|
||||
# var to true **and** it's not one of the market data
|
||||
# endpoints since we basically never want to display the
|
||||
# test net feeds, we only are using it for testing order
|
||||
# ctl machinery B)
|
||||
venue_key += '_testnet'
|
||||
|
||||
sesh, path = self.venue_sesh[venue_key]
|
||||
|
||||
meth: Callable = getattr(sesh, method)
|
||||
resp = await meth(
|
||||
path=path + endpoint,
|
||||
params=params,
|
||||
timeout=float('inf'),
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def _cache_pairs(
|
||||
self,
|
||||
venue: str,
|
||||
|
||||
) -> None:
|
||||
# lookup internal mkt-specific pair table to update
|
||||
pair_table: dict[str, Pair] = self._venue2pairs[venue]
|
||||
asset_table: dict[str, Asset] = self._venue2assets[venue]
|
||||
|
||||
# make API request(s)
|
||||
resp = await self._api(
|
||||
'exchangeInfo',
|
||||
params={}, # NOTE: retrieve all symbols by default
|
||||
# XXX: MUST explicitly pass the routing venue since we
|
||||
# don't know the routing mode but want to cache market
|
||||
# infos across all venues
|
||||
venue=venue,
|
||||
allow_testnet=False, # XXX: never use testnet for symbol lookups
|
||||
)
|
||||
mkt_pairs = resp['symbols']
|
||||
if not mkt_pairs:
|
||||
raise SymbolNotFound(f'No market pairs found!?:\n{resp}')
|
||||
|
||||
pairs_view_subtable: dict[str, Pair] = {}
|
||||
|
||||
for item in mkt_pairs:
|
||||
filters_ls: list = item.pop('filters', False)
|
||||
if filters_ls:
|
||||
filters = {}
|
||||
for entry in filters_ls:
|
||||
ftype = entry['filterType']
|
||||
filters[ftype] = entry
|
||||
|
||||
item['filters'] = filters
|
||||
|
||||
pair_type: Type = PAIRTYPES[venue]
|
||||
pair: Pair = pair_type(**item)
|
||||
pair_table[pair.symbol.upper()] = pair
|
||||
|
||||
# update an additional top-level-cross-venue-table
|
||||
# `._pairs: ChainMap` for search B0
|
||||
pairs_view_subtable[pair.bs_fqme] = pair
|
||||
|
||||
if venue == 'spot':
|
||||
if (name := pair.quoteAsset) not in asset_table:
|
||||
asset_table[name] = Asset(
|
||||
name=name,
|
||||
atype='crypto_currency',
|
||||
tx_tick=digits_to_dec(pair.quoteAssetPrecision),
|
||||
)
|
||||
|
||||
if (name := pair.baseAsset) not in asset_table:
|
||||
asset_table[name] = Asset(
|
||||
name=name,
|
||||
atype='crypto_currency',
|
||||
tx_tick=digits_to_dec(pair.baseAssetPrecision),
|
||||
)
|
||||
|
||||
# NOTE: make merged view of all market-type pairs but
|
||||
# use market specific `Pair.bs_fqme` for keys!
|
||||
# this allows searching for market pairs with different
|
||||
# suffixes easily, for ex. `BTCUSDT.USDTM.PERP` will show
|
||||
# up when a user uses the search endpoint with pattern
|
||||
# `btc` B)
|
||||
self._pairs.maps.append(pairs_view_subtable)
|
||||
|
||||
if venue == 'spot':
|
||||
return
|
||||
|
||||
assets: list[dict] = resp.get('assets', ())
|
||||
for entry in assets:
|
||||
name: str = entry['asset']
|
||||
asset_table[name] = self._venue2assets['spot'].get(name)
|
||||
|
||||
async def exch_info(
|
||||
self,
|
||||
sym: str | None = None,
|
||||
|
||||
venue: MarketType | None = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
'''
|
||||
Fresh exchange-pairs info query for symbol ``sym: str``.
|
||||
|
||||
Depending on `mkt_type` different api eps are used:
|
||||
- spot:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
- usd futes:
|
||||
https://binance-docs.github.io/apidocs/futures/en/#check-server-time
|
||||
- coin futes:
|
||||
https://binance-docs.github.io/apidocs/delivery/en/#exchange-information
|
||||
|
||||
'''
|
||||
pair_table: dict[str, Pair] = self._venue2pairs[
|
||||
venue or self.mkt_mode
|
||||
]
|
||||
if cached_pair := pair_table.get(sym):
|
||||
return cached_pair
|
||||
|
||||
venues: list[str] = ['spot', 'usdtm_futes']
|
||||
if venue:
|
||||
venues: list[str] = [venue]
|
||||
|
||||
# batch per-venue download of all exchange infos
|
||||
async with trio.open_nursery() as rn:
|
||||
for ven in venues:
|
||||
rn.start_soon(
|
||||
self._cache_pairs,
|
||||
ven,
|
||||
)
|
||||
|
||||
return pair_table[sym] if sym else self._pairs
|
||||
|
||||
# TODO: unused except by `brokers.core.search_symbols()`?
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
|
||||
fq_pairs: dict = await self.exch_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
fq_pairs,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0]['symbol']: item[0]
|
||||
for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str,
|
||||
|
||||
start_dt: datetime | None = None,
|
||||
end_dt: datetime | None = None,
|
||||
|
||||
as_np: bool = True,
|
||||
|
||||
) -> list[tuple] | np.ndarray:
|
||||
|
||||
# NOTE: diff market-venues have diff datums limits:
|
||||
# - spot max is 1k
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
# - usdm futes max is 1500
|
||||
# https://binance-docs.github.io/apidocs/futures/en/#kline-candlestick-data
|
||||
limits: dict[str, int] = {
|
||||
'spot': 1000,
|
||||
'usdtm_futes': 1500,
|
||||
}
|
||||
limit = limits[self.mkt_mode]
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = now('UTC').add(minutes=1)
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_time = binance_timestamp(start_dt)
|
||||
end_time = binance_timestamp(end_dt)
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-data
|
||||
bars = await self._api(
|
||||
'klines',
|
||||
params={
|
||||
'symbol': symbol.upper(),
|
||||
'interval': '1m',
|
||||
'startTime': start_time,
|
||||
'endTime': end_time,
|
||||
'limit': limit
|
||||
},
|
||||
allow_testnet=False,
|
||||
)
|
||||
new_bars: list[tuple] = []
|
||||
for i, bar_list in enumerate(bars):
|
||||
|
||||
bar = OHLC(*bar_list)
|
||||
bar.typecast()
|
||||
|
||||
row = []
|
||||
for j, (name, ftype) in enumerate(def_iohlcv_fields[1:]):
|
||||
|
||||
# TODO: maybe we should go nanoseconds on all
|
||||
# history time stamps?
|
||||
if name == 'time':
|
||||
# convert to epoch seconds: float
|
||||
row.append(bar.time / 1000.0)
|
||||
|
||||
else:
|
||||
row.append(getattr(bar, name))
|
||||
|
||||
new_bars.append((i,) + tuple(row))
|
||||
|
||||
if not as_np:
|
||||
return bars
|
||||
|
||||
return np.array(
|
||||
new_bars,
|
||||
dtype=def_iohlcv_fields,
|
||||
)
|
||||
|
||||
# TODO: maybe drop? Do we need this if we can simply request it
|
||||
# over the user stream wss?
|
||||
# async def get_positions(
|
||||
# self,
|
||||
# symbol: str,
|
||||
# recv_window: int = 60000
|
||||
|
||||
# ) -> tuple:
|
||||
|
||||
# positions = {}
|
||||
# volumes = {}
|
||||
|
||||
# params = dict([
|
||||
# ('symbol', symbol),
|
||||
# ('recvWindow', recv_window),
|
||||
# ('timestamp', binance_timestamp(now()))
|
||||
# ])
|
||||
# resp = await self._api(
|
||||
# 'allOrders',
|
||||
# params=params,
|
||||
# signed=True
|
||||
# )
|
||||
# log.info(f'done. len {len(resp)}')
|
||||
|
||||
# return positions, volumes
|
||||
|
||||
async def get_deposits(
|
||||
self,
|
||||
recv_window: int = 60000
|
||||
) -> list:
|
||||
|
||||
# TODO: can't we drop this since normal dicts are
|
||||
# ordered implicitly in mordern python?
|
||||
params = dict([
|
||||
('recvWindow', recv_window),
|
||||
('timestamp', binance_timestamp(now()))
|
||||
])
|
||||
return await self._sapi(
|
||||
'capital/deposit/hisrec',
|
||||
params=params,
|
||||
signed=True,
|
||||
)
|
||||
|
||||
async def get_withdrawls(
|
||||
self,
|
||||
recv_window: int = 60000
|
||||
|
||||
) -> list:
|
||||
|
||||
params = dict([
|
||||
('recvWindow', recv_window),
|
||||
('timestamp', binance_timestamp(now()))
|
||||
])
|
||||
return await self._sapi(
|
||||
'capital/withdraw/history',
|
||||
params=params,
|
||||
signed=True,
|
||||
)
|
||||
|
||||
async def get_open_orders(
|
||||
self,
|
||||
symbol: str | None = None,
|
||||
|
||||
) -> list[Order]:
|
||||
'''
|
||||
Get all open orders for venue-account.
|
||||
|
||||
WARNING: apparently not specifying the symbol is given
|
||||
a much heavier API "weight" meaning you shouldn't call it
|
||||
often to avoid getting throttled as per:
|
||||
|
||||
'https://binance-docs.github.io/apidocs/futures/en/#current-all-open-orders-user_data
|
||||
|
||||
|
||||
'''
|
||||
params: dict[str, Any] = {
|
||||
'timestamp': binance_timestamp(now()),
|
||||
}
|
||||
if symbol is not None:
|
||||
params['symbol'] = symbol
|
||||
|
||||
resp = await self._api(
|
||||
'openOrders',
|
||||
params=params,
|
||||
signed=True,
|
||||
method='get',
|
||||
allow_testnet=True,
|
||||
)
|
||||
# figure out which venue (in FQME terms) we're using
|
||||
# since that normally maps 1-to-1 with the account (right?)
|
||||
venue: str = self.mkt_mode.rstrip('_futes')
|
||||
|
||||
orders: list[Order] = []
|
||||
for entry in resp:
|
||||
oid: str = entry['clientOrderId']
|
||||
symbol: str = entry['symbol']
|
||||
|
||||
# build out a fqme-styled key that should map to a pair
|
||||
# entry in `._pairs` cross-venue table.
|
||||
bs_mktid, _, expiry = entry['symbol'].partition('_')
|
||||
bs_mktid += f'.{venue.upper()}'
|
||||
|
||||
if expiry:
|
||||
bs_mktid += f'.{expiry}'
|
||||
else:
|
||||
bs_mktid += '.PERP'
|
||||
|
||||
# should never key error if we've got it right B)
|
||||
pair: Pair = self._pairs[bs_mktid]
|
||||
|
||||
orders.append(
|
||||
Order(
|
||||
oid=oid,
|
||||
symbol=pair.bs_fqme.lower(),
|
||||
|
||||
action=entry['side'].lower(),
|
||||
price=float(entry['price']),
|
||||
size=float(entry['origQty']),
|
||||
|
||||
exec_mode='live',
|
||||
account=f'binance.{venue}',
|
||||
)
|
||||
)
|
||||
return orders
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
side: str, # sell / buy
|
||||
quantity: float,
|
||||
price: float,
|
||||
|
||||
oid: int | None = None,
|
||||
tif: str = 'GTC',
|
||||
recv_window: int = 60000,
|
||||
|
||||
# iceberg_quantity: float | None = None,
|
||||
resp_type: str = 'ACK',
|
||||
|
||||
# TODO: this is probably useful for doing stops, maybe we
|
||||
# can set it only on dark-stops?
|
||||
# close_all: bool = False,
|
||||
|
||||
modify: bool = False,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Submit or modify a live limit order to ze binance.
|
||||
|
||||
For modify see:
|
||||
- spot:
|
||||
- futes https://binance-docs.github.io/apidocs/futures/en/#modify-order-trade
|
||||
|
||||
'''
|
||||
# lookup the binance-native symbol from search table
|
||||
bs_mktid: str = self._pairs[symbol.upper()].symbol
|
||||
params: dict = dict([
|
||||
('symbol', bs_mktid),
|
||||
('side', side.upper()),
|
||||
('type', 'LIMIT'),
|
||||
('timeInForce', tif),
|
||||
('quantity', quantity),
|
||||
('price', price),
|
||||
('recvWindow', recv_window),
|
||||
('newOrderRespType', resp_type),
|
||||
('timestamp', binance_timestamp(now()))
|
||||
|
||||
# ('closeAll', close_all),
|
||||
])
|
||||
|
||||
method: str = 'post'
|
||||
|
||||
# NOTE: modifies only require diff key for user oid:
|
||||
# https://binance-docs.github.io/apidocs/futures/en/#modify-order-trade
|
||||
if modify:
|
||||
assert oid
|
||||
params['origClientOrderId'] = oid
|
||||
method: str = 'put'
|
||||
|
||||
elif oid:
|
||||
params['newClientOrderId'] = oid
|
||||
|
||||
log.info(
|
||||
'Submitting ReST order request:\n'
|
||||
f'{pformat(params)}'
|
||||
)
|
||||
resp = await self._api(
|
||||
'order',
|
||||
params=params,
|
||||
signed=True,
|
||||
method=method,
|
||||
venue=self.mkt_mode,
|
||||
allow_testnet=True,
|
||||
)
|
||||
|
||||
# ensure our id is tracked by them
|
||||
if (
|
||||
oid
|
||||
and not modify
|
||||
):
|
||||
assert oid == resp['clientOrderId']
|
||||
|
||||
reqid: str = resp['orderId']
|
||||
return reqid
|
||||
|
||||
async def submit_cancel(
|
||||
self,
|
||||
symbol: str,
|
||||
oid: str,
|
||||
|
||||
recv_window: int = 60000
|
||||
|
||||
) -> None:
|
||||
bs_mktid: str = self._pairs[symbol.upper()].symbol
|
||||
params = dict([
|
||||
('symbol', bs_mktid),
|
||||
# ('orderId', oid),
|
||||
('origClientOrderId', oid),
|
||||
('recvWindow', recv_window),
|
||||
('timestamp', binance_timestamp(now()))
|
||||
])
|
||||
|
||||
log.cancel(
|
||||
'Submitting ReST order cancel: {oid}\n'
|
||||
f'{pformat(params)}'
|
||||
)
|
||||
await self._api(
|
||||
'order',
|
||||
params=params,
|
||||
signed=True,
|
||||
method='delete',
|
||||
allow_testnet=True,
|
||||
)
|
||||
|
||||
async def get_listen_key(self) -> str:
|
||||
|
||||
resp = await self._api(
|
||||
# 'userDataStream', # spot
|
||||
'listenKey',
|
||||
params={},
|
||||
method='post',
|
||||
signed=True,
|
||||
allow_testnet=True,
|
||||
)
|
||||
return resp['listenKey']
|
||||
|
||||
async def keep_alive_key(self, listen_key: str) -> None:
|
||||
await self._api(
|
||||
# 'userDataStream',
|
||||
'listenKey',
|
||||
params={'listenKey': listen_key},
|
||||
method='put',
|
||||
allow_testnet=True,
|
||||
)
|
||||
|
||||
async def close_listen_key(self, listen_key: str) -> None:
|
||||
await self._api(
|
||||
# 'userDataStream',
|
||||
'listenKey',
|
||||
params={'listenKey': listen_key},
|
||||
method='delete',
|
||||
allow_testnet=True,
|
||||
)
|
||||
|
||||
@acm
|
||||
async def manage_listen_key(self):
|
||||
|
||||
async def periodic_keep_alive(
|
||||
self,
|
||||
listen_key: str,
|
||||
timeout=60 * 29 # 29 minutes
|
||||
):
|
||||
while True:
|
||||
await trio.sleep(timeout)
|
||||
await self.keep_alive_key(listen_key)
|
||||
|
||||
key = await self.get_listen_key()
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(periodic_keep_alive, self, key)
|
||||
yield key
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
await self.close_listen_key(key)
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> Client:
|
||||
|
||||
client = Client()
|
||||
await client.exch_info()
|
||||
log.info(
|
||||
f'{client} in {client.mkt_mode} mode: caching exchange infos..\n'
|
||||
'Cached multi-market pairs:\n'
|
||||
f'spot: {len(client._spot_pairs)}\n'
|
||||
f'usdtm_futes: {len(client._ufutes_pairs)}\n'
|
||||
f'Total: {len(client._pairs)}\n'
|
||||
)
|
||||
|
||||
yield client
|
||||
|
|
@ -1,674 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez (aka ze jefe)
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Live order control B)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
)
|
||||
import time
|
||||
from time import time_ns
|
||||
|
||||
from bidict import bidict
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from piker.accounting import (
|
||||
Asset,
|
||||
# MktPair,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
get_logger,
|
||||
)
|
||||
from piker.data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
from piker.brokers import (
|
||||
open_cached_client,
|
||||
BrokerError,
|
||||
)
|
||||
from piker.clearing import OrderDialogs
|
||||
from piker.clearing._messages import (
|
||||
BrokerdOrder,
|
||||
BrokerdOrderAck,
|
||||
BrokerdStatus,
|
||||
BrokerdPosition,
|
||||
BrokerdFill,
|
||||
BrokerdCancel,
|
||||
BrokerdError,
|
||||
Status,
|
||||
Order,
|
||||
)
|
||||
from .venues import (
|
||||
Pair,
|
||||
_futes_ws,
|
||||
_testnet_futes_ws,
|
||||
)
|
||||
from .api import Client
|
||||
|
||||
log = get_logger('piker.brokers.binance')
|
||||
|
||||
|
||||
async def handle_order_requests(
|
||||
ems_order_stream: tractor.MsgStream,
|
||||
client: Client,
|
||||
dids: bidict[str, str],
|
||||
dialogs: OrderDialogs,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Receive order requests from `emsd`, translate tramsit API calls and transmit.
|
||||
|
||||
'''
|
||||
msg: dict | BrokerdOrder | BrokerdCancel
|
||||
async for msg in ems_order_stream:
|
||||
log.info(f'Rx order request:\n{pformat(msg)}')
|
||||
match msg:
|
||||
case {
|
||||
'action': 'cancel',
|
||||
}:
|
||||
cancel = BrokerdCancel(**msg)
|
||||
existing: BrokerdOrder | None = dialogs.get(cancel.oid)
|
||||
if not existing:
|
||||
log.error(
|
||||
f'NO Existing order-dialog for {cancel.oid}!?'
|
||||
)
|
||||
await ems_order_stream.send(BrokerdError(
|
||||
oid=cancel.oid,
|
||||
|
||||
# TODO: do we need the symbol?
|
||||
# https://github.com/pikers/piker/issues/514
|
||||
symbol='unknown',
|
||||
|
||||
reason=(
|
||||
'Invalid `binance` order request dialog oid',
|
||||
)
|
||||
))
|
||||
continue
|
||||
|
||||
else:
|
||||
symbol: str = existing['symbol']
|
||||
try:
|
||||
await client.submit_cancel(
|
||||
symbol,
|
||||
cancel.oid,
|
||||
)
|
||||
except BrokerError as be:
|
||||
await ems_order_stream.send(
|
||||
BrokerdError(
|
||||
oid=msg['oid'],
|
||||
symbol=symbol,
|
||||
reason=(
|
||||
'`binance` CANCEL failed:\n'
|
||||
f'{be}'
|
||||
))
|
||||
)
|
||||
continue
|
||||
|
||||
case {
|
||||
'account': ('binance.usdtm' | 'binance.spot') as account,
|
||||
'action': action,
|
||||
} if action in {'buy', 'sell'}:
|
||||
|
||||
# validate
|
||||
order = BrokerdOrder(**msg)
|
||||
oid: str = order.oid # emsd order id
|
||||
modify: bool = False
|
||||
|
||||
# NOTE: check and report edits
|
||||
if existing := dialogs.get(order.oid):
|
||||
log.info(
|
||||
f'Existing order for {oid} updated:\n'
|
||||
f'{pformat(existing.maps[-1])} -> {pformat(msg)}'
|
||||
)
|
||||
modify = True
|
||||
|
||||
# only add new msg AFTER the existing check
|
||||
dialogs.add_msg(oid, msg)
|
||||
|
||||
else:
|
||||
# XXX NOTE: update before the ack!
|
||||
# track latest request state such that map
|
||||
# lookups start at the most recent msg and then
|
||||
# scan reverse-chronologically.
|
||||
dialogs.add_msg(oid, msg)
|
||||
|
||||
# XXX: ACK the request **immediately** before sending
|
||||
# the api side request to ensure the ems maps the oid ->
|
||||
# reqid correctly!
|
||||
resp = BrokerdOrderAck(
|
||||
oid=oid, # ems order request id
|
||||
reqid=oid, # our custom int mapping
|
||||
account='binance', # piker account
|
||||
)
|
||||
await ems_order_stream.send(resp)
|
||||
|
||||
# call our client api to submit the order
|
||||
# NOTE: modifies only require diff key for user oid:
|
||||
# https://binance-docs.github.io/apidocs/futures/en/#modify-order-trade
|
||||
try:
|
||||
reqid = await client.submit_limit(
|
||||
symbol=order.symbol,
|
||||
side=order.action,
|
||||
quantity=order.size,
|
||||
price=order.price,
|
||||
oid=oid,
|
||||
modify=modify,
|
||||
)
|
||||
|
||||
# SMH they do gen their own order id: ints..
|
||||
# assert reqid == order.oid
|
||||
dids[order.oid] = reqid
|
||||
|
||||
except BrokerError as be:
|
||||
await ems_order_stream.send(
|
||||
BrokerdError(
|
||||
oid=msg['oid'],
|
||||
symbol=msg['symbol'],
|
||||
reason=(
|
||||
'`binance` request failed:\n'
|
||||
f'{be}'
|
||||
))
|
||||
)
|
||||
continue
|
||||
|
||||
case _:
|
||||
account = msg.get('account')
|
||||
if account not in {'binance.spot', 'binance.futes'}:
|
||||
log.error(
|
||||
'Order request does not have a valid binance account name?\n'
|
||||
'Only one of\n'
|
||||
'- `binance.spot` or,\n'
|
||||
'- `binance.usdtm`\n'
|
||||
'is currently valid!'
|
||||
)
|
||||
await ems_order_stream.send(
|
||||
BrokerdError(
|
||||
oid=msg['oid'],
|
||||
symbol=msg['symbol'],
|
||||
reason=(
|
||||
f'Invalid `binance` broker request msg:\n{msg}'
|
||||
))
|
||||
)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_trade_dialog(
|
||||
ctx: tractor.Context,
|
||||
|
||||
) -> AsyncIterator[dict[str, Any]]:
|
||||
|
||||
# TODO: how do we set this from the EMS such that
|
||||
# positions are loaded from the correct venue on the user
|
||||
# stream at startup? (that is in an attempt to support both
|
||||
# spot and futes markets?)
|
||||
# - I guess we just want to instead start 2 separate user
|
||||
# stream tasks right? unless we want another actor pool?
|
||||
# XXX: see issue: <urlhere>
|
||||
venue_name: str = 'futes'
|
||||
venue_mode: str = 'usdtm_futes'
|
||||
account_name: str = 'usdtm'
|
||||
use_testnet: bool = False
|
||||
|
||||
async with open_cached_client('binance') as client:
|
||||
subconf: dict = client.conf[venue_name]
|
||||
use_testnet = subconf.get('use_testnet', False)
|
||||
|
||||
# XXX: if no futes.api_key or spot.api_key has been set we
|
||||
# always fall back to the paper engine!
|
||||
if not subconf.get('api_key'):
|
||||
await ctx.started('paper')
|
||||
return
|
||||
|
||||
async with (
|
||||
open_cached_client('binance') as client,
|
||||
):
|
||||
client.mkt_mode: str = venue_mode
|
||||
|
||||
# TODO: map these wss urls depending on spot or futes
|
||||
# setting passed when this task is spawned?
|
||||
wss_url: str = _futes_ws if not use_testnet else _testnet_futes_ws
|
||||
|
||||
wss: NoBsWs
|
||||
async with (
|
||||
client.manage_listen_key() as listen_key,
|
||||
open_autorecon_ws(f'{wss_url}/?listenKey={listen_key}') as wss,
|
||||
):
|
||||
nsid: int = time_ns()
|
||||
await wss.send_msg({
|
||||
# "method": "SUBSCRIBE",
|
||||
"method": "REQUEST",
|
||||
"params":
|
||||
[
|
||||
f"{listen_key}@account",
|
||||
f"{listen_key}@balance",
|
||||
f"{listen_key}@position",
|
||||
|
||||
# TODO: does this even work!? seems to cause
|
||||
# a hang on the first msg..? lelelel.
|
||||
# f"{listen_key}@order",
|
||||
],
|
||||
"id": nsid
|
||||
})
|
||||
|
||||
with trio.fail_after(6):
|
||||
msg = await wss.recv_msg()
|
||||
assert msg['id'] == nsid
|
||||
|
||||
# TODO: load other market wide data / statistics:
|
||||
# - OI: https://binance-docs.github.io/apidocs/futures/en/#open-interest
|
||||
# - OI stats: https://binance-docs.github.io/apidocs/futures/en/#open-interest-statistics
|
||||
accounts: bidict[str, str] = bidict({'binance.usdtm': None})
|
||||
balances: dict[Asset, float] = {}
|
||||
positions: list[BrokerdPosition] = []
|
||||
|
||||
for resp_dict in msg['result']:
|
||||
resp: dict = resp_dict['res']
|
||||
req: str = resp_dict['req']
|
||||
|
||||
# @account response should be something like:
|
||||
# {'accountAlias': 'sRFzFzAuuXsR',
|
||||
# 'canDeposit': True,
|
||||
# 'canTrade': True,
|
||||
# 'canWithdraw': True,
|
||||
# 'feeTier': 0}
|
||||
if 'account' in req:
|
||||
# NOTE: fill in the hash-like key/alias binance
|
||||
# provides for the account.
|
||||
alias: str = resp['accountAlias']
|
||||
accounts['binance.usdtm'] = alias
|
||||
|
||||
# @balance response:
|
||||
# {'accountAlias': 'sRFzFzAuuXsR',
|
||||
# 'balances': [{'asset': 'BTC',
|
||||
# 'availableBalance': '0.00000000',
|
||||
# 'balance': '0.00000000',
|
||||
# 'crossUnPnl': '0.00000000',
|
||||
# 'crossWalletBalance': '0.00000000',
|
||||
# 'maxWithdrawAmount': '0.00000000',
|
||||
# 'updateTime': 0}]
|
||||
# ...
|
||||
# }
|
||||
elif 'balance' in req:
|
||||
for entry in resp['balances']:
|
||||
name: str = entry['asset']
|
||||
balance: float = float(entry['balance'])
|
||||
last_update_t: int = entry['updateTime']
|
||||
|
||||
spot_asset: Asset = client._venue2assets['spot'][name]
|
||||
|
||||
if balance > 0:
|
||||
balances[spot_asset] = (balance, last_update_t)
|
||||
# await tractor.breakpoint()
|
||||
|
||||
# @position response:
|
||||
# {'positions': [{'entryPrice': '0.0',
|
||||
# 'isAutoAddMargin': False,
|
||||
# 'isolatedMargin': '0',
|
||||
# 'leverage': 20,
|
||||
# 'liquidationPrice': '0',
|
||||
# 'marginType': 'CROSSED',
|
||||
# 'markPrice': '0.60289650',
|
||||
# 'markPrice': '0.00000000',
|
||||
# 'maxNotionalValue': '25000',
|
||||
# 'notional': '0',
|
||||
# 'positionAmt': '0',
|
||||
# 'positionSide': 'BOTH',
|
||||
# 'symbol': 'ETHUSDT_230630',
|
||||
# 'unRealizedProfit': '0.00000000',
|
||||
# 'updateTime': 1672741444894}
|
||||
# ...
|
||||
# }
|
||||
elif 'position' in req:
|
||||
for entry in resp['positions']:
|
||||
bs_mktid: str = entry['symbol']
|
||||
entry_size: float = float(entry['positionAmt'])
|
||||
|
||||
pair: Pair | None = client._venue2pairs[
|
||||
venue_mode
|
||||
].get(bs_mktid)
|
||||
if (
|
||||
pair
|
||||
and entry_size > 0
|
||||
):
|
||||
entry_price: float = float(entry['entryPrice'])
|
||||
|
||||
ppmsg = BrokerdPosition(
|
||||
broker='binance',
|
||||
account=f'binance.{account_name}',
|
||||
|
||||
# TODO: maybe we should be passing back
|
||||
# a `MktPair` here?
|
||||
symbol=pair.bs_fqme.lower() + '.binance',
|
||||
|
||||
size=entry_size,
|
||||
avg_price=entry_price,
|
||||
)
|
||||
positions.append(ppmsg)
|
||||
|
||||
if pair is None:
|
||||
log.warning(
|
||||
f'`{bs_mktid}` Position entry but no market pair?\n'
|
||||
f'{pformat(entry)}\n'
|
||||
)
|
||||
|
||||
await ctx.started((
|
||||
positions,
|
||||
list(accounts)
|
||||
))
|
||||
|
||||
# TODO: package more state tracking into the dialogs API?
|
||||
# - hmm maybe we could include `OrderDialogs.dids:
|
||||
# bidict` as part of the interface and then ask for
|
||||
# a reqid field to be passed at init?
|
||||
# |-> `OrderDialog(reqid_field='orderId')` kinda thing?
|
||||
# - also maybe bundle in some kind of dialog to account
|
||||
# table?
|
||||
dialogs = OrderDialogs()
|
||||
dids: dict[str, int] = bidict()
|
||||
|
||||
# TODO: further init setup things to get full EMS and
|
||||
# .accounting support B)
|
||||
# - live order loading via user stream subscription and
|
||||
# update to the order dialog table.
|
||||
# - MAKE SURE we add live orders loaded during init
|
||||
# into the dialogs table to ensure they can be
|
||||
# cancelled, meaning we can do a symbol lookup.
|
||||
# - position loading using `piker.accounting` subsys
|
||||
# and comparison with binance's own position calcs.
|
||||
# - load pps and accounts using accounting apis, write
|
||||
# the ledger and account files
|
||||
# - table: PpTable
|
||||
# - ledger: TransactionLedger
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as tn,
|
||||
ctx.open_stream() as ems_stream,
|
||||
):
|
||||
# deliver all pre-exist open orders to EMS thus syncing
|
||||
# state with existing live limits reported by them.
|
||||
order: Order
|
||||
for order in await client.get_open_orders():
|
||||
status_msg = Status(
|
||||
time_ns=time.time_ns(),
|
||||
resp='open',
|
||||
oid=order.oid,
|
||||
reqid=order.oid,
|
||||
|
||||
# embedded order info
|
||||
req=order,
|
||||
src='binance',
|
||||
)
|
||||
dialogs.add_msg(order.oid, order.to_dict())
|
||||
await ems_stream.send(status_msg)
|
||||
|
||||
tn.start_soon(
|
||||
handle_order_requests,
|
||||
ems_stream,
|
||||
client,
|
||||
dids,
|
||||
dialogs,
|
||||
)
|
||||
tn.start_soon(
|
||||
handle_order_updates,
|
||||
venue_mode,
|
||||
account_name,
|
||||
client,
|
||||
ems_stream,
|
||||
wss,
|
||||
dialogs,
|
||||
|
||||
)
|
||||
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
||||
async def handle_order_updates(
|
||||
venue: str,
|
||||
account_name: str,
|
||||
client: Client,
|
||||
ems_stream: tractor.MsgStream,
|
||||
wss: NoBsWs,
|
||||
dialogs: OrderDialogs,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Main msg handling loop for all things order management.
|
||||
|
||||
This code is broken out to make the context explicit and state
|
||||
variables defined in the signature clear to the reader.
|
||||
|
||||
'''
|
||||
async for msg in wss:
|
||||
log.info(f'Rx USERSTREAM msg:\n{pformat(msg)}')
|
||||
match msg:
|
||||
|
||||
# ORDER update
|
||||
# spot: https://binance-docs.github.io/apidocs/spot/en/#payload-balance-update
|
||||
# futes: https://binance-docs.github.io/apidocs/futures/en/#event-order-update
|
||||
# futes: https://binance-docs.github.io/apidocs/futures/en/#event-balance-and-position-update
|
||||
# {'o': {
|
||||
# 'L': '0',
|
||||
# 'N': 'USDT',
|
||||
# 'R': False,
|
||||
# 'S': 'BUY',
|
||||
# 'T': 1687028772484,
|
||||
# 'X': 'NEW',
|
||||
# 'a': '0',
|
||||
# 'ap': '0',
|
||||
# 'b': '7012.06520',
|
||||
# 'c': '518d4122-8d3e-49b0-9a1e-1fabe6f62e4c',
|
||||
# 'cp': False,
|
||||
# 'f': 'GTC',
|
||||
# 'i': 3376956924,
|
||||
# 'l': '0',
|
||||
# 'm': False,
|
||||
# 'n': '0',
|
||||
# 'o': 'LIMIT',
|
||||
# 'ot': 'LIMIT',
|
||||
# 'p': '21136.80',
|
||||
# 'pP': False,
|
||||
# 'ps': 'BOTH',
|
||||
# 'q': '0.047',
|
||||
# 'rp': '0',
|
||||
# 's': 'BTCUSDT',
|
||||
# 'si': 0,
|
||||
# 'sp': '0',
|
||||
# 'ss': 0,
|
||||
# 't': 0,
|
||||
# 'wt': 'CONTRACT_PRICE',
|
||||
# 'x': 'NEW',
|
||||
# 'z': '0'}
|
||||
# }
|
||||
case {
|
||||
# 'e': 'executionReport',
|
||||
'e': 'ORDER_TRADE_UPDATE',
|
||||
'T': int(epoch_ms),
|
||||
'o': {
|
||||
's': bs_mktid,
|
||||
|
||||
# XXX NOTE XXX see special ids for market
|
||||
# events or margin calls:
|
||||
# // special client order id:
|
||||
# // starts with "autoclose-": liquidation order
|
||||
# // "adl_autoclose": ADL auto close order
|
||||
# // "settlement_autoclose-": settlement order
|
||||
# for delisting or delivery
|
||||
'c': oid,
|
||||
# 'i': reqid, # binance internal int id
|
||||
|
||||
# prices
|
||||
'a': submit_price,
|
||||
'ap': avg_price,
|
||||
'L': fill_price,
|
||||
|
||||
# sizing
|
||||
'q': req_size,
|
||||
'l': clear_size_filled, # this event
|
||||
'z': accum_size_filled, # accum
|
||||
|
||||
# commissions
|
||||
'n': cost,
|
||||
'N': cost_asset,
|
||||
|
||||
# state
|
||||
'S': side,
|
||||
'X': status,
|
||||
},
|
||||
} as order_msg:
|
||||
log.info(
|
||||
f'{status} for {side} ORDER oid: {oid}\n'
|
||||
f'bs_mktid: {bs_mktid}\n\n'
|
||||
|
||||
f'order size: {req_size}\n'
|
||||
f'cleared size: {clear_size_filled}\n'
|
||||
f'accum filled size: {accum_size_filled}\n\n'
|
||||
|
||||
f'submit price: {submit_price}\n'
|
||||
f'fill_price: {fill_price}\n'
|
||||
f'avg clearing price: {avg_price}\n\n'
|
||||
|
||||
f'cost: {cost}@{cost_asset}\n'
|
||||
)
|
||||
|
||||
# status remap from binance to piker's
|
||||
# status set:
|
||||
# - NEW
|
||||
# - PARTIALLY_FILLED
|
||||
# - FILLED
|
||||
# - CANCELED
|
||||
# - EXPIRED
|
||||
# https://binance-docs.github.io/apidocs/futures/en/#event-order-update
|
||||
|
||||
req_size: float = float(req_size)
|
||||
accum_size_filled: float = float(accum_size_filled)
|
||||
fill_price: float = float(fill_price)
|
||||
|
||||
match status:
|
||||
case 'PARTIALLY_FILLED' | 'FILLED':
|
||||
status = 'fill'
|
||||
|
||||
fill_msg = BrokerdFill(
|
||||
time_ns=time_ns(),
|
||||
# reqid=reqid,
|
||||
reqid=oid,
|
||||
|
||||
# just use size value for now?
|
||||
# action=action,
|
||||
size=clear_size_filled,
|
||||
price=fill_price,
|
||||
|
||||
# TODO: maybe capture more msg data
|
||||
# i.e fees?
|
||||
broker_details={'name': 'broker'} | order_msg,
|
||||
broker_time=time.time(),
|
||||
)
|
||||
await ems_stream.send(fill_msg)
|
||||
|
||||
if accum_size_filled == req_size:
|
||||
status = 'closed'
|
||||
dialogs.pop(oid)
|
||||
|
||||
case 'NEW':
|
||||
status = 'open'
|
||||
|
||||
case 'EXPIRED':
|
||||
status = 'canceled'
|
||||
dialogs.pop(oid)
|
||||
|
||||
case _:
|
||||
status = status.lower()
|
||||
|
||||
resp = BrokerdStatus(
|
||||
time_ns=time_ns(),
|
||||
# reqid=reqid,
|
||||
reqid=oid,
|
||||
|
||||
# TODO: i feel like we don't need to make the
|
||||
# ems and upstream clients aware of this?
|
||||
# account='binance.usdtm',
|
||||
|
||||
status=status,
|
||||
|
||||
filled=accum_size_filled,
|
||||
remaining=req_size - accum_size_filled,
|
||||
broker_details={
|
||||
'name': 'binance',
|
||||
'broker_time': epoch_ms / 1000.
|
||||
}
|
||||
)
|
||||
await ems_stream.send(resp)
|
||||
|
||||
# ACCOUNT and POSITION update B)
|
||||
# {
|
||||
# 'E': 1687036749218,
|
||||
# 'e': 'ACCOUNT_UPDATE'
|
||||
# 'T': 1687036749215,
|
||||
# 'a': {'B': [{'a': 'USDT',
|
||||
# 'bc': '0',
|
||||
# 'cw': '1267.48920735',
|
||||
# 'wb': '1410.90245576'}],
|
||||
# 'P': [{'cr': '-3292.10973007',
|
||||
# 'ep': '26349.90000',
|
||||
# 'iw': '143.41324841',
|
||||
# 'ma': 'USDT',
|
||||
# 'mt': 'isolated',
|
||||
# 'pa': '0.038',
|
||||
# 'ps': 'BOTH',
|
||||
# 's': 'BTCUSDT',
|
||||
# 'up': '5.17555453'}],
|
||||
# 'm': 'ORDER'},
|
||||
# }
|
||||
case {
|
||||
'T': int(epoch_ms),
|
||||
'e': 'ACCOUNT_UPDATE',
|
||||
'a': {
|
||||
'P': [{
|
||||
's': bs_mktid,
|
||||
'pa': pos_amount,
|
||||
'ep': entry_price,
|
||||
}],
|
||||
},
|
||||
}:
|
||||
# real-time relay position updates back to EMS
|
||||
pair: Pair | None = client._venue2pairs[venue].get(bs_mktid)
|
||||
ppmsg = BrokerdPosition(
|
||||
broker='binance',
|
||||
account=f'binance.{account_name}',
|
||||
|
||||
# TODO: maybe we should be passing back
|
||||
# a `MktPair` here?
|
||||
symbol=pair.bs_fqme.lower() + '.binance',
|
||||
|
||||
size=float(pos_amount),
|
||||
avg_price=float(entry_price),
|
||||
)
|
||||
await ems_stream.send(ppmsg)
|
||||
|
||||
case _:
|
||||
log.warning(
|
||||
'Unhandled event:\n'
|
||||
f'{pformat(msg)}'
|
||||
)
|
||||
|
|
@ -1,499 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Real-time and historical data feed endpoints.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
aclosing,
|
||||
)
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
import itertools
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Generator,
|
||||
)
|
||||
import time
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
from pendulum import (
|
||||
from_timestamp,
|
||||
)
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
|
||||
from piker.brokers import (
|
||||
open_cached_client,
|
||||
)
|
||||
from piker._cacheables import (
|
||||
async_lifo_cache,
|
||||
)
|
||||
from piker.accounting import (
|
||||
Asset,
|
||||
DerivTypes,
|
||||
MktPair,
|
||||
unpack_fqme,
|
||||
digits_to_dec,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
from piker.data.validate import FeedInit
|
||||
from piker.data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
DataUnavailable,
|
||||
get_logger,
|
||||
)
|
||||
|
||||
from .api import (
|
||||
Client,
|
||||
)
|
||||
from .venues import (
|
||||
Pair,
|
||||
FutesPair,
|
||||
get_api_eps,
|
||||
)
|
||||
|
||||
log = get_logger('piker.brokers.binance')
|
||||
|
||||
|
||||
class L1(Struct):
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams
|
||||
|
||||
update_id: int
|
||||
sym: str
|
||||
|
||||
bid: float
|
||||
bsize: float
|
||||
ask: float
|
||||
asize: float
|
||||
|
||||
|
||||
# validation type
|
||||
class AggTrade(Struct, frozen=True):
|
||||
e: str # Event type
|
||||
E: int # Event time
|
||||
s: str # Symbol
|
||||
a: int # Aggregate trade ID
|
||||
p: float # Price
|
||||
q: float # Quantity
|
||||
f: int # First trade ID
|
||||
l: int # noqa Last trade ID
|
||||
T: int # Trade time
|
||||
m: bool # Is the buyer the market maker?
|
||||
M: bool | None = None # Ignore
|
||||
|
||||
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
) -> AsyncGenerator[NoBsWs, dict]:
|
||||
|
||||
# TODO: match syntax here!
|
||||
msg: dict[str, Any]
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
# for l1 streams binance doesn't add an event type field so
|
||||
# identify those messages by matching keys
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams
|
||||
case {
|
||||
# NOTE: this is never an old value it seems, so
|
||||
# they are always sending real L1 spread updates.
|
||||
'u': upid, # update id
|
||||
's': sym,
|
||||
'b': bid,
|
||||
'B': bsize,
|
||||
'a': ask,
|
||||
'A': asize,
|
||||
}:
|
||||
# TODO: it would be super nice to have a `L1` piker type
|
||||
# which "renders" incremental tick updates from a packed
|
||||
# msg-struct:
|
||||
# - backend msgs after packed into the type such that we
|
||||
# can reduce IPC usage but without each backend having
|
||||
# to do that incremental update logic manually B)
|
||||
# - would it maybe be more efficient to use this instead?
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#diff-depth-stream
|
||||
l1 = L1(
|
||||
update_id=upid,
|
||||
sym=sym,
|
||||
bid=bid,
|
||||
bsize=bsize,
|
||||
ask=ask,
|
||||
asize=asize,
|
||||
)
|
||||
# for speed probably better to only specifically
|
||||
# cast fields we need in numerical form?
|
||||
# l1.typecast()
|
||||
|
||||
# repack into piker's tick-quote format
|
||||
yield 'l1', {
|
||||
'symbol': l1.sym,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'bid',
|
||||
'price': float(l1.bid),
|
||||
'size': float(l1.bsize),
|
||||
},
|
||||
{
|
||||
'type': 'bsize',
|
||||
'price': float(l1.bid),
|
||||
'size': float(l1.bsize),
|
||||
},
|
||||
{
|
||||
'type': 'ask',
|
||||
'price': float(l1.ask),
|
||||
'size': float(l1.asize),
|
||||
},
|
||||
{
|
||||
'type': 'asize',
|
||||
'price': float(l1.ask),
|
||||
'size': float(l1.asize),
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#aggregate-trade-streams
|
||||
case {
|
||||
'e': 'aggTrade',
|
||||
}:
|
||||
# NOTE: this is purely for a definition,
|
||||
# ``msgspec.Struct`` does not runtime-validate until you
|
||||
# decode/encode, see:
|
||||
# https://jcristharif.com/msgspec/structs.html#type-validation
|
||||
msg = AggTrade(**msg) # TODO: should we .copy() ?
|
||||
piker_quote: dict = {
|
||||
'symbol': msg.s,
|
||||
'last': float(msg.p),
|
||||
'brokerd_ts': time.time(),
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': float(msg.p),
|
||||
'size': float(msg.q),
|
||||
'broker_ts': msg.T,
|
||||
}],
|
||||
}
|
||||
yield 'trade', piker_quote
|
||||
|
||||
|
||||
def make_sub(pairs: list[str], sub_name: str, uid: int) -> dict[str, str]:
|
||||
'''
|
||||
Create a request subscription packet dict.
|
||||
|
||||
- spot:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#live-subscribing-unsubscribing-to-streams
|
||||
|
||||
- futes:
|
||||
https://binance-docs.github.io/apidocs/futures/en/#websocket-market-streams
|
||||
|
||||
'''
|
||||
return {
|
||||
'method': 'SUBSCRIBE',
|
||||
'params': [
|
||||
f'{pair.lower()}@{sub_name}'
|
||||
for pair in pairs
|
||||
],
|
||||
'id': uid
|
||||
}
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
mkt: MktPair,
|
||||
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: datetime | None = None,
|
||||
start_dt: datetime | None = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
if timeframe != 60:
|
||||
raise DataUnavailable('Only 1m bars are supported')
|
||||
|
||||
# TODO: better wrapping for venue / mode?
|
||||
# - eventually logic for usd vs. coin settled futes
|
||||
# based on `MktPair.src` type/value?
|
||||
# - maybe something like `async with
|
||||
# Client.use_venue('usdtm_futes')`
|
||||
if mkt.type_key in DerivTypes:
|
||||
client.mkt_mode = 'usdtm_futes'
|
||||
else:
|
||||
client.mkt_mode = 'spot'
|
||||
|
||||
# NOTE: always query using their native symbology!
|
||||
mktid: str = mkt.bs_mktid
|
||||
array = await client.bars(
|
||||
mktid,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
times = array['time']
|
||||
if (
|
||||
end_dt is None
|
||||
):
|
||||
inow = round(time.time())
|
||||
if (inow - times[-1]) > 60:
|
||||
await tractor.breakpoint()
|
||||
|
||||
start_dt = from_timestamp(times[0])
|
||||
end_dt = from_timestamp(times[-1])
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
@async_lifo_cache()
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
|
||||
# uppercase since kraken bs_mktid is always upper
|
||||
if 'binance' not in fqme:
|
||||
fqme += '.binance'
|
||||
|
||||
bs_fqme, _, broker = fqme.rpartition('.')
|
||||
broker, mkt_ep, venue, expiry = unpack_fqme(fqme)
|
||||
|
||||
# NOTE: see the `FutesPair.bs_fqme: str` implementation
|
||||
# to understand the reverse market info lookup below.
|
||||
mkt_mode = venue = venue.lower() or 'spot'
|
||||
_atype: str = ''
|
||||
if (
|
||||
venue
|
||||
and 'spot' not in venue.lower()
|
||||
|
||||
# XXX: catch all in case user doesn't know which
|
||||
# venue they want (usdtm vs. coinm) and we can choose
|
||||
# a default (via config?) once we support coin-m APIs.
|
||||
or 'perp' in bs_fqme.lower()
|
||||
):
|
||||
mkt_mode: str = f'{venue.lower()}_futes'
|
||||
if 'perp' in expiry:
|
||||
_atype = 'perpetual_future'
|
||||
|
||||
else:
|
||||
_atype = 'future'
|
||||
|
||||
async with open_cached_client(
|
||||
'binance',
|
||||
) as client:
|
||||
|
||||
# switch mode depending on input pattern parsing
|
||||
client.mkt_mode = mkt_mode
|
||||
|
||||
pair_str: str = mkt_ep.upper()
|
||||
pair: Pair = await client.exch_info(pair_str)
|
||||
|
||||
if 'futes' in mkt_mode:
|
||||
assert isinstance(pair, FutesPair)
|
||||
|
||||
mkt = MktPair(
|
||||
dst=Asset(
|
||||
name=pair.baseAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.baseAssetPrecision),
|
||||
),
|
||||
src=Asset(
|
||||
name=pair.quoteAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.quoteAssetPrecision),
|
||||
),
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=pair.symbol,
|
||||
expiry=expiry,
|
||||
venue=venue,
|
||||
broker='binance',
|
||||
_atype=_atype,
|
||||
)
|
||||
both = mkt, pair
|
||||
return both
|
||||
|
||||
|
||||
@acm
|
||||
async def subscribe(
|
||||
ws: NoBsWs,
|
||||
symbols: list[str],
|
||||
|
||||
# defined once at import time to keep a global state B)
|
||||
iter_subids: Generator[int, None, None] = itertools.count(),
|
||||
|
||||
):
|
||||
# setup subs
|
||||
|
||||
subid: int = next(iter_subids)
|
||||
|
||||
# trade data (aka L1)
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#symbol-order-book-ticker
|
||||
l1_sub = make_sub(symbols, 'bookTicker', subid)
|
||||
await ws.send_msg(l1_sub)
|
||||
|
||||
# aggregate (each order clear by taker **not** by maker)
|
||||
# trades data:
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#aggregate-trade-streams
|
||||
agg_trades_sub = make_sub(symbols, 'aggTrade', subid)
|
||||
await ws.send_msg(agg_trades_sub)
|
||||
|
||||
# might get ack from ws server, or maybe some
|
||||
# other msg still in transit..
|
||||
res = await ws.recv_msg()
|
||||
subid: str | None = res.get('id')
|
||||
if subid:
|
||||
assert res['id'] == subid
|
||||
|
||||
yield
|
||||
|
||||
subs = []
|
||||
for sym in symbols:
|
||||
subs.append("{sym}@aggTrade")
|
||||
subs.append("{sym}@bookTicker")
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
"method": "UNSUBSCRIBE",
|
||||
"params": subs,
|
||||
"id": subid,
|
||||
})
|
||||
|
||||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
||||
async with (
|
||||
send_chan as send_chan,
|
||||
open_cached_client('binance') as client,
|
||||
):
|
||||
init_msgs: list[FeedInit] = []
|
||||
for sym in symbols:
|
||||
mkt, pair = await get_mkt_info(sym)
|
||||
|
||||
# build out init msgs according to latest spec
|
||||
init_msgs.append(
|
||||
FeedInit(mkt_info=mkt)
|
||||
)
|
||||
|
||||
wss_url: str = get_api_eps(client.mkt_mode)[1] # 2nd elem is wss url
|
||||
|
||||
# TODO: for sanity, but remove eventually Xp
|
||||
if 'future' in mkt.type_key:
|
||||
assert 'fstream' in wss_url
|
||||
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
url=wss_url,
|
||||
fixture=partial(
|
||||
subscribe,
|
||||
symbols=[mkt.bs_mktid],
|
||||
),
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
aclosing(stream_messages(ws)) as msg_gen,
|
||||
):
|
||||
# log.info('WAITING ON FIRST LIVE QUOTE..')
|
||||
typ, quote = await anext(msg_gen)
|
||||
|
||||
# pull a first quote and deliver
|
||||
while typ != 'trade':
|
||||
typ, quote = await anext(msg_gen)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
# signal to caller feed is ready for consumption
|
||||
feed_is_live.set()
|
||||
|
||||
# import time
|
||||
# last = time.time()
|
||||
|
||||
# XXX NOTE: can't include the `.binance` suffix
|
||||
# or the sampling loop will not broadcast correctly
|
||||
# since `bus._subscribers.setdefault(bs_fqme, set())`
|
||||
# is used inside `.data.open_feed_bus()` !!!
|
||||
topic: str = mkt.bs_fqme
|
||||
|
||||
# start streaming
|
||||
async for typ, quote in msg_gen:
|
||||
|
||||
# period = time.time() - last
|
||||
# hz = 1/period if period else float('inf')
|
||||
# if hz > 60:
|
||||
# log.info(f'Binance quotez : {hz}')
|
||||
await send_chan.send({topic: quote})
|
||||
# last = time.time()
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> Client:
|
||||
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
fqpairs_cache = await client.exch_info()
|
||||
# TODO: maybe we should deliver the cache
|
||||
# so that client's can always do a local-lookup-first
|
||||
# style try and then update async as (new) match results
|
||||
# are delivered from here?
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
pattern: str
|
||||
async for pattern in stream:
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
fqpairs_cache,
|
||||
score_cutoff=50,
|
||||
)
|
||||
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
item[0].bs_fqme: item[0]
|
||||
for item in matches
|
||||
})
|
||||
|
|
@ -1,214 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez (aka ze jefe)
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Per market data-type definitions and schemas types.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Literal,
|
||||
)
|
||||
from decimal import Decimal
|
||||
|
||||
from msgspec import field
|
||||
|
||||
from piker.data.types import Struct
|
||||
|
||||
|
||||
# API endpoint paths by venue / sub-API
|
||||
_domain: str = 'binance.com'
|
||||
_spot_url = f'https://api.{_domain}'
|
||||
_futes_url = f'https://fapi.{_domain}'
|
||||
|
||||
# WEBsocketz
|
||||
# NOTE XXX: see api docs which show diff addr?
|
||||
# https://developers.binance.com/docs/binance-trading-api/websocket_api#general-api-information
|
||||
_spot_ws: str = 'wss://stream.binance.com/ws'
|
||||
# or this one? ..
|
||||
# 'wss://ws-api.binance.com:443/ws-api/v3',
|
||||
|
||||
# https://binance-docs.github.io/apidocs/futures/en/#websocket-market-streams
|
||||
_futes_ws: str = f'wss://fstream.{_domain}/ws'
|
||||
_auth_futes_ws: str = 'wss://fstream-auth.{_domain}/ws'
|
||||
|
||||
# test nets
|
||||
# NOTE: spot test network only allows certain ep sets:
|
||||
# https://testnet.binance.vision/
|
||||
# https://www.binance.com/en/support/faq/how-to-test-my-functions-on-binance-testnet-ab78f9a1b8824cf0a106b4229c76496d
|
||||
_testnet_spot_url: str = 'https://testnet.binance.vision/api'
|
||||
_testnet_spot_ws: str = 'wss://testnet.binance.vision/ws'
|
||||
# or this one? ..
|
||||
# 'wss://testnet.binance.vision/ws-api/v3'
|
||||
|
||||
_testnet_futes_url: str = 'https://testnet.binancefuture.com'
|
||||
_testnet_futes_ws: str = 'wss://stream.binancefuture.com/ws'
|
||||
|
||||
|
||||
MarketType = Literal[
|
||||
'spot',
|
||||
# 'margin',
|
||||
'usdtm_futes',
|
||||
# 'coin_futes',
|
||||
]
|
||||
|
||||
|
||||
def get_api_eps(venue: MarketType) -> tuple[str, str]:
|
||||
'''
|
||||
Return API ep root paths per venue.
|
||||
|
||||
'''
|
||||
return {
|
||||
'spot': (
|
||||
_spot_url,
|
||||
_spot_ws,
|
||||
),
|
||||
'usdtm_futes': (
|
||||
_futes_url,
|
||||
_futes_ws,
|
||||
),
|
||||
}[venue]
|
||||
|
||||
|
||||
class Pair(Struct, frozen=True, kw_only=True):
|
||||
symbol: str
|
||||
status: str
|
||||
orderTypes: list[str]
|
||||
|
||||
# src
|
||||
quoteAsset: str
|
||||
quotePrecision: int
|
||||
|
||||
# dst
|
||||
baseAsset: str
|
||||
baseAssetPrecision: int
|
||||
|
||||
filters: dict[
|
||||
str,
|
||||
str | int | float,
|
||||
] = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
# XXX: lul, after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
step_size: str = self.filters['PRICE_FILTER']['tickSize'].rstrip('0')
|
||||
return Decimal(step_size)
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
step_size: str = self.filters['LOT_SIZE']['stepSize'].rstrip('0')
|
||||
return Decimal(step_size)
|
||||
|
||||
@property
|
||||
def bs_fqme(self) -> str:
|
||||
return self.symbol
|
||||
|
||||
|
||||
class SpotPair(Pair, frozen=True):
|
||||
|
||||
cancelReplaceAllowed: bool
|
||||
allowTrailingStop: bool
|
||||
quoteAssetPrecision: int
|
||||
|
||||
baseCommissionPrecision: int
|
||||
quoteCommissionPrecision: int
|
||||
|
||||
icebergAllowed: bool
|
||||
ocoAllowed: bool
|
||||
quoteOrderQtyMarketAllowed: bool
|
||||
isSpotTradingAllowed: bool
|
||||
isMarginTradingAllowed: bool
|
||||
|
||||
defaultSelfTradePreventionMode: str
|
||||
allowedSelfTradePreventionModes: list[str]
|
||||
permissions: list[str]
|
||||
|
||||
@property
|
||||
def bs_fqme(self) -> str:
|
||||
return f'{self.symbol}.SPOT'
|
||||
|
||||
|
||||
|
||||
class FutesPair(Pair):
|
||||
symbol: str # 'BTCUSDT',
|
||||
pair: str # 'BTCUSDT',
|
||||
baseAssetPrecision: int # 8,
|
||||
contractType: str # 'PERPETUAL',
|
||||
deliveryDate: int # 4133404800000,
|
||||
liquidationFee: float # '0.012500',
|
||||
maintMarginPercent: float # '2.5000',
|
||||
marginAsset: str # 'USDT',
|
||||
marketTakeBound: float # '0.05',
|
||||
maxMoveOrderLimit: int # 10000,
|
||||
onboardDate: int # 1569398400000,
|
||||
pricePrecision: int # 2,
|
||||
quantityPrecision: int # 3,
|
||||
quoteAsset: str # 'USDT',
|
||||
quotePrecision: int # 8,
|
||||
requiredMarginPercent: float # '5.0000',
|
||||
settlePlan: int # 0,
|
||||
timeInForce: list[str] # ['GTC', 'IOC', 'FOK', 'GTX'],
|
||||
triggerProtect: float # '0.0500',
|
||||
underlyingSubType: list[str] # ['PoW'],
|
||||
underlyingType: str # 'COIN'
|
||||
|
||||
# NOTE: for compat with spot pairs and `MktPair.src: Asset`
|
||||
# processing..
|
||||
@property
|
||||
def quoteAssetPrecision(self) -> int:
|
||||
return self.quotePrecision
|
||||
|
||||
@property
|
||||
def bs_fqme(self) -> str:
|
||||
symbol: str = self.symbol
|
||||
ctype: str = self.contractType
|
||||
margin: str = self.marginAsset
|
||||
|
||||
match ctype:
|
||||
case 'PERPETUAL':
|
||||
return f'{symbol}.{margin}M.PERP'
|
||||
|
||||
case 'CURRENT_QUARTER':
|
||||
pair, _, expiry = symbol.partition('_')
|
||||
return f'{pair}.{margin}M.{expiry}'
|
||||
|
||||
case '':
|
||||
subtype: list[str] = self.underlyingSubType
|
||||
if not subtype:
|
||||
if self.status == 'PENDING_TRADING':
|
||||
return f'{symbol}.{margin}M.PENDING'
|
||||
|
||||
match subtype[0]:
|
||||
case 'DEFI':
|
||||
return f'{symbol}.{subtype}.PERP'
|
||||
|
||||
# XXX: yeah no clue then..
|
||||
return f'{symbol}.WTF.PWNED.BBQ'
|
||||
|
||||
|
||||
|
||||
PAIRTYPES: dict[MarketType, Pair] = {
|
||||
'spot': SpotPair,
|
||||
'usdtm_futes': FutesPair,
|
||||
|
||||
# TODO: support coin-margined venue:
|
||||
# https://binance-docs.github.io/apidocs/delivery/en/#change-log
|
||||
# 'coinm_futes': CoinFutesPair,
|
||||
}
|
||||
|
|
@ -21,7 +21,6 @@ import os
|
|||
from functools import partial
|
||||
from operator import attrgetter
|
||||
from operator import itemgetter
|
||||
from types import ModuleType
|
||||
|
||||
import click
|
||||
import trio
|
||||
|
|
@ -29,173 +28,20 @@ import tractor
|
|||
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..service import (
|
||||
maybe_spawn_brokerd,
|
||||
maybe_open_pikerd,
|
||||
)
|
||||
from ..brokers import (
|
||||
core,
|
||||
get_brokermod,
|
||||
data,
|
||||
)
|
||||
DEFAULT_BROKER = 'binance'
|
||||
from ..log import get_console_log, colorize_json, get_logger
|
||||
from .._daemon import maybe_spawn_brokerd, maybe_open_pikerd
|
||||
from ..brokers import core, get_brokermod, data
|
||||
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
_config_dir = click.get_app_dir('piker')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
|
||||
|
||||
OK = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
FAIL = '\033[91m'
|
||||
ENDC = '\033[0m'
|
||||
|
||||
|
||||
def print_ok(s: str, **kwargs):
|
||||
print(OK + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def print_error(s: str, **kwargs):
|
||||
print(FAIL + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def get_method(client, meth_name: str):
|
||||
print(f'checking client for method \'{meth_name}\'...', end='', flush=True)
|
||||
method = getattr(client, meth_name, None)
|
||||
assert method
|
||||
print_ok('found!.')
|
||||
return method
|
||||
|
||||
|
||||
async def run_method(client, meth_name: str, **kwargs):
|
||||
method = get_method(client, meth_name)
|
||||
print('running...', end='', flush=True)
|
||||
result = await method(**kwargs)
|
||||
print_ok(f'done! result: {type(result)}')
|
||||
return result
|
||||
|
||||
|
||||
async def run_test(broker_name: str):
|
||||
brokermod = get_brokermod(broker_name)
|
||||
total = 0
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
print('getting client...', end='', flush=True)
|
||||
if not hasattr(brokermod, 'get_client'):
|
||||
print_error('fail! no \'get_client\' context manager found.')
|
||||
return
|
||||
|
||||
async with brokermod.get_client(is_brokercheck=True) as client:
|
||||
print_ok('done! inside client context.')
|
||||
|
||||
# check for methods present on brokermod
|
||||
method_list = [
|
||||
'backfill_bars',
|
||||
'get_client',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
|
||||
]
|
||||
|
||||
for method in method_list:
|
||||
print(
|
||||
f'checking brokermod for method \'{method}\'...',
|
||||
end='', flush=True)
|
||||
if not hasattr(brokermod, method):
|
||||
print_error(f'fail! method \'{method}\' not found.')
|
||||
failed += 1
|
||||
else:
|
||||
print_ok('done!')
|
||||
passed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
# for private methods only check is present
|
||||
method_list = [
|
||||
'get_balances',
|
||||
'get_assets',
|
||||
'get_trades',
|
||||
'get_xfers',
|
||||
'submit_limit',
|
||||
'submit_cancel',
|
||||
'search_symbols',
|
||||
]
|
||||
|
||||
for method_name in method_list:
|
||||
try:
|
||||
get_method(client, method_name)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
syms = await run_method(client, 'symbol_info')
|
||||
total += 1
|
||||
|
||||
if len(syms) == 0:
|
||||
raise BaseException('Empty Symbol list?')
|
||||
|
||||
passed += 1
|
||||
|
||||
first_sym = tuple(syms.keys())[0]
|
||||
|
||||
method_list = [
|
||||
('cache_symbols', {}),
|
||||
('search_symbols', {'pattern': first_sym[:-1]}),
|
||||
('bars', {'symbol': first_sym})
|
||||
]
|
||||
|
||||
for method_name, method_kwargs in method_list:
|
||||
try:
|
||||
await run_method(client, method_name, **method_kwargs)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
print(f'total: {total}, passed: {passed}, failed: {failed}')
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('broker', nargs=1, required=True)
|
||||
@click.pass_obj
|
||||
def brokercheck(config, broker):
|
||||
'''
|
||||
Test broker apis for completeness.
|
||||
|
||||
'''
|
||||
async def bcheck_main():
|
||||
async with maybe_spawn_brokerd(broker) as portal:
|
||||
await portal.run(run_test, broker)
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(run_test, broker)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--keys', '-k', multiple=True,
|
||||
help='Return results only for these keys')
|
||||
help='Return results only for these keys')
|
||||
@click.argument('meth', nargs=1)
|
||||
@click.argument('kwargs', nargs=-1)
|
||||
@click.pass_obj
|
||||
|
|
@ -242,7 +88,7 @@ def quote(config, tickers):
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermod = list(config['brokermods'].values())[0]
|
||||
brokermod = config['brokermods'][0]
|
||||
|
||||
quotes = trio.run(partial(core.stocks_quote, brokermod, tickers))
|
||||
if not quotes:
|
||||
|
|
@ -269,7 +115,7 @@ def bars(config, symbol, count):
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermod = list(config['brokermods'].values())[0]
|
||||
brokermod = config['brokermods'][0]
|
||||
|
||||
# broker backend should return at the least a
|
||||
# list of candle dictionaries
|
||||
|
|
@ -304,7 +150,7 @@ def record(config, rate, name, dhost, filename):
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermod = list(config['brokermods'].values())[0]
|
||||
brokermod = config['brokermods'][0]
|
||||
loglevel = config['loglevel']
|
||||
log = config['log']
|
||||
|
||||
|
|
@ -369,7 +215,7 @@ def optsquote(config, symbol, date):
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermod = list(config['brokermods'].values())[0]
|
||||
brokermod = config['brokermods'][0]
|
||||
|
||||
quotes = trio.run(
|
||||
partial(
|
||||
|
|
@ -386,70 +232,26 @@ def optsquote(config, symbol, date):
|
|||
@cli.command()
|
||||
@click.argument('tickers', nargs=-1, required=True)
|
||||
@click.pass_obj
|
||||
def mkt_info(
|
||||
config: dict,
|
||||
tickers: list[str],
|
||||
):
|
||||
def symbol_info(config, tickers):
|
||||
'''
|
||||
Print symbol quotes to the console
|
||||
|
||||
'''
|
||||
from msgspec.json import encode, decode
|
||||
from ..accounting import MktPair
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
)
|
||||
|
||||
# global opts
|
||||
brokermods: dict[str, ModuleType] = config['brokermods']
|
||||
brokermod = config['brokermods'][0]
|
||||
|
||||
mkts: list[MktPair] = []
|
||||
async def main():
|
||||
|
||||
async with open_piker_runtime(
|
||||
name='mkt_info_query',
|
||||
# loglevel=loglevel,
|
||||
debug_mode=True,
|
||||
|
||||
) as (_, _):
|
||||
for fqme in tickers:
|
||||
bs_fqme, _, broker = fqme.rpartition('.')
|
||||
brokermod: ModuleType = brokermods[broker]
|
||||
mkt, bs_pair = await core.mkt_info(
|
||||
brokermod,
|
||||
bs_fqme,
|
||||
)
|
||||
mkts.append((mkt, bs_pair))
|
||||
|
||||
trio.run(main)
|
||||
|
||||
if not mkts:
|
||||
log.error(
|
||||
f'No market info could be found for {tickers}'
|
||||
)
|
||||
quotes = trio.run(partial(core.symbol_info, brokermod, tickers))
|
||||
if not quotes:
|
||||
log.error(f"No quotes could be found for {tickers}?")
|
||||
return
|
||||
|
||||
if len(mkts) < len(tickers):
|
||||
syms = tuple(map(itemgetter('fqme'), mkts))
|
||||
if len(quotes) < len(tickers):
|
||||
syms = tuple(map(itemgetter('symbol'), quotes))
|
||||
for ticker in tickers:
|
||||
if ticker not in syms:
|
||||
log.warn(f"Could not find symbol {ticker}?")
|
||||
brokermod.log.warn(f"Could not find symbol {ticker}?")
|
||||
|
||||
|
||||
# TODO: use ``rich.Table`` intead here!
|
||||
for mkt, bs_pair in mkts:
|
||||
click.echo(
|
||||
'\n'
|
||||
'----------------------------------------------------\n'
|
||||
f'{type(bs_pair)}\n'
|
||||
'----------------------------------------------------\n'
|
||||
f'{colorize_json(bs_pair.to_dict())}\n'
|
||||
'----------------------------------------------------\n'
|
||||
f'as piker `MktPair` with fqme: {mkt.fqme}\n'
|
||||
'----------------------------------------------------\n'
|
||||
# NOTE: roundtrip to json codec for console print
|
||||
f'{colorize_json(decode(encode(mkt)))}'
|
||||
)
|
||||
click.echo(colorize_json(quotes))
|
||||
|
||||
|
||||
@cli.command()
|
||||
|
|
@ -461,7 +263,7 @@ def search(config, pattern):
|
|||
|
||||
'''
|
||||
# global opts
|
||||
brokermods = list(config['brokermods'].values())
|
||||
brokermods = config['brokermods']
|
||||
|
||||
# define tractor entrypoint
|
||||
async def main(func):
|
||||
|
|
@ -485,37 +287,3 @@ def search(config, pattern):
|
|||
return
|
||||
|
||||
click.echo(colorize_json(quotes))
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('section', required=False)
|
||||
@click.argument('value', required=False)
|
||||
@click.option('--delete', '-d', flag_value=True, help='Delete section')
|
||||
@click.pass_obj
|
||||
def brokercfg(config, section, value, delete):
|
||||
"""If invoked with no arguments, open an editor to edit broker configs file
|
||||
or get / update an individual section.
|
||||
"""
|
||||
from .. import config
|
||||
|
||||
if section:
|
||||
conf, path = config.load()
|
||||
|
||||
if not delete:
|
||||
if value:
|
||||
config.set_value(conf, section, value)
|
||||
|
||||
click.echo(
|
||||
colorize_json(
|
||||
config.get_value(conf, section))
|
||||
)
|
||||
else:
|
||||
config.del_value(conf, section)
|
||||
|
||||
config.write(config=conf)
|
||||
|
||||
else:
|
||||
conf, path = config.load(raw=True)
|
||||
config.write(
|
||||
raw=click.edit(text=conf)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -26,11 +26,13 @@ from typing import List, Dict, Any, Optional
|
|||
|
||||
import trio
|
||||
|
||||
from ._util import log
|
||||
from ..log import get_logger
|
||||
from . import get_brokermod
|
||||
from ..service import maybe_spawn_brokerd
|
||||
from . import open_cached_client
|
||||
from ..accounting import MktPair
|
||||
from .._daemon import maybe_spawn_brokerd
|
||||
from .._cacheables import open_cached_client
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def api(brokername: str, methname: str, **kwargs) -> dict:
|
||||
|
|
@ -117,19 +119,15 @@ async def bars(
|
|||
return await client.bars(symbol, **kwargs)
|
||||
|
||||
|
||||
async def mkt_info(
|
||||
async def symbol_info(
|
||||
brokermod: ModuleType,
|
||||
fqme: str,
|
||||
symbol: str,
|
||||
**kwargs,
|
||||
|
||||
) -> MktPair:
|
||||
'''
|
||||
Return MktPair info from broker including src and dst assets.
|
||||
|
||||
'''
|
||||
return await brokermod.get_mkt_info(
|
||||
fqme.replace(brokermod.name, '')
|
||||
)
|
||||
) -> Dict[str, Dict[str, Dict[str, Any]]]:
|
||||
"""Return symbol info from broker.
|
||||
"""
|
||||
async with brokermod.get_client() as client:
|
||||
return await client.symbol_info(symbol, **kwargs)
|
||||
|
||||
|
||||
async def search_w_brokerd(name: str, pattern: str) -> dict:
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@ import tractor
|
|||
from tractor.experimental import msgpub
|
||||
from async_generator import asynccontextmanager
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, get_console_log
|
||||
from . import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def wait_for_network(
|
||||
net_func: Callable,
|
||||
sleep: int = 1
|
||||
|
|
@ -227,28 +227,26 @@ async def get_cached_feed(
|
|||
|
||||
@tractor.stream
|
||||
async def start_quote_stream(
|
||||
stream: tractor.Context, # marks this as a streaming func
|
||||
ctx: tractor.Context, # marks this as a streaming func
|
||||
broker: str,
|
||||
symbols: List[Any],
|
||||
feed_type: str = 'stock',
|
||||
rate: int = 3,
|
||||
) -> None:
|
||||
'''
|
||||
Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
"""Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
pattern.
|
||||
|
||||
Spawns new quoter tasks for each broker backend on-demand.
|
||||
Since most brokers seems to support batch quote requests we
|
||||
limit to one task per process (for now).
|
||||
|
||||
'''
|
||||
"""
|
||||
# XXX: why do we need this again?
|
||||
get_console_log(tractor.current_actor().loglevel)
|
||||
|
||||
# pull global vars from local actor
|
||||
symbols = list(symbols)
|
||||
log.info(
|
||||
f"{stream.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
f"{ctx.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
# another actor task may have already created it
|
||||
async with get_cached_feed(broker) as feed:
|
||||
|
||||
|
|
@ -292,13 +290,13 @@ async def start_quote_stream(
|
|||
assert fquote['displayable']
|
||||
payload[sym] = fquote
|
||||
|
||||
await stream.send_yield(payload)
|
||||
await ctx.send_yield(payload)
|
||||
|
||||
await stream_poll_requests(
|
||||
|
||||
# ``trionics.msgpub`` required kwargs
|
||||
task_name=feed_type,
|
||||
ctx=stream,
|
||||
ctx=ctx,
|
||||
topics=symbols,
|
||||
packetizer=feed.mod.packetizer,
|
||||
|
||||
|
|
@ -321,11 +319,9 @@ async def call_client(
|
|||
|
||||
|
||||
class DataFeed:
|
||||
'''
|
||||
Data feed client for streaming symbol data from and making API
|
||||
client calls to a (remote) ``brokerd`` daemon.
|
||||
|
||||
'''
|
||||
"""Data feed client for streaming symbol data from and making API client calls
|
||||
to a (remote) ``brokerd`` daemon.
|
||||
"""
|
||||
_allowed = ('stock', 'option')
|
||||
|
||||
def __init__(self, portal, brokermod):
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
``deribit`` backend
|
||||
------------------
|
||||
pretty good liquidity crypto derivatives, uses custom json rpc over ws for
|
||||
client methods, then `cryptofeed` for data streams.
|
||||
|
||||
status
|
||||
******
|
||||
- supports option charts
|
||||
- no order support yet
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[deribit]
|
||||
key_id = 'XXXXXXXX'
|
||||
key_secret = 'Xx_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
|
||||
To obtain an api id and secret you need to create an account, which can be a
|
||||
real market account over at:
|
||||
|
||||
- deribit.com (requires KYC for deposit address)
|
||||
|
||||
Or a testnet account over at:
|
||||
|
||||
- test.deribit.com
|
||||
|
||||
For testnet once the account is created here is how you deposit fake crypto to
|
||||
try it out:
|
||||
|
||||
1) Go to Wallet:
|
||||
|
||||
.. figure:: assets/0_wallet.png
|
||||
:align: center
|
||||
:target: assets/0_wallet.png
|
||||
:alt: wallet page
|
||||
|
||||
2) Then click on the elipsis menu and select deposit
|
||||
|
||||
.. figure:: assets/1_wallet_select_deposit.png
|
||||
:align: center
|
||||
:target: assets/1_wallet_select_deposit.png
|
||||
:alt: wallet deposit page
|
||||
|
||||
3) This will take you to the deposit address page
|
||||
|
||||
.. figure:: assets/2_gen_deposit_addr.png
|
||||
:align: center
|
||||
:target: assets/2_gen_deposit_addr.png
|
||||
:alt: generate deposit address page
|
||||
|
||||
4) After clicking generate you should see the address, copy it and go to the
|
||||
`coin faucet <https://test.deribit.com/dericoin/BTC/deposit>`_ and send fake
|
||||
coins to that address.
|
||||
|
||||
.. figure:: assets/3_deposit_address.png
|
||||
:align: center
|
||||
:target: assets/3_deposit_address.png
|
||||
:alt: generated address
|
||||
|
||||
5) Back in the deposit address page you should see the deposit in your history
|
||||
|
||||
.. figure:: assets/4_wallet_deposit_history.png
|
||||
:align: center
|
||||
:target: assets/4_wallet_deposit_history.png
|
||||
:alt: wallet deposit history
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
|
||||
from piker.log import get_logger
|
||||
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
# backfill_bars,
|
||||
)
|
||||
# from .broker import (
|
||||
# open_trade_dialog,
|
||||
# norm_trade_records,
|
||||
# )
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
# 'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
# 'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
# 'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
# annotation to let backend agnostic code
|
||||
# know if ``brokerd`` should be spawned with
|
||||
# ``tractor``'s aio mode.
|
||||
_infect_asyncio: bool = True
|
||||
|
|
@ -1,660 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
import asyncio
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
import time
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
)
|
||||
|
||||
import pendulum
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
from tractor.trionics import (
|
||||
broadcast_receiver,
|
||||
maybe_open_context
|
||||
)
|
||||
from tractor import to_asyncio
|
||||
# XXX WOOPS XD
|
||||
# yeah you'll need to install it since it was removed in #489 by
|
||||
# accident; well i thought we had removed all usage..
|
||||
from cryptofeed import FeedHandler
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT,
|
||||
L1_BOOK, TRADES,
|
||||
OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
from piker.data.types import Struct
|
||||
from piker.data import def_iohlcv_fields
|
||||
from piker.data._web_bs import (
|
||||
open_jsonrpc_session
|
||||
)
|
||||
|
||||
|
||||
from piker import config
|
||||
from piker.log import get_logger
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
_url = 'https://www.deribit.com'
|
||||
_ws_url = 'wss://www.deribit.com/ws/api/v2'
|
||||
_testnet_ws_url = 'wss://test.deribit.com/ws/api/v2'
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
id: int
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
usIn: int
|
||||
usOut: int
|
||||
usDiff: int
|
||||
testnet: bool
|
||||
|
||||
class JSONRPCChannel(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
method: str
|
||||
params: dict
|
||||
|
||||
|
||||
class KLinesResult(Struct):
|
||||
close: list[float]
|
||||
cost: list[float]
|
||||
high: list[float]
|
||||
low: list[float]
|
||||
open: list[float]
|
||||
status: str
|
||||
ticks: list[int]
|
||||
volume: list[float]
|
||||
|
||||
class Trade(Struct):
|
||||
trade_seq: int
|
||||
trade_id: str
|
||||
timestamp: int
|
||||
tick_direction: int
|
||||
price: float
|
||||
mark_price: float
|
||||
iv: float
|
||||
instrument_name: str
|
||||
index_price: float
|
||||
direction: str
|
||||
combo_trade_id: Optional[int] = 0,
|
||||
combo_id: Optional[str] = '',
|
||||
amount: float
|
||||
|
||||
class LastTradesResult(Struct):
|
||||
trades: list[Trade]
|
||||
has_more: bool
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def deribit_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
def str_to_cb_sym(name: str) -> Symbol:
|
||||
base, strike_price, expiry_date, option_type = name.split('-')
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'put':
|
||||
option_type = PUT
|
||||
elif option_type == 'call':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date,
|
||||
expiry_normalize=False)
|
||||
|
||||
|
||||
def piker_sym_to_cb_sym(name: str) -> Symbol:
|
||||
base, expiry_date, strike_price, option_type = tuple(
|
||||
name.upper().split('-'))
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'P':
|
||||
option_type = PUT
|
||||
elif option_type == 'C':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date.upper())
|
||||
|
||||
|
||||
def cb_sym_to_deribit_inst(sym: Symbol):
|
||||
# cryptofeed normalized
|
||||
cb_norm = ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z']
|
||||
|
||||
# deribit specific
|
||||
months = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
|
||||
|
||||
exp = sym.expiry_date
|
||||
|
||||
# YYMDD
|
||||
# 01234
|
||||
year, month, day = (
|
||||
exp[:2], months[cb_norm.index(exp[2:3])], exp[3:])
|
||||
|
||||
otype = 'C' if sym.option_type == CALL else 'P'
|
||||
|
||||
return f'{sym.base}-{day}{month}{year}-{sym.strike_price}-{otype}'
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
|
||||
section = conf.get('deribit')
|
||||
|
||||
# TODO: document why we send this, basically because logging params for cryptofeed
|
||||
conf['log'] = {}
|
||||
conf['log']['disabled'] = True
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for deribit in {path}')
|
||||
|
||||
return conf
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
def __init__(self, json_rpc: Callable) -> None:
|
||||
self._pairs: dict[str, Any] = None
|
||||
|
||||
config = get_config().get('deribit', {})
|
||||
|
||||
if ('key_id' in config) and ('key_secret' in config):
|
||||
self._key_id = config['key_id']
|
||||
self._key_secret = config['key_secret']
|
||||
|
||||
else:
|
||||
self._key_id = None
|
||||
self._key_secret = None
|
||||
|
||||
self.json_rpc = json_rpc
|
||||
|
||||
@property
|
||||
def currencies(self):
|
||||
return ['btc', 'eth', 'sol', 'usd']
|
||||
|
||||
async def get_balances(self, kind: str = 'option') -> dict[str, float]:
|
||||
"""Return the set of positions for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_positions', params={
|
||||
'currency': currency.upper(),
|
||||
'kind': kind})
|
||||
|
||||
balances[currency] = resp.result
|
||||
|
||||
return balances
|
||||
|
||||
async def get_assets(self) -> dict[str, float]:
|
||||
"""Return the set of asset balances for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_account_summary', params={
|
||||
'currency': currency.upper()})
|
||||
|
||||
balances[currency] = resp.result['balance']
|
||||
|
||||
return balances
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float
|
||||
) -> dict:
|
||||
"""Place an order
|
||||
"""
|
||||
params = {
|
||||
'instrument_name': symbol.upper(),
|
||||
'amount': size,
|
||||
'type': 'limit',
|
||||
'price': price,
|
||||
}
|
||||
resp = await self.json_rpc(
|
||||
f'private/{action}', params)
|
||||
|
||||
return resp.result
|
||||
|
||||
async def submit_cancel(self, oid: str):
|
||||
"""Send cancel request for order id
|
||||
"""
|
||||
resp = await self.json_rpc(
|
||||
'private/cancel', {'order_id': oid})
|
||||
return resp.result
|
||||
|
||||
async def symbol_info(
|
||||
self,
|
||||
instrument: Optional[str] = None,
|
||||
currency: str = 'btc', # BTC, ETH, SOL, USDC
|
||||
kind: str = 'option',
|
||||
expired: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""Get symbol info for the exchange.
|
||||
|
||||
"""
|
||||
if self._pairs:
|
||||
return self._pairs
|
||||
|
||||
# will retrieve all symbols by default
|
||||
params = {
|
||||
'currency': currency.upper(),
|
||||
'kind': kind,
|
||||
'expired': str(expired).lower()
|
||||
}
|
||||
|
||||
resp = await self.json_rpc('public/get_instruments', params)
|
||||
results = resp.result
|
||||
|
||||
instruments = {
|
||||
item['instrument_name'].lower(): item
|
||||
for item in results
|
||||
}
|
||||
|
||||
if instrument is not None:
|
||||
return instruments[instrument]
|
||||
else:
|
||||
return instruments
|
||||
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
data,
|
||||
score_cutoff=35,
|
||||
limit=limit
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0]['instrument_name'].lower(): item[0]
|
||||
for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str,
|
||||
start_dt: Optional[datetime] = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
limit: int = 1000,
|
||||
as_np: bool = True,
|
||||
) -> dict:
|
||||
instrument = symbol
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_time = deribit_timestamp(start_dt)
|
||||
end_time = deribit_timestamp(end_dt)
|
||||
|
||||
# https://docs.deribit.com/#public-get_tradingview_chart_data
|
||||
resp = await self.json_rpc(
|
||||
'public/get_tradingview_chart_data',
|
||||
params={
|
||||
'instrument_name': instrument.upper(),
|
||||
'start_timestamp': start_time,
|
||||
'end_timestamp': end_time,
|
||||
'resolution': '1'
|
||||
})
|
||||
|
||||
result = KLinesResult(**resp.result)
|
||||
new_bars = []
|
||||
for i in range(len(result.close)):
|
||||
|
||||
_open = result.open[i]
|
||||
high = result.high[i]
|
||||
low = result.low[i]
|
||||
close = result.close[i]
|
||||
volume = result.volume[i]
|
||||
|
||||
row = [
|
||||
(start_time + (i * (60 * 1000))) / 1000.0, # time
|
||||
result.open[i],
|
||||
result.high[i],
|
||||
result.low[i],
|
||||
result.close[i],
|
||||
result.volume[i],
|
||||
0
|
||||
]
|
||||
|
||||
new_bars.append((i,) + tuple(row))
|
||||
|
||||
array = np.array(new_bars, dtype=def_iohlcv_fields) if as_np else klines
|
||||
return array
|
||||
|
||||
async def last_trades(
|
||||
self,
|
||||
instrument: str,
|
||||
count: int = 10
|
||||
):
|
||||
resp = await self.json_rpc(
|
||||
'public/get_last_trades_by_instrument',
|
||||
params={
|
||||
'instrument_name': instrument,
|
||||
'count': count
|
||||
})
|
||||
|
||||
return LastTradesResult(**resp.result)
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client(
|
||||
is_brokercheck: bool = False
|
||||
) -> Client:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_jsonrpc_session(
|
||||
_testnet_ws_url, dtype=JSONRPCResult) as json_rpc
|
||||
):
|
||||
client = Client(json_rpc)
|
||||
|
||||
_refresh_token: Optional[str] = None
|
||||
_access_token: Optional[str] = None
|
||||
|
||||
async def _auth_loop(
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED
|
||||
):
|
||||
"""Background task that adquires a first access token and then will
|
||||
refresh the access token while the nursery isn't cancelled.
|
||||
|
||||
https://docs.deribit.com/?python#authentication-2
|
||||
"""
|
||||
renew_time = 10
|
||||
access_scope = 'trade:read_write'
|
||||
_expiry_time = time.time()
|
||||
got_access = False
|
||||
nonlocal _refresh_token
|
||||
nonlocal _access_token
|
||||
|
||||
while True:
|
||||
if time.time() - _expiry_time < renew_time:
|
||||
# if we are close to token expiry time
|
||||
|
||||
if _refresh_token != None:
|
||||
# if we have a refresh token already dont need to send
|
||||
# secret
|
||||
params = {
|
||||
'grant_type': 'refresh_token',
|
||||
'refresh_token': _refresh_token,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
else:
|
||||
# we don't have refresh token, send secret to initialize
|
||||
params = {
|
||||
'grant_type': 'client_credentials',
|
||||
'client_id': client._key_id,
|
||||
'client_secret': client._key_secret,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
resp = await json_rpc('public/auth', params)
|
||||
result = resp.result
|
||||
|
||||
_expiry_time = time.time() + result['expires_in']
|
||||
_refresh_token = result['refresh_token']
|
||||
|
||||
if 'access_token' in result:
|
||||
_access_token = result['access_token']
|
||||
|
||||
if not got_access:
|
||||
# first time this loop runs we must indicate task is
|
||||
# started, we have auth
|
||||
got_access = True
|
||||
task_status.started()
|
||||
|
||||
else:
|
||||
await trio.sleep(renew_time / 2)
|
||||
|
||||
# if we have client creds launch auth loop
|
||||
if client._key_id is not None:
|
||||
await n.start(_auth_loop)
|
||||
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
@acm
|
||||
async def open_feed_handler():
|
||||
fh = FeedHandler(config=get_config())
|
||||
yield fh
|
||||
await to_asyncio.run_task(fh.stop_async)
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_feed_handler() -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_context(
|
||||
acm_func=open_feed_handler,
|
||||
key='feedhandler',
|
||||
) as (cache_hit, fh):
|
||||
yield fh
|
||||
|
||||
|
||||
async def aio_price_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _trade(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('trade', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'last': data,
|
||||
'broker_ts': time.time(),
|
||||
'data': data.to_dict(),
|
||||
'receipt': receipt_timestamp
|
||||
}))
|
||||
|
||||
async def _l1(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('l1', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'ticks': [
|
||||
{'type': 'bid',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'bsize',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'ask',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)},
|
||||
{'type': 'asize',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)}
|
||||
]
|
||||
}))
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[TRADES, L1_BOOK],
|
||||
symbols=[piker_sym_to_cb_sym(instrument)],
|
||||
callbacks={
|
||||
TRADES: _trade,
|
||||
L1_BOOK: _l1
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_price_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_price_feed,
|
||||
kwargs={
|
||||
'instrument': instrument
|
||||
},
|
||||
key=f'{instrument}-price',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
|
||||
|
||||
|
||||
async def aio_order_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _fill(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
async def _order_info(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[FILLS, ORDER_INFO],
|
||||
symbols=[instrument.upper()],
|
||||
callbacks={
|
||||
FILLS: _fill,
|
||||
ORDER_INFO: _order_info,
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_order_feed(
|
||||
instrument: list[str]
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_order_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_order_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_order_feed,
|
||||
kwargs={
|
||||
'instrument': instrument,
|
||||
'fh': fh
|
||||
},
|
||||
key=f'{instrument}-order',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 169 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 132 KiB |
|
|
@ -1,185 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, Callable
|
||||
import time
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import pendulum
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
|
||||
from piker.brokers import open_cached_client
|
||||
from piker.log import get_logger, get_console_log
|
||||
from piker.data import ShmArray
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataUnavailable,
|
||||
)
|
||||
|
||||
from cryptofeed import FeedHandler
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT, L1_BOOK, TRADES, OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
from .api import (
|
||||
Client, Trade,
|
||||
get_config,
|
||||
str_to_cb_sym, piker_sym_to_cb_sym, cb_sym_to_deribit_inst,
|
||||
maybe_open_price_feed
|
||||
)
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
mkt: MktPair,
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
fnstrument: str = mkt.bs_fqme
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
array = await client.bars(
|
||||
instrument,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
if len(array) == 0:
|
||||
raise DataUnavailable
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym = symbols[0]
|
||||
|
||||
async with (
|
||||
open_cached_client('deribit') as client,
|
||||
send_chan as send_chan
|
||||
):
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
sym: {
|
||||
'symbol_info': {
|
||||
'asset_type': 'option',
|
||||
'price_tick_size': 0.0005
|
||||
},
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
nsym = piker_sym_to_cb_sym(sym)
|
||||
|
||||
async with maybe_open_price_feed(sym) as stream:
|
||||
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
last_trades = (await client.last_trades(
|
||||
cb_sym_to_deribit_inst(nsym), count=1)).trades
|
||||
|
||||
if len(last_trades) == 0:
|
||||
last_trade = None
|
||||
async for typ, quote in stream:
|
||||
if typ == 'trade':
|
||||
last_trade = Trade(**(quote['data']))
|
||||
break
|
||||
|
||||
else:
|
||||
last_trade = Trade(**(last_trades[0]))
|
||||
|
||||
first_quote = {
|
||||
'symbol': sym,
|
||||
'last': last_trade.price,
|
||||
'brokerd_ts': last_trade.timestamp,
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': last_trade.price,
|
||||
'size': last_trade.amount,
|
||||
'broker_ts': last_trade.timestamp
|
||||
}]
|
||||
}
|
||||
task_status.started((init_msgs, first_quote))
|
||||
|
||||
feed_is_live.set()
|
||||
|
||||
async for typ, quote in stream:
|
||||
topic = quote['symbol']
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> Client:
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# repack in dict form
|
||||
await stream.send(
|
||||
await client.search_symbols(pattern))
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
``ib`` backend
|
||||
--------------
|
||||
more or less the "everything broker" for traditional and international
|
||||
markets. they are the "go to" provider for automatic retail trading
|
||||
and we interface to their APIs using the `ib_insync` project.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
currently there is not yet full support for:
|
||||
- options charting and trading
|
||||
- paxos based crypto rt feeds and trading
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib]
|
||||
hosts = [
|
||||
"127.0.0.1",
|
||||
]
|
||||
# TODO: when we eventually spawn gateways in our
|
||||
# container, we can just dynamically allocate these
|
||||
# using IBC.
|
||||
ports = [
|
||||
4002,
|
||||
4003,
|
||||
4006,
|
||||
4001,
|
||||
7497,
|
||||
]
|
||||
|
||||
# XXX: for a paper account the flex web query service
|
||||
# is not supported so you have to manually download
|
||||
# and XML report and put it in a location that can be
|
||||
# accessed by the ``brokerd.ib`` backend code for parsing.
|
||||
flex_token = '1111111111111111'
|
||||
flex_trades_query_id = '6969696' # live accounts only?
|
||||
|
||||
# 3rd party web-api token
|
||||
# (XXX: not sure if this works yet)
|
||||
trade_log_token = '111111111111111'
|
||||
|
||||
# when clients are being scanned this determines
|
||||
# which clients are preferred to be used for data feeds
|
||||
# based on account names which are detected as active
|
||||
# on each client.
|
||||
prefer_data_account = [
|
||||
# this has to be first in order to make data work with dual paper + live
|
||||
'main',
|
||||
'algopaper',
|
||||
]
|
||||
|
||||
[ib.accounts]
|
||||
main = 'U69696969'
|
||||
algopaper = 'DU9696969'
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_ib_algopaper.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
["0000e1a7.630f5e5a.01.01"]
|
||||
secType = "FUT"
|
||||
conId = 515416577
|
||||
symbol = "MNQ"
|
||||
lastTradeDateOrContractMonth = "20221216"
|
||||
strike = 0.0
|
||||
right = ""
|
||||
multiplier = "2"
|
||||
exchange = "GLOBEX"
|
||||
primaryExchange = ""
|
||||
currency = "USD"
|
||||
localSymbol = "MNQZ2"
|
||||
tradingClass = "MNQ"
|
||||
includeExpired = false
|
||||
secIdType = ""
|
||||
secId = ""
|
||||
comboLegsDescrip = ""
|
||||
comboLegs = []
|
||||
execId = "0000e1a7.630f5e5a.01.01"
|
||||
time = 1661972086.0
|
||||
acctNumber = "DU69696969"
|
||||
side = "BOT"
|
||||
shares = 1.0
|
||||
price = 12372.75
|
||||
permId = 441472655
|
||||
clientId = 6116
|
||||
orderId = 985
|
||||
liquidation = 0
|
||||
cumQty = 1.0
|
||||
avgPrice = 12372.75
|
||||
orderRef = ""
|
||||
evRule = ""
|
||||
evMultiplier = 0.0
|
||||
modelCode = ""
|
||||
lastLiquidity = 1
|
||||
broker_time = 1661972086.0
|
||||
name = "ib"
|
||||
commission = 0.57
|
||||
realizedPNL = 243.41
|
||||
yield_ = 0.0
|
||||
yieldRedemptionDate = 0
|
||||
listingExchange = "GLOBEX"
|
||||
date = "2022-08-31T18:54:46+00:00"
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib.algopaper."mnq.globex.20221216"]
|
||||
size = -1.0
|
||||
ppu = 12423.630576923071
|
||||
bs_mktid = 515416577
|
||||
expiry = "2022-12-16T00:00:00+00:00"
|
||||
clears = [
|
||||
{ dt = "2022-08-31T18:54:46+00:00", ppu = 12423.630576923071, accum_size = -19.0, price = 12372.75, size = 1.0, cost = 0.57, tid = "0000e1a7.630f5e5a.01.01" },
|
||||
]
|
||||
|
|
@ -20,10 +20,15 @@ Interactive Brokers API backend.
|
|||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
- ``data.py`` for real-time data feed endpoints
|
||||
|
||||
- ``client.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
- ``report.py`` for the hackery to build manual pp calcs
|
||||
to avoid ib's absolute bullshit FIFO style position
|
||||
tracking..
|
||||
|
||||
"""
|
||||
from .api import (
|
||||
get_client,
|
||||
|
|
@ -33,38 +38,23 @@ from .feed import (
|
|||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
open_trade_dialog,
|
||||
)
|
||||
from .ledger import (
|
||||
norm_trade_records,
|
||||
)
|
||||
from .broker import trades_dialogue
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
'norm_trade_records',
|
||||
'open_trade_dialog',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
]
|
||||
|
||||
_brokerd_mods: list[str] = [
|
||||
'api',
|
||||
'broker',
|
||||
]
|
||||
|
||||
_datad_mods: list[str] = [
|
||||
'feed',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = (
|
||||
_brokerd_mods
|
||||
+
|
||||
_datad_mods
|
||||
)
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
|
|
|
|||
|
|
@ -1,191 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
"FLEX" report processing utils.
|
||||
|
||||
"""
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
from .api import (
|
||||
get_config,
|
||||
log,
|
||||
)
|
||||
from piker.accounting import (
|
||||
open_trade_ledger,
|
||||
)
|
||||
|
||||
|
||||
def parse_flex_dt(
|
||||
record: str,
|
||||
) -> pendulum.datetime:
|
||||
'''
|
||||
Parse stupid flex record datetime stamps for the `dateTime` field..
|
||||
|
||||
'''
|
||||
date, ts = record.split(';')
|
||||
dt = pendulum.parse(date)
|
||||
ts = f'{ts[:2]}:{ts[2:4]}:{ts[4:]}'
|
||||
tsdt = pendulum.parse(ts)
|
||||
return dt.set(hour=tsdt.hour, minute=tsdt.minute, second=tsdt.second)
|
||||
|
||||
|
||||
def flex_records_to_ledger_entries(
|
||||
accounts: bidict,
|
||||
trade_entries: list[object],
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Convert flex report entry objects into ``dict`` form, pretty much
|
||||
straight up without modification except add a `pydatetime` field
|
||||
from the parsed timestamp.
|
||||
|
||||
'''
|
||||
trades_by_account = {}
|
||||
for t in trade_entries:
|
||||
entry = t.__dict__
|
||||
|
||||
# XXX: LOL apparently ``toml`` has a bug
|
||||
# where a section key error will show up in the write
|
||||
# if you leave a table key as an `int`? So i guess
|
||||
# cast to strs for all keys..
|
||||
|
||||
# oddly for some so-called "BookTrade" entries
|
||||
# this field seems to be blank, no cuckin clue.
|
||||
# trade['ibExecID']
|
||||
tid = str(entry.get('ibExecID') or entry['tradeID'])
|
||||
# date = str(entry['tradeDate'])
|
||||
|
||||
# XXX: is it going to cause problems if a account name
|
||||
# get's lost? The user should be able to find it based
|
||||
# on the actual exec history right?
|
||||
acctid = accounts[str(entry['accountId'])]
|
||||
|
||||
# probably a flex record with a wonky non-std timestamp..
|
||||
dt = entry['pydatetime'] = parse_flex_dt(entry['dateTime'])
|
||||
entry['datetime'] = str(dt)
|
||||
|
||||
if not tid:
|
||||
# this is likely some kind of internal adjustment
|
||||
# transaction, likely one of the following:
|
||||
# - an expiry event that will show a "book trade" indicating
|
||||
# some adjustment to cash balances: zeroing or itm settle.
|
||||
# - a manual cash balance position adjustment likely done by
|
||||
# the user from the accounts window in TWS where they can
|
||||
# manually set the avg price and size:
|
||||
# https://api.ibkr.com/lib/cstools/faq/web1/index.html#/tag/DTWS_ADJ_AVG_COST
|
||||
log.warning(f'Skipping ID-less ledger entry:\n{pformat(entry)}')
|
||||
continue
|
||||
|
||||
trades_by_account.setdefault(
|
||||
acctid, {}
|
||||
)[tid] = entry
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_account[acctid] = dict(sorted(
|
||||
trades_by_account[acctid].items(),
|
||||
key=lambda entry: entry[1]['pydatetime'],
|
||||
))
|
||||
|
||||
return trades_by_account
|
||||
|
||||
|
||||
def load_flex_trades(
|
||||
path: str | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
|
||||
from ib_insync import flexreport, util
|
||||
|
||||
conf = get_config()
|
||||
|
||||
if not path:
|
||||
# load ``brokers.toml`` and try to get the flex
|
||||
# token and query id that must be previously defined
|
||||
# by the user.
|
||||
token = conf.get('flex_token')
|
||||
if not token:
|
||||
raise ValueError(
|
||||
'You must specify a ``flex_token`` field in your'
|
||||
'`brokers.toml` in order load your trade log, see our'
|
||||
'intructions for how to set this up here:\n'
|
||||
'PUT LINK HERE!'
|
||||
)
|
||||
|
||||
qid = conf['flex_trades_query_id']
|
||||
|
||||
# TODO: hack this into our logging
|
||||
# system like we do with the API client..
|
||||
util.logToConsole()
|
||||
|
||||
# TODO: rewrite the query part of this with async..httpx?
|
||||
report = flexreport.FlexReport(
|
||||
token=token,
|
||||
queryId=qid,
|
||||
)
|
||||
|
||||
else:
|
||||
# XXX: another project we could potentially look at,
|
||||
# https://pypi.org/project/ibflex/
|
||||
report = flexreport.FlexReport(path=path)
|
||||
|
||||
trade_entries = report.extract('Trade')
|
||||
ln = len(trade_entries)
|
||||
log.info(f'Loaded {ln} trades from flex query')
|
||||
|
||||
trades_by_account = flex_records_to_ledger_entries(
|
||||
conf['accounts'].inverse, # reverse map to user account names
|
||||
trade_entries,
|
||||
)
|
||||
|
||||
ledger_dict: dict | None = None
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_id = trades_by_account[acctid]
|
||||
|
||||
with open_trade_ledger('ib', acctid) as ledger_dict:
|
||||
tid_delta = set(trades_by_id) - set(ledger_dict)
|
||||
log.info(
|
||||
'New trades detected\n'
|
||||
f'{pformat(tid_delta)}'
|
||||
)
|
||||
if tid_delta:
|
||||
sorted_delta = dict(sorted(
|
||||
{tid: trades_by_id[tid] for tid in tid_delta}.items(),
|
||||
key=lambda entry: entry[1].pop('pydatetime'),
|
||||
))
|
||||
ledger_dict.update(sorted_delta)
|
||||
|
||||
return ledger_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
import os
|
||||
|
||||
args = sys.argv
|
||||
if len(args) > 1:
|
||||
args = args[1:]
|
||||
for arg in args:
|
||||
path = os.path.abspath(arg)
|
||||
load_flex_trades(path=path)
|
||||
else:
|
||||
# expect brokers.toml to have an entry and
|
||||
# pull from the web service.
|
||||
load_flex_trades()
|
||||
|
|
@ -1,260 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
``ib`` utilities and hacks suitable for use in the backend and/or as
|
||||
runnable script-programs.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from functools import partial
|
||||
from typing import (
|
||||
Literal,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
import subprocess
|
||||
|
||||
import tractor
|
||||
|
||||
from piker.brokers._util import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .api import Client
|
||||
from ib_insync import IB
|
||||
|
||||
log = get_logger('piker.brokers.ib')
|
||||
|
||||
_reset_tech: Literal[
|
||||
'vnc',
|
||||
'i3ipc_xdotool',
|
||||
|
||||
# TODO: in theory we can use a different linux DE API or
|
||||
# some other type of similar window scanning/mgmt client
|
||||
# (on other OSs) to do the same.
|
||||
|
||||
] = 'vnc'
|
||||
|
||||
|
||||
async def data_reset_hack(
|
||||
# vnc_host: str,
|
||||
client: Client,
|
||||
reset_type: Literal['data', 'connection'],
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Run key combos for resetting data feeds and yield back to caller
|
||||
when complete.
|
||||
|
||||
NOTE: this is a linux-only hack around!
|
||||
|
||||
There are multiple "techs" you can use depending on your infra setup:
|
||||
|
||||
- if running ib-gw in a container with a VNC server running the most
|
||||
performant method is the `'vnc'` option.
|
||||
|
||||
- if running ib-gw/tws locally, and you are using `i3` you can use
|
||||
the ``i3ipc`` lib and ``xdotool`` to send the appropriate click
|
||||
and key-combos automatically to your local desktop's java X-apps.
|
||||
|
||||
https://interactivebrokers.github.io/tws-api/historical_limitations.html#pacing_violations
|
||||
|
||||
TODOs:
|
||||
- a return type that hopefully determines if the hack was
|
||||
successful.
|
||||
- other OS support?
|
||||
- integration with ``ib-gw`` run in docker + Xorg?
|
||||
- is it possible to offer a local server that can be accessed by
|
||||
a client? Would be sure be handy for running native java blobs
|
||||
that need to be wrangle.
|
||||
|
||||
'''
|
||||
ib_client: IB = client.ib
|
||||
|
||||
# look up any user defined vnc socket address mapped from
|
||||
# a particular API socket port.
|
||||
api_port: str = str(ib_client.client.port)
|
||||
vnc_host: str
|
||||
vnc_port: int
|
||||
vnc_host, vnc_port = client.conf['vnc_addrs'].get(
|
||||
api_port,
|
||||
('localhost', 3003)
|
||||
)
|
||||
|
||||
no_setup_msg:str = (
|
||||
f'No data reset hack test setup for {vnc_host}!\n'
|
||||
'See setup @\n'
|
||||
'https://github.com/pikers/piker/tree/master/piker/brokers/ib'
|
||||
)
|
||||
global _reset_tech
|
||||
|
||||
match _reset_tech:
|
||||
case 'vnc':
|
||||
try:
|
||||
await tractor.to_asyncio.run_task(
|
||||
partial(
|
||||
vnc_click_hack,
|
||||
host=vnc_host,
|
||||
port=vnc_port,
|
||||
)
|
||||
)
|
||||
except OSError:
|
||||
if vnc_host != 'localhost':
|
||||
log.warning(no_setup_msg)
|
||||
return False
|
||||
|
||||
try:
|
||||
import i3ipc # noqa (since a deps dynamic check)
|
||||
except ModuleNotFoundError:
|
||||
log.warning(no_setup_msg)
|
||||
return False
|
||||
|
||||
try:
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
_reset_tech = 'i3ipc_xdotool'
|
||||
return True
|
||||
except OSError:
|
||||
log.exception(no_setup_msg)
|
||||
return False
|
||||
|
||||
case 'i3ipc_xdotool':
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
|
||||
case _ as tech:
|
||||
raise RuntimeError(f'{tech} is not supported for reset tech!?')
|
||||
|
||||
# we don't really need the ``xdotool`` approach any more B)
|
||||
return True
|
||||
|
||||
|
||||
async def vnc_click_hack(
|
||||
host: str,
|
||||
port: int,
|
||||
reset_type: str = 'data'
|
||||
) -> None:
|
||||
'''
|
||||
Reset the data or network connection for the VNC attached
|
||||
ib gateway using magic combos.
|
||||
|
||||
'''
|
||||
try:
|
||||
import asyncvnc
|
||||
except ModuleNotFoundError:
|
||||
log.warning(
|
||||
"In order to leverage `piker`'s built-in data reset hacks, install "
|
||||
"the `asyncvnc` project: https://github.com/barneygale/asyncvnc"
|
||||
)
|
||||
return
|
||||
|
||||
# two different hot keys which trigger diff types of reset
|
||||
# requests B)
|
||||
key = {
|
||||
'data': 'f',
|
||||
'connection': 'r'
|
||||
}[reset_type]
|
||||
|
||||
async with asyncvnc.connect(
|
||||
host,
|
||||
port=port,
|
||||
|
||||
# TODO: doesn't work see:
|
||||
# https://github.com/barneygale/asyncvnc/issues/7
|
||||
# password='ibcansmbz',
|
||||
|
||||
) as client:
|
||||
|
||||
# move to middle of screen
|
||||
# 640x1800
|
||||
client.mouse.move(
|
||||
x=500,
|
||||
y=500,
|
||||
)
|
||||
client.mouse.click()
|
||||
client.keyboard.press('Ctrl', 'Alt', key) # keys are stacked
|
||||
|
||||
|
||||
def i3ipc_xdotool_manual_click_hack() -> None:
|
||||
'''
|
||||
Do the data reset hack but expecting a local X-window using `xdotool`.
|
||||
|
||||
'''
|
||||
import i3ipc
|
||||
i3 = i3ipc.Connection()
|
||||
|
||||
# TODO: might be worth offering some kinda api for grabbing
|
||||
# the window id from the pid?
|
||||
# https://stackoverflow.com/a/2250879
|
||||
t = i3.get_tree()
|
||||
|
||||
orig_win_id = t.find_focused().window
|
||||
|
||||
# for tws
|
||||
win_names: list[str] = [
|
||||
'Interactive Brokers', # tws running in i3
|
||||
'IB Gateway', # gw running in i3
|
||||
# 'IB', # gw running in i3 (newer version?)
|
||||
]
|
||||
|
||||
try:
|
||||
for name in win_names:
|
||||
results = t.find_titled(name)
|
||||
print(f'results for {name}: {results}')
|
||||
if results:
|
||||
con = results[0]
|
||||
print(f'Resetting data feed for {name}')
|
||||
win_id = str(con.window)
|
||||
w, h = con.rect.width, con.rect.height
|
||||
|
||||
# TODO: seems to be a few libs for python but not sure
|
||||
# if they support all the sub commands we need, order of
|
||||
# most recent commit history:
|
||||
# https://github.com/rr-/pyxdotool
|
||||
# https://github.com/ShaneHutter/pyxdotool
|
||||
# https://github.com/cphyc/pyxdotool
|
||||
|
||||
# TODO: only run the reconnect (2nd) kc on a detected
|
||||
# disconnect?
|
||||
for key_combo, timeout in [
|
||||
# only required if we need a connection reset.
|
||||
# ('ctrl+alt+r', 12),
|
||||
# data feed reset.
|
||||
('ctrl+alt+f', 6)
|
||||
]:
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', win_id,
|
||||
|
||||
# move mouse to bottom left of window (where
|
||||
# there should be nothing to click).
|
||||
'mousemove_relative', '--sync', str(w-4), str(h-4),
|
||||
|
||||
# NOTE: we may need to stick a `--retry 3` in here..
|
||||
'click', '--window', win_id,
|
||||
'--repeat', '3', '1',
|
||||
|
||||
# hackzorzes
|
||||
'key', key_combo,
|
||||
],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# re-activate and focus original window
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', str(orig_win_id),
|
||||
'click', '--window', str(orig_win_id), '1',
|
||||
])
|
||||
except subprocess.TimeoutExpired:
|
||||
log.exception('xdotool timed out?')
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,250 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Trade transaction accounting and normalization.
|
||||
|
||||
'''
|
||||
from bisect import insort
|
||||
from decimal import Decimal
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
|
||||
from piker.accounting import (
|
||||
dec_digits,
|
||||
digits_to_dec,
|
||||
Transaction,
|
||||
MktPair,
|
||||
)
|
||||
from ._flex_reports import parse_flex_dt
|
||||
from ._util import log
|
||||
|
||||
|
||||
def norm_trade_records(
|
||||
ledger: dict[str, Any],
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Normalize a flex report or API retrieved executions
|
||||
ledger into our standard record format.
|
||||
|
||||
'''
|
||||
records: list[Transaction] = []
|
||||
|
||||
for tid, record in ledger.items():
|
||||
conid = record.get('conId') or record['conid']
|
||||
comms = record.get('commission')
|
||||
if comms is None:
|
||||
comms = -1*record['ibCommission']
|
||||
|
||||
price = record.get('price') or record['tradePrice']
|
||||
|
||||
# the api doesn't do the -/+ on the quantity for you but flex
|
||||
# records do.. are you fucking serious ib...!?
|
||||
size = record.get('quantity') or record['shares'] * {
|
||||
'BOT': 1,
|
||||
'SLD': -1,
|
||||
}[record['side']]
|
||||
|
||||
exch = record['exchange']
|
||||
lexch = record.get('listingExchange')
|
||||
|
||||
# NOTE: remove null values since `tomlkit` can't serialize
|
||||
# them to file.
|
||||
dnc = record.pop('deltaNeutralContract', False)
|
||||
if dnc is not None:
|
||||
record['deltaNeutralContract'] = dnc
|
||||
|
||||
suffix = lexch or exch
|
||||
symbol = record['symbol']
|
||||
|
||||
# likely an opts contract record from a flex report..
|
||||
# TODO: no idea how to parse ^ the strike part from flex..
|
||||
# (00010000 any, or 00007500 tsla, ..)
|
||||
# we probably must do the contract lookup for this?
|
||||
if ' ' in symbol or '--' in exch:
|
||||
underlying, _, tail = symbol.partition(' ')
|
||||
suffix = exch = 'opt'
|
||||
expiry = tail[:6]
|
||||
# otype = tail[6]
|
||||
# strike = tail[7:]
|
||||
|
||||
print(f'skipping opts contract {symbol}')
|
||||
continue
|
||||
|
||||
# timestamping is way different in API records
|
||||
dtstr = record.get('datetime')
|
||||
date = record.get('date')
|
||||
flex_dtstr = record.get('dateTime')
|
||||
|
||||
if dtstr or date:
|
||||
dt = pendulum.parse(dtstr or date)
|
||||
|
||||
elif flex_dtstr:
|
||||
# probably a flex record with a wonky non-std timestamp..
|
||||
dt = parse_flex_dt(record['dateTime'])
|
||||
|
||||
# special handling of symbol extraction from
|
||||
# flex records using some ad-hoc schema parsing.
|
||||
asset_type: str = record.get(
|
||||
'assetCategory'
|
||||
) or record.get('secType', 'STK')
|
||||
|
||||
# TODO: XXX: WOA this is kinda hacky.. probably
|
||||
# should figure out the correct future pair key more
|
||||
# explicitly and consistently?
|
||||
if asset_type == 'FUT':
|
||||
# (flex) ledger entries don't have any simple 3-char key?
|
||||
symbol = record['symbol'][:3]
|
||||
asset_type: str = 'future'
|
||||
|
||||
elif asset_type == 'STK':
|
||||
asset_type: str = 'stock'
|
||||
|
||||
# try to build out piker fqme from record.
|
||||
expiry = (
|
||||
record.get('lastTradeDateOrContractMonth')
|
||||
or record.get('expiry')
|
||||
)
|
||||
|
||||
if expiry:
|
||||
expiry = str(expiry).strip(' ')
|
||||
suffix = f'{exch}.{expiry}'
|
||||
expiry = pendulum.parse(expiry)
|
||||
|
||||
# src: str = record['currency']
|
||||
price_tick: Decimal = digits_to_dec(dec_digits(price))
|
||||
|
||||
pair = MktPair.from_fqme(
|
||||
fqme=f'{symbol}.{suffix}.ib',
|
||||
bs_mktid=str(conid),
|
||||
_atype=str(asset_type), # XXX: can't serlialize `tomlkit.String`
|
||||
|
||||
price_tick=price_tick,
|
||||
# NOTE: for "legacy" assets, volume is normally discreet, not
|
||||
# a float, but we keep a digit in case the suitz decide
|
||||
# to get crazy and change it; we'll be kinda ready
|
||||
# schema-wise..
|
||||
size_tick='1',
|
||||
)
|
||||
|
||||
fqme = pair.fqme
|
||||
|
||||
# NOTE: for flex records the normal fields for defining an fqme
|
||||
# sometimes won't be available so we rely on two approaches for
|
||||
# the "reverse lookup" of piker style fqme keys:
|
||||
# - when dealing with API trade records received from
|
||||
# `IB.trades()` we do a contract lookup at he time of processing
|
||||
# - when dealing with flex records, it is assumed the record
|
||||
# is at least a day old and thus the TWS position reporting system
|
||||
# should already have entries if the pps are still open, in
|
||||
# which case, we can pull the fqme from that table (see
|
||||
# `trades_dialogue()` above).
|
||||
insort(
|
||||
records,
|
||||
Transaction(
|
||||
fqme=fqme,
|
||||
sym=pair,
|
||||
tid=tid,
|
||||
size=size,
|
||||
price=price,
|
||||
cost=comms,
|
||||
dt=dt,
|
||||
expiry=expiry,
|
||||
bs_mktid=str(conid),
|
||||
),
|
||||
key=lambda t: t.dt
|
||||
)
|
||||
|
||||
return {r.tid: r for r in records}
|
||||
|
||||
|
||||
def api_trades_to_ledger_entries(
|
||||
accounts: bidict[str, str],
|
||||
|
||||
# TODO: maybe we should just be passing through the
|
||||
# ``ib_insync.order.Trade`` instance directly here
|
||||
# instead of pre-casting to dicts?
|
||||
trade_entries: list[dict],
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Convert API execution objects entry objects into ``dict`` form,
|
||||
pretty much straight up without modification except add
|
||||
a `pydatetime` field from the parsed timestamp.
|
||||
|
||||
'''
|
||||
trades_by_account = {}
|
||||
for t in trade_entries:
|
||||
# NOTE: example of schema we pull from the API client.
|
||||
# {
|
||||
# 'commissionReport': CommissionReport(...
|
||||
# 'contract': {...
|
||||
# 'execution': Execution(...
|
||||
# 'time': 1654801166.0
|
||||
# }
|
||||
|
||||
# flatten all sub-dicts and values into one top level entry.
|
||||
entry = {}
|
||||
for section, val in t.items():
|
||||
match section:
|
||||
case 'contract' | 'execution' | 'commissionReport':
|
||||
# sub-dict cases
|
||||
entry.update(val)
|
||||
|
||||
case 'time':
|
||||
# ib has wack ns timestamps, or is that us?
|
||||
continue
|
||||
|
||||
case _:
|
||||
entry[section] = val
|
||||
|
||||
tid = str(entry['execId'])
|
||||
dt = pendulum.from_timestamp(entry['time'])
|
||||
# TODO: why isn't this showing seconds in the str?
|
||||
entry['pydatetime'] = dt
|
||||
entry['datetime'] = str(dt)
|
||||
acctid = accounts[entry['acctNumber']]
|
||||
|
||||
if not tid:
|
||||
# this is likely some kind of internal adjustment
|
||||
# transaction, likely one of the following:
|
||||
# - an expiry event that will show a "book trade" indicating
|
||||
# some adjustment to cash balances: zeroing or itm settle.
|
||||
# - a manual cash balance position adjustment likely done by
|
||||
# the user from the accounts window in TWS where they can
|
||||
# manually set the avg price and size:
|
||||
# https://api.ibkr.com/lib/cstools/faq/web1/index.html#/tag/DTWS_ADJ_AVG_COST
|
||||
log.warning(f'Skipping ID-less ledger entry:\n{pformat(entry)}')
|
||||
continue
|
||||
|
||||
trades_by_account.setdefault(
|
||||
acctid, {}
|
||||
)[tid] = entry
|
||||
|
||||
# sort entries in output by python based datetime
|
||||
for acctid in trades_by_account:
|
||||
trades_by_account[acctid] = dict(sorted(
|
||||
trades_by_account[acctid].items(),
|
||||
key=lambda entry: entry[1].pop('pydatetime'),
|
||||
))
|
||||
|
||||
return trades_by_account
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,64 +0,0 @@
|
|||
``kraken`` backend
|
||||
------------------
|
||||
though they don't have the most liquidity of all the cexes they sure are
|
||||
accommodating to those of us who appreciate a little ``xmr``.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken]
|
||||
accounts.spot = 'spot'
|
||||
key_descr = "spot"
|
||||
api_key = "69696969696969696696969696969696969696969696969696969696"
|
||||
secret = "BOOBSBOOBSBOOBSBOOBSBOOBSSMBZ69696969696969669969696969696"
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_kraken_spot.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[TFJBKK-SMBZS-VJ4UWS]
|
||||
ordertxid = "SMBZSA-7CNQU-3HWLNJ"
|
||||
postxid = "SMBZSE-M7IF5-CFI7LT"
|
||||
pair = "XXMRZEUR"
|
||||
time = 1655691993.4133966
|
||||
type = "buy"
|
||||
ordertype = "limit"
|
||||
price = "103.97000000"
|
||||
cost = "499.99999977"
|
||||
fee = "0.80000000"
|
||||
vol = "4.80907954"
|
||||
margin = "0.00000000"
|
||||
misc = ""
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken.spot."xmreur.kraken"]
|
||||
size = 4.80907954
|
||||
ppu = 103.97000000
|
||||
bs_mktid = "XXMRZEUR"
|
||||
clears = [
|
||||
{ tid = "TFJBKK-SMBZS-VJ4UWS", cost = 0.8, price = 103.97, size = 4.80907954, dt = "2022-05-20T02:26:33.413397+00:00" },
|
||||
]
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken backend.
|
||||
|
||||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
'''
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
get_mkt_info,
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
open_trade_dialog,
|
||||
norm_trade_records,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
'get_mkt_info',
|
||||
'open_trade_dialog',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
|
@ -1,671 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken web API wrapping.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import itertools
|
||||
from typing import (
|
||||
Any,
|
||||
Union,
|
||||
)
|
||||
import time
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import asks
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import urllib.parse
|
||||
import hashlib
|
||||
import hmac
|
||||
import base64
|
||||
import trio
|
||||
|
||||
from piker import config
|
||||
from piker.data.types import Struct
|
||||
from piker.data import def_iohlcv_fields
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
digits_to_dec,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
)
|
||||
from piker.accounting import Transaction
|
||||
from piker.log import get_logger
|
||||
|
||||
log = get_logger('piker.brokers.kraken')
|
||||
|
||||
# <uri>/<version>/
|
||||
_url = 'https://api.kraken.com/0'
|
||||
# TODO: this is the only backend providing this right?
|
||||
# in which case we should drop it from the defaults and
|
||||
# instead make a custom fields descr in this module!
|
||||
_show_wap_in_history = True
|
||||
_symbol_info_translation: dict[str, str] = {
|
||||
'tick_decimals': 'pair_decimals',
|
||||
}
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
section = conf.get('kraken')
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for kraken in {path}')
|
||||
return {}
|
||||
|
||||
return section
|
||||
|
||||
|
||||
def get_kraken_signature(
|
||||
urlpath: str,
|
||||
data: dict[str, Any],
|
||||
secret: str
|
||||
) -> str:
|
||||
postdata = urllib.parse.urlencode(data)
|
||||
encoded = (str(data['nonce']) + postdata).encode()
|
||||
message = urlpath.encode() + hashlib.sha256(encoded).digest()
|
||||
|
||||
mac = hmac.new(base64.b64decode(secret), message, hashlib.sha512)
|
||||
sigdigest = base64.b64encode(mac.digest())
|
||||
return sigdigest.decode()
|
||||
|
||||
|
||||
class InvalidKey(ValueError):
|
||||
'''
|
||||
EAPI:Invalid key
|
||||
This error is returned when the API key used for the call is
|
||||
either expired or disabled, please review the API key in your
|
||||
Settings -> API tab of account management or generate a new one
|
||||
and update your application.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
# https://www.kraken.com/features/api#get-tradable-pairs
|
||||
class Pair(Struct):
|
||||
altname: str # alternate pair name
|
||||
wsname: str # WebSocket pair name (if available)
|
||||
aclass_base: str # asset class of base component
|
||||
base: str # asset id of base component
|
||||
aclass_quote: str # asset class of quote component
|
||||
quote: str # asset id of quote component
|
||||
lot: str # volume lot size
|
||||
|
||||
cost_decimals: int
|
||||
costmin: float
|
||||
pair_decimals: int # scaling decimal places for pair
|
||||
lot_decimals: int # scaling decimal places for volume
|
||||
|
||||
# amount to multiply lot volume by to get currency volume
|
||||
lot_multiplier: float
|
||||
|
||||
# array of leverage amounts available when buying
|
||||
leverage_buy: list[int]
|
||||
# array of leverage amounts available when selling
|
||||
leverage_sell: list[int]
|
||||
|
||||
# fee schedule array in [volume, percent fee] tuples
|
||||
fees: list[tuple[int, float]]
|
||||
|
||||
# maker fee schedule array in [volume, percent fee] tuples (if on
|
||||
# maker/taker)
|
||||
fees_maker: list[tuple[int, float]]
|
||||
|
||||
fee_volume_currency: str # volume discount currency
|
||||
margin_call: str # margin call level
|
||||
margin_stop: str # stop-out/liquidation margin level
|
||||
ordermin: float # minimum order volume for pair
|
||||
tick_size: float # min price step size
|
||||
status: str
|
||||
|
||||
short_position_limit: float = 0
|
||||
long_position_limit: float = float('inf')
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.pair_decimals)
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.lot_decimals)
|
||||
|
||||
@property
|
||||
def bs_fqme(self) -> str:
|
||||
return f'{self.symbol}.SPOT'
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
# symbol mapping from all names to the altname
|
||||
_ntable: dict[str, str] = {}
|
||||
|
||||
# 2-way map of symbol names to their "alt names" ffs XD
|
||||
_altnames: bidict[str, str] = bidict()
|
||||
|
||||
_pairs: dict[str, Pair] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: dict[str, str],
|
||||
name: str = '',
|
||||
api_key: str = '',
|
||||
secret: str = ''
|
||||
) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._sesh.headers.update({
|
||||
'User-Agent':
|
||||
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
|
||||
})
|
||||
self._name = name
|
||||
self._api_key = api_key
|
||||
self._secret = secret
|
||||
|
||||
self.conf: dict[str, str] = config
|
||||
self.assets: dict[str, Asset] = {}
|
||||
|
||||
@property
|
||||
def pairs(self) -> dict[str, Pair]:
|
||||
if self._pairs is None:
|
||||
raise RuntimeError(
|
||||
"Make sure to run `cache_symbols()` on startup!"
|
||||
)
|
||||
# retreive and cache all symbols
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def _public(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
) -> dict[str, Any]:
|
||||
resp = await self._sesh.post(
|
||||
path=f'/public/{method}',
|
||||
json=data,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def _private(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
uri_path: str
|
||||
) -> dict[str, Any]:
|
||||
headers = {
|
||||
'Content-Type':
|
||||
'application/x-www-form-urlencoded',
|
||||
'API-Key':
|
||||
self._api_key,
|
||||
'API-Sign':
|
||||
get_kraken_signature(uri_path, data, self._secret)
|
||||
}
|
||||
resp = await self._sesh.post(
|
||||
path=f'/private/{method}',
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def endpoint(
|
||||
self,
|
||||
method: str,
|
||||
data: dict[str, Any]
|
||||
|
||||
) -> dict[str, Any]:
|
||||
uri_path = f'/0/private/{method}'
|
||||
data['nonce'] = str(int(1000*time.time()))
|
||||
return await self._private(method, data, uri_path)
|
||||
|
||||
async def get_balances(
|
||||
self,
|
||||
) -> dict[str, float]:
|
||||
'''
|
||||
Return the set of asset balances for this account
|
||||
by symbol.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'Balance',
|
||||
{},
|
||||
)
|
||||
by_bsmktid = resp['result']
|
||||
|
||||
# TODO: we need to pull out the "asset" decimals
|
||||
# data and return a `decimal.Decimal` instead here!
|
||||
# using the underlying Asset
|
||||
return {
|
||||
self._altnames[sym].lower(): float(bal)
|
||||
for sym, bal in by_bsmktid.items()
|
||||
}
|
||||
|
||||
async def get_assets(self) -> dict[str, Asset]:
|
||||
'''
|
||||
Load and cache all asset infos and pack into
|
||||
our native ``Asset`` struct.
|
||||
|
||||
https://docs.kraken.com/rest/#tag/Market-Data/operation/getAssetInfo
|
||||
|
||||
return msg:
|
||||
"asset1": {
|
||||
"aclass": "string",
|
||||
"altname": "string",
|
||||
"decimals": 0,
|
||||
"display_decimals": 0,
|
||||
"collateral_value": 0,
|
||||
"status": "string"
|
||||
}
|
||||
|
||||
'''
|
||||
resp = await self._public('Assets', {})
|
||||
assets = resp['result']
|
||||
|
||||
for bs_mktid, info in assets.items():
|
||||
altname = self._altnames[bs_mktid] = info['altname']
|
||||
aclass: str = info['aclass']
|
||||
|
||||
self.assets[bs_mktid] = Asset(
|
||||
name=altname.lower(),
|
||||
atype=f'crypto_{aclass}',
|
||||
tx_tick=digits_to_dec(info['decimals']),
|
||||
info=info,
|
||||
)
|
||||
|
||||
return self.assets
|
||||
|
||||
async def get_trades(
|
||||
self,
|
||||
fetch_limit: int | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Get the trades (aka cleared orders) history from the rest endpoint:
|
||||
https://docs.kraken.com/rest/#operation/getTradeHistory
|
||||
|
||||
'''
|
||||
ofs = 0
|
||||
trades_by_id: dict[str, Any] = {}
|
||||
|
||||
for i in itertools.count():
|
||||
if (
|
||||
fetch_limit
|
||||
and i >= fetch_limit
|
||||
):
|
||||
break
|
||||
|
||||
# increment 'ofs' pagination offset
|
||||
ofs = i*50
|
||||
|
||||
resp = await self.endpoint(
|
||||
'TradesHistory',
|
||||
{'ofs': ofs},
|
||||
)
|
||||
by_id = resp['result']['trades']
|
||||
trades_by_id.update(by_id)
|
||||
|
||||
# can get up to 50 results per query, see:
|
||||
# https://docs.kraken.com/rest/#tag/User-Data/operation/getTradeHistory
|
||||
if (
|
||||
len(by_id) < 50
|
||||
):
|
||||
err = resp.get('error')
|
||||
if err:
|
||||
raise BrokerError(err)
|
||||
|
||||
# we know we received the max amount of
|
||||
# trade results so there may be more history.
|
||||
# catch the end of the trades
|
||||
count = resp['result']['count']
|
||||
break
|
||||
|
||||
# santity check on update
|
||||
assert count == len(trades_by_id.values())
|
||||
return trades_by_id
|
||||
|
||||
async def get_xfers(
|
||||
self,
|
||||
asset: str,
|
||||
src_asset: str = '',
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Get asset balance transfer transactions.
|
||||
|
||||
Currently only withdrawals are supported.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'WithdrawStatus',
|
||||
{'asset': asset},
|
||||
)
|
||||
try:
|
||||
xfers: list[dict] = resp['result']
|
||||
except KeyError:
|
||||
log.exception(f'Kraken suxxx: {resp}')
|
||||
return []
|
||||
|
||||
# eg. resp schema:
|
||||
# 'result': [{'method': 'Bitcoin', 'aclass': 'currency', 'asset':
|
||||
# 'XXBT', 'refid': 'AGBJRMB-JHD2M4-NDI3NR', 'txid':
|
||||
# 'b95d66d3bb6fd76cbccb93f7639f99a505cb20752c62ea0acc093a0e46547c44',
|
||||
# 'info': 'bc1qc8enqjekwppmw3g80p56z5ns7ze3wraqk5rl9z',
|
||||
# 'amount': '0.00300726', 'fee': '0.00001000', 'time':
|
||||
# 1658347714, 'status': 'Success'}]}
|
||||
|
||||
trans: dict[str, Transaction] = {}
|
||||
for entry in xfers:
|
||||
|
||||
# look up the normalized name and asset info
|
||||
asset_key = entry['asset']
|
||||
asset = self.assets[asset_key]
|
||||
asset_key = self._altnames[asset_key].lower()
|
||||
|
||||
# XXX: this is in the asset units (likely) so it isn't
|
||||
# quite the same as a commisions cost necessarily..)
|
||||
cost = float(entry['fee'])
|
||||
|
||||
fqme = asset_key + '.kraken'
|
||||
|
||||
tx = Transaction(
|
||||
fqme=fqme,
|
||||
sym=asset,
|
||||
tid=entry['txid'],
|
||||
dt=pendulum.from_timestamp(entry['time']),
|
||||
bs_mktid=f'{asset_key}{src_asset}',
|
||||
size=-1*(
|
||||
float(entry['amount'])
|
||||
+
|
||||
cost
|
||||
),
|
||||
# since this will be treated as a "sell" it
|
||||
# shouldn't be needed to compute the be price.
|
||||
price='NaN',
|
||||
|
||||
# XXX: see note above
|
||||
cost=cost,
|
||||
)
|
||||
trans[tx.tid] = tx
|
||||
|
||||
return trans
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str = None,
|
||||
validate: bool = False # set True test call without a real submission
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
|
||||
'''
|
||||
# Build common data dict for common keys from both endpoints
|
||||
data = {
|
||||
"pair": symbol,
|
||||
"price": str(price),
|
||||
"validate": validate
|
||||
}
|
||||
if reqid is None:
|
||||
# Build order data for kraken api
|
||||
data |= {
|
||||
"ordertype": "limit",
|
||||
"type": action,
|
||||
"volume": str(size),
|
||||
}
|
||||
return await self.endpoint('AddOrder', data)
|
||||
|
||||
else:
|
||||
# Edit order data for kraken api
|
||||
data["txid"] = reqid
|
||||
return await self.endpoint('EditOrder', data)
|
||||
|
||||
async def submit_cancel(
|
||||
self,
|
||||
reqid: str,
|
||||
) -> dict:
|
||||
'''
|
||||
Send cancel request for order id ``reqid``.
|
||||
|
||||
'''
|
||||
# txid is a transaction id given by kraken
|
||||
return await self.endpoint('CancelOrder', {"txid": reqid})
|
||||
|
||||
async def pair_info(
|
||||
self,
|
||||
pair_patt: str | None = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
'''
|
||||
Query for a tradeable asset pair (info), or all if no input
|
||||
pattern is provided.
|
||||
|
||||
https://docs.kraken.com/rest/#tag/Market-Data/operation/getTradableAssetPairs
|
||||
|
||||
'''
|
||||
# get all pairs by default, or filter
|
||||
# to whatever pattern is provided as input.
|
||||
pairs: dict[str, str] | None = None
|
||||
if pair_patt is not None:
|
||||
pairs = {'pair': pair_patt}
|
||||
|
||||
resp = await self._public(
|
||||
'AssetPairs',
|
||||
pairs,
|
||||
)
|
||||
err = resp['error']
|
||||
if err:
|
||||
raise SymbolNotFound(pair_patt)
|
||||
|
||||
pairs: dict[str, Pair] = {
|
||||
|
||||
key: Pair(**data)
|
||||
for key, data in resp['result'].items()
|
||||
}
|
||||
# always cache so we can possibly do faster lookup
|
||||
self._pairs.update(pairs)
|
||||
|
||||
if pair_patt is not None:
|
||||
return next(iter(pairs.items()))[1]
|
||||
|
||||
return pairs
|
||||
|
||||
async def cache_symbols(self) -> dict:
|
||||
'''
|
||||
Load all market pair info build and cache it for downstream use.
|
||||
|
||||
A ``._ntable: dict[str, str]`` is available for mapping the
|
||||
websocket pair name-keys and their http endpoint API (smh)
|
||||
equivalents to the "alternative name" which is generally the one
|
||||
we actually want to use XD
|
||||
|
||||
'''
|
||||
if not self._pairs:
|
||||
pairs = await self.pair_info()
|
||||
assert self._pairs == pairs
|
||||
|
||||
# table of all ws and rest keys to their alt-name values.
|
||||
ntable: dict[str, str] = {}
|
||||
|
||||
for rest_key in list(pairs.keys()):
|
||||
|
||||
pair: Pair = pairs[rest_key]
|
||||
altname = pair.altname
|
||||
wsname = pair.wsname
|
||||
ntable[altname] = ntable[rest_key] = ntable[wsname] = altname
|
||||
|
||||
# register the pair under all monikers, a giant flat
|
||||
# surjection of all possible names to each info obj.
|
||||
self._pairs[altname] = self._pairs[wsname] = pair
|
||||
|
||||
self._ntable.update(ntable)
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Search for a symbol by "alt name"..
|
||||
|
||||
It is expected that the ``Client._pairs`` table
|
||||
gets populated before conducting the underlying fuzzy-search
|
||||
over the pair-key set.
|
||||
|
||||
'''
|
||||
if not len(self._pairs):
|
||||
await self.cache_symbols()
|
||||
assert self._pairs, '`Client.cache_symbols()` was never called!?'
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
self._pairs,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0].altname: item[0] for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str = 'XBTUSD',
|
||||
|
||||
# UTC 2017-07-02 12:53:20
|
||||
since: Union[int, datetime] | None = None,
|
||||
count: int = 720, # <- max allowed per query
|
||||
as_np: bool = True,
|
||||
|
||||
) -> dict:
|
||||
|
||||
if since is None:
|
||||
since = pendulum.now('UTC').start_of('minute').subtract(
|
||||
minutes=count).timestamp()
|
||||
|
||||
elif isinstance(since, int):
|
||||
since = pendulum.from_timestamp(since).timestamp()
|
||||
|
||||
else: # presumably a pendulum datetime
|
||||
since = since.timestamp()
|
||||
|
||||
# UTC 2017-07-02 12:53:20 is oldest seconds value
|
||||
since = str(max(1499000000, int(since)))
|
||||
json = await self._public(
|
||||
'OHLC',
|
||||
data={
|
||||
'pair': symbol,
|
||||
'since': since,
|
||||
},
|
||||
)
|
||||
try:
|
||||
res = json['result']
|
||||
res.pop('last')
|
||||
bars = next(iter(res.values()))
|
||||
|
||||
new_bars = []
|
||||
|
||||
first = bars[0]
|
||||
last_nz_vwap = first[-3]
|
||||
if last_nz_vwap == 0:
|
||||
# use close if vwap is zero
|
||||
last_nz_vwap = first[-4]
|
||||
|
||||
# convert all fields to native types
|
||||
for i, bar in enumerate(bars):
|
||||
# normalize weird zero-ed vwap values..cmon kraken..
|
||||
# indicates vwap didn't change since last bar
|
||||
vwap = float(bar.pop(-3))
|
||||
if vwap != 0:
|
||||
last_nz_vwap = vwap
|
||||
if vwap == 0:
|
||||
vwap = last_nz_vwap
|
||||
|
||||
# re-insert vwap as the last of the fields
|
||||
bar.append(vwap)
|
||||
|
||||
new_bars.append(
|
||||
(i,) + tuple(
|
||||
ftype(bar[j]) for j, (name, ftype) in enumerate(
|
||||
def_iohlcv_fields[1:]
|
||||
)
|
||||
)
|
||||
)
|
||||
array = np.array(new_bars, dtype=def_iohlcv_fields) if as_np else bars
|
||||
return array
|
||||
except KeyError:
|
||||
errmsg = json['error'][0]
|
||||
|
||||
if 'not found' in errmsg:
|
||||
raise SymbolNotFound(errmsg + f': {symbol}')
|
||||
|
||||
elif 'Too many requests' in errmsg:
|
||||
raise DataThrottle(f'{symbol}')
|
||||
|
||||
else:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
@classmethod
|
||||
def normalize_symbol(
|
||||
cls,
|
||||
ticker: str
|
||||
) -> tuple[str, Pair]:
|
||||
'''
|
||||
Normalize symbol names to to a 3x3 pair from the global
|
||||
definition map which we build out from the data retreived from
|
||||
the 'AssetPairs' endpoint, see methods above.
|
||||
|
||||
'''
|
||||
try:
|
||||
return cls._ntable[ticker]
|
||||
except KeyError as ke:
|
||||
raise SymbolNotFound(f'kraken has no {ke.args[0]}')
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> Client:
|
||||
|
||||
conf = get_config()
|
||||
if conf:
|
||||
client = Client(
|
||||
conf,
|
||||
name=conf['key_descr'],
|
||||
api_key=conf['api_key'],
|
||||
secret=conf['secret']
|
||||
)
|
||||
else:
|
||||
client = Client({})
|
||||
|
||||
# at startup, load all symbols, and asset info in
|
||||
# batch requests.
|
||||
async with trio.open_nursery() as nurse:
|
||||
nurse.start_soon(client.get_assets)
|
||||
await client.cache_symbols()
|
||||
|
||||
yield client
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,507 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Real-time and historical data feed endpoints.
|
||||
|
||||
'''
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
aclosing,
|
||||
)
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
AsyncGenerator,
|
||||
Callable,
|
||||
Optional,
|
||||
)
|
||||
import time
|
||||
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import pendulum
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
MktPair,
|
||||
unpack_fqme,
|
||||
)
|
||||
from piker.brokers import (
|
||||
open_cached_client,
|
||||
SymbolNotFound,
|
||||
)
|
||||
from piker._cacheables import (
|
||||
async_lifo_cache,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
DataUnavailable,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
from piker.data.validate import FeedInit
|
||||
from piker.data._web_bs import open_autorecon_ws, NoBsWs
|
||||
from .api import (
|
||||
log,
|
||||
Client,
|
||||
Pair,
|
||||
)
|
||||
|
||||
|
||||
class OHLC(Struct, frozen=True):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://docs.kraken.com/websockets/#message-ohlc
|
||||
|
||||
'''
|
||||
chan_id: int # internal kraken id
|
||||
chan_name: str # eg. ohlc-1 (name-interval)
|
||||
pair: str # fx pair
|
||||
|
||||
# unpacked from array
|
||||
time: float # Begin time of interval, in seconds since epoch
|
||||
etime: float # End time of interval, in seconds since epoch
|
||||
open: float # Open price of interval
|
||||
high: float # High price within interval
|
||||
low: float # Low price within interval
|
||||
close: float # Close price of interval
|
||||
vwap: float # Volume weighted average price within interval
|
||||
volume: float # Accumulated volume **within interval**
|
||||
count: int # Number of trades within interval
|
||||
|
||||
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Message stream parser and heartbeat handler.
|
||||
|
||||
Deliver ws subscription messages as well as handle heartbeat logic
|
||||
though a single async generator.
|
||||
|
||||
'''
|
||||
last_hb: float = 0
|
||||
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
case {'event': 'heartbeat'}:
|
||||
now = time.time()
|
||||
delay = now - last_hb
|
||||
last_hb = now
|
||||
|
||||
# XXX: why tf is this not printing without --tl flag?
|
||||
log.debug(f"Heartbeat after {delay}")
|
||||
# print(f"Heartbeat after {delay}")
|
||||
|
||||
continue
|
||||
|
||||
case _:
|
||||
# passthrough sub msgs
|
||||
yield msg
|
||||
|
||||
|
||||
async def process_data_feed_msgs(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Parse and pack data feed messages.
|
||||
|
||||
'''
|
||||
async with aclosing(stream_messages(ws)) as ws_stream:
|
||||
async for msg in ws_stream:
|
||||
match msg:
|
||||
case {
|
||||
'errorMessage': errmsg
|
||||
}:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
case {
|
||||
'event': 'subscriptionStatus',
|
||||
} as sub:
|
||||
log.info(
|
||||
'WS subscription is active:\n'
|
||||
f'{sub}'
|
||||
)
|
||||
continue
|
||||
|
||||
case [
|
||||
chan_id,
|
||||
*payload_array,
|
||||
chan_name,
|
||||
pair
|
||||
]:
|
||||
if 'ohlc' in chan_name:
|
||||
array: list = payload_array[0]
|
||||
ohlc = OHLC(
|
||||
chan_id,
|
||||
chan_name,
|
||||
pair,
|
||||
*map(float, array[:-1]),
|
||||
count=array[-1],
|
||||
)
|
||||
yield 'ohlc', ohlc.copy()
|
||||
|
||||
elif 'spread' in chan_name:
|
||||
|
||||
bid, ask, ts, bsize, asize = map(
|
||||
float, payload_array[0])
|
||||
|
||||
# TODO: really makes you think IB has a horrible API...
|
||||
quote = {
|
||||
'symbol': pair.replace('/', ''),
|
||||
'ticks': [
|
||||
{'type': 'bid', 'price': bid, 'size': bsize},
|
||||
{'type': 'bsize', 'price': bid, 'size': bsize},
|
||||
|
||||
{'type': 'ask', 'price': ask, 'size': asize},
|
||||
{'type': 'asize', 'price': ask, 'size': asize},
|
||||
],
|
||||
}
|
||||
yield 'l1', quote
|
||||
|
||||
# elif 'book' in msg[-2]:
|
||||
# chan_id, *payload_array, chan_name, pair = msg
|
||||
# print(msg)
|
||||
|
||||
case {
|
||||
'connectionID': conid,
|
||||
'event': 'systemStatus',
|
||||
'status': 'online',
|
||||
'version': ver,
|
||||
}:
|
||||
log.info(
|
||||
f'Established {ver} ws connection with id: {conid}'
|
||||
)
|
||||
continue
|
||||
|
||||
case _:
|
||||
print(f'UNHANDLED MSG: {msg}')
|
||||
# yield msg
|
||||
|
||||
|
||||
def normalize(ohlc: OHLC) -> dict:
|
||||
'''
|
||||
Norm an `OHLC` msg to piker's minimal (live-)quote schema.
|
||||
|
||||
'''
|
||||
quote = ohlc.to_dict()
|
||||
quote['broker_ts'] = quote['time']
|
||||
quote['brokerd_ts'] = time.time()
|
||||
quote['symbol'] = quote['pair'] = quote['pair'].replace('/', '')
|
||||
quote['last'] = quote['close']
|
||||
quote['bar_wap'] = ohlc.vwap
|
||||
return quote
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
mkt: MktPair,
|
||||
|
||||
) -> AsyncGenerator[Callable, None]:
|
||||
|
||||
symbol: str = mkt.bs_mktid
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# lol, kraken won't send any more then the "last"
|
||||
# 720 1m bars.. so we have to just ignore further
|
||||
# requests of this type..
|
||||
queries: int = 0
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
nonlocal queries
|
||||
if (
|
||||
queries > 0
|
||||
or timeframe != 60
|
||||
):
|
||||
raise DataUnavailable(
|
||||
'Only a single query for 1m bars supported')
|
||||
|
||||
count = 0
|
||||
while count <= 3:
|
||||
try:
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
since=end_dt,
|
||||
)
|
||||
count += 1
|
||||
queries += 1
|
||||
break
|
||||
except DataThrottle:
|
||||
log.warning(f'kraken OHLC throttle for {symbol}')
|
||||
await trio.sleep(1)
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 1, 'rate': 1}
|
||||
|
||||
|
||||
@async_lifo_cache()
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
'''
|
||||
Query for and return a `MktPair` and backend-native `Pair` (or
|
||||
wtv else) info.
|
||||
|
||||
If more then one fqme is provided return a ``dict`` of native
|
||||
key-strs to `MktPair`s.
|
||||
|
||||
'''
|
||||
venue: str = 'spot'
|
||||
expiry: str = ''
|
||||
if '.kraken' in fqme:
|
||||
broker, pair, venue, expiry = unpack_fqme(fqme)
|
||||
venue: str = venue or 'spot'
|
||||
|
||||
if venue != 'spot':
|
||||
raise SymbolNotFound(
|
||||
'kraken only supports spot markets right now!\n'
|
||||
f'{fqme}\n'
|
||||
)
|
||||
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# uppercase since kraken bs_mktid is always upper
|
||||
bs_fqme, _, broker = fqme.partition('.')
|
||||
pair_str: str = bs_fqme.upper()
|
||||
bs_mktid: str = Client.normalize_symbol(pair_str)
|
||||
pair: Pair = await client.pair_info(pair_str)
|
||||
|
||||
assets = client.assets
|
||||
dst_asset: Asset = assets[pair.base]
|
||||
src_asset: Asset = assets[pair.quote]
|
||||
|
||||
mkt = MktPair(
|
||||
dst=dst_asset,
|
||||
src=src_asset,
|
||||
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
expiry=expiry,
|
||||
venue=venue or 'spot',
|
||||
|
||||
# TODO: futes
|
||||
# _atype=_atype,
|
||||
|
||||
broker='kraken',
|
||||
)
|
||||
return mkt, pair
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# backend specific
|
||||
sub_type: str = 'ohlc',
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Subscribe for ohlc stream of quotes for ``pairs``.
|
||||
|
||||
``pairs`` must be formatted <crypto_symbol>/<fiat_symbol>.
|
||||
|
||||
'''
|
||||
|
||||
ws_pairs: list[str] = []
|
||||
init_msgs: list[FeedInit] = []
|
||||
|
||||
async with (
|
||||
send_chan as send_chan,
|
||||
):
|
||||
for sym_str in symbols:
|
||||
mkt, pair = await get_mkt_info(sym_str)
|
||||
init_msgs.append(
|
||||
FeedInit(mkt_info=mkt)
|
||||
)
|
||||
|
||||
ws_pairs.append(pair.wsname)
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: NoBsWs):
|
||||
|
||||
# XXX: setup subs
|
||||
# https://docs.kraken.com/websockets/#message-subscribe
|
||||
# specific logic for this in kraken's sync client:
|
||||
# https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
|
||||
ohlc_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'ohlc',
|
||||
'interval': 1,
|
||||
},
|
||||
}
|
||||
|
||||
# TODO: we want to eventually allow unsubs which should
|
||||
# be completely fine to request from a separate task
|
||||
# since internally the ws methods appear to be FIFO
|
||||
# locked.
|
||||
await ws.send_msg(ohlc_sub)
|
||||
|
||||
# trade data (aka L1)
|
||||
l1_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'spread',
|
||||
# 'depth': 10}
|
||||
},
|
||||
}
|
||||
|
||||
# pull a first quote and deliver
|
||||
await ws.send_msg(l1_sub)
|
||||
|
||||
yield
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
'pair': ws_pairs,
|
||||
'event': 'unsubscribe',
|
||||
'subscription': ['ohlc', 'spread'],
|
||||
})
|
||||
|
||||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
# see the tips on reconnection logic:
|
||||
# https://support.kraken.com/hc/en-us/articles/360044504011-WebSocket-API-unexpected-disconnections-from-market-data-feeds
|
||||
ws: NoBsWs
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
'wss://ws.kraken.com/',
|
||||
fixture=subscribe,
|
||||
reset_after=20,
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
# NOTE: not sure this actually works XD particularly
|
||||
# if we call `ws._connect()` manally in the streaming
|
||||
# async gen..
|
||||
aclosing(process_data_feed_msgs(ws)) as msg_gen,
|
||||
):
|
||||
# pull a first quote and deliver
|
||||
typ, ohlc_last = await anext(msg_gen)
|
||||
quote = normalize(ohlc_last)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
feed_is_live.set()
|
||||
|
||||
# keep start of last interval for volume tracking
|
||||
last_interval_start: float = ohlc_last.etime
|
||||
|
||||
# start streaming
|
||||
topic: str = mkt.bs_fqme
|
||||
async for typ, quote in msg_gen:
|
||||
match typ:
|
||||
|
||||
# TODO: can get rid of all this by using
|
||||
# ``trades`` subscription..? Not sure why this
|
||||
# wasn't used originally? (music queues) zoltannn..
|
||||
# https://docs.kraken.com/websockets/#message-trade
|
||||
case 'ohlc':
|
||||
# generate tick values to match time & sales pane:
|
||||
# https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m
|
||||
volume = quote.volume
|
||||
|
||||
# new OHLC sample interval
|
||||
if quote.etime > last_interval_start:
|
||||
last_interval_start: float = quote.etime
|
||||
tick_volume: float = volume
|
||||
|
||||
else:
|
||||
# this is the tick volume *within the interval*
|
||||
tick_volume: float = volume - ohlc_last.volume
|
||||
|
||||
ohlc_last = quote
|
||||
last = quote.close
|
||||
|
||||
quote = normalize(quote)
|
||||
ticks = quote.setdefault(
|
||||
'ticks',
|
||||
[],
|
||||
)
|
||||
if tick_volume:
|
||||
ticks.append({
|
||||
'type': 'trade',
|
||||
'price': last,
|
||||
'size': tick_volume,
|
||||
})
|
||||
|
||||
case 'l1':
|
||||
# passthrough quote msg
|
||||
pass
|
||||
|
||||
case _:
|
||||
log.warning(f'Unknown WSS message: {typ}, {quote}')
|
||||
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
|
||||
) -> Client:
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started(cache)
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
cache,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
pair[0].altname: pair[0]
|
||||
for pair in matches
|
||||
})
|
||||
|
|
@ -1,909 +0,0 @@
|
|||
# Copyright (C) (in stewardship for pikers)
|
||||
# - Jared Goldman
|
||||
# - Tyler Goodlet
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kucoin broker backend
|
||||
|
||||
'''
|
||||
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
aclosing,
|
||||
)
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import base64
|
||||
import hmac
|
||||
import hashlib
|
||||
import time
|
||||
from functools import partial
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Literal,
|
||||
AsyncGenerator,
|
||||
)
|
||||
import wsproto
|
||||
from uuid import uuid4
|
||||
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
from trio_typing import TaskStatus
|
||||
import asks
|
||||
from bidict import bidict
|
||||
import numpy as np
|
||||
import pendulum
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
digits_to_dec,
|
||||
MktPair,
|
||||
)
|
||||
from piker import config
|
||||
from piker.brokers import (
|
||||
open_cached_client,
|
||||
)
|
||||
from piker._cacheables import (
|
||||
async_lifo_cache,
|
||||
)
|
||||
from piker.log import get_logger
|
||||
from piker.data.validate import FeedInit
|
||||
from piker.data.types import Struct
|
||||
from piker.data import def_iohlcv_fields
|
||||
from piker.data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
from ._util import DataUnavailable
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class KucoinMktPair(Struct, frozen=True):
|
||||
'''
|
||||
Kucoin's pair format:
|
||||
https://docs.kucoin.com/#get-symbols-list
|
||||
|
||||
'''
|
||||
baseCurrency: str
|
||||
baseIncrement: float
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return Decimal(str(self.baseIncrement))
|
||||
|
||||
baseMaxSize: float
|
||||
baseMinSize: float
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return Decimal(str(self.baseMinSize))
|
||||
|
||||
enableTrading: bool
|
||||
feeCurrency: str
|
||||
isMarginEnabled: bool
|
||||
market: str
|
||||
minFunds: float
|
||||
name: str
|
||||
priceIncrement: float
|
||||
priceLimitRate: float
|
||||
quoteCurrency: str
|
||||
quoteIncrement: float
|
||||
quoteMaxSize: float
|
||||
quoteMinSize: float
|
||||
symbol: str # our bs_mktid, kucoin's internal id
|
||||
|
||||
|
||||
class AccountTrade(Struct, frozen=True):
|
||||
'''
|
||||
Historical trade format:
|
||||
https://docs.kucoin.com/#get-account-ledgers
|
||||
|
||||
'''
|
||||
id: str
|
||||
currency: str
|
||||
amount: float
|
||||
fee: float
|
||||
balance: float
|
||||
accountType: str
|
||||
bizType: str
|
||||
direction: Literal['in', 'out']
|
||||
createdAt: float
|
||||
context: list[str]
|
||||
|
||||
|
||||
class AccountResponse(Struct, frozen=True):
|
||||
'''
|
||||
https://docs.kucoin.com/#get-account-ledgers
|
||||
|
||||
'''
|
||||
currentPage: int
|
||||
pageSize: int
|
||||
totalNum: int
|
||||
totalPage: int
|
||||
items: list[AccountTrade]
|
||||
|
||||
|
||||
class KucoinTrade(Struct, frozen=True):
|
||||
'''
|
||||
Real-time trade format:
|
||||
https://docs.kucoin.com/#symbol-ticker
|
||||
|
||||
'''
|
||||
bestAsk: float
|
||||
bestAskSize: float
|
||||
bestBid: float
|
||||
bestBidSize: float
|
||||
price: float
|
||||
sequence: float
|
||||
size: float
|
||||
time: float
|
||||
|
||||
|
||||
class KucoinL2(Struct, frozen=True):
|
||||
'''
|
||||
Real-time L2 order book format:
|
||||
https://docs.kucoin.com/#level2-5-best-ask-bid-orders
|
||||
|
||||
'''
|
||||
|
||||
asks: list[list[float]]
|
||||
bids: list[list[float]]
|
||||
timestamp: float
|
||||
|
||||
|
||||
class Currency(Struct, frozen=True):
|
||||
'''
|
||||
Currency (asset) info:
|
||||
https://docs.kucoin.com/#get-currencies
|
||||
|
||||
'''
|
||||
currency: str
|
||||
name: str
|
||||
fullName: str
|
||||
precision: int
|
||||
confirms: int
|
||||
contractAddress: str
|
||||
withdrawalMinSize: str
|
||||
withdrawalMinFee: str
|
||||
isWithdrawEnabled: bool
|
||||
isDepositEnabled: bool
|
||||
isMarginEnabled: bool
|
||||
isDebitEnabled: bool
|
||||
|
||||
|
||||
class BrokerConfig(Struct, frozen=True):
|
||||
key_id: str
|
||||
key_secret: str
|
||||
key_passphrase: str
|
||||
|
||||
|
||||
def get_config() -> BrokerConfig | None:
|
||||
conf, _ = config.load()
|
||||
|
||||
section = conf.get('kucoin')
|
||||
|
||||
if section is None:
|
||||
log.warning('No config section found for kucoin in config')
|
||||
return None
|
||||
|
||||
return BrokerConfig(**section).copy()
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self) -> None:
|
||||
self._config: BrokerConfig | None = get_config()
|
||||
self._pairs: dict[str, KucoinMktPair] = {}
|
||||
self._fqmes2mktids: bidict[str, str] = bidict()
|
||||
self._bars: list[list[float]] = []
|
||||
self._currencies: dict[str, Currency] = {}
|
||||
|
||||
def _gen_auth_req_headers(
|
||||
self,
|
||||
action: Literal['POST', 'GET'],
|
||||
endpoint: str,
|
||||
api: str = 'v2',
|
||||
|
||||
) -> dict[str, str | bytes]:
|
||||
'''
|
||||
Generate authenticated request headers
|
||||
https://docs.kucoin.com/#authentication
|
||||
|
||||
'''
|
||||
|
||||
if not self._config:
|
||||
raise ValueError(
|
||||
'No config found when trying to send authenticated request')
|
||||
|
||||
str_to_sign = (
|
||||
str(int(time.time() * 1000))
|
||||
+ action + f'/api/{api}/{endpoint.lstrip("/")}'
|
||||
)
|
||||
|
||||
signature = base64.b64encode(
|
||||
hmac.new(
|
||||
self._config.key_secret.encode('utf-8'),
|
||||
str_to_sign.encode('utf-8'),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
)
|
||||
|
||||
passphrase = base64.b64encode(
|
||||
hmac.new(
|
||||
self._config.key_secret.encode('utf-8'),
|
||||
self._config.key_passphrase.encode('utf-8'),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
)
|
||||
|
||||
return {
|
||||
'KC-API-SIGN': signature,
|
||||
'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000),
|
||||
'KC-API-KEY': self._config.key_id,
|
||||
'KC-API-PASSPHRASE': passphrase,
|
||||
# XXX: Even if using the v1 api - this stays the same
|
||||
'KC-API-KEY-VERSION': '2',
|
||||
}
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
action: Literal['POST', 'GET'],
|
||||
endpoint: str,
|
||||
api: str = 'v2',
|
||||
headers: dict = {},
|
||||
) -> Any:
|
||||
'''
|
||||
Generic request wrapper for Kucoin API
|
||||
|
||||
'''
|
||||
if self._config:
|
||||
headers = self._gen_auth_req_headers(
|
||||
action,
|
||||
endpoint,
|
||||
api,
|
||||
)
|
||||
|
||||
api_url = f'https://api.kucoin.com/api/{api}/{endpoint}'
|
||||
|
||||
res = await asks.request(action, api_url, headers=headers)
|
||||
|
||||
json = res.json()
|
||||
if 'data' in json:
|
||||
return json['data']
|
||||
else:
|
||||
log.error(
|
||||
f'Error making request to {api_url} ->\n'
|
||||
f'{pformat(res)}'
|
||||
)
|
||||
return json['msg']
|
||||
|
||||
async def _get_ws_token(
|
||||
self,
|
||||
private: bool = False,
|
||||
) -> tuple[str, int] | None:
|
||||
'''
|
||||
Fetch ws token needed for sub access:
|
||||
https://docs.kucoin.com/#apply-connect-token
|
||||
returns a token and the interval we must ping
|
||||
the server at to keep the connection alive
|
||||
|
||||
'''
|
||||
token_type = 'private' if private else 'public'
|
||||
try:
|
||||
data: dict[str, Any] | None = await self._request(
|
||||
'POST',
|
||||
endpoint=f'bullet-{token_type}',
|
||||
api='v1'
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(
|
||||
f'Error making request for Kucoin ws token -> {str(e)}')
|
||||
return None
|
||||
|
||||
if data and 'token' in data:
|
||||
# ping_interval is in ms
|
||||
ping_interval: int = data['instanceServers'][0]['pingInterval']
|
||||
return data['token'], ping_interval
|
||||
elif data:
|
||||
log.error(
|
||||
'Error making request for Kucoin ws token'
|
||||
f'{data.json()["msg"]}'
|
||||
)
|
||||
|
||||
async def get_currencies(
|
||||
self,
|
||||
update: bool = False,
|
||||
) -> dict[str, Currency]:
|
||||
'''
|
||||
Retrieve all "currency" info:
|
||||
https://docs.kucoin.com/#get-currencies
|
||||
|
||||
We use this for creating piker-interal ``Asset``s.
|
||||
|
||||
'''
|
||||
if (
|
||||
not self._currencies
|
||||
or update
|
||||
):
|
||||
currencies: dict[str, Currency] = {}
|
||||
entries: list[dict] = await self._request(
|
||||
'GET',
|
||||
api='v1',
|
||||
endpoint='currencies',
|
||||
)
|
||||
for entry in entries:
|
||||
curr = Currency(**entry).copy()
|
||||
currencies[curr.name] = curr
|
||||
|
||||
self._currencies.update(currencies)
|
||||
|
||||
return self._currencies
|
||||
|
||||
async def _get_pairs(
|
||||
self,
|
||||
) -> tuple[
|
||||
dict[str, KucoinMktPair],
|
||||
bidict[str, KucoinMktPair],
|
||||
]:
|
||||
entries = await self._request('GET', 'symbols')
|
||||
log.info(f' {len(entries)} Kucoin market pairs fetched')
|
||||
|
||||
pairs: dict[str, KucoinMktPair] = {}
|
||||
fqmes2mktids: bidict[str, str] = bidict()
|
||||
for item in entries:
|
||||
pair = pairs[item['name']] = KucoinMktPair(**item)
|
||||
fqmes2mktids[
|
||||
item['name'].lower().replace('-', '')
|
||||
] = pair.name
|
||||
|
||||
return pairs, fqmes2mktids
|
||||
|
||||
async def cache_pairs(
|
||||
self,
|
||||
update: bool = False,
|
||||
|
||||
) -> dict[str, KucoinMktPair]:
|
||||
'''
|
||||
Get request all market pairs and store in a local cache.
|
||||
|
||||
Also create a table of piker style fqme -> kucoin symbols.
|
||||
|
||||
'''
|
||||
if (
|
||||
not self._pairs
|
||||
or update
|
||||
):
|
||||
pairs, fqmes = await self._get_pairs()
|
||||
self._pairs.update(pairs)
|
||||
self._fqmes2mktids.update(fqmes)
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = 30,
|
||||
|
||||
) -> dict[str, KucoinMktPair]:
|
||||
'''
|
||||
Use fuzzy search to match against all market names.
|
||||
|
||||
'''
|
||||
data = await self.cache_pairs()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern, data, score_cutoff=35, limit=limit
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0].name: item[0] for item in matches}
|
||||
|
||||
async def last_trades(self, sym: str) -> list[AccountTrade]:
|
||||
trades = await self._request(
|
||||
'GET',
|
||||
endpoint=f'accounts/ledgers?currency={sym}',
|
||||
api='v1'
|
||||
)
|
||||
trades = AccountResponse(**trades)
|
||||
return trades.items
|
||||
|
||||
async def _get_bars(
|
||||
self,
|
||||
fqme: str,
|
||||
|
||||
start_dt: datetime | None = None,
|
||||
end_dt: datetime | None = None,
|
||||
limit: int = 1000,
|
||||
as_np: bool = True,
|
||||
type: str = '1min',
|
||||
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Get OHLC data and convert to numpy array for perffff:
|
||||
https://docs.kucoin.com/#get-klines
|
||||
|
||||
Kucoin bar data format:
|
||||
[
|
||||
'1545904980', //Start time of the candle cycle 0
|
||||
'0.058', //opening price 1
|
||||
'0.049', //closing price 2
|
||||
'0.058', //highest price 3
|
||||
'0.049', //lowest price 4
|
||||
'0.018', //Transaction volume 5
|
||||
'0.000945' //Transaction amount 6
|
||||
],
|
||||
|
||||
piker ohlc numpy array format:
|
||||
[
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
]
|
||||
|
||||
'''
|
||||
# Generate generic end and start time if values not passed
|
||||
# Currently gives us 12hrs of data
|
||||
if (
|
||||
end_dt is None
|
||||
and start_dt is None
|
||||
):
|
||||
end_dt = pendulum.now('UTC').add(minutes=1)
|
||||
start_dt = end_dt.start_of('minute').subtract(minutes=limit)
|
||||
|
||||
if (
|
||||
start_dt
|
||||
and end_dt is None
|
||||
):
|
||||
# just set end to limit's worth in future
|
||||
end_dt = start_dt.start_of('minute').add(minutes=limit)
|
||||
|
||||
else:
|
||||
start_dt = end_dt.start_of('minute').subtract(minutes=limit)
|
||||
|
||||
start_dt = int(start_dt.timestamp())
|
||||
end_dt = int(end_dt.timestamp())
|
||||
|
||||
kucoin_sym = self._fqmes2mktids[fqme]
|
||||
|
||||
url = (
|
||||
f'market/candles?type={type}'
|
||||
f'&symbol={kucoin_sym}'
|
||||
f'&startAt={start_dt}'
|
||||
f'&endAt={end_dt}'
|
||||
)
|
||||
|
||||
for i in range(10):
|
||||
data: list[list[str]] | dict = await self._request(
|
||||
'GET',
|
||||
url,
|
||||
api='v1',
|
||||
)
|
||||
|
||||
if not isinstance(data, list):
|
||||
# Do a gradual backoff if Kucoin is rate limiting us
|
||||
backoff_interval = i
|
||||
log.warn(
|
||||
f'History call failed, backing off for {backoff_interval}s'
|
||||
)
|
||||
await trio.sleep(backoff_interval)
|
||||
else:
|
||||
bars: list[list[str]] = data
|
||||
break
|
||||
|
||||
new_bars = []
|
||||
reversed_bars = bars[::-1]
|
||||
|
||||
# Convert from kucoin format to piker format
|
||||
for i, bar in enumerate(reversed_bars):
|
||||
new_bars.append(
|
||||
(
|
||||
# index
|
||||
i,
|
||||
# time
|
||||
int(bar[0]),
|
||||
# open
|
||||
float(bar[1]),
|
||||
# high
|
||||
float(bar[3]),
|
||||
# low
|
||||
float(bar[4]),
|
||||
# close
|
||||
float(bar[2]),
|
||||
# volume
|
||||
float(bar[5]),
|
||||
# bar_wap
|
||||
# 0.0,
|
||||
)
|
||||
)
|
||||
|
||||
array = np.array(
|
||||
new_bars, dtype=def_iohlcv_fields) if as_np else bars
|
||||
return array
|
||||
|
||||
|
||||
def fqme_to_kucoin_sym(
|
||||
fqme: str,
|
||||
pairs: dict[str, KucoinMktPair],
|
||||
|
||||
) -> str:
|
||||
pair_data = pairs[fqme]
|
||||
return pair_data.baseCurrency + '-' + pair_data.quoteCurrency
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> AsyncGenerator[Client, None]:
|
||||
client = Client()
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(client.cache_pairs)
|
||||
await client.get_currencies()
|
||||
|
||||
yield client
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> None:
|
||||
async with open_cached_client('kucoin') as client:
|
||||
# load all symbols locally for fast search
|
||||
await client.cache_pairs()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
async for pattern in stream:
|
||||
await stream.send(await client.search_symbols(pattern))
|
||||
log.info('Kucoin symbol search opened')
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ping_task(
|
||||
ws: wsproto.WSConnection,
|
||||
ping_interval, connect_id
|
||||
) -> AsyncGenerator[None, None]:
|
||||
'''
|
||||
Spawn a non-blocking task that pings the ws
|
||||
server every ping_interval so Kucoin doesn't drop
|
||||
our connection
|
||||
|
||||
'''
|
||||
async with trio.open_nursery() as n:
|
||||
# TODO: cache this task so it's only called once
|
||||
async def ping_server():
|
||||
while True:
|
||||
await trio.sleep((ping_interval - 1000) / 1000)
|
||||
await ws.send_msg({'id': connect_id, 'type': 'ping'})
|
||||
|
||||
log.info('Starting ping task for kucoin ws connection')
|
||||
n.start_soon(ping_server)
|
||||
|
||||
yield
|
||||
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
@async_lifo_cache()
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, KucoinMktPair]:
|
||||
'''
|
||||
Query for and return a `MktPair` and `KucoinMktPair`.
|
||||
|
||||
'''
|
||||
async with open_cached_client('kucoin') as client:
|
||||
# split off any fqme broker part
|
||||
bs_fqme, _, broker = fqme.partition('.')
|
||||
|
||||
pairs: dict[str, KucoinMktPair] = await client.cache_pairs()
|
||||
|
||||
try:
|
||||
# likely search result key which is already in native mkt symbol form
|
||||
pair: KucoinMktPair = pairs[bs_fqme]
|
||||
bs_mktid: str = bs_fqme
|
||||
|
||||
except KeyError:
|
||||
|
||||
# likely a piker-style fqme from API request or CLI
|
||||
bs_mktid: str = client._fqmes2mktids[bs_fqme]
|
||||
pair: KucoinMktPair = pairs[bs_mktid]
|
||||
|
||||
# symbology sanity
|
||||
assert bs_mktid == pair.symbol
|
||||
|
||||
assets: dict[str, Currency] = client._currencies
|
||||
|
||||
# TODO: maybe just do this processing in
|
||||
# a .get_assets() method (see kraken)?
|
||||
src: Currency = assets[pair.quoteCurrency]
|
||||
src_asset = Asset(
|
||||
name=src.name,
|
||||
atype='crypto_currency',
|
||||
tx_tick=digits_to_dec(src.precision),
|
||||
info=src.to_dict(),
|
||||
)
|
||||
dst: Currency = assets[pair.baseCurrency]
|
||||
dst_asset = Asset(
|
||||
name=dst.name,
|
||||
atype='crypto_currency',
|
||||
tx_tick=digits_to_dec(dst.precision),
|
||||
info=dst.to_dict(),
|
||||
)
|
||||
mkt = MktPair(
|
||||
dst=dst_asset,
|
||||
src=src_asset,
|
||||
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
broker='kucoin',
|
||||
)
|
||||
return mkt, pair
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[dict, dict]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Required piker api to stream real-time data.
|
||||
Where the rubber hits the road baby
|
||||
|
||||
'''
|
||||
init_msgs: list[FeedInit] = []
|
||||
|
||||
async with open_cached_client('kucoin') as client:
|
||||
|
||||
log.info(f'Starting up quote stream(s) for {symbols}')
|
||||
for sym_str in symbols:
|
||||
mkt, pair = await get_mkt_info(sym_str)
|
||||
init_msgs.append(
|
||||
FeedInit(mkt_info=mkt)
|
||||
)
|
||||
|
||||
ws: NoBsWs
|
||||
token, ping_interval = await client._get_ws_token()
|
||||
connect_id = str(uuid4())
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
(
|
||||
f'wss://ws-api-spot.kucoin.com/?'
|
||||
f'token={token}&[connectId={connect_id}]'
|
||||
),
|
||||
fixture=partial(
|
||||
subscribe,
|
||||
connect_id=connect_id,
|
||||
bs_mktid=pair.symbol,
|
||||
),
|
||||
) as ws,
|
||||
open_ping_task(ws, ping_interval, connect_id),
|
||||
aclosing(stream_messages(ws, sym_str)) as msg_gen,
|
||||
):
|
||||
typ, quote = await anext(msg_gen)
|
||||
|
||||
while typ != 'trade':
|
||||
# take care to not unblock here until we get a real
|
||||
# trade quote
|
||||
typ, quote = await anext(msg_gen)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
feed_is_live.set()
|
||||
|
||||
async for typ, msg in msg_gen:
|
||||
await send_chan.send({sym_str: msg})
|
||||
|
||||
|
||||
@acm
|
||||
async def subscribe(
|
||||
ws: NoBsWs,
|
||||
connect_id,
|
||||
bs_mktid,
|
||||
|
||||
# subs are filled in with `bs_mktid` from avbove
|
||||
topics: list[str] = [
|
||||
'/market/ticker:{bs_mktid}', # clearing events
|
||||
'/spotMarket/level2Depth5:{bs_mktid}', # level 2
|
||||
],
|
||||
|
||||
) -> AsyncGenerator[None, None]:
|
||||
|
||||
eps: list[str] = []
|
||||
for topic in topics:
|
||||
ep: str = topic.format(bs_mktid=bs_mktid)
|
||||
eps.append(ep)
|
||||
await ws.send_msg(
|
||||
{
|
||||
'id': connect_id,
|
||||
'type': 'subscribe',
|
||||
'topic': ep,
|
||||
'privateChannel': False,
|
||||
'response': True,
|
||||
}
|
||||
)
|
||||
|
||||
welcome_msg = await ws.recv_msg()
|
||||
log.info(f'WS welcome: {welcome_msg}')
|
||||
|
||||
for _ in topics:
|
||||
ack_msg = await ws.recv_msg()
|
||||
log.info(f'Sub ACK: {ack_msg}')
|
||||
|
||||
yield
|
||||
|
||||
# unsub
|
||||
if ws.connected():
|
||||
log.info(f'Unsubscribing to {bs_mktid} feed')
|
||||
for ep in eps:
|
||||
await ws.send_msg(
|
||||
{
|
||||
'id': connect_id,
|
||||
'type': 'unsubscribe',
|
||||
'topic': ep,
|
||||
'privateChannel': False,
|
||||
'response': True,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
sym: str,
|
||||
|
||||
) -> AsyncGenerator[tuple[str, dict], None]:
|
||||
'''
|
||||
Core (live) feed msg handler: relay market events
|
||||
to the piker-ized tick-stream format.
|
||||
|
||||
'''
|
||||
last_trade_ts: float = 0
|
||||
|
||||
dict_msg: dict[str, Any]
|
||||
async for dict_msg in ws:
|
||||
match dict_msg:
|
||||
case {
|
||||
'subject': 'trade.ticker',
|
||||
'data': trade_data_dict,
|
||||
}:
|
||||
trade_data = KucoinTrade(**trade_data_dict)
|
||||
|
||||
# XXX: Filter out duplicate messages as ws feed will
|
||||
# send duplicate market state
|
||||
# https://docs.kucoin.com/#level2-5-best-ask-bid-orders
|
||||
if trade_data.time == last_trade_ts:
|
||||
continue
|
||||
|
||||
last_trade_ts = trade_data.time
|
||||
|
||||
yield 'trade', {
|
||||
'symbol': sym,
|
||||
'last': trade_data.price,
|
||||
'brokerd_ts': last_trade_ts,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'trade',
|
||||
'price': float(trade_data.price),
|
||||
'size': float(trade_data.size),
|
||||
'broker_ts': last_trade_ts,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
case {
|
||||
'subject': 'level2',
|
||||
'data': trade_data_dict,
|
||||
}:
|
||||
l2_data = KucoinL2(**trade_data_dict)
|
||||
first_ask = l2_data.asks[0]
|
||||
first_bid = l2_data.bids[0]
|
||||
yield 'l1', {
|
||||
'symbol': sym,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'bid',
|
||||
'price': float(first_bid[0]),
|
||||
'size': float(first_bid[1]),
|
||||
},
|
||||
{
|
||||
'type': 'bsize',
|
||||
'price': float(first_bid[0]),
|
||||
'size': float(first_bid[1]),
|
||||
},
|
||||
{
|
||||
'type': 'ask',
|
||||
'price': float(first_ask[0]),
|
||||
'size': float(first_ask[1]),
|
||||
},
|
||||
{
|
||||
'type': 'asize',
|
||||
'price': float(first_ask[0]),
|
||||
'size': float(first_ask[1]),
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
case {'type': 'pong'}:
|
||||
# resp to ping task req
|
||||
continue
|
||||
|
||||
case _:
|
||||
log.warn(f'Unhandled message: {dict_msg}')
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
mkt: MktPair,
|
||||
|
||||
) -> AsyncGenerator[Callable, None]:
|
||||
|
||||
symbol: str = mkt.bs_fqme
|
||||
|
||||
async with open_cached_client('kucoin') as client:
|
||||
log.info('Attempting to open kucoin history client')
|
||||
|
||||
async def get_ohlc_history(
|
||||
timeframe: float,
|
||||
end_dt: datetime | None = None,
|
||||
start_dt: datetime | None = None,
|
||||
) -> tuple[
|
||||
np.ndarray, datetime
|
||||
| None, datetime
|
||||
| None
|
||||
]: # start # end
|
||||
if timeframe != 60:
|
||||
raise DataUnavailable('Only 1m bars are supported')
|
||||
|
||||
array = await client._get_bars(
|
||||
symbol,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
|
||||
times = array['time']
|
||||
|
||||
if not len(times):
|
||||
raise DataUnavailable(
|
||||
f'No more history before {start_dt}?'
|
||||
)
|
||||
|
||||
if end_dt is None:
|
||||
inow = round(time.time())
|
||||
|
||||
print(
|
||||
f'difference in time between load and processing'
|
||||
f'{inow - times[-1]}'
|
||||
)
|
||||
|
||||
start_dt = pendulum.from_timestamp(times[0])
|
||||
end_dt = pendulum.from_timestamp(times[-1])
|
||||
|
||||
log.info('History succesfully fetched baby')
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc_history, {}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -40,17 +40,13 @@ import wrapt
|
|||
import asks
|
||||
|
||||
from ..calc import humanize, percent_change
|
||||
from . import open_cached_client
|
||||
from piker._cacheables import async_lifo_cache
|
||||
from .._cacheables import open_cached_client, async_lifo_cache
|
||||
from .. import config
|
||||
from ._util import resproc, BrokerError, SymbolNotFound
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, colorize_json, get_console_log
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_use_practice_account = False
|
||||
_refresh_token_ep = 'https://{}login.questrade.com/oauth2/'
|
||||
|
|
|
|||
|
|
@ -27,13 +27,12 @@ from typing import List
|
|||
from async_generator import asynccontextmanager
|
||||
import asks
|
||||
|
||||
from ._util import (
|
||||
resproc,
|
||||
BrokerError,
|
||||
log,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._util import resproc, BrokerError
|
||||
from ..calc import percent_change
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_service_ep = 'https://api.robinhood.com'
|
||||
|
||||
|
||||
|
|
@ -66,10 +65,8 @@ class Client:
|
|||
self.api = _API(self._sess)
|
||||
|
||||
def _zip_in_order(self, symbols: [str], quotes: List[dict]):
|
||||
return {
|
||||
quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, quotes)
|
||||
}
|
||||
return {quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, results_dict)}
|
||||
|
||||
async def quote(self, symbols: [str]):
|
||||
"""Retrieve quotes for a list of ``symbols``.
|
||||
|
|
|
|||
|
|
@ -18,23 +18,3 @@
|
|||
Market machinery for order executions, book, management.
|
||||
|
||||
"""
|
||||
from ..log import get_logger
|
||||
from ._client import (
|
||||
open_ems,
|
||||
OrderClient,
|
||||
)
|
||||
from ._ems import (
|
||||
open_brokerd_dialog,
|
||||
)
|
||||
from ._util import OrderDialogs
|
||||
|
||||
|
||||
__all__ = [
|
||||
'open_ems',
|
||||
'OrderClient',
|
||||
'open_brokerd_dialog',
|
||||
'OrderDialogs',
|
||||
|
||||
]
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
|
|||
|
|
@ -22,10 +22,54 @@ from enum import Enum
|
|||
from typing import Optional
|
||||
|
||||
from bidict import bidict
|
||||
from pydantic import BaseModel, validator
|
||||
|
||||
from ._pos import Position
|
||||
from . import MktPair
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
from ._messages import BrokerdPosition, Status
|
||||
|
||||
|
||||
class Position(BaseModel):
|
||||
'''
|
||||
Basic pp (personal position) model with attached fills history.
|
||||
|
||||
This type should be IPC wire ready?
|
||||
|
||||
'''
|
||||
symbol: Symbol
|
||||
|
||||
# last size and avg entry price
|
||||
size: float
|
||||
avg_price: float # TODO: contextual pricing
|
||||
|
||||
# ordered record of known constituent trade messages
|
||||
fills: list[Status] = []
|
||||
|
||||
def update_from_msg(
|
||||
self,
|
||||
msg: BrokerdPosition,
|
||||
|
||||
) -> None:
|
||||
|
||||
# XXX: better place to do this?
|
||||
symbol = self.symbol
|
||||
|
||||
lot_size_digits = symbol.lot_size_digits
|
||||
avg_price, size = (
|
||||
round(msg['avg_price'], ndigits=symbol.tick_size_digits),
|
||||
round(msg['size'], ndigits=lot_size_digits),
|
||||
)
|
||||
|
||||
self.avg_price = avg_price
|
||||
self.size = size
|
||||
|
||||
@property
|
||||
def dsize(self) -> float:
|
||||
'''
|
||||
The "dollar" size of the pp, normally in trading (fiat) unit
|
||||
terms.
|
||||
|
||||
'''
|
||||
return self.avg_price * self.size
|
||||
|
||||
|
||||
_size_units = bidict({
|
||||
|
|
@ -40,9 +84,34 @@ SizeUnit = Enum(
|
|||
)
|
||||
|
||||
|
||||
class Allocator(Struct):
|
||||
class Allocator(BaseModel):
|
||||
|
||||
mkt: MktPair
|
||||
class Config:
|
||||
validate_assignment = True
|
||||
copy_on_model_validation = False
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
# required to get the account validator lookup working?
|
||||
extra = 'allow'
|
||||
underscore_attrs_are_private = False
|
||||
|
||||
symbol: Symbol
|
||||
account: Optional[str] = 'paper'
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
size_unit: str = 'currency'
|
||||
_size_units: dict[str, Optional[str]] = _size_units
|
||||
|
||||
@validator('size_unit', pre=True)
|
||||
def maybe_lookup_key(cls, v):
|
||||
# apply the corresponding enum key for the text "description" value
|
||||
if v not in _size_units:
|
||||
return _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
return v
|
||||
|
||||
# TODO: if we ever want ot support non-uniform entry-slot-proportion
|
||||
# "sizes"
|
||||
|
|
@ -51,28 +120,6 @@ class Allocator(Struct):
|
|||
units_limit: float
|
||||
currency_limit: float
|
||||
slots: int
|
||||
account: Optional[str] = 'paper'
|
||||
|
||||
_size_units: bidict[str, Optional[str]] = _size_units
|
||||
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
_size_unit: str = 'currency'
|
||||
|
||||
@property
|
||||
def size_unit(self) -> str:
|
||||
return self._size_unit
|
||||
|
||||
@size_unit.setter
|
||||
def size_unit(self, v: str) -> Optional[str]:
|
||||
if v not in _size_units:
|
||||
v = _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
self._size_unit = v
|
||||
return v
|
||||
|
||||
def step_sizes(
|
||||
self,
|
||||
|
|
@ -93,13 +140,10 @@ class Allocator(Struct):
|
|||
else:
|
||||
return self.units_limit
|
||||
|
||||
def limit_info(self) -> tuple[str, float]:
|
||||
return self.size_unit, self.limit()
|
||||
|
||||
def next_order_info(
|
||||
self,
|
||||
|
||||
# we only need a startup size for exit calcs, we can then
|
||||
# we only need a startup size for exit calcs, we can the
|
||||
# determine how large slots should be if the initial pp size was
|
||||
# larger then the current live one, and the live one is smaller
|
||||
# then the initial config settings.
|
||||
|
|
@ -114,8 +158,8 @@ class Allocator(Struct):
|
|||
depending on position / order entry config.
|
||||
|
||||
'''
|
||||
mkt: MktPair = self.mkt
|
||||
ld: int = mkt.size_tick_digits
|
||||
sym = self.symbol
|
||||
ld = sym.lot_size_digits
|
||||
|
||||
size_unit = self.size_unit
|
||||
live_size = live_pp.size
|
||||
|
|
@ -125,13 +169,13 @@ class Allocator(Struct):
|
|||
u_per_slot, currency_per_slot = self.step_sizes()
|
||||
|
||||
if size_unit == 'units':
|
||||
slot_size: float = u_per_slot
|
||||
l_sub_pp: float = self.units_limit - abs_live_size
|
||||
slot_size = u_per_slot
|
||||
l_sub_pp = self.units_limit - abs_live_size
|
||||
|
||||
elif size_unit == 'currency':
|
||||
live_cost_basis: float = abs_live_size * live_pp.ppu
|
||||
slot_size: float = currency_per_slot / price
|
||||
l_sub_pp: float = (self.currency_limit - live_cost_basis) / price
|
||||
live_cost_basis = abs_live_size * live_pp.avg_price
|
||||
slot_size = currency_per_slot / price
|
||||
l_sub_pp = (self.currency_limit - live_cost_basis) / price
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
|
|
@ -140,20 +184,12 @@ class Allocator(Struct):
|
|||
|
||||
# an entry (adding-to or starting a pp)
|
||||
if (
|
||||
action == 'buy' and live_size > 0 or
|
||||
action == 'sell' and live_size < 0 or
|
||||
live_size == 0
|
||||
or (
|
||||
action == 'buy'
|
||||
and live_size > 0
|
||||
)
|
||||
or (
|
||||
action == 'sell'
|
||||
and live_size < 0
|
||||
)
|
||||
):
|
||||
order_size = min(
|
||||
slot_size,
|
||||
max(l_sub_pp, 0),
|
||||
)
|
||||
|
||||
order_size = min(slot_size, l_sub_pp)
|
||||
|
||||
# an exit (removing-from or going to net-zero pp)
|
||||
else:
|
||||
|
|
@ -169,7 +205,7 @@ class Allocator(Struct):
|
|||
if size_unit == 'currency':
|
||||
# compute the "projected" limit's worth of units at the
|
||||
# current pp (weighted) price:
|
||||
slot_size = currency_per_slot / live_pp.ppu
|
||||
slot_size = currency_per_slot / live_pp.avg_price
|
||||
|
||||
else:
|
||||
slot_size = u_per_slot
|
||||
|
|
@ -184,7 +220,7 @@ class Allocator(Struct):
|
|||
order_size = max(slotted_pp, slot_size)
|
||||
|
||||
if (
|
||||
abs_live_size < slot_size
|
||||
abs_live_size < slot_size or
|
||||
|
||||
# NOTE: front/back "loading" heurstic:
|
||||
# if the remaining pp is in between 0-1.5x a slot's
|
||||
|
|
@ -193,17 +229,14 @@ class Allocator(Struct):
|
|||
# **without** going past a net-zero pp. if the pp is
|
||||
# > 1.5x a slot size, then front load: exit a slot's and
|
||||
# expect net-zero to be acquired on the final exit.
|
||||
or slot_size < pp_size < round((1.5*slot_size), ndigits=ld)
|
||||
or (
|
||||
slot_size < pp_size < round((1.5*slot_size), ndigits=ld) or
|
||||
|
||||
# underlying requires discrete (int) units (eg. stocks)
|
||||
# and thus our slot size (based on our limit) would
|
||||
# exit a fractional unit's worth so, presuming we aren't
|
||||
# supporting a fractional-units-style broker, we need
|
||||
# exit the final unit.
|
||||
ld == 0
|
||||
and abs_live_size == 1
|
||||
)
|
||||
# underlying requires discrete (int) units (eg. stocks)
|
||||
# and thus our slot size (based on our limit) would
|
||||
# exit a fractional unit's worth so, presuming we aren't
|
||||
# supporting a fractional-units-style broker, we need
|
||||
# exit the final unit.
|
||||
ld == 0 and abs_live_size == 1
|
||||
):
|
||||
order_size = abs_live_size
|
||||
|
||||
|
|
@ -211,15 +244,9 @@ class Allocator(Struct):
|
|||
if order_size < slot_size:
|
||||
# compute a fractional slots size to display
|
||||
slots_used = self.slots_used(
|
||||
Position(
|
||||
mkt=mkt,
|
||||
size=order_size,
|
||||
ppu=price,
|
||||
bs_mktid=mkt.bs_mktid,
|
||||
)
|
||||
Position(symbol=sym, size=order_size, avg_price=price)
|
||||
)
|
||||
|
||||
# TODO: render an actual ``Executable`` type here?
|
||||
return {
|
||||
'size': abs(round(order_size, ndigits=ld)),
|
||||
'size_digits': ld,
|
||||
|
|
@ -244,8 +271,8 @@ class Allocator(Struct):
|
|||
abs_pp_size = abs(pp.size)
|
||||
|
||||
if self.size_unit == 'currency':
|
||||
# live_currency_size = size or (abs_pp_size * pp.ppu)
|
||||
live_currency_size = abs_pp_size * pp.ppu
|
||||
# live_currency_size = size or (abs_pp_size * pp.avg_price)
|
||||
live_currency_size = abs_pp_size * pp.avg_price
|
||||
prop = live_currency_size / self.currency_limit
|
||||
|
||||
else:
|
||||
|
|
@ -257,15 +284,23 @@ class Allocator(Struct):
|
|||
return round(prop * self.slots)
|
||||
|
||||
|
||||
_derivs = (
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
)
|
||||
|
||||
|
||||
def mk_allocator(
|
||||
|
||||
mkt: MktPair,
|
||||
symbol: Symbol,
|
||||
startup_pp: Position,
|
||||
|
||||
# default allocation settings
|
||||
defaults: dict[str, float] = {
|
||||
'account': None, # select paper by default
|
||||
# 'size_unit': 'currency',
|
||||
'size_unit': 'currency',
|
||||
'units_limit': 400,
|
||||
'currency_limit': 5e3,
|
||||
'slots': 4,
|
||||
|
|
@ -283,9 +318,42 @@ def mk_allocator(
|
|||
'currency_limit': 6e3,
|
||||
'slots': 6,
|
||||
}
|
||||
|
||||
defaults.update(user_def)
|
||||
|
||||
return Allocator(
|
||||
mkt=mkt,
|
||||
alloc = Allocator(
|
||||
symbol=symbol,
|
||||
**defaults,
|
||||
)
|
||||
|
||||
asset_type = symbol.type_key
|
||||
|
||||
# specific configs by asset class / type
|
||||
|
||||
if asset_type in _derivs:
|
||||
# since it's harder to know how currency "applies" in this case
|
||||
# given leverage properties
|
||||
alloc.size_unit = '# units'
|
||||
|
||||
# set units limit to slots size thus making make the next
|
||||
# entry step 1.0
|
||||
alloc.units_limit = alloc.slots
|
||||
|
||||
# if the current position is already greater then the limit
|
||||
# settings, increase the limit to the current position
|
||||
if alloc.size_unit == 'currency':
|
||||
startup_size = startup_pp.size * startup_pp.avg_price
|
||||
|
||||
if startup_size > alloc.currency_limit:
|
||||
alloc.currency_limit = round(startup_size, ndigits=2)
|
||||
|
||||
else:
|
||||
startup_size = abs(startup_pp.size)
|
||||
|
||||
if startup_size > alloc.units_limit:
|
||||
alloc.units_limit = startup_size
|
||||
|
||||
if asset_type in _derivs:
|
||||
alloc.slots = alloc.units_limit
|
||||
|
||||
return alloc
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,284 +18,211 @@
|
|||
Orders and execution client API.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from typing import Dict
|
||||
from pprint import pformat
|
||||
from typing import TYPE_CHECKING
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import trio
|
||||
import tractor
|
||||
from tractor.trionics import broadcast_receiver
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..data.types import Struct
|
||||
from ..service import maybe_open_emsd
|
||||
from ._messages import (
|
||||
Order,
|
||||
Cancel,
|
||||
BrokerdPosition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._messages import (
|
||||
Status,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._ems import _emsd_main
|
||||
from .._daemon import maybe_open_emsd
|
||||
from ._messages import Order, Cancel
|
||||
|
||||
|
||||
class OrderClient(Struct):
|
||||
'''
|
||||
EMS-client-side order book ctl and tracking.
|
||||
log = get_logger(__name__)
|
||||
|
||||
(A)sync API for submitting orders and alerts to the `emsd` service;
|
||||
this is the main control for execution management from client code.
|
||||
|
||||
@dataclass
|
||||
class OrderBook:
|
||||
'''EMS-client-side order book ctl and tracking.
|
||||
|
||||
A style similar to "model-view" is used here where this api is
|
||||
provided as a supervised control for an EMS actor which does all the
|
||||
hard/fast work of talking to brokers/exchanges to conduct
|
||||
executions.
|
||||
|
||||
Currently, this is mostly for keeping local state to match the EMS
|
||||
and use received events to trigger graphics updates.
|
||||
|
||||
'''
|
||||
# IPC stream to `emsd` actor
|
||||
_ems_stream: tractor.MsgStream
|
||||
|
||||
# mem channels used to relay order requests to the EMS daemon
|
||||
_to_relay_task: trio.abc.SendChannel
|
||||
_from_sync_order_client: trio.abc.ReceiveChannel
|
||||
_to_ems: trio.abc.SendChannel
|
||||
_from_order_book: trio.abc.ReceiveChannel
|
||||
|
||||
# history table
|
||||
_sent_orders: dict[str, Order] = {}
|
||||
_sent_orders: Dict[str, Order] = field(default_factory=dict)
|
||||
_ready_to_receive: trio.Event = trio.Event()
|
||||
|
||||
def send_nowait(
|
||||
def send(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
msg: Order,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Sync version of ``.send()``.
|
||||
|
||||
'''
|
||||
) -> dict:
|
||||
self._sent_orders[msg.oid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
return msg
|
||||
|
||||
async def send(
|
||||
def update(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Send a new order msg async to the `emsd` service.
|
||||
|
||||
'''
|
||||
self._sent_orders[msg.oid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def update_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Sync version of ``.update()``.
|
||||
|
||||
'''
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
return msg
|
||||
msg = cmd.dict()
|
||||
msg.update(data)
|
||||
self._sent_orders[uuid] = Order(**msg)
|
||||
self._to_ems.send_nowait(msg)
|
||||
return cmd
|
||||
|
||||
async def update(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
) -> dict:
|
||||
'''
|
||||
Update an existing order dialog with a msg updated from
|
||||
``update`` kwargs.
|
||||
def cancel(self, uuid: str) -> bool:
|
||||
"""Cancel an order (or alert) in the EMS.
|
||||
|
||||
'''
|
||||
"""
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def _mk_cancel_msg(
|
||||
self,
|
||||
uuid: str,
|
||||
) -> Cancel:
|
||||
cmd = self._sent_orders.get(uuid)
|
||||
if not cmd:
|
||||
log.error(
|
||||
f'Unknown order {uuid}!?\n'
|
||||
f'Maybe there is a stale entry or line?\n'
|
||||
f'You should report this as a bug!'
|
||||
)
|
||||
return
|
||||
|
||||
fqme = str(cmd.symbol)
|
||||
return Cancel(
|
||||
msg = Cancel(
|
||||
oid=uuid,
|
||||
symbol=fqme,
|
||||
symbol=cmd.symbol,
|
||||
)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
|
||||
|
||||
_orders: OrderBook = None
|
||||
|
||||
|
||||
def get_orders(
|
||||
emsd_uid: tuple[str, str] = None
|
||||
) -> OrderBook:
|
||||
""""
|
||||
OrderBook singleton factory per actor.
|
||||
|
||||
"""
|
||||
if emsd_uid is not None:
|
||||
# TODO: read in target emsd's active book on startup
|
||||
pass
|
||||
|
||||
global _orders
|
||||
|
||||
if _orders is None:
|
||||
size = 100
|
||||
tx, rx = trio.open_memory_channel(size)
|
||||
brx = broadcast_receiver(rx, size)
|
||||
|
||||
# setup local ui event streaming channels for request/resp
|
||||
# streamging with EMS daemon
|
||||
_orders = OrderBook(
|
||||
_to_ems=tx,
|
||||
_from_order_book=brx,
|
||||
)
|
||||
|
||||
def cancel_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Sync version of ``.cancel()``.
|
||||
|
||||
'''
|
||||
self._to_relay_task.send_nowait(
|
||||
self._mk_cancel_msg(uuid)
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Cancel an already existintg order (or alert) dialog.
|
||||
|
||||
'''
|
||||
await self._ems_stream.send(
|
||||
self._mk_cancel_msg(uuid)
|
||||
)
|
||||
return _orders
|
||||
|
||||
|
||||
# TODO: we can get rid of this relay loop once we move
|
||||
# order_mode inputs to async code!
|
||||
async def relay_order_cmds_from_sync_code(
|
||||
|
||||
async def relay_orders_from_sync_code(
|
||||
|
||||
client: OrderClient,
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Order submission relay task: deliver orders sent from synchronous (UI)
|
||||
code to the EMS via ``OrderClient._from_sync_order_client``.
|
||||
"""
|
||||
Order streaming task: deliver orders transmitted from UI
|
||||
to downstream consumers.
|
||||
|
||||
This is run in the UI actor (usually the one running Qt but could be
|
||||
any other client service code). This process simply delivers order
|
||||
messages to the above ``_to_relay_task`` send channel (from sync code using
|
||||
messages to the above ``_to_ems`` send channel (from sync code using
|
||||
``.send_nowait()``), these values are pulled from the channel here
|
||||
and relayed to any consumer(s) that called this function using
|
||||
a ``tractor`` portal.
|
||||
|
||||
This effectively makes order messages look like they're being
|
||||
"pushed" from the parent to the EMS where local sync code is likely
|
||||
doing the pushing from some non-async UI handler.
|
||||
doing the pushing from some UI.
|
||||
|
||||
'''
|
||||
async with (
|
||||
client._from_sync_order_client.subscribe() as sync_order_cmds
|
||||
):
|
||||
async for cmd in sync_order_cmds:
|
||||
sym = cmd.symbol
|
||||
msg = pformat(cmd.to_dict())
|
||||
|
||||
if sym == symbol_key:
|
||||
log.info(f'Send order cmd:\n{msg}')
|
||||
"""
|
||||
book = get_orders()
|
||||
async with book._from_order_book.subscribe() as orders_stream:
|
||||
async for cmd in orders_stream:
|
||||
if cmd['symbol'] == symbol_key:
|
||||
log.info(f'Send order cmd:\n{pformat(cmd)}')
|
||||
# send msg over IPC / wire
|
||||
await to_ems_stream.send(cmd)
|
||||
|
||||
else:
|
||||
log.warning(
|
||||
f'Ignoring unmatched order cmd for {sym} != {symbol_key}:'
|
||||
f'\n{msg}'
|
||||
)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ems(
|
||||
fqme: str,
|
||||
mode: str = 'live',
|
||||
loglevel: str = 'error',
|
||||
fqsn: str,
|
||||
|
||||
) -> tuple[
|
||||
OrderClient, # client
|
||||
tractor.MsgStream, # order ctl stream
|
||||
dict[
|
||||
# brokername, acctid
|
||||
tuple[str, str],
|
||||
dict[str, BrokerdPosition],
|
||||
],
|
||||
list[str],
|
||||
dict[str, Status],
|
||||
]:
|
||||
) -> (
|
||||
OrderBook,
|
||||
tractor.MsgStream,
|
||||
dict,
|
||||
):
|
||||
'''
|
||||
(Maybe) spawn an EMS-daemon (emsd), deliver an `OrderClient` for
|
||||
requesting orders/alerts and a `trades_stream` which delivers all
|
||||
response-msgs.
|
||||
Spawn an EMS daemon and begin sending orders and receiving
|
||||
alerts.
|
||||
|
||||
This is a "client side" entrypoint which may spawn the `emsd` service
|
||||
if it can't be discovered and generally speaking is the lowest level
|
||||
broker control client-API.
|
||||
This EMS tries to reduce most broker's terrible order entry apis to
|
||||
a very simple protocol built on a few easy to grok and/or
|
||||
"rantsy" premises:
|
||||
|
||||
- most users will prefer "dark mode" where orders are not submitted
|
||||
to a broker until and execution condition is triggered
|
||||
(aka client-side "hidden orders")
|
||||
|
||||
- Brokers over-complicate their apis and generally speaking hire
|
||||
poor designers to create them. We're better off using creating a super
|
||||
minimal, schema-simple, request-event-stream protocol to unify all the
|
||||
existing piles of shit (and shocker, it'll probably just end up
|
||||
looking like a decent crypto exchange's api)
|
||||
|
||||
- all order types can be implemented with client-side limit orders
|
||||
|
||||
- we aren't reinventing a wheel in this case since none of these
|
||||
brokers are exposing FIX protocol; it is they doing the re-invention.
|
||||
|
||||
|
||||
TODO: make some fancy diagrams using mermaid.io
|
||||
|
||||
the possible set of responses from the stream is currently:
|
||||
- 'dark_submitted', 'broker_submitted'
|
||||
- 'dark_cancelled', 'broker_cancelled'
|
||||
- 'dark_executed', 'broker_executed'
|
||||
- 'broker_filled'
|
||||
|
||||
'''
|
||||
# TODO: prolly hand in the `MktPair` instance directly here as well!
|
||||
from piker.accounting import unpack_fqme
|
||||
broker, mktep, venue, suffix = unpack_fqme(fqme)
|
||||
# wait for service to connect back to us signalling
|
||||
# ready for order commands
|
||||
book = get_orders()
|
||||
|
||||
async with maybe_open_emsd(
|
||||
broker,
|
||||
loglevel=loglevel,
|
||||
) as portal:
|
||||
from ..data._source import unpack_fqsn
|
||||
broker, symbol, suffix = unpack_fqsn(fqsn)
|
||||
|
||||
async with maybe_open_emsd(broker) as portal:
|
||||
|
||||
from ._ems import _emsd_main
|
||||
async with (
|
||||
# connect to emsd
|
||||
portal.open_context(
|
||||
_emsd_main,
|
||||
fqme=fqme,
|
||||
exec_mode=mode,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (
|
||||
ctx,
|
||||
(
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
),
|
||||
_emsd_main,
|
||||
fqsn=fqsn,
|
||||
|
||||
) as (ctx, (positions, accounts)),
|
||||
|
||||
# open 2-way trade command stream
|
||||
ctx.open_stream() as trades_stream,
|
||||
):
|
||||
size: int = 100 # what should this be?
|
||||
tx, rx = trio.open_memory_channel(size)
|
||||
brx = broadcast_receiver(rx, size)
|
||||
|
||||
# setup local ui event streaming channels for request/resp
|
||||
# streamging with EMS daemon
|
||||
client = OrderClient(
|
||||
_ems_stream=trades_stream,
|
||||
_to_relay_task=tx,
|
||||
_from_sync_order_client=brx,
|
||||
)
|
||||
|
||||
client._ems_stream = trades_stream
|
||||
|
||||
# start sync code order msg delivery task
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(
|
||||
relay_orders_from_sync_code,
|
||||
client,
|
||||
fqme,
|
||||
relay_order_cmds_from_sync_code,
|
||||
fqsn,
|
||||
trades_stream
|
||||
)
|
||||
|
||||
yield (
|
||||
client,
|
||||
trades_stream,
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
|
||||
# stop the sync-msg-relay task on exit.
|
||||
n.cancel_scope.cancel()
|
||||
yield book, trades_stream, positions, accounts
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -15,161 +15,108 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Clearing sub-system message and protocols.
|
||||
Clearing system messagingn types and protocols.
|
||||
|
||||
"""
|
||||
# from collections import (
|
||||
# ChainMap,
|
||||
# deque,
|
||||
# )
|
||||
from typing import (
|
||||
Literal,
|
||||
)
|
||||
from typing import Optional, Union
|
||||
|
||||
from msgspec import field
|
||||
# TODO: try out just encoding/send direction for now?
|
||||
# import msgspec
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
|
||||
|
||||
# TODO: a composite for tracking msg flow on 2-legged
|
||||
# dialogs.
|
||||
# class Dialog(ChainMap):
|
||||
# '''
|
||||
# Msg collection abstraction to easily track the state changes of
|
||||
# a msg flow in one high level, query-able and immutable construct.
|
||||
|
||||
# The main use case is to query data from a (long-running)
|
||||
# msg-transaction-sequence
|
||||
|
||||
|
||||
# '''
|
||||
# def update(
|
||||
# self,
|
||||
# msg,
|
||||
# ) -> None:
|
||||
# self.maps.insert(0, msg.to_dict())
|
||||
|
||||
# def flatten(self) -> dict:
|
||||
# return dict(self)
|
||||
|
||||
|
||||
# TODO: ``msgspec`` stuff worth paying attention to:
|
||||
# - schema evolution:
|
||||
# https://jcristharif.com/msgspec/usage.html#schema-evolution
|
||||
# - for eg. ``BrokerdStatus``, instead just have separate messages?
|
||||
# - use literals for a common msg determined by diff keys?
|
||||
# - https://jcristharif.com/msgspec/usage.html#literal
|
||||
|
||||
# --------------
|
||||
# Client -> emsd
|
||||
# --------------
|
||||
|
||||
class Order(Struct):
|
||||
|
||||
# TODO: ideally we can combine these 2 fields into
|
||||
# 1 and just use the size polarity to determine a buy/sell.
|
||||
# i would like to see this become more like
|
||||
# https://jcristharif.com/msgspec/usage.html#literal
|
||||
# action: Literal[
|
||||
# 'live',
|
||||
# 'dark',
|
||||
# 'alert',
|
||||
# ]
|
||||
|
||||
action: Literal[
|
||||
'buy',
|
||||
'sell',
|
||||
'alert',
|
||||
]
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: Literal[
|
||||
'dark',
|
||||
'live',
|
||||
# 'paper', no right?
|
||||
]
|
||||
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
# TODO: figure out how to optionally typecast this to `MktPair`?
|
||||
symbol: str # | MktPair
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float # -ve is "sell", +ve is "buy"
|
||||
|
||||
brokers: list[str] = []
|
||||
|
||||
|
||||
class Cancel(Struct):
|
||||
'''
|
||||
Cancel msg for removing a dark (ems triggered) or
|
||||
class Cancel(BaseModel):
|
||||
'''Cancel msg for removing a dark (ems triggered) or
|
||||
broker-submitted (live) trigger/order.
|
||||
|
||||
'''
|
||||
action: str = 'cancel'
|
||||
oid: str # uuid4
|
||||
symbol: str
|
||||
action: str = 'cancel'
|
||||
|
||||
|
||||
# --------------
|
||||
class Order(BaseModel):
|
||||
|
||||
action: str # {'buy', 'sell', 'alert'}
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
symbol: Union[str, Symbol]
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float
|
||||
brokers: list[str]
|
||||
|
||||
# Assigned once initial ack is received
|
||||
# ack_time_ns: Optional[int] = None
|
||||
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: str # {'dark', 'live', 'paper'}
|
||||
|
||||
class Config:
|
||||
# just for pre-loading a ``Symbol`` when used
|
||||
# in the order mode staging process
|
||||
arbitrary_types_allowed = True
|
||||
# don't copy this model instance when used in
|
||||
# a recursive model
|
||||
copy_on_model_validation = False
|
||||
|
||||
# Client <- emsd
|
||||
# --------------
|
||||
# update msgs from ems which relay state change info
|
||||
# from the active clearing engine.
|
||||
|
||||
class Status(Struct):
|
||||
|
||||
time_ns: int
|
||||
oid: str # uuid4 ems-order dialog id
|
||||
|
||||
resp: Literal[
|
||||
'pending', # acked by broker but not yet open
|
||||
'open',
|
||||
'dark_open', # dark/algo triggered order is open in ems clearing loop
|
||||
'triggered', # above triggered order sent to brokerd, or an alert closed
|
||||
'closed', # fully cleared all size/units
|
||||
'fill', # partial execution
|
||||
'canceled',
|
||||
'error',
|
||||
]
|
||||
class Status(BaseModel):
|
||||
|
||||
name: str = 'status'
|
||||
oid: str # uuid4
|
||||
time_ns: int
|
||||
|
||||
# {
|
||||
# 'dark_submitted',
|
||||
# 'dark_cancelled',
|
||||
# 'dark_triggered',
|
||||
|
||||
# 'broker_submitted',
|
||||
# 'broker_cancelled',
|
||||
# 'broker_executed',
|
||||
# 'broker_filled',
|
||||
# 'broker_errored',
|
||||
|
||||
# 'alert_submitted',
|
||||
# 'alert_triggered',
|
||||
|
||||
# }
|
||||
resp: str # "response", see above
|
||||
|
||||
# symbol: str
|
||||
|
||||
# trigger info
|
||||
trigger_price: Optional[float] = None
|
||||
# price: float
|
||||
|
||||
# broker: Optional[str] = None
|
||||
|
||||
# this maps normally to the ``BrokerdOrder.reqid`` below, an id
|
||||
# normally allocated internally by the backend broker routing system
|
||||
reqid: int | str | None = None
|
||||
broker_reqid: Optional[Union[int, str]] = None
|
||||
|
||||
# the (last) source order/request msg if provided
|
||||
# (eg. the Order/Cancel which causes this msg) and
|
||||
# acts as a back-reference to the corresponding
|
||||
# request message which was the source of this msg.
|
||||
req: Order | None = None
|
||||
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
src: str | None = None
|
||||
|
||||
# set when a cancel request msg was set for this order flow dialog
|
||||
# but the brokerd dialog isn't yet in a cancelled state.
|
||||
cancel_called: bool = False
|
||||
|
||||
# for relaying a boxed brokerd-dialog-side msg data "through" the
|
||||
# ems layer to clients.
|
||||
# for relaying backend msg data "through" the ems layer
|
||||
brokerd_msg: dict = {}
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd -> brokerd
|
||||
# ---------------
|
||||
# requests *sent* from ems to respective backend broker daemon
|
||||
|
||||
class BrokerdCancel(Struct):
|
||||
class BrokerdCancel(BaseModel):
|
||||
|
||||
action: str = 'cancel'
|
||||
oid: str # piker emsd order id
|
||||
time_ns: int
|
||||
|
||||
|
|
@ -180,39 +127,34 @@ class BrokerdCancel(Struct):
|
|||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: int | str | None = None
|
||||
action: str = 'cancel'
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
|
||||
class BrokerdOrder(Struct):
|
||||
class BrokerdOrder(BaseModel):
|
||||
|
||||
action: str # {buy, sell}
|
||||
oid: str
|
||||
account: str
|
||||
time_ns: int
|
||||
|
||||
symbol: str # fqme
|
||||
price: float
|
||||
size: float
|
||||
|
||||
# TODO: if we instead rely on a +ve/-ve size to determine
|
||||
# the action we more or less don't need this field right?
|
||||
action: str = '' # {buy, sell}
|
||||
|
||||
# "broker request id": broker specific/internal order id if this is
|
||||
# None, creates a new order otherwise if the id is valid the backend
|
||||
# api must modify the existing matching order. If the broker allows
|
||||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: int | str | None = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
symbol: str # symbol.<providername> ?
|
||||
price: float
|
||||
size: float
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd <- brokerd
|
||||
# ---------------
|
||||
# requests *received* to ems from broker backend
|
||||
|
||||
class BrokerdOrderAck(Struct):
|
||||
|
||||
class BrokerdOrderAck(BaseModel):
|
||||
'''
|
||||
Immediate reponse to a brokerd order request providing the broker
|
||||
specific unique order id so that the EMS can associate this
|
||||
|
|
@ -220,96 +162,102 @@ class BrokerdOrderAck(Struct):
|
|||
``.oid`` (which is a uuid4).
|
||||
|
||||
'''
|
||||
name: str = 'ack'
|
||||
|
||||
# defined and provided by backend
|
||||
reqid: int | str
|
||||
reqid: Union[int, str]
|
||||
|
||||
# emsd id originally sent in matching request msg
|
||||
oid: str
|
||||
account: str = ''
|
||||
name: str = 'ack'
|
||||
|
||||
|
||||
class BrokerdStatus(Struct):
|
||||
class BrokerdStatus(BaseModel):
|
||||
|
||||
time_ns: int
|
||||
reqid: int | str
|
||||
status: Literal[
|
||||
'open',
|
||||
'canceled',
|
||||
'pending',
|
||||
'error',
|
||||
'closed',
|
||||
]
|
||||
|
||||
# TODO: do we need this?
|
||||
account: str | None = None,
|
||||
name: str = 'status'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
|
||||
# XXX: should be best effort set for every update
|
||||
account: str = ''
|
||||
|
||||
# {
|
||||
# 'submitted',
|
||||
# 'cancelled',
|
||||
# 'filled',
|
||||
# }
|
||||
status: str
|
||||
|
||||
filled: float = 0.0
|
||||
reason: str = ''
|
||||
remaining: float = 0.0
|
||||
|
||||
# external: bool = False
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
external: bool = False
|
||||
|
||||
# XXX: not required schema as of yet
|
||||
broker_details: dict = field(default_factory=lambda: {
|
||||
broker_details: dict = {
|
||||
'name': '',
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
class BrokerdFill(Struct):
|
||||
class BrokerdFill(BaseModel):
|
||||
'''
|
||||
A single message indicating a "fill-details" event from the
|
||||
broker if avaiable.
|
||||
A single message indicating a "fill-details" event from the broker
|
||||
if avaiable.
|
||||
|
||||
'''
|
||||
# brokerd timestamp required for order mode arrow placement on x-axis
|
||||
# TODO: maybe int if we force ns?
|
||||
# we need to normalize this somehow since backends will use their
|
||||
# own format and likely across many disparate epoch clocks...
|
||||
name: str = 'fill'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
broker_time: float
|
||||
reqid: int | str
|
||||
|
||||
# order exeuction related
|
||||
action: str
|
||||
size: float
|
||||
price: float
|
||||
|
||||
name: str = 'fill'
|
||||
action: str | None = None
|
||||
broker_details: dict = {} # meta-data (eg. commisions etc.)
|
||||
|
||||
# brokerd timestamp required for order mode arrow placement on x-axis
|
||||
|
||||
class BrokerdError(Struct):
|
||||
# TODO: maybe int if we force ns?
|
||||
# we need to normalize this somehow since backends will use their
|
||||
# own format and likely across many disparate epoch clocks...
|
||||
broker_time: float
|
||||
|
||||
|
||||
class BrokerdError(BaseModel):
|
||||
'''
|
||||
Optional error type that can be relayed to emsd for error handling.
|
||||
|
||||
This is still a TODO thing since we're not sure how to employ it yet.
|
||||
|
||||
'''
|
||||
name: str = 'error'
|
||||
oid: str
|
||||
reason: str
|
||||
|
||||
# TODO: drop this right?
|
||||
symbol: str | None = None
|
||||
|
||||
# if no brokerd order request was actually submitted (eg. we errored
|
||||
# at the ``pikerd`` layer) then there will be ``reqid`` allocated.
|
||||
reqid: int | str | None = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
name: str = 'error'
|
||||
symbol: str
|
||||
reason: str
|
||||
broker_details: dict = {}
|
||||
|
||||
|
||||
class BrokerdPosition(Struct):
|
||||
'''
|
||||
Position update event from brokerd.
|
||||
class BrokerdPosition(BaseModel):
|
||||
'''Position update event from brokerd.
|
||||
|
||||
'''
|
||||
name: str = 'position'
|
||||
|
||||
broker: str
|
||||
account: str
|
||||
symbol: str
|
||||
currency: str
|
||||
size: float
|
||||
avg_price: float
|
||||
currency: str = ''
|
||||
name: str = 'position'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,85 +14,58 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Fake trading: a full forward testing simulation engine.
|
||||
"""
|
||||
Fake trading for forward testing.
|
||||
|
||||
We can real-time emulate any mkt conditions you want bruddr B)
|
||||
Just slide us the model que quieres..
|
||||
|
||||
'''
|
||||
from collections import defaultdict
|
||||
from contextlib import asynccontextmanager as acm
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from operator import itemgetter
|
||||
import itertools
|
||||
import time
|
||||
from typing import (
|
||||
Callable,
|
||||
)
|
||||
from typing import Tuple, Optional, Callable
|
||||
import uuid
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import trio
|
||||
import tractor
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..brokers import get_brokermod
|
||||
from .. import data
|
||||
from ..data.types import Struct
|
||||
from ..accounting._mktinfo import (
|
||||
MktPair,
|
||||
)
|
||||
from ..accounting import (
|
||||
Position,
|
||||
PpTable,
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
open_pps,
|
||||
)
|
||||
from ..data import iterticks
|
||||
from ..accounting import unpack_fqme
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ..data._normalize import iterticks
|
||||
from ..data._source import unpack_fqsn
|
||||
from ..log import get_logger
|
||||
from ._messages import (
|
||||
BrokerdCancel,
|
||||
BrokerdOrder,
|
||||
BrokerdOrderAck,
|
||||
BrokerdStatus,
|
||||
BrokerdFill,
|
||||
BrokerdPosition,
|
||||
BrokerdError,
|
||||
BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
|
||||
BrokerdFill, BrokerdPosition, BrokerdError
|
||||
)
|
||||
|
||||
|
||||
class PaperBoi(Struct):
|
||||
'''
|
||||
Emulates a broker order client providing approximately the same API
|
||||
and delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements (eg open, closed, fill msgs).
|
||||
log = get_logger(__name__)
|
||||
|
||||
'''
|
||||
|
||||
@dataclass
|
||||
class PaperBoi:
|
||||
"""
|
||||
Emulates a broker order client providing the same API and
|
||||
delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements.
|
||||
|
||||
"""
|
||||
broker: str
|
||||
|
||||
ems_trades_stream: tractor.MsgStream
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
|
||||
# map of paper "live" orders which be used
|
||||
# to simulate fills based on paper engine settings
|
||||
_buys: defaultdict[str, bidict]
|
||||
_sells: defaultdict[str, bidict]
|
||||
_buys: bidict
|
||||
_sells: bidict
|
||||
_reqids: bidict
|
||||
_mkts: dict[str, MktPair] = {}
|
||||
_positions: dict[str, BrokerdPosition]
|
||||
|
||||
# init edge case L1 spread
|
||||
last_ask: tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: tuple[float, float] = (0, 0)
|
||||
last_ask: Tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: Tuple[float, float] = (0, 0)
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
|
|
@ -101,51 +74,48 @@ class PaperBoi(Struct):
|
|||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str | None,
|
||||
|
||||
reqid: Optional[str],
|
||||
) -> int:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
"""Place an order and return integer request id provided by client.
|
||||
|
||||
"""
|
||||
is_modify: bool = False
|
||||
if reqid is None:
|
||||
reqid = str(uuid.uuid4())
|
||||
|
||||
else:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = self._reqids[reqid]
|
||||
assert old_price != price
|
||||
is_modify = True
|
||||
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
'''
|
||||
if action == 'alert':
|
||||
# bypass all fill simulation
|
||||
return reqid
|
||||
|
||||
entry = self._reqids.get(reqid)
|
||||
if entry:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = entry
|
||||
else:
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
# TODO: net latency model
|
||||
# we checkpoint here quickly particulalry
|
||||
# for dark orders since we want the dark_executed
|
||||
# to trigger first thus creating a lookup entry
|
||||
# in the broker trades event processing loop
|
||||
await trio.sleep(0.01)
|
||||
await trio.sleep(0.05)
|
||||
|
||||
if (
|
||||
action == 'sell'
|
||||
and size > 0
|
||||
):
|
||||
if action == 'sell':
|
||||
size = -size
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='open',
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='submitted',
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
filled=0.0,
|
||||
reason='paper_trigger',
|
||||
remaining=size,
|
||||
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
# if we're already a clearing price simulate an immediate fill
|
||||
if (
|
||||
|
|
@ -153,28 +123,28 @@ class PaperBoi(Struct):
|
|||
) or (
|
||||
action == 'sell' and (clear_price := self.last_bid[0]) >= price
|
||||
):
|
||||
await self.fake_fill(
|
||||
symbol,
|
||||
clear_price,
|
||||
size,
|
||||
action,
|
||||
reqid,
|
||||
oid,
|
||||
)
|
||||
await self.fake_fill(symbol, clear_price, size, action, reqid, oid)
|
||||
|
||||
# register this submissions as a paper live order
|
||||
else:
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
# register this submissions as a paper live order
|
||||
|
||||
# submit order to book simulation fill loop
|
||||
if action == 'buy':
|
||||
orders = self._buys
|
||||
|
||||
elif action == 'sell':
|
||||
orders = self._sells
|
||||
|
||||
# {symbol -> bidict[oid, (<price data>)]}
|
||||
orders[symbol][oid] = (price, size, reqid, action)
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
|
||||
if is_modify:
|
||||
# remove any existing order for the old price
|
||||
orders[symbol].pop((oid, old_price))
|
||||
|
||||
# buys/sells: (symbol -> (price -> order))
|
||||
orders.setdefault(symbol, {})[(oid, price)] = (size, reqid, action)
|
||||
|
||||
return reqid
|
||||
|
||||
|
|
@ -187,26 +157,26 @@ class PaperBoi(Struct):
|
|||
oid, symbol, action, price = self._reqids[reqid]
|
||||
|
||||
if action == 'buy':
|
||||
self._buys[symbol].pop(oid, None)
|
||||
self._buys[symbol].pop((oid, price))
|
||||
elif action == 'sell':
|
||||
self._sells[symbol].pop(oid, None)
|
||||
self._sells[symbol].pop((oid, price))
|
||||
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.01)
|
||||
await trio.sleep(0.05)
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='canceled',
|
||||
account='paper',
|
||||
status='cancelled',
|
||||
oid=oid,
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
async def fake_fill(
|
||||
self,
|
||||
|
||||
fqme: str,
|
||||
symbol: str,
|
||||
price: float,
|
||||
size: float,
|
||||
action: str, # one of {'buy', 'sell'}
|
||||
|
|
@ -220,21 +190,21 @@ class PaperBoi(Struct):
|
|||
remaining: float = 0,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Pretend to fill a broker order @ price and size.
|
||||
"""Pretend to fill a broker order @ price and size.
|
||||
|
||||
'''
|
||||
"""
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.01)
|
||||
fill_time_ns = time.time_ns()
|
||||
fill_time_s = time.time()
|
||||
await trio.sleep(0.05)
|
||||
|
||||
msg = BrokerdFill(
|
||||
|
||||
fill_msg = BrokerdFill(
|
||||
reqid=reqid,
|
||||
time_ns=fill_time_ns,
|
||||
time_ns=time.time_ns(),
|
||||
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
|
||||
broker_time=datetime.now().timestamp(),
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
|
|
@ -244,67 +214,79 @@ class PaperBoi(Struct):
|
|||
'name': self.broker + '_paper',
|
||||
},
|
||||
)
|
||||
log.info(f'Fake filling order:\n{fill_msg}')
|
||||
await self.ems_trades_stream.send(fill_msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
if order_complete:
|
||||
|
||||
msg = BrokerdStatus(
|
||||
|
||||
reqid=reqid,
|
||||
time_ns=time.time_ns(),
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='closed',
|
||||
|
||||
status='filled',
|
||||
filled=size,
|
||||
remaining=0 if order_complete else remaining,
|
||||
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
'oid': oid,
|
||||
},
|
||||
'name': self.broker,
|
||||
},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
# NOTE: for paper we set the "bs_mktid" as just the fqme since
|
||||
# we don't actually have any unique backend symbol ourselves
|
||||
# other then this thing, our fqme address.
|
||||
bs_mktid: str = fqme
|
||||
t = Transaction(
|
||||
fqme=fqme,
|
||||
sym=self._mkts[fqme],
|
||||
tid=oid,
|
||||
size=size,
|
||||
price=price,
|
||||
cost=0, # TODO: cost model
|
||||
dt=pendulum.from_timestamp(fill_time_s),
|
||||
bs_mktid=bs_mktid,
|
||||
# lookup any existing position
|
||||
token = f'{symbol}.{self.broker}'
|
||||
pp_msg = self._positions.setdefault(
|
||||
token,
|
||||
BrokerdPosition(
|
||||
broker=self.broker,
|
||||
account='paper',
|
||||
symbol=symbol,
|
||||
# TODO: we need to look up the asset currency from
|
||||
# broker info. i guess for crypto this can be
|
||||
# inferred from the pair?
|
||||
currency='',
|
||||
size=0.0,
|
||||
avg_price=0,
|
||||
)
|
||||
)
|
||||
|
||||
# update in-mem ledger and pos table
|
||||
self.ledger.update_from_t(t)
|
||||
self.ppt.update_from_trans({oid: t})
|
||||
# "avg position price" calcs
|
||||
# TODO: eventually it'd be nice to have a small set of routines
|
||||
# to do this stuff from a sequence of cleared orders to enable
|
||||
# so called "contextual positions".
|
||||
new_size = size + pp_msg.size
|
||||
|
||||
# transmit pp msg to ems
|
||||
pp = self.ppt.pps[bs_mktid]
|
||||
pp_msg = BrokerdPosition(
|
||||
broker=self.broker,
|
||||
account='paper',
|
||||
symbol=fqme,
|
||||
# old size minus the new size gives us size differential with
|
||||
# +ve -> increase in pp size
|
||||
# -ve -> decrease in pp size
|
||||
size_diff = abs(new_size) - abs(pp_msg.size)
|
||||
|
||||
size=pp.size,
|
||||
avg_price=pp.ppu,
|
||||
if new_size == 0:
|
||||
pp_msg.avg_price = 0
|
||||
|
||||
# TODO: we need to look up the asset currency from
|
||||
# broker info. i guess for crypto this can be
|
||||
# inferred from the pair?
|
||||
# currency=bs_mktid,
|
||||
)
|
||||
# write all updates to filesys immediately
|
||||
# (adds latency but that works for simulation anyway)
|
||||
self.ledger.write_config()
|
||||
self.ppt.write_config()
|
||||
elif size_diff > 0:
|
||||
# only update the "average position price" when the position
|
||||
# size increases not when it decreases (i.e. the position is
|
||||
# being made smaller)
|
||||
pp_msg.avg_price = (
|
||||
abs(size) * price + pp_msg.avg_price * abs(pp_msg.size)
|
||||
) / abs(new_size)
|
||||
|
||||
await self.ems_trades_stream.send(pp_msg)
|
||||
pp_msg.size = new_size
|
||||
|
||||
await self.ems_trades_stream.send(pp_msg.dict())
|
||||
|
||||
|
||||
async def simulate_fills(
|
||||
quote_stream: tractor.MsgStream, # noqa
|
||||
quote_stream: 'tractor.ReceiveStream', # noqa
|
||||
client: PaperBoi,
|
||||
|
||||
) -> None:
|
||||
|
||||
# TODO: more machinery to better simulate real-world market things:
|
||||
|
|
@ -324,116 +306,61 @@ async def simulate_fills(
|
|||
|
||||
# this stream may eventually contain multiple symbols
|
||||
async for quotes in quote_stream:
|
||||
|
||||
for sym, quote in quotes.items():
|
||||
|
||||
for tick in iterticks(
|
||||
quote,
|
||||
# dark order price filter(s)
|
||||
types=('ask', 'bid', 'trade', 'last')
|
||||
):
|
||||
tick_price = tick['price']
|
||||
# print(tick)
|
||||
tick_price = tick.get('price')
|
||||
ttype = tick['type']
|
||||
|
||||
buys: bidict[str, tuple] = client._buys[sym]
|
||||
iter_buys = reversed(sorted(
|
||||
buys.values(),
|
||||
key=itemgetter(0),
|
||||
))
|
||||
if ttype in ('ask',):
|
||||
|
||||
def buy_on_ask(our_price):
|
||||
return tick_price <= our_price
|
||||
client.last_ask = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_ask[1]),
|
||||
)
|
||||
|
||||
sells: bidict[str, tuple] = client._sells[sym]
|
||||
iter_sells = sorted(
|
||||
sells.values(),
|
||||
key=itemgetter(0)
|
||||
)
|
||||
orders = client._buys.get(sym, {})
|
||||
|
||||
def sell_on_bid(our_price):
|
||||
return tick_price >= our_price
|
||||
book_sequence = reversed(
|
||||
sorted(orders.keys(), key=itemgetter(1)))
|
||||
|
||||
match tick:
|
||||
def pred(our_price):
|
||||
return tick_price < our_price
|
||||
|
||||
# on an ask queue tick, only clear buy entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'ask',
|
||||
}:
|
||||
client.last_ask = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_ask[1]),
|
||||
)
|
||||
elif ttype in ('bid',):
|
||||
|
||||
iter_entries = zip(
|
||||
iter_buys,
|
||||
itertools.repeat(buy_on_ask)
|
||||
)
|
||||
client.last_bid = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_bid[1]),
|
||||
)
|
||||
|
||||
# on a bid queue tick, only clear sell entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'bid',
|
||||
}:
|
||||
client.last_bid = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_bid[1]),
|
||||
)
|
||||
orders = client._sells.get(sym, {})
|
||||
book_sequence = sorted(orders.keys(), key=itemgetter(1))
|
||||
|
||||
iter_entries = zip(
|
||||
iter_sells,
|
||||
itertools.repeat(sell_on_bid)
|
||||
)
|
||||
def pred(our_price):
|
||||
return tick_price > our_price
|
||||
|
||||
# TODO: fix this block, though it definitely
|
||||
# costs a lot more CPU-wise
|
||||
# - doesn't seem like clears are happening still on
|
||||
# "resting" limit orders?
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': ('trade' | 'last'),
|
||||
}:
|
||||
# in the clearing price / last price case we
|
||||
# want to iterate both sides of our book for
|
||||
# clears since we don't know which direction the
|
||||
# price is going to move (especially with HFT)
|
||||
# and thus we simply interleave both sides (buys
|
||||
# and sells) until one side clears and then
|
||||
# break until the next tick?
|
||||
def interleave():
|
||||
for pair in zip(
|
||||
iter_buys,
|
||||
iter_sells,
|
||||
):
|
||||
for order_info, pred in zip(
|
||||
pair,
|
||||
itertools.cycle([buy_on_ask, sell_on_bid]),
|
||||
):
|
||||
yield order_info, pred
|
||||
elif ttype in ('trade', 'last'):
|
||||
# TODO: simulate actual book queues and our orders
|
||||
# place in it, might require full L2 data?
|
||||
continue
|
||||
|
||||
iter_entries = interleave()
|
||||
# iterate book prices descending
|
||||
for oid, our_price in book_sequence:
|
||||
if pred(our_price):
|
||||
|
||||
# NOTE: all other (non-clearable) tick event types
|
||||
# - we don't want to sping the simulated clear loop
|
||||
# below unecessarily and further don't want to pop
|
||||
# simulated live orders prematurely.
|
||||
case _:
|
||||
continue
|
||||
|
||||
# iterate all potentially clearable book prices
|
||||
# in FIFO order per side.
|
||||
for order_info, pred in iter_entries:
|
||||
(our_price, size, reqid, action) = order_info
|
||||
|
||||
# print(order_info)
|
||||
clearable = pred(our_price)
|
||||
if clearable:
|
||||
# pop and retreive order info
|
||||
oid = {
|
||||
'buy': buys,
|
||||
'sell': sells
|
||||
}[action].inverse.pop(order_info)
|
||||
# retreive order info
|
||||
(size, reqid, action) = orders.pop((oid, our_price))
|
||||
|
||||
# clearing price would have filled entirely
|
||||
await client.fake_fill(
|
||||
fqme=sym,
|
||||
symbol=sym,
|
||||
# todo slippage to determine fill price
|
||||
price=tick_price,
|
||||
size=size,
|
||||
|
|
@ -441,6 +368,9 @@ async def simulate_fills(
|
|||
reqid=reqid,
|
||||
oid=oid,
|
||||
)
|
||||
else:
|
||||
# prices are iterated in sorted order so we're done
|
||||
break
|
||||
|
||||
|
||||
async def handle_order_requests(
|
||||
|
|
@ -450,236 +380,119 @@ async def handle_order_requests(
|
|||
|
||||
) -> None:
|
||||
|
||||
request_msg: dict
|
||||
# order_request: dict
|
||||
async for request_msg in ems_order_stream:
|
||||
match request_msg:
|
||||
case {'action': ('buy' | 'sell')}:
|
||||
order = BrokerdOrder(**request_msg)
|
||||
account = order.account
|
||||
|
||||
# error on bad inputs
|
||||
reason = None
|
||||
if account != 'paper':
|
||||
reason = f'No account found:`{account}` (paper only)?'
|
||||
action = request_msg['action']
|
||||
|
||||
elif order.size == 0:
|
||||
reason = 'Invalid size: 0'
|
||||
if action in {'buy', 'sell'}:
|
||||
|
||||
if reason:
|
||||
log.error(reason)
|
||||
await ems_order_stream.send(BrokerdError(
|
||||
oid=order.oid,
|
||||
symbol=order.symbol,
|
||||
reason=reason,
|
||||
))
|
||||
continue
|
||||
|
||||
reqid = order.reqid or str(uuid.uuid4())
|
||||
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
oid=order.oid,
|
||||
reqid=reqid,
|
||||
account='paper'
|
||||
)
|
||||
account = request_msg['account']
|
||||
if account != 'paper':
|
||||
log.error(
|
||||
'This is a paper account, only a `paper` selection is valid'
|
||||
)
|
||||
await ems_order_stream.send(BrokerdError(
|
||||
oid=request_msg['oid'],
|
||||
symbol=request_msg['symbol'],
|
||||
reason=f'Paper only. No account found: `{account}` ?',
|
||||
).dict())
|
||||
continue
|
||||
|
||||
# call our client api to submit the order
|
||||
reqid = await client.submit_limit(
|
||||
# validate
|
||||
order = BrokerdOrder(**request_msg)
|
||||
|
||||
# call our client api to submit the order
|
||||
reqid = await client.submit_limit(
|
||||
|
||||
oid=order.oid,
|
||||
symbol=order.symbol,
|
||||
price=order.price,
|
||||
action=order.action,
|
||||
size=order.size,
|
||||
|
||||
# XXX: by default 0 tells ``ib_insync`` methods that
|
||||
# there is no existing order so ask the client to create
|
||||
# a new one (which it seems to do by allocating an int
|
||||
# counter - collision prone..)
|
||||
reqid=order.reqid,
|
||||
)
|
||||
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
|
||||
# ems order request id
|
||||
oid=order.oid,
|
||||
symbol=f'{order.symbol}.{client.broker}',
|
||||
price=order.price,
|
||||
action=order.action,
|
||||
size=order.size,
|
||||
# XXX: by default 0 tells ``ib_insync`` methods that
|
||||
# there is no existing order so ask the client to create
|
||||
# a new one (which it seems to do by allocating an int
|
||||
# counter - collision prone..)
|
||||
|
||||
# broker specific request id
|
||||
reqid=reqid,
|
||||
)
|
||||
log.info(f'Submitted paper LIMIT {reqid}:\n{order}')
|
||||
|
||||
case {'action': 'cancel'}:
|
||||
msg = BrokerdCancel(**request_msg)
|
||||
await client.submit_cancel(
|
||||
reqid=msg.reqid
|
||||
)
|
||||
).dict()
|
||||
)
|
||||
|
||||
case _:
|
||||
log.error(f'Unknown order command: {request_msg}')
|
||||
elif action == 'cancel':
|
||||
msg = BrokerdCancel(**request_msg)
|
||||
|
||||
await client.submit_cancel(
|
||||
reqid=msg.reqid
|
||||
)
|
||||
|
||||
_reqids: bidict[str, tuple] = {}
|
||||
_buys: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
_sells: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
else:
|
||||
log.error(f'Unknown order command: {request_msg}')
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_trade_dialog(
|
||||
async def trades_dialogue(
|
||||
|
||||
ctx: tractor.Context,
|
||||
broker: str,
|
||||
fqme: str | None = None, # if empty, we only boot broker mode
|
||||
loglevel: str = 'warning',
|
||||
fqsn: str,
|
||||
loglevel: str = None,
|
||||
|
||||
) -> None:
|
||||
tractor.log.get_console_log(loglevel)
|
||||
|
||||
# enable piker.clearing console log for *this* subactor
|
||||
get_console_log(loglevel)
|
||||
async with (
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
with (
|
||||
open_pps(
|
||||
broker,
|
||||
'paper',
|
||||
write_on_exit=True,
|
||||
) as ppt,
|
||||
data.open_feed(
|
||||
[fqsn],
|
||||
loglevel=loglevel,
|
||||
) as feed,
|
||||
|
||||
open_trade_ledger(
|
||||
broker,
|
||||
'paper',
|
||||
) as ledger
|
||||
):
|
||||
# NOTE: retreive market(pair) info from the backend broker
|
||||
# since ledger entries (in their backend native format) often
|
||||
# don't contain necessary market info per trade record entry..
|
||||
# - if no fqme was passed in, we presume we're running in
|
||||
# "ledger-sync-only mode" and thus we load mkt info for
|
||||
# each symbol found in the ledger to a ppt table manually.
|
||||
|
||||
# TODO: how to process ledger info from backends?
|
||||
# - should we be rolling our own actor-cached version of these
|
||||
# client API refs or using portal IPC to send requests to the
|
||||
# existing brokerd daemon?
|
||||
# - alternatively we can possibly expect and use
|
||||
# a `.broker.norm_trade_records()` ep?
|
||||
brokermod = get_brokermod(broker)
|
||||
gmi = getattr(brokermod, 'get_mkt_info', None)
|
||||
|
||||
# update all transactions with mkt info before
|
||||
# loading any pps
|
||||
mkt_by_fqme: dict[str, MktPair] = {}
|
||||
if fqme:
|
||||
bs_fqme, _, broker = fqme.rpartition('.')
|
||||
mkt, _ = await brokermod.get_mkt_info(bs_fqme)
|
||||
mkt_by_fqme[mkt.fqme] = mkt
|
||||
|
||||
# for each sym in the ledger load it's `MktPair` info
|
||||
for tid, txdict in ledger.data.items():
|
||||
l_fqme: str = txdict.get('fqme') or txdict['fqsn']
|
||||
|
||||
if (
|
||||
gmi
|
||||
and l_fqme not in mkt_by_fqme
|
||||
):
|
||||
mkt, pair = await brokermod.get_mkt_info(
|
||||
l_fqme.rstrip(f'.{broker}'),
|
||||
)
|
||||
mkt_by_fqme[l_fqme] = mkt
|
||||
|
||||
# if an ``fqme: str`` input was provided we only
|
||||
# need a ``MktPair`` for that one market, since we're
|
||||
# running in real simulated-clearing mode, not just ledger
|
||||
# syncing.
|
||||
if (
|
||||
fqme is not None
|
||||
and fqme in mkt_by_fqme
|
||||
):
|
||||
break
|
||||
|
||||
# update pos table from ledger history and provide a ``MktPair``
|
||||
# lookup for internal position accounting calcs.
|
||||
ppt.update_from_trans(ledger.to_trans(mkt_by_fqme=mkt_by_fqme))
|
||||
|
||||
pp_msgs: list[BrokerdPosition] = []
|
||||
pos: Position
|
||||
token: str # f'{symbol}.{self.broker}'
|
||||
for token, pos in ppt.pps.items():
|
||||
pp_msgs.append(BrokerdPosition(
|
||||
broker=broker,
|
||||
account='paper',
|
||||
symbol=pos.mkt.fqme,
|
||||
size=pos.size,
|
||||
avg_price=pos.ppu,
|
||||
))
|
||||
|
||||
await ctx.started((
|
||||
pp_msgs,
|
||||
['paper'],
|
||||
))
|
||||
|
||||
# write new positions state in case ledger was
|
||||
# newer then that tracked in pps.toml
|
||||
ppt.write_config()
|
||||
|
||||
# exit early since no fqme was passed,
|
||||
# normally this case is just to load
|
||||
# positions "offline".
|
||||
if fqme is None:
|
||||
log.warning(
|
||||
'Paper engine only running in position delivery mode!\n'
|
||||
'NO SIMULATED CLEARING LOOP IS ACTIVE!'
|
||||
)
|
||||
await trio.sleep_forever()
|
||||
return
|
||||
# TODO: load paper positions per broker from .toml config file
|
||||
# and pass as symbol to position data mapping: ``dict[str, dict]``
|
||||
# await ctx.started(all_positions)
|
||||
await ctx.started(({}, {'paper',}))
|
||||
|
||||
async with (
|
||||
data.open_feed(
|
||||
[fqme],
|
||||
loglevel=loglevel,
|
||||
) as feed,
|
||||
ctx.open_stream() as ems_stream,
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
# sanity check all the mkt infos
|
||||
for fqme, flume in feed.flumes.items():
|
||||
assert mkt_by_fqme[fqme] == flume.mkt
|
||||
|
||||
async with (
|
||||
ctx.open_stream() as ems_stream,
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
client = PaperBoi(
|
||||
broker=broker,
|
||||
ems_trades_stream=ems_stream,
|
||||
ppt=ppt,
|
||||
ledger=ledger,
|
||||
client = PaperBoi(
|
||||
broker,
|
||||
ems_stream,
|
||||
_buys={},
|
||||
_sells={},
|
||||
|
||||
_buys=_buys,
|
||||
_sells=_sells,
|
||||
_reqids=_reqids,
|
||||
_reqids={},
|
||||
|
||||
_mkts=mkt_by_fqme,
|
||||
# TODO: load paper positions from ``positions.toml``
|
||||
_positions={},
|
||||
)
|
||||
|
||||
)
|
||||
n.start_soon(handle_order_requests, client, ems_stream)
|
||||
|
||||
n.start_soon(
|
||||
handle_order_requests,
|
||||
client,
|
||||
ems_stream,
|
||||
)
|
||||
|
||||
# paper engine simulator clearing task
|
||||
await simulate_fills(feed.streams[broker], client)
|
||||
# paper engine simulator clearing task
|
||||
await simulate_fills(feed.stream, client)
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_paperboi(
|
||||
fqme: str | None = None,
|
||||
broker: str | None = None,
|
||||
loglevel: str | None = None,
|
||||
fqsn: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> Callable:
|
||||
'''
|
||||
|
|
@ -687,39 +500,28 @@ async def open_paperboi(
|
|||
its context.
|
||||
|
||||
'''
|
||||
if not fqme:
|
||||
assert broker, 'One of `broker` or `fqme` is required siss..!'
|
||||
else:
|
||||
broker, _, _, _ = unpack_fqme(fqme)
|
||||
|
||||
we_spawned: bool = False
|
||||
broker, symbol, expiry = unpack_fqsn(fqsn)
|
||||
service_name = f'paperboi.{broker}'
|
||||
|
||||
async with (
|
||||
tractor.find_actor(service_name) as portal,
|
||||
tractor.open_nursery() as an,
|
||||
tractor.open_nursery() as tn,
|
||||
):
|
||||
# NOTE: only spawn if no paperboi already is up since we likely
|
||||
# don't need more then one actor for simulated order clearing
|
||||
# per broker-backend.
|
||||
# only spawn if no paperboi already is up
|
||||
# (we likely don't need more then one proc for basic
|
||||
# simulated order clearing)
|
||||
if portal is None:
|
||||
log.info('Starting new paper-engine actor')
|
||||
portal = await an.start_actor(
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
we_spawned = True
|
||||
|
||||
async with portal.open_context(
|
||||
open_trade_dialog,
|
||||
broker=broker,
|
||||
fqme=fqme,
|
||||
loglevel=loglevel,
|
||||
trades_dialogue,
|
||||
broker=broker,
|
||||
fqsn=fqsn,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (ctx, first):
|
||||
yield ctx, first
|
||||
|
||||
# tear down connection and any spawned actor on exit
|
||||
await ctx.cancel()
|
||||
if we_spawned:
|
||||
await portal.cancel_actor()
|
||||
yield ctx, first
|
||||
|
|
|
|||
|
|
@ -1,96 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Sub-sys module commons.
|
||||
|
||||
"""
|
||||
from collections import ChainMap
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
subsys: str = 'piker.clearing'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
||||
|
||||
# TODO: use this in other backends like kraken which currently has
|
||||
# a less formalized version more or less:
|
||||
# `apiflows[reqid].maps.append(status_msg.to_dict())`
|
||||
class OrderDialogs(Struct):
|
||||
'''
|
||||
Order control dialog (and thus transaction) tracking via
|
||||
message recording.
|
||||
|
||||
Allows easily recording messages associated with a given set of
|
||||
order control transactions and looking up the latest field
|
||||
state using the entire (reverse chronological) msg flow.
|
||||
|
||||
'''
|
||||
_flows: dict[str, ChainMap] = {}
|
||||
|
||||
def add_msg(
|
||||
self,
|
||||
oid: str,
|
||||
msg: dict,
|
||||
) -> None:
|
||||
|
||||
# NOTE: manually enter a new map on the first msg add to
|
||||
# avoid creating one with an empty dict first entry in
|
||||
# `ChainMap.maps` which is the default if none passed at
|
||||
# init.
|
||||
cm: ChainMap = self._flows.get(oid)
|
||||
if cm:
|
||||
cm.maps.insert(0, msg)
|
||||
else:
|
||||
cm = ChainMap(msg)
|
||||
self._flows[oid] = cm
|
||||
|
||||
# TODO: wrap all this in the `collections.abc.Mapping` interface?
|
||||
def get(
|
||||
self,
|
||||
oid: str,
|
||||
|
||||
) -> ChainMap[str, Any]:
|
||||
'''
|
||||
Return the dialog `ChainMap` for provided id.
|
||||
|
||||
'''
|
||||
return self._flows.get(oid, None)
|
||||
|
||||
def pop(
|
||||
self,
|
||||
oid: str,
|
||||
|
||||
) -> ChainMap[str, Any]:
|
||||
'''
|
||||
Pop and thus remove the `ChainMap` containing the msg flow
|
||||
for the given order id.
|
||||
|
||||
'''
|
||||
if (flow := self._flows.pop(oid, None)) is None:
|
||||
log.warning(f'No flow found for oid: {oid}')
|
||||
|
||||
return flow
|
||||
|
|
@ -19,108 +19,75 @@ CLI commons.
|
|||
|
||||
'''
|
||||
import os
|
||||
from contextlib import AsyncExitStack
|
||||
from types import ModuleType
|
||||
from pprint import pformat
|
||||
|
||||
import click
|
||||
import trio
|
||||
import tractor
|
||||
|
||||
from ..log import (
|
||||
get_console_log,
|
||||
get_logger,
|
||||
colorize_json,
|
||||
)
|
||||
from ..log import get_console_log, get_logger, colorize_json
|
||||
from ..brokers import get_brokermod
|
||||
from ..service import (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
from .._daemon import _tractor_kwargs
|
||||
from .. import config
|
||||
|
||||
|
||||
log = get_logger('piker.cli')
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--pdb', is_flag=True, help='Enable tractor debug mode')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.option('--host', '-h', default='127.0.0.1', help='Host address to bind')
|
||||
@click.option(
|
||||
'--tsdb',
|
||||
is_flag=True,
|
||||
help='Enable local ``marketstore`` instance'
|
||||
)
|
||||
@click.option(
|
||||
'--es',
|
||||
is_flag=True,
|
||||
help='Enable local ``elasticsearch`` instance'
|
||||
)
|
||||
def pikerd(
|
||||
loglevel: str,
|
||||
host: str,
|
||||
port: int,
|
||||
tl: bool,
|
||||
pdb: bool,
|
||||
tsdb: bool,
|
||||
es: bool,
|
||||
):
|
||||
def pikerd(loglevel, host, tl, pdb, tsdb):
|
||||
'''
|
||||
Spawn the piker broker-daemon.
|
||||
|
||||
'''
|
||||
log = get_console_log(loglevel, name='cli')
|
||||
from .._daemon import open_pikerd
|
||||
log = get_console_log(loglevel)
|
||||
|
||||
if pdb:
|
||||
log.warning((
|
||||
"\n"
|
||||
"!!! YOU HAVE ENABLED DAEMON DEBUG MODE !!!\n"
|
||||
"When a `piker` daemon crashes it will block the "
|
||||
"task-thread until resumed from console!\n"
|
||||
"!!! You have enabled daemon DEBUG mode !!!\n"
|
||||
"If a daemon crashes it will likely block"
|
||||
" the service until resumed from console!\n"
|
||||
"\n"
|
||||
))
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
|
||||
from .. import service
|
||||
|
||||
async def main():
|
||||
service_mngr: service.Services
|
||||
|
||||
async with (
|
||||
service.open_pikerd(
|
||||
open_pikerd(
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
registry_addr=reg_addr,
|
||||
|
||||
) as service_mngr, # normally delivers a ``Services`` handle
|
||||
|
||||
AsyncExitStack() as stack,
|
||||
), # normally delivers a ``Services`` handle
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
if tsdb:
|
||||
dname, conf = await stack.enter_async_context(
|
||||
service.marketstore.start_ahab_daemon(
|
||||
service_mngr,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
)
|
||||
log.info(f'TSDB `{dname}` up with conf:\n{conf}')
|
||||
from piker.data._ahab import start_ahab
|
||||
from piker.data.marketstore import start_marketstore
|
||||
|
||||
log.info('Spawning `marketstore` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await n.start(
|
||||
start_ahab,
|
||||
'marketstored',
|
||||
start_marketstore,
|
||||
|
||||
if es:
|
||||
dname, conf = await stack.enter_async_context(
|
||||
service.elastic.start_ahab_daemon(
|
||||
service_mngr,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
)
|
||||
log.info(f'DB `{dname}` up with conf:\n{conf}')
|
||||
log.info(
|
||||
f'`marketstore` up!\n'
|
||||
f'`marketstored` pid: {pid}\n'
|
||||
f'docker container id: {cid}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
|
@ -130,50 +97,25 @@ def pikerd(
|
|||
@click.group(context_settings=config._context_defaults)
|
||||
@click.option(
|
||||
'--brokers', '-b',
|
||||
default=None,
|
||||
default=[DEFAULT_BROKER],
|
||||
multiple=True,
|
||||
help='Broker backend to use'
|
||||
)
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--configdir', '-c', help='Configuration directory')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.pass_context
|
||||
def cli(
|
||||
ctx: click.Context,
|
||||
brokers: list[str],
|
||||
loglevel: str,
|
||||
tl: bool,
|
||||
configdir: str,
|
||||
host: str,
|
||||
port: int,
|
||||
|
||||
) -> None:
|
||||
def cli(ctx, brokers, loglevel, tl, configdir):
|
||||
if configdir is not None:
|
||||
assert os.path.isdir(configdir), f"`{configdir}` is not a valid path"
|
||||
config._override_config_dir(configdir)
|
||||
|
||||
# TODO: for typer see
|
||||
# https://typer.tiangolo.com/tutorial/commands/context/
|
||||
ctx.ensure_object(dict)
|
||||
|
||||
if not brokers:
|
||||
# (try to) load all (supposedly) supported data/broker backends
|
||||
from piker.brokers import __brokers__
|
||||
brokers = __brokers__
|
||||
|
||||
brokermods: dict[str, ModuleType] = {
|
||||
broker: get_brokermod(broker) for broker in brokers
|
||||
}
|
||||
assert brokermods
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
if len(brokers) == 1:
|
||||
brokermods = [get_brokermod(brokers[0])]
|
||||
else:
|
||||
brokermods = [get_brokermod(broker) for broker in brokers]
|
||||
|
||||
ctx.obj.update({
|
||||
'brokers': brokers,
|
||||
|
|
@ -183,7 +125,6 @@ def cli(
|
|||
'log': get_console_log(loglevel),
|
||||
'confdir': config._config_dir,
|
||||
'wl_path': config._watchlists_data_path,
|
||||
'registry_addr': reg_addr,
|
||||
})
|
||||
|
||||
# allow enabling same loglevel in ``tractor`` machinery
|
||||
|
|
@ -193,53 +134,38 @@ def cli(
|
|||
|
||||
@cli.command()
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.argument('ports', nargs=-1, required=False)
|
||||
@click.argument('names', nargs=-1, required=False)
|
||||
@click.pass_obj
|
||||
def services(config, tl, ports):
|
||||
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
_default_registry_port,
|
||||
_default_registry_host,
|
||||
)
|
||||
|
||||
host = _default_registry_host
|
||||
if not ports:
|
||||
ports = [_default_registry_port]
|
||||
def services(config, tl, names):
|
||||
|
||||
async def list_services():
|
||||
nonlocal host
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
),
|
||||
tractor.get_arbiter(
|
||||
host=host,
|
||||
port=ports[0]
|
||||
) as portal
|
||||
):
|
||||
|
||||
async with tractor.get_arbiter(
|
||||
*_tractor_kwargs['arbiter_addr']
|
||||
) as portal:
|
||||
registry = await portal.run_from_ns('self', 'get_registry')
|
||||
json_d = {}
|
||||
for key, socket in registry.items():
|
||||
# name, uuid = uid
|
||||
host, port = socket
|
||||
json_d[key] = f'{host}:{port}'
|
||||
click.echo(f"{colorize_json(json_d)}")
|
||||
|
||||
trio.run(list_services)
|
||||
tractor.run(
|
||||
list_services,
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
arbiter_addr=_tractor_kwargs['arbiter_addr'],
|
||||
)
|
||||
|
||||
|
||||
def _load_clis() -> None:
|
||||
from ..service import marketstore # noqa
|
||||
from ..service import elastic # noqa
|
||||
from ..data import marketstore # noqa
|
||||
from ..data import cli # noqa
|
||||
from ..brokers import cli # noqa
|
||||
from ..ui import cli # noqa
|
||||
from ..watchlists import cli # noqa
|
||||
|
||||
# typer implemented
|
||||
from ..storage import cli # noqa
|
||||
from ..accounting import cli # noqa
|
||||
|
||||
|
||||
# load downstream cli modules
|
||||
_load_clis()
|
||||
|
|
|
|||
295
piker/config.py
295
piker/config.py
|
|
@ -15,42 +15,27 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Platform configuration (files) mgmt.
|
||||
Broker configuration mgmt.
|
||||
|
||||
"""
|
||||
import platform
|
||||
import sys
|
||||
import os
|
||||
from os.path import dirname
|
||||
import shutil
|
||||
from typing import (
|
||||
Callable,
|
||||
MutableMapping,
|
||||
)
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from bidict import bidict
|
||||
import tomlkit
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
import toml
|
||||
|
||||
from .log import get_logger
|
||||
|
||||
log = get_logger('broker-config')
|
||||
|
||||
|
||||
# XXX NOTE: taken from ``click`` since apparently they have some
|
||||
# taken from ``click`` since apparently they have some
|
||||
# super weirdness with sigint and sudo..no clue
|
||||
# we're probably going to slowly just modify it to our own version over
|
||||
# time..
|
||||
def get_app_dir(
|
||||
app_name: str,
|
||||
roaming: bool = True,
|
||||
force_posix: bool = False,
|
||||
|
||||
) -> str:
|
||||
def get_app_dir(app_name, roaming=True, force_posix=False):
|
||||
r"""Returns the config folder for the application. The default behavior
|
||||
is to return whatever is most appropriate for the operating system.
|
||||
|
||||
|
|
@ -89,30 +74,7 @@ def get_app_dir(
|
|||
def _posixify(name):
|
||||
return "-".join(name.split()).lower()
|
||||
|
||||
# NOTE: for testing with `pytest` we leverage the `tmp_dir`
|
||||
# fixture to generate (and clean up) a test-request-specific
|
||||
# directory for isolated configuration files such that,
|
||||
# - multiple tests can run (possibly in parallel) without data races
|
||||
# on the config state,
|
||||
# - we don't need to ever worry about leaking configs into the
|
||||
# system thus avoiding needing to manage config cleaup fixtures or
|
||||
# other bothers (since obviously `tmp_dir` cleans up after itself).
|
||||
#
|
||||
# In order to "pass down" the test dir path to all (sub-)actors in
|
||||
# the actor tree we preload the root actor's runtime vars state (an
|
||||
# internal mechanism for inheriting state down an actor tree in
|
||||
# `tractor`) with the testing dir and check for it whenever we
|
||||
# detect `pytest` is being used (which it isn't under normal
|
||||
# operation).
|
||||
if "pytest" in sys.modules:
|
||||
import tractor
|
||||
actor = tractor.current_actor(err_on_no_runtime=False)
|
||||
if actor: # runtime is up
|
||||
rvs = tractor._state._runtime_vars
|
||||
testdirpath = Path(rvs['piker_vars']['piker_test_dir'])
|
||||
assert testdirpath.exists(), 'piker test harness might be borked!?'
|
||||
app_name = str(testdirpath)
|
||||
|
||||
# if WIN:
|
||||
if platform.system() == 'Windows':
|
||||
key = "APPDATA" if roaming else "LOCALAPPDATA"
|
||||
folder = os.environ.get(key)
|
||||
|
|
@ -132,33 +94,28 @@ def get_app_dir(
|
|||
)
|
||||
|
||||
|
||||
_click_config_dir: Path = Path(get_app_dir('piker'))
|
||||
_config_dir: Path = _click_config_dir
|
||||
_parent_user: str = os.environ.get('SUDO_USER')
|
||||
_config_dir = _click_config_dir = get_app_dir('piker')
|
||||
_parent_user = os.environ.get('SUDO_USER')
|
||||
|
||||
if _parent_user:
|
||||
non_root_user_dir = Path(
|
||||
os.path.expanduser(f'~{_parent_user}')
|
||||
non_root_user_dir = os.path.expanduser(
|
||||
f'~{_parent_user}'
|
||||
)
|
||||
root: str = 'root'
|
||||
_ccds: str = str(_click_config_dir) # click config dir string
|
||||
i_tail: int = int(_ccds.rfind(root) + len(root))
|
||||
root = 'root'
|
||||
_config_dir = (
|
||||
non_root_user_dir
|
||||
/
|
||||
Path(_ccds[i_tail+1:]) # +1 to capture trailing '/'
|
||||
non_root_user_dir +
|
||||
_click_config_dir[
|
||||
_click_config_dir.rfind(root) + len(root):
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_conf_names: set[str] = {
|
||||
'conf', # god config
|
||||
'brokers', # sec backend deatz
|
||||
'watchlists', # (user defined) market lists
|
||||
'brokers',
|
||||
'trades',
|
||||
'watchlists',
|
||||
}
|
||||
|
||||
# TODO: probably drop all this super legacy, questrade specific,
|
||||
# config stuff XD ?
|
||||
_watchlists_data_path: Path = _config_dir / Path('watchlists.json')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
_context_defaults = dict(
|
||||
default_map={
|
||||
# Questrade specific quote poll rates
|
||||
|
|
@ -172,14 +129,6 @@ _context_defaults = dict(
|
|||
)
|
||||
|
||||
|
||||
class ConfigurationError(Exception):
|
||||
'Misconfigured settings, likely in a TOML file.'
|
||||
|
||||
|
||||
class NoSignature(ConfigurationError):
|
||||
'No credentials setup for broker backend!'
|
||||
|
||||
|
||||
def _override_config_dir(
|
||||
path: str
|
||||
) -> None:
|
||||
|
|
@ -194,129 +143,75 @@ def _conf_fn_w_ext(
|
|||
return f'{name}.toml'
|
||||
|
||||
|
||||
def get_conf_dir() -> Path:
|
||||
'''
|
||||
Return the user configuration directory ``Path``
|
||||
on the local filesystem.
|
||||
|
||||
'''
|
||||
return _config_dir
|
||||
|
||||
|
||||
def get_conf_path(
|
||||
conf_name: str = 'brokers',
|
||||
|
||||
) -> Path:
|
||||
'''
|
||||
Return the top-level default config path normally under
|
||||
``~/.config/piker`` on linux for a given ``conf_name``, the config
|
||||
name.
|
||||
) -> str:
|
||||
"""Return the default config path normally under
|
||||
``~/.config/piker`` on linux.
|
||||
|
||||
Contains files such as:
|
||||
- brokers.toml
|
||||
- watchlists.toml
|
||||
- trades.toml
|
||||
|
||||
# maybe coming soon ;)
|
||||
- signals.toml
|
||||
- strats.toml
|
||||
|
||||
'''
|
||||
if 'account.' not in conf_name:
|
||||
assert str(conf_name) in _conf_names
|
||||
|
||||
"""
|
||||
assert conf_name in _conf_names
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
return _config_dir / Path(fn)
|
||||
return os.path.join(
|
||||
_config_dir,
|
||||
fn,
|
||||
)
|
||||
|
||||
|
||||
def repodir() -> Path:
|
||||
def repodir():
|
||||
'''
|
||||
Return the abspath as ``Path`` to the git repo's root dir.
|
||||
Return the abspath to the repo directory.
|
||||
|
||||
'''
|
||||
repodir: Path = Path(__file__).absolute().parent.parent
|
||||
confdir: Path = repodir / 'config'
|
||||
|
||||
if not confdir.is_dir():
|
||||
# prolly inside stupid GH actions CI..
|
||||
repodir: Path = Path(os.environ.get('GITHUB_WORKSPACE'))
|
||||
confdir: Path = repodir / 'config'
|
||||
|
||||
assert confdir.is_dir(), f'{confdir} DNE, {repodir} is likely incorrect!'
|
||||
return repodir
|
||||
dirpath = os.path.abspath(
|
||||
# we're 3 levels down in **this** module file
|
||||
dirname(dirname(os.path.realpath(__file__)))
|
||||
)
|
||||
return dirpath
|
||||
|
||||
|
||||
def load(
|
||||
conf_name: str = 'brokers', # appended with .toml suffix
|
||||
path: Path | None = None,
|
||||
conf_name: str = 'brokers',
|
||||
path: str = None
|
||||
|
||||
decode: Callable[
|
||||
[str | bytes,],
|
||||
MutableMapping,
|
||||
] = tomllib.loads,
|
||||
|
||||
touch_if_dne: bool = False,
|
||||
|
||||
**tomlkws,
|
||||
|
||||
) -> tuple[dict, Path]:
|
||||
) -> (dict, str):
|
||||
'''
|
||||
Load config file by name.
|
||||
|
||||
If desired config is not in the top level piker-user config path then
|
||||
pass the ``path: Path`` explicitly.
|
||||
|
||||
'''
|
||||
# create the $HOME/.config/piker dir if dne
|
||||
if not _config_dir.is_dir():
|
||||
_config_dir.mkdir(
|
||||
parents=True,
|
||||
exist_ok=True,
|
||||
)
|
||||
|
||||
path_provided: bool = path is not None
|
||||
path: Path = path or get_conf_path(conf_name)
|
||||
|
||||
if (
|
||||
not path.is_file()
|
||||
and touch_if_dne
|
||||
):
|
||||
# only do a template if no path provided,
|
||||
# just touch an empty file with same name.
|
||||
if path_provided:
|
||||
with path.open(mode='x'):
|
||||
pass
|
||||
|
||||
# try to copy in a template config to the user's dir if one
|
||||
# exists.
|
||||
else:
|
||||
fn: str = _conf_fn_w_ext(conf_name)
|
||||
template: Path = repodir() / 'config' / fn
|
||||
if template.is_file():
|
||||
shutil.copyfile(template, path)
|
||||
|
||||
elif fn and template:
|
||||
assert template.is_file(), f'{template} is not a file!?'
|
||||
|
||||
assert path.is_file(), f'Config file {path} not created!?'
|
||||
|
||||
with path.open(mode='r') as fp:
|
||||
config: dict = decode(
|
||||
fp.read(),
|
||||
**tomlkws,
|
||||
path = path or get_conf_path(conf_name)
|
||||
if not os.path.isfile(path):
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
|
||||
template = os.path.join(
|
||||
repodir(),
|
||||
'config',
|
||||
fn
|
||||
)
|
||||
# try to copy in a template config to the user's directory
|
||||
# if one exists.
|
||||
if os.path.isfile(template):
|
||||
shutil.copyfile(template, path)
|
||||
|
||||
config = toml.load(path)
|
||||
log.debug(f"Read config file {path}")
|
||||
return config, path
|
||||
|
||||
|
||||
def write(
|
||||
config: dict, # toml config as dict
|
||||
|
||||
name: str | None = None,
|
||||
path: Path | None = None,
|
||||
fail_empty: bool = True,
|
||||
|
||||
**toml_kwargs,
|
||||
name: str = 'brokers',
|
||||
path: str = None,
|
||||
|
||||
) -> None:
|
||||
''''
|
||||
|
|
@ -325,37 +220,29 @@ def write(
|
|||
Create a ``brokers.ini`` file if one does not exist.
|
||||
|
||||
'''
|
||||
if name:
|
||||
path: Path = path or get_conf_path(name)
|
||||
dirname: Path = path.parent
|
||||
if not dirname.is_dir():
|
||||
log.debug(f"Creating config dir {_config_dir}")
|
||||
dirname.mkdir()
|
||||
path = path or get_conf_path(name)
|
||||
dirname = os.path.dirname(path)
|
||||
if not os.path.isdir(dirname):
|
||||
log.debug(f"Creating config dir {_config_dir}")
|
||||
os.makedirs(dirname)
|
||||
|
||||
if (
|
||||
not config
|
||||
and fail_empty
|
||||
):
|
||||
if not config:
|
||||
raise ValueError(
|
||||
"Watch out you're trying to write a blank config!"
|
||||
)
|
||||
"Watch out you're trying to write a blank config!")
|
||||
|
||||
log.debug(
|
||||
f"Writing config `{name}` file to:\n"
|
||||
f"{path}"
|
||||
)
|
||||
with path.open(mode='w') as fp:
|
||||
return tomlkit.dump( # preserve style on write B)
|
||||
config,
|
||||
fp,
|
||||
**toml_kwargs,
|
||||
)
|
||||
with open(path, 'w') as cf:
|
||||
return toml.dump(config, cf)
|
||||
|
||||
|
||||
def load_accounts(
|
||||
providers: list[str] | None = None
|
||||
|
||||
) -> bidict[str, str | None]:
|
||||
providers: Optional[list[str]] = None
|
||||
|
||||
) -> bidict[str, Optional[str]]:
|
||||
|
||||
conf, path = load()
|
||||
accounts = bidict()
|
||||
|
|
@ -378,51 +265,3 @@ def load_accounts(
|
|||
accounts['paper'] = None
|
||||
|
||||
return accounts
|
||||
|
||||
|
||||
# XXX: Recursive getting & setting
|
||||
|
||||
def get_value(_dict, _section):
|
||||
subs = _section.split('.')
|
||||
if len(subs) > 1:
|
||||
return get_value(
|
||||
_dict[subs[0]],
|
||||
'.'.join(subs[1:]),
|
||||
)
|
||||
|
||||
else:
|
||||
return _dict[_section]
|
||||
|
||||
|
||||
def set_value(_dict, _section, val):
|
||||
subs = _section.split('.')
|
||||
if len(subs) > 1:
|
||||
if subs[0] not in _dict:
|
||||
_dict[subs[0]] = {}
|
||||
|
||||
return set_value(
|
||||
_dict[subs[0]],
|
||||
'.'.join(subs[1:]),
|
||||
val
|
||||
)
|
||||
|
||||
else:
|
||||
_dict[_section] = val
|
||||
|
||||
|
||||
def del_value(_dict, _section):
|
||||
subs = _section.split('.')
|
||||
if len(subs) > 1:
|
||||
if subs[0] not in _dict:
|
||||
return
|
||||
|
||||
return del_value(
|
||||
_dict[subs[0]],
|
||||
'.'.join(subs[1:])
|
||||
)
|
||||
|
||||
else:
|
||||
if _section not in _dict:
|
||||
return
|
||||
|
||||
del _dict[_section]
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ and storing data from your brokers as well as
|
|||
sharing live streams over a network.
|
||||
|
||||
"""
|
||||
from .ticktools import iterticks
|
||||
from ._normalize import iterticks
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
attach_shm_array,
|
||||
|
|
@ -30,20 +30,13 @@ from ._sharedmem import (
|
|||
get_shm_token,
|
||||
ShmArray,
|
||||
)
|
||||
from ._source import (
|
||||
def_iohlcv_fields,
|
||||
def_ohlcv_fields,
|
||||
)
|
||||
from .feed import (
|
||||
Feed,
|
||||
open_feed,
|
||||
_setup_persistent_brokerd,
|
||||
)
|
||||
from .flows import Flume
|
||||
|
||||
|
||||
__all__ = [
|
||||
'Flume',
|
||||
'Feed',
|
||||
'open_feed',
|
||||
'ShmArray',
|
||||
'iterticks',
|
||||
|
|
@ -51,6 +44,5 @@ __all__ = [
|
|||
'attach_shm_array',
|
||||
'open_shm_array',
|
||||
'get_shm_token',
|
||||
'def_iohlcv_fields',
|
||||
'def_ohlcv_fields',
|
||||
'_setup_persistent_brokerd',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,385 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Supervisor for docker with included specific-image service helpers.
|
||||
|
||||
'''
|
||||
import os
|
||||
import time
|
||||
from typing import (
|
||||
Optional,
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
from tractor.msg import NamespacePath
|
||||
import docker
|
||||
import json
|
||||
from docker.models.containers import Container as DockerContainer
|
||||
from docker.errors import (
|
||||
DockerException,
|
||||
APIError,
|
||||
)
|
||||
from requests.exceptions import ConnectionError, ReadTimeout
|
||||
|
||||
from ..log import get_logger, get_console_log
|
||||
from .. import config
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DockerNotStarted(Exception):
|
||||
'Prolly you dint start da daemon bruh'
|
||||
|
||||
|
||||
class ContainerError(RuntimeError):
|
||||
'Error reported via app-container logging level'
|
||||
|
||||
|
||||
@acm
|
||||
async def open_docker(
|
||||
url: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> docker.DockerClient:
|
||||
|
||||
client: Optional[docker.DockerClient] = None
|
||||
try:
|
||||
client = docker.DockerClient(
|
||||
base_url=url,
|
||||
**kwargs
|
||||
) if url else docker.from_env(**kwargs)
|
||||
|
||||
yield client
|
||||
|
||||
except (
|
||||
DockerException,
|
||||
APIError,
|
||||
) as err:
|
||||
|
||||
def unpack_msg(err: Exception) -> str:
|
||||
args = getattr(err, 'args', None)
|
||||
if args:
|
||||
return args
|
||||
else:
|
||||
return str(err)
|
||||
|
||||
# could be more specific so let's check if it's just perms.
|
||||
if err.args:
|
||||
errs = err.args
|
||||
for err in errs:
|
||||
msg = unpack_msg(err)
|
||||
if 'PermissionError' in msg:
|
||||
raise DockerException('You dint run as root yo!')
|
||||
|
||||
elif 'FileNotFoundError' in msg:
|
||||
raise DockerNotStarted('Did you start da service sister?')
|
||||
|
||||
# not perms?
|
||||
raise
|
||||
|
||||
finally:
|
||||
if client:
|
||||
client.close()
|
||||
|
||||
|
||||
class Container:
|
||||
'''
|
||||
Wrapper around a ``docker.models.containers.Container`` to include
|
||||
log capture and relay through our native logging system and helper
|
||||
method(s) for cancellation/teardown.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
cntr: DockerContainer,
|
||||
) -> None:
|
||||
|
||||
self.cntr = cntr
|
||||
# log msg de-duplication
|
||||
self.seen_so_far = set()
|
||||
|
||||
async def process_logs_until(
|
||||
self,
|
||||
patt: str,
|
||||
bp_on_msg: bool = False,
|
||||
) -> bool:
|
||||
'''
|
||||
Attempt to capture container log messages and relay through our
|
||||
native logging system.
|
||||
|
||||
'''
|
||||
seen_so_far = self.seen_so_far
|
||||
|
||||
while True:
|
||||
logs = self.cntr.logs()
|
||||
entries = logs.decode().split('\n')
|
||||
for entry in entries:
|
||||
|
||||
# ignore null lines
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
try:
|
||||
record = json.loads(entry.strip())
|
||||
except json.JSONDecodeError:
|
||||
if 'Error' in entry:
|
||||
raise RuntimeError(entry)
|
||||
raise
|
||||
|
||||
msg = record['msg']
|
||||
level = record['level']
|
||||
if msg and entry not in seen_so_far:
|
||||
seen_so_far.add(entry)
|
||||
if bp_on_msg:
|
||||
await tractor.breakpoint()
|
||||
|
||||
getattr(log, level, log.error)(f'{msg}')
|
||||
|
||||
# print(f'level: {level}')
|
||||
if level in ('error', 'fatal'):
|
||||
raise ContainerError(msg)
|
||||
|
||||
if patt in msg:
|
||||
return True
|
||||
|
||||
# do a checkpoint so we don't block if cancelled B)
|
||||
await trio.sleep(0.01)
|
||||
|
||||
return False
|
||||
|
||||
def try_signal(
|
||||
self,
|
||||
signal: str = 'SIGINT',
|
||||
|
||||
) -> bool:
|
||||
try:
|
||||
# XXX: market store doesn't seem to shutdown nicely all the
|
||||
# time with this (maybe because there are still open grpc
|
||||
# connections?) noticably after client connections have been
|
||||
# made or are in use/teardown. It works just fine if you
|
||||
# just start and stop the container tho?..
|
||||
log.cancel(f'SENDING {signal} to {self.cntr.id}')
|
||||
self.cntr.kill(signal)
|
||||
return True
|
||||
|
||||
except docker.errors.APIError as err:
|
||||
if 'is not running' in err.explanation:
|
||||
return False
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
stop_msg: str,
|
||||
) -> None:
|
||||
|
||||
cid = self.cntr.id
|
||||
# first try a graceful cancel
|
||||
log.cancel(
|
||||
f'SIGINT cancelling container: {cid}\n'
|
||||
f'waiting on stop msg: "{stop_msg}"'
|
||||
)
|
||||
self.try_signal('SIGINT')
|
||||
|
||||
start = time.time()
|
||||
for _ in range(30):
|
||||
|
||||
with trio.move_on_after(0.5) as cs:
|
||||
cs.shield = True
|
||||
await self.process_logs_until(stop_msg)
|
||||
|
||||
# if we aren't cancelled on above checkpoint then we
|
||||
# assume we read the expected stop msg and terminated.
|
||||
break
|
||||
|
||||
try:
|
||||
log.info(f'Polling for container shutdown:\n{cid}')
|
||||
|
||||
if self.cntr.status not in {'exited', 'not-running'}:
|
||||
self.cntr.wait(
|
||||
timeout=0.1,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
break
|
||||
|
||||
except (
|
||||
ReadTimeout,
|
||||
):
|
||||
log.info(f'Still waiting on container:\n{cid}')
|
||||
continue
|
||||
|
||||
except (
|
||||
docker.errors.APIError,
|
||||
ConnectionError,
|
||||
):
|
||||
log.exception('Docker connection failure')
|
||||
break
|
||||
else:
|
||||
delay = time.time() - start
|
||||
log.error(
|
||||
f'Failed to kill container {cid} after {delay}s\n'
|
||||
'sending SIGKILL..'
|
||||
)
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
log.cancel(f'Container stopped: {cid}')
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_ahabd(
|
||||
ctx: tractor.Context,
|
||||
endpoint: str, # ns-pointer str-msg-type
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
get_console_log('info', name=__name__)
|
||||
|
||||
async with open_docker() as client:
|
||||
|
||||
# TODO: eventually offer a config-oriented API to do the mounts,
|
||||
# params, etc. passing to ``Containter.run()``?
|
||||
# call into endpoint for container config/init
|
||||
ep_func = NamespacePath(endpoint).load_ref()
|
||||
(
|
||||
dcntr,
|
||||
cntr_config,
|
||||
start_msg,
|
||||
stop_msg,
|
||||
) = ep_func(client)
|
||||
cntr = Container(dcntr)
|
||||
|
||||
with trio.move_on_after(1):
|
||||
found = await cntr.process_logs_until(start_msg)
|
||||
|
||||
if not found and cntr not in client.containers.list():
|
||||
raise RuntimeError(
|
||||
'Failed to start `marketstore` check logs deats'
|
||||
)
|
||||
|
||||
await ctx.started((
|
||||
cntr.cntr.id,
|
||||
os.getpid(),
|
||||
cntr_config,
|
||||
))
|
||||
|
||||
try:
|
||||
|
||||
# TODO: we might eventually want a proxy-style msg-prot here
|
||||
# to allow remote control of containers without needing
|
||||
# callers to have root perms?
|
||||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
with trio.CancelScope(shield=True):
|
||||
await cntr.cancel(stop_msg)
|
||||
|
||||
|
||||
async def start_ahab(
|
||||
service_name: str,
|
||||
endpoint: Callable[docker.DockerClient, DockerContainer],
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.Event,
|
||||
dict[str, Any],
|
||||
],
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Start a ``docker`` container supervisor with given service name.
|
||||
|
||||
Currently the actor calling this task should normally be started
|
||||
with root permissions (until we decide to use something that doesn't
|
||||
require this, like docker's rootless mode or some wrapper project) but
|
||||
te root perms are de-escalated after the docker supervisor sub-actor
|
||||
is started.
|
||||
|
||||
'''
|
||||
cn_ready = trio.Event()
|
||||
try:
|
||||
async with tractor.open_nursery(
|
||||
loglevel='runtime',
|
||||
) as tn:
|
||||
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
|
||||
# TODO: we have issues with this on teardown
|
||||
# where ``tractor`` tries to issue ``os.kill()``
|
||||
# and hits perms errors since the root process
|
||||
# doesn't any longer have root perms..
|
||||
|
||||
# de-escalate root perms to the original user
|
||||
# after the docker supervisor actor is spawned.
|
||||
if config._parent_user:
|
||||
import pwd
|
||||
os.setuid(
|
||||
pwd.getpwnam(
|
||||
config._parent_user
|
||||
)[2] # named user's uid
|
||||
)
|
||||
|
||||
async with portal.open_context(
|
||||
open_ahabd,
|
||||
endpoint=str(NamespacePath.from_ref(endpoint)),
|
||||
) as (ctx, first):
|
||||
|
||||
cid, pid, cntr_config = first
|
||||
|
||||
task_status.started((
|
||||
cn_ready,
|
||||
cntr_config,
|
||||
(cid, pid),
|
||||
))
|
||||
|
||||
await trio.sleep_forever()
|
||||
|
||||
# since we demoted root perms in this parent
|
||||
# we'll get a perms error on proc cleanup in
|
||||
# ``tractor`` nursery exit. just make sure
|
||||
# the child is terminated and don't raise the
|
||||
# error if so.
|
||||
|
||||
# TODO: we could also consider adding
|
||||
# a ``tractor.ZombieDetected`` or something that we could raise
|
||||
# if we find the child didn't terminate.
|
||||
except PermissionError:
|
||||
log.warning('Failed to cancel root permsed container')
|
||||
|
||||
except (
|
||||
trio.MultiError,
|
||||
) as err:
|
||||
for subexc in err.exceptions:
|
||||
if isinstance(subexc, PermissionError):
|
||||
log.warning('Failed to cancel root perms-ed container')
|
||||
return
|
||||
else:
|
||||
raise
|
||||
|
|
@ -1,831 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Pre-(path)-graphics formatted x/y nd/1d rendering subsystem.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
from msgspec import field
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ._sharedmem import (
|
||||
ShmArray,
|
||||
)
|
||||
from ._pathops import (
|
||||
path_arrays_from_ohlc,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._dataviz import (
|
||||
Viz,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
|
||||
class IncrementalFormatter(msgspec.Struct):
|
||||
'''
|
||||
Incrementally updating, pre-path-graphics tracking, formatter.
|
||||
|
||||
Allows tracking source data state in an updateable pre-graphics
|
||||
``np.ndarray`` format (in local process memory) as well as
|
||||
incrementally rendering from that format **to** 1d x/y for path
|
||||
generation using ``pg.functions.arrayToQPath()``.
|
||||
|
||||
'''
|
||||
shm: ShmArray
|
||||
viz: Viz
|
||||
|
||||
# the value to be multiplied any any index into the x/y_1d arrays
|
||||
# given the input index is based on the original source data array.
|
||||
flat_index_ratio: float = 1
|
||||
|
||||
@property
|
||||
def index_field(self) -> 'str':
|
||||
'''
|
||||
Value (``str``) used to look up the "index series" from the
|
||||
underlying source ``numpy`` struct-array; delegate directly to
|
||||
the managing ``Viz``.
|
||||
|
||||
'''
|
||||
return self.viz.index_field
|
||||
|
||||
# Incrementally updated xy ndarray formatted data, a pre-1d
|
||||
# format which is updated and cached independently of the final
|
||||
# pre-graphics-path 1d format.
|
||||
x_nd: Optional[np.ndarray] = None
|
||||
y_nd: Optional[np.ndarray] = None
|
||||
|
||||
@property
|
||||
def xy_nd(self) -> tuple[np.ndarray, np.ndarray]:
|
||||
return (
|
||||
self.x_nd[self.xy_slice],
|
||||
self.y_nd[self.xy_slice],
|
||||
)
|
||||
|
||||
@property
|
||||
def xy_slice(self) -> slice:
|
||||
return slice(
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
)
|
||||
|
||||
# indexes which slice into the above arrays (which are allocated
|
||||
# based on source data shm input size) and allow retrieving
|
||||
# incrementally updated data.
|
||||
xy_nd_start: int | None = None
|
||||
xy_nd_stop: int | None = None
|
||||
|
||||
# TODO: eventually incrementally update 1d-pre-graphics path data?
|
||||
x_1d: np.ndarray | None = None
|
||||
y_1d: np.ndarray | None = None
|
||||
|
||||
# incremental view-change state(s) tracking
|
||||
_last_vr: tuple[float, float] | None = None
|
||||
_last_ivdr: tuple[float, float] | None = None
|
||||
|
||||
@property
|
||||
def index_step_size(self) -> float:
|
||||
'''
|
||||
Readonly value computed on first ``.diff()`` call.
|
||||
|
||||
'''
|
||||
return self.viz.index_step()
|
||||
|
||||
def diff(
|
||||
self,
|
||||
new_read: tuple[np.ndarray],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
# TODO:
|
||||
# - can the renderer just call ``Viz.read()`` directly? unpack
|
||||
# latest source data read
|
||||
# - eventually maybe we can implement some kind of
|
||||
# transform on the ``QPainterPath`` that will more or less
|
||||
# detect the diff in "elements" terms? update diff state since
|
||||
# we've now rendered paths.
|
||||
(
|
||||
xfirst,
|
||||
xlast,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
) = new_read
|
||||
|
||||
index = array['index']
|
||||
|
||||
# if the first index in the read array is 0 then
|
||||
# it means the source buffer has bee completely backfilled to
|
||||
# available space.
|
||||
src_start = index[0]
|
||||
src_stop = index[-1] + 1
|
||||
|
||||
# these are the "formatted output data" indices
|
||||
# for the pre-graphics arrays.
|
||||
nd_start = self.xy_nd_start
|
||||
nd_stop = self.xy_nd_stop
|
||||
|
||||
if (
|
||||
nd_start is None
|
||||
):
|
||||
assert nd_stop is None
|
||||
|
||||
# setup to do a prepend of all existing src history
|
||||
nd_start = self.xy_nd_start = src_stop
|
||||
# set us in a zero-to-append state
|
||||
nd_stop = self.xy_nd_stop = src_stop
|
||||
|
||||
# compute the length diffs between the first/last index entry in
|
||||
# the input data and the last indexes we have on record from the
|
||||
# last time we updated the curve index.
|
||||
prepend_length = int(nd_start - src_start)
|
||||
append_length = int(src_stop - nd_stop)
|
||||
|
||||
# blah blah blah
|
||||
# do diffing for prepend, append and last entry
|
||||
return (
|
||||
slice(src_start, nd_start),
|
||||
prepend_length,
|
||||
append_length,
|
||||
slice(nd_stop, src_stop),
|
||||
)
|
||||
|
||||
def _track_inview_range(
|
||||
self,
|
||||
view_range: tuple[int, int],
|
||||
|
||||
) -> bool:
|
||||
# if a view range is passed, plan to draw the
|
||||
# source ouput that's "in view" of the chart.
|
||||
vl, vr = view_range
|
||||
zoom_or_append = False
|
||||
last_vr = self._last_vr
|
||||
|
||||
# incremental in-view data update.
|
||||
if last_vr:
|
||||
lvl, lvr = last_vr # relative slice indices
|
||||
|
||||
# TODO: detecting more specifically the interaction changes
|
||||
# last_ivr = self._last_ivdr or (vl, vr)
|
||||
# al, ar = last_ivr # abs slice indices
|
||||
# left_change = abs(x_iv[0] - al) >= 1
|
||||
# right_change = abs(x_iv[-1] - ar) >= 1
|
||||
|
||||
# likely a zoom/pan view change or data append update
|
||||
if (
|
||||
(vr - lvr) > 2
|
||||
or vl < lvl
|
||||
|
||||
# append / prepend update
|
||||
# we had an append update where the view range
|
||||
# didn't change but the data-viewed (shifted)
|
||||
# underneath, so we need to redraw.
|
||||
# or left_change and right_change and last_vr == view_range
|
||||
|
||||
# not (left_change and right_change) and ivr
|
||||
# (
|
||||
# or abs(x_iv[ivr] - livr) > 1
|
||||
):
|
||||
zoom_or_append = True
|
||||
|
||||
self._last_vr = view_range
|
||||
|
||||
return zoom_or_append
|
||||
|
||||
def format_to_1d(
|
||||
self,
|
||||
new_read: tuple,
|
||||
array_key: str,
|
||||
profiler: Profiler,
|
||||
|
||||
slice_to_inview: bool = True,
|
||||
force_full_realloc: bool = False,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
shm = self.shm
|
||||
|
||||
(
|
||||
_,
|
||||
_,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
|
||||
) = new_read
|
||||
|
||||
(
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
append_len,
|
||||
post_slice,
|
||||
) = self.diff(new_read)
|
||||
|
||||
# we first need to allocate xy data arrays
|
||||
# from the source data.
|
||||
if (
|
||||
self.y_nd is None
|
||||
or force_full_realloc
|
||||
):
|
||||
self.xy_nd_start = shm._first.value
|
||||
self.xy_nd_stop = shm._last.value
|
||||
self.x_nd, self.y_nd = self.allocate_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
)
|
||||
profiler('allocated xy history')
|
||||
|
||||
# once allocated we do incremental pre/append
|
||||
# updates from the diff with the source buffer.
|
||||
else:
|
||||
if prepend_len:
|
||||
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
# this is the pre-sliced, "normally expected"
|
||||
# new data that an updater would normally be
|
||||
# expected to process, however in some cases (like
|
||||
# step curves) the updater routine may want to do
|
||||
# the source history-data reading itself, so we pass
|
||||
# both here.
|
||||
shm._array[pre_slice],
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=False,
|
||||
)
|
||||
|
||||
self.xy_nd_start -= prepend_len
|
||||
profiler('prepended xy history: {prepend_length}')
|
||||
|
||||
if append_len:
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
shm._array[post_slice],
|
||||
post_slice,
|
||||
append_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=True,
|
||||
)
|
||||
self.xy_nd_stop += append_len
|
||||
profiler('appened xy history: {append_length}')
|
||||
# sanity
|
||||
# slice_ln = post_slice.stop - post_slice.start
|
||||
# assert append_len == slice_ln
|
||||
|
||||
view_changed: bool = False
|
||||
view_range: tuple[int, int] = (ivl, ivr)
|
||||
if slice_to_inview:
|
||||
view_changed = self._track_inview_range(view_range)
|
||||
array = in_view
|
||||
profiler(f'{self.viz.name} view range slice {view_range}')
|
||||
|
||||
# TODO: we need to check if the last-datum-in-view is true and
|
||||
# if so only slice to the 2nd last datumonly slice to the 2nd
|
||||
# last datum.
|
||||
# hist = array[:slice_to_head]
|
||||
|
||||
# XXX: WOA WTF TRACTOR DEBUGGING BUGGG
|
||||
# assert 0
|
||||
|
||||
# xy-path data transform: convert source data to a format
|
||||
# able to be passed to a `QPainterPath` rendering routine.
|
||||
if not len(array):
|
||||
# XXX: this might be why the profiler only has exits?
|
||||
return
|
||||
|
||||
# TODO: hist here should be the pre-sliced
|
||||
# x/y_data in the case where allocate_xy is
|
||||
# defined?
|
||||
x_1d, y_1d, connect = self.format_xy_nd_to_1d(
|
||||
array,
|
||||
array_key,
|
||||
view_range,
|
||||
)
|
||||
# cache/save last 1d outputs for use by other
|
||||
# readers (eg. `Viz.draw_last_datum()` in the
|
||||
# only-draw-last-uppx case).
|
||||
self.x_1d = x_1d
|
||||
self.y_1d = y_1d
|
||||
|
||||
# app_tres = None
|
||||
# if append_len:
|
||||
# appended = array[-append_len-1:slice_to_head]
|
||||
# app_tres = self.format_xy_nd_to_1d(
|
||||
# appended,
|
||||
# array_key,
|
||||
# (
|
||||
# view_range[1] - append_len + slice_to_head,
|
||||
# view_range[1]
|
||||
# ),
|
||||
# )
|
||||
# # assert (len(appended) - 1) == append_len
|
||||
# # assert len(appended) == append_len
|
||||
# print(
|
||||
# f'{self.viz.name} APPEND LEN: {append_len}\n'
|
||||
# f'{self.viz.name} APPENDED: {appended}\n'
|
||||
# f'{self.viz.name} app_tres: {app_tres}\n'
|
||||
# )
|
||||
|
||||
# update the last "in view data range"
|
||||
if len(x_1d):
|
||||
self._last_ivdr = x_1d[0], x_1d[-1]
|
||||
|
||||
profiler('.format_to_1d()')
|
||||
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
connect,
|
||||
prepend_len,
|
||||
append_len,
|
||||
view_changed,
|
||||
# app_tres,
|
||||
)
|
||||
|
||||
###############################
|
||||
# Sub-type override interface #
|
||||
###############################
|
||||
|
||||
x_offset: np.ndarray = np.array([0])
|
||||
|
||||
# optional pre-graphics xy formatted data which
|
||||
# is incrementally updated in sync with the source data.
|
||||
# XXX: was ``.allocate_xy()``
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert the structured-array ``src_shm`` format to
|
||||
a equivalently shaped (and field-less) ``np.ndarray``.
|
||||
|
||||
Eg. a 4 field x N struct-array => (N, 4)
|
||||
|
||||
'''
|
||||
y_nd = src_shm._array[data_field].copy()
|
||||
x_nd = (
|
||||
src_shm._array[self.index_field].copy()
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
return x_nd, y_nd
|
||||
|
||||
# XXX: was ``.update_xy()``
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write pushed data to flattened copy
|
||||
y_nd_new = new_from_src[data_field]
|
||||
self.y_nd[read_slc] = y_nd_new
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name.upper()}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{list(x_nd[-s:])}\n'
|
||||
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{list(y_nd[-s:])}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new[0]}\n'
|
||||
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
# XXX: was ``.format_xy()``
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # 1d x
|
||||
np.ndarray, # 1d y
|
||||
np.ndarray | str, # connection array/style
|
||||
]:
|
||||
'''
|
||||
Default xy-nd array to 1d pre-graphics-path render routine.
|
||||
|
||||
Return single field column data verbatim
|
||||
|
||||
'''
|
||||
# NOTE: we don't include the very last datum which is filled in
|
||||
# normally by another graphics object.
|
||||
x_1d = array[self.index_field][:-1]
|
||||
y_1d = array[array_key][:-1]
|
||||
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# x_nd = list(self.x_nd[self.xy_slice][-s:-1])
|
||||
# y_nd = list(self.y_nd[self.xy_slice][-s:-1])
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# f'XY data:\n'
|
||||
# f'x: {x_nd}\n'
|
||||
# f'y: {y_nd}\n\n'
|
||||
# f'x_1d: {list(x_1d[-s:])}\n'
|
||||
# f'y_1d: {list(y_1d[-s:])}\n\n'
|
||||
|
||||
# )
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
|
||||
# 1d connection array or style-key to
|
||||
# ``pg.functions.arrayToQPath()``
|
||||
'all',
|
||||
)
|
||||
|
||||
|
||||
class OHLCBarsFmtr(IncrementalFormatter):
|
||||
x_offset: np.ndarray = np.array([
|
||||
-0.5,
|
||||
0,
|
||||
0,
|
||||
0.5,
|
||||
])
|
||||
|
||||
fields: list[str] = field(
|
||||
default_factory=lambda: ['open', 'high', 'low', 'close']
|
||||
)
|
||||
flat_index_ratio: float = 4
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
ohlc_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input struct-array holding OHLC samples into a pair of
|
||||
flattened x, y arrays with the same size (datums wise) as the source
|
||||
data.
|
||||
|
||||
'''
|
||||
y_nd = ohlc_shm.ustruct(self.fields)
|
||||
|
||||
# generate an flat-interpolated x-domain
|
||||
x_nd = (
|
||||
np.broadcast_to(
|
||||
ohlc_shm._array[self.index_field][:, None],
|
||||
(
|
||||
ohlc_shm._array.size,
|
||||
# 4, # only ohlc
|
||||
y_nd.shape[1],
|
||||
),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
assert y_nd.any()
|
||||
|
||||
# write pushed data to flattened copy
|
||||
return (
|
||||
x_nd,
|
||||
y_nd,
|
||||
)
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write newly pushed data to flattened copy
|
||||
# a struct-arr is always passed in.
|
||||
new_y_nd = rfn.structured_to_unstructured(
|
||||
new_from_src[self.fields]
|
||||
)
|
||||
self.y_nd[read_slc] = new_y_nd
|
||||
|
||||
# generate same-valued-per-row x support based on y shape
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = np.broadcast_to(
|
||||
new_from_src[self.index_field][:, None],
|
||||
new_y_nd.shape,
|
||||
) + self.x_offset
|
||||
|
||||
# TODO: can we drop this frame and just use the above?
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
start: int = 0, # XXX: do we need this?
|
||||
# 0.5 is no overlap between arms, 1.0 is full overlap
|
||||
w: float = 0.16,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
More or less direct proxy to the ``numba``-fied
|
||||
``path_arrays_from_ohlc()`` (above) but with closed in kwargs
|
||||
for line spacing.
|
||||
|
||||
'''
|
||||
x, y, c = path_arrays_from_ohlc(
|
||||
array[:-1],
|
||||
start,
|
||||
bar_w=self.index_step_size,
|
||||
bar_gap=w * self.index_step_size,
|
||||
|
||||
# XXX: don't ask, due to a ``numba`` bug..
|
||||
use_time_index=(self.index_field == 'time'),
|
||||
)
|
||||
return x, y, c
|
||||
|
||||
|
||||
class OHLCBarsAsCurveFmtr(OHLCBarsFmtr):
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
# TODO: in the case of an existing ``.update_xy()``
|
||||
# should we be passing in array as an xy arrays tuple?
|
||||
|
||||
# 2 more datum-indexes to capture zero at end
|
||||
x_flat = self.x_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
y_flat = self.y_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
|
||||
# slice to view
|
||||
ivl, ivr = vr
|
||||
x_iv_flat = x_flat[ivl:ivr]
|
||||
y_iv_flat = y_flat[ivl:ivr]
|
||||
|
||||
# reshape to 1d for graphics rendering
|
||||
y_iv = y_iv_flat.reshape(-1)
|
||||
x_iv = x_iv_flat.reshape(-1)
|
||||
|
||||
return x_iv, y_iv, 'all'
|
||||
|
||||
|
||||
class StepCurveFmtr(IncrementalFormatter):
|
||||
|
||||
x_offset: np.ndarray = np.array([
|
||||
0,
|
||||
1,
|
||||
])
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input 1d shm array to a "step array" format
|
||||
for use by path graphics generation.
|
||||
|
||||
'''
|
||||
i = shm._array[self.index_field].copy()
|
||||
out = shm._array[data_field].copy()
|
||||
|
||||
x_out = (
|
||||
np.broadcast_to(
|
||||
i[:, None],
|
||||
(i.size, 2),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# fill out Nx2 array to hold each step's left + right vertices.
|
||||
y_out = np.empty(
|
||||
x_out.shape,
|
||||
dtype=out.dtype,
|
||||
)
|
||||
# fill in (current) values from source shm buffer
|
||||
y_out[:] = out[:, np.newaxis]
|
||||
|
||||
# TODO: pretty sure we can drop this?
|
||||
# start y at origin level
|
||||
# y_out[0, 0] = 0
|
||||
# y_out[self.xy_nd_start] = 0
|
||||
return x_out, y_out
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
array_key: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
slice,
|
||||
]:
|
||||
# NOTE: for a step curve we slice from one datum prior
|
||||
# to the current "update slice" to get the previous
|
||||
# "level".
|
||||
#
|
||||
# why this is needed,
|
||||
# - the current new append slice will often have a zero
|
||||
# value in the latest datum-step (at least for zero-on-new
|
||||
# cases like vlm in the) as per configuration of the FSP
|
||||
# engine.
|
||||
# - we need to look back a datum to get the last level which
|
||||
# will be used to terminate/complete the last step x-width
|
||||
# which will be set to pair with the last x-index THIS MEANS
|
||||
#
|
||||
# XXX: this means WE CAN'T USE the append slice since we need to
|
||||
# "look backward" one step to get the needed back-to-zero level
|
||||
# and the update data in ``new_from_src`` will only contain the
|
||||
# latest new data.
|
||||
back_1 = slice(
|
||||
read_slc.start - 1,
|
||||
read_slc.stop,
|
||||
)
|
||||
|
||||
to_write = src_shm._array[back_1]
|
||||
y_nd_new = self.y_nd[back_1]
|
||||
y_nd_new[:] = to_write[array_key][:, None]
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field][:, None]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# XXX: uncomment for debugging
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'dolla_vlm' in name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{x_nd[-s:]}\n'
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{y_nd[-s:]}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new}\n'
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
last_t, last = array[-1][[self.index_field, array_key]]
|
||||
|
||||
start = self.xy_nd_start
|
||||
stop = self.xy_nd_stop
|
||||
|
||||
x_step = self.x_nd[start:stop]
|
||||
y_step = self.y_nd[start:stop]
|
||||
|
||||
# slice out in-view data
|
||||
ivl, ivr = vr
|
||||
|
||||
# NOTE: add an extra step to get the vertical-line-down-to-zero
|
||||
# adjacent to the last-datum graphic (filled rect).
|
||||
x_step_iv = x_step[ivl:ivr+1]
|
||||
y_step_iv = y_step[ivl:ivr+1]
|
||||
|
||||
# flatten to 1d
|
||||
x_1d = x_step_iv.reshape(x_step_iv.size)
|
||||
y_1d = y_step_iv.reshape(y_step_iv.size)
|
||||
|
||||
# debugging
|
||||
# if y_1d.any():
|
||||
# s = 6
|
||||
# print(
|
||||
# f'x_step_iv:\n{x_step_iv[-s:]}\n'
|
||||
# f'y_step_iv:\n{y_step_iv[-s:]}\n\n'
|
||||
# f'x_1d:\n{x_1d[-s:]}\n'
|
||||
# f'y_1d:\n{y_1d[-s:]}\n'
|
||||
# )
|
||||
|
||||
return x_1d, y_1d, 'all'
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Stream format enforcement.
|
||||
|
||||
'''
|
||||
from itertools import chain
|
||||
from typing import AsyncIterator
|
||||
|
||||
|
||||
def iterticks(
|
||||
quote: dict,
|
||||
types: tuple[str] = (
|
||||
'trade',
|
||||
'dark_trade',
|
||||
),
|
||||
deduplicate_darks: bool = False,
|
||||
|
||||
) -> AsyncIterator:
|
||||
'''
|
||||
Iterate through ticks delivered per quote cycle.
|
||||
|
||||
'''
|
||||
if deduplicate_darks:
|
||||
assert 'dark_trade' in types
|
||||
|
||||
# print(f"{quote}\n\n")
|
||||
ticks = quote.get('ticks', ())
|
||||
trades = {}
|
||||
darks = {}
|
||||
|
||||
if ticks:
|
||||
|
||||
# do a first pass and attempt to remove duplicate dark
|
||||
# trades with the same tick signature.
|
||||
if deduplicate_darks:
|
||||
for tick in ticks:
|
||||
ttype = tick.get('type')
|
||||
|
||||
time = tick.get('time', None)
|
||||
if time:
|
||||
sig = (
|
||||
time,
|
||||
tick['price'],
|
||||
tick['size']
|
||||
)
|
||||
|
||||
if ttype == 'dark_trade':
|
||||
darks[sig] = tick
|
||||
|
||||
elif ttype == 'trade':
|
||||
trades[sig] = tick
|
||||
|
||||
# filter duplicates
|
||||
for sig, tick in trades.items():
|
||||
tick = darks.pop(sig, None)
|
||||
if tick:
|
||||
ticks.remove(tick)
|
||||
# print(f'DUPLICATE {tick}')
|
||||
|
||||
# re-insert ticks
|
||||
ticks.extend(list(chain(trades.values(), darks.values())))
|
||||
|
||||
for tick in ticks:
|
||||
# print(f"{quote['symbol']}: {tick}")
|
||||
ttype = tick.get('type')
|
||||
if ttype in types:
|
||||
yield tick
|
||||
|
|
@ -1,291 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Super fast ``QPainterPath`` generation related operator routines.
|
||||
|
||||
"""
|
||||
from math import (
|
||||
ceil,
|
||||
floor,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
from numba import (
|
||||
# types,
|
||||
njit,
|
||||
float64,
|
||||
int64,
|
||||
# optional,
|
||||
)
|
||||
|
||||
# TODO: for ``numba`` typing..
|
||||
# from ._source import numba_ohlc_dtype
|
||||
from ._m4 import ds_m4
|
||||
from .._profile import (
|
||||
Profiler,
|
||||
pg_profile_enabled,
|
||||
ms_slower_then,
|
||||
)
|
||||
|
||||
|
||||
def xy_downsample(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
|
||||
x_spacer: float = 0.5,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
float,
|
||||
float,
|
||||
]:
|
||||
'''
|
||||
Downsample 1D (flat ``numpy.ndarray``) arrays using M4 given an input
|
||||
``uppx`` (units-per-pixel) and add space between discreet datums.
|
||||
|
||||
'''
|
||||
# downsample whenever more then 1 pixels per datum can be shown.
|
||||
# always refresh data bounds until we get diffing
|
||||
# working properly, see above..
|
||||
m4_out = ds_m4(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
)
|
||||
|
||||
if m4_out is not None:
|
||||
bins, x, y, ymn, ymx = m4_out
|
||||
# flatten output to 1d arrays suitable for path-graphics generation.
|
||||
x = np.broadcast_to(x[:, None], y.shape)
|
||||
x = (x + np.array(
|
||||
[-x_spacer, 0, 0, x_spacer]
|
||||
)).flatten()
|
||||
y = y.flatten()
|
||||
|
||||
return x, y, ymn, ymx
|
||||
|
||||
# XXX: we accept a None output for the case where the input range
|
||||
# to ``ds_m4()`` is bad (-ve) and we want to catch and debug
|
||||
# that (seemingly super rare) circumstance..
|
||||
return None
|
||||
|
||||
|
||||
@njit(
|
||||
# NOTE: need to construct this manually for readonly
|
||||
# arrays, see https://github.com/numba/numba/issues/4511
|
||||
# (
|
||||
# types.Array(
|
||||
# numba_ohlc_dtype,
|
||||
# 1,
|
||||
# 'C',
|
||||
# readonly=True,
|
||||
# ),
|
||||
# int64,
|
||||
# types.unicode_type,
|
||||
# optional(float64),
|
||||
# ),
|
||||
nogil=True
|
||||
)
|
||||
def path_arrays_from_ohlc(
|
||||
data: np.ndarray,
|
||||
start: int64,
|
||||
bar_w: float64,
|
||||
bar_gap: float64 = 0.16,
|
||||
use_time_index: bool = True,
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
Generate an array of lines objects from input ohlc data.
|
||||
|
||||
'''
|
||||
size = int(data.shape[0] * 6)
|
||||
|
||||
# XXX: see this for why the dtype might have to be defined outside
|
||||
# the routine.
|
||||
# https://github.com/numba/numba/issues/4098#issuecomment-493914533
|
||||
x = np.zeros(
|
||||
shape=size,
|
||||
dtype=float64,
|
||||
)
|
||||
y, c = x.copy(), x.copy()
|
||||
|
||||
half_w: float = bar_w/2
|
||||
|
||||
# TODO: report bug for assert @
|
||||
# /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
|
||||
for i, q in enumerate(data[start:], start):
|
||||
|
||||
open = q['open']
|
||||
high = q['high']
|
||||
low = q['low']
|
||||
close = q['close']
|
||||
|
||||
if use_time_index:
|
||||
index = float64(q['time'])
|
||||
else:
|
||||
index = float64(q['index'])
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index = float64(q[index_field])
|
||||
# AND this (probably)
|
||||
# open, high, low, close, index = q[
|
||||
# ['open', 'high', 'low', 'close', 'index']]
|
||||
|
||||
istart = i * 6
|
||||
istop = istart + 6
|
||||
|
||||
# x,y detail the 6 points which connect all vertexes of a ohlc bar
|
||||
mid: float = index + half_w
|
||||
x[istart:istop] = (
|
||||
index + bar_gap,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
index + bar_w - bar_gap,
|
||||
)
|
||||
y[istart:istop] = (
|
||||
open,
|
||||
open,
|
||||
low,
|
||||
high,
|
||||
close,
|
||||
close,
|
||||
)
|
||||
|
||||
# specifies that the first edge is never connected to the
|
||||
# prior bars last edge thus providing a small "gap"/"space"
|
||||
# between bars determined by ``bar_gap``.
|
||||
c[istart:istop] = (1, 1, 1, 1, 1, 0)
|
||||
|
||||
return x, y, c
|
||||
|
||||
|
||||
def hl2mxmn(
|
||||
ohlc: np.ndarray,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Convert a OHLC struct-array containing 'high'/'low' columns
|
||||
to a "joined" max/min 1-d array.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
hls = ohlc[[
|
||||
'low',
|
||||
'high',
|
||||
]]
|
||||
|
||||
mxmn = np.empty(2*hls.size, dtype=np.float64)
|
||||
x = np.empty(2*hls.size, dtype=np.float64)
|
||||
trace_hl(hls, mxmn, x, index[0])
|
||||
x = x + index[0]
|
||||
|
||||
return mxmn, x
|
||||
|
||||
|
||||
@njit(
|
||||
# TODO: the type annots..
|
||||
# float64[:](float64[:],),
|
||||
)
|
||||
def trace_hl(
|
||||
hl: 'np.ndarray',
|
||||
out: np.ndarray,
|
||||
x: np.ndarray,
|
||||
start: int,
|
||||
|
||||
# the "offset" values in the x-domain which
|
||||
# place the 2 output points around each ``int``
|
||||
# master index.
|
||||
margin: float = 0.43,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
"Trace" the outline of the high-low values of an ohlc sequence
|
||||
as a line such that the maximum deviation (aka disperaion) between
|
||||
bars if preserved.
|
||||
|
||||
This routine is expected to modify input arrays in-place.
|
||||
|
||||
'''
|
||||
last_l = hl['low'][0]
|
||||
last_h = hl['high'][0]
|
||||
|
||||
for i in range(hl.size):
|
||||
row = hl[i]
|
||||
l, h = row['low'], row['high']
|
||||
|
||||
up_diff = h - last_l
|
||||
down_diff = last_h - l
|
||||
|
||||
if up_diff > down_diff:
|
||||
out[2*i + 1] = h
|
||||
out[2*i] = last_l
|
||||
else:
|
||||
out[2*i + 1] = l
|
||||
out[2*i] = last_h
|
||||
|
||||
last_l = l
|
||||
last_h = h
|
||||
|
||||
x[2*i] = int(i) - margin
|
||||
x[2*i + 1] = int(i) + margin
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def ohlc_flatten(
|
||||
ohlc: np.ndarray,
|
||||
use_mxmn: bool = True,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
'''
|
||||
Convert an OHLCV struct-array into a flat ready-for-line-plotting
|
||||
1-d array that is 4 times the size with x-domain values distributed
|
||||
evenly (by 0.5 steps) over each index.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
|
||||
if use_mxmn:
|
||||
# traces a line optimally over highs to lows
|
||||
# using numba. NOTE: pretty sure this is faster
|
||||
# and looks about the same as the below output.
|
||||
flat, x = hl2mxmn(ohlc)
|
||||
|
||||
else:
|
||||
flat = rfn.structured_to_unstructured(
|
||||
ohlc[['open', 'high', 'low', 'close']]
|
||||
).flatten()
|
||||
|
||||
x = np.linspace(
|
||||
start=index[0] - 0.5,
|
||||
stop=index[-1] + 0.5,
|
||||
num=len(flat),
|
||||
)
|
||||
return x, flat
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -27,22 +27,29 @@ from multiprocessing.shared_memory import SharedMemory, _USE_POSIX
|
|||
if _USE_POSIX:
|
||||
from _posixshmem import shm_unlink
|
||||
|
||||
# import msgspec
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
import tractor
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ._util import log
|
||||
from ._source import def_iohlcv_fields
|
||||
from .types import Struct
|
||||
from ..log import get_logger
|
||||
from ._source import base_iohlc_dtype
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# how much is probably dependent on lifestyle
|
||||
_secs_in_day = int(60 * 60 * 24)
|
||||
# we try for a buncha times, but only on a run-every-other-day kinda week.
|
||||
_days_worth = 16
|
||||
_default_size = _days_worth * _secs_in_day
|
||||
# where to start the new data append index
|
||||
_rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
|
||||
|
||||
|
||||
def cuckoff_mantracker():
|
||||
'''
|
||||
Disable all ``multiprocessing``` "resource tracking" machinery since
|
||||
it's an absolute multi-threaded mess of non-SC madness.
|
||||
|
||||
'''
|
||||
from multiprocessing import resource_tracker as mantracker
|
||||
|
||||
# Tell the "resource tracker" thing to fuck off.
|
||||
|
|
@ -61,6 +68,7 @@ def cuckoff_mantracker():
|
|||
mantracker._resource_tracker = ManTracker()
|
||||
mantracker.register = mantracker._resource_tracker.register
|
||||
mantracker.ensure_running = mantracker._resource_tracker.ensure_running
|
||||
# ensure_running = mantracker._resource_tracker.ensure_running
|
||||
mantracker.unregister = mantracker._resource_tracker.unregister
|
||||
mantracker.getfd = mantracker._resource_tracker.getfd
|
||||
|
||||
|
|
@ -99,39 +107,36 @@ class SharedInt:
|
|||
log.warning(f'Shm for {name} already unlinked?')
|
||||
|
||||
|
||||
class _Token(Struct, frozen=True):
|
||||
class _Token(BaseModel):
|
||||
'''
|
||||
Internal represenation of a shared memory "token"
|
||||
which can be used to key a system wide post shm entry.
|
||||
|
||||
'''
|
||||
class Config:
|
||||
frozen = True
|
||||
|
||||
shm_name: str # this servers as a "key" value
|
||||
shm_first_index_name: str
|
||||
shm_last_index_name: str
|
||||
dtype_descr: tuple
|
||||
size: int # in struct-array index / row terms
|
||||
|
||||
@property
|
||||
def dtype(self) -> np.dtype:
|
||||
return np.dtype(list(map(tuple, self.dtype_descr))).descr
|
||||
|
||||
def as_msg(self):
|
||||
return self.to_dict()
|
||||
return self.dict()
|
||||
|
||||
@classmethod
|
||||
def from_msg(cls, msg: dict) -> _Token:
|
||||
if isinstance(msg, _Token):
|
||||
return msg
|
||||
|
||||
# TODO: native struct decoding
|
||||
# return _token_dec.decode(msg)
|
||||
|
||||
msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
|
||||
return _Token(**msg)
|
||||
|
||||
|
||||
# _token_dec = msgspec.msgpack.Decoder(_Token)
|
||||
|
||||
# TODO: this api?
|
||||
# _known_tokens = tractor.ActorVar('_shm_tokens', {})
|
||||
# _known_tokens = tractor.ContextStack('_known_tokens', )
|
||||
|
|
@ -150,7 +155,6 @@ def get_shm_token(key: str) -> _Token:
|
|||
|
||||
def _make_token(
|
||||
key: str,
|
||||
size: int,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
) -> _Token:
|
||||
'''
|
||||
|
|
@ -158,13 +162,12 @@ def _make_token(
|
|||
to access a shared array.
|
||||
|
||||
'''
|
||||
dtype = def_iohlcv_fields if dtype is None else dtype
|
||||
dtype = base_iohlc_dtype if dtype is None else dtype
|
||||
return _Token(
|
||||
shm_name=key,
|
||||
shm_first_index_name=key + "_first",
|
||||
shm_last_index_name=key + "_last",
|
||||
dtype_descr=tuple(np.dtype(dtype).descr),
|
||||
size=size,
|
||||
dtype_descr=np.dtype(dtype).descr
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -216,7 +219,6 @@ class ShmArray:
|
|||
shm_first_index_name=self._first._shm.name,
|
||||
shm_last_index_name=self._last._shm.name,
|
||||
dtype_descr=tuple(self._array.dtype.descr),
|
||||
size=self._len,
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
@ -248,6 +250,7 @@ class ShmArray:
|
|||
# to load an empty array..
|
||||
if len(a) == 0 and self._post_init:
|
||||
raise RuntimeError('Empty array race condition hit!?')
|
||||
# breakpoint()
|
||||
|
||||
return a
|
||||
|
||||
|
|
@ -257,7 +260,7 @@ class ShmArray:
|
|||
|
||||
# type that all field values will be cast to
|
||||
# in the returned view.
|
||||
common_dtype: np.dtype = float,
|
||||
common_dtype: np.dtype = np.float,
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
|
|
@ -312,7 +315,7 @@ class ShmArray:
|
|||
field_map: Optional[dict[str, str]] = None,
|
||||
prepend: bool = False,
|
||||
update_first: bool = True,
|
||||
start: int | None = None,
|
||||
start: Optional[int] = None,
|
||||
|
||||
) -> int:
|
||||
'''
|
||||
|
|
@ -354,11 +357,7 @@ class ShmArray:
|
|||
# tries to access ``.array`` (which due to the index
|
||||
# overlap will be empty). Pretty sure we've fixed it now
|
||||
# but leaving this here as a reminder.
|
||||
if (
|
||||
prepend
|
||||
and update_first
|
||||
and length
|
||||
):
|
||||
if prepend and update_first and length:
|
||||
assert index < self._first.value
|
||||
|
||||
if (
|
||||
|
|
@ -432,10 +431,10 @@ class ShmArray:
|
|||
|
||||
|
||||
def open_shm_array(
|
||||
size: int,
|
||||
key: str | None = None,
|
||||
dtype: np.dtype | None = None,
|
||||
append_start_index: int | None = None,
|
||||
|
||||
key: Optional[str] = None,
|
||||
size: int = _default_size,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
readonly: bool = False,
|
||||
|
||||
) -> ShmArray:
|
||||
|
|
@ -465,8 +464,7 @@ def open_shm_array(
|
|||
|
||||
token = _make_token(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
dtype=dtype
|
||||
)
|
||||
|
||||
# create single entry arrays for storing an first and last indices
|
||||
|
|
@ -500,13 +498,10 @@ def open_shm_array(
|
|||
# ``ShmArray._start.value: int = 0`` and the yet-to-be written
|
||||
# real-time section will start at ``ShmArray.index: int``.
|
||||
|
||||
# this sets the index to nearly 2/3rds into the the length of
|
||||
# the buffer leaving at least a "days worth of second samples"
|
||||
# for the real-time section.
|
||||
if append_start_index is None:
|
||||
append_start_index = round(size * 0.616)
|
||||
|
||||
last.value = first.value = append_start_index
|
||||
# this sets the index to 3/4 of the length of the buffer
|
||||
# leaving a "days worth of second samples" for the real-time
|
||||
# section.
|
||||
last.value = first.value = _rt_buffer_start
|
||||
|
||||
shmarr = ShmArray(
|
||||
array,
|
||||
|
|
@ -520,15 +515,16 @@ def open_shm_array(
|
|||
|
||||
# "unlink" created shm on process teardown by
|
||||
# pushing teardown calls onto actor context stack
|
||||
stack = tractor.current_actor().lifetime_stack
|
||||
stack.callback(shmarr.close)
|
||||
stack.callback(shmarr.destroy)
|
||||
|
||||
tractor._actor._lifetime_stack.callback(shmarr.close)
|
||||
tractor._actor._lifetime_stack.callback(shmarr.destroy)
|
||||
|
||||
return shmarr
|
||||
|
||||
|
||||
def attach_shm_array(
|
||||
token: tuple[str, str, tuple[str, str]],
|
||||
size: int = _default_size,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> ShmArray:
|
||||
|
|
@ -567,7 +563,7 @@ def attach_shm_array(
|
|||
raise _err
|
||||
|
||||
shmarr = np.ndarray(
|
||||
(token.size,),
|
||||
(size,),
|
||||
dtype=token.dtype,
|
||||
buffer=shm.buf
|
||||
)
|
||||
|
|
@ -606,18 +602,15 @@ def attach_shm_array(
|
|||
if key not in _known_tokens:
|
||||
_known_tokens[key] = token
|
||||
|
||||
# "close" attached shm on actor teardown
|
||||
tractor.current_actor().lifetime_stack.callback(sha.close)
|
||||
# "close" attached shm on process teardown
|
||||
tractor._actor._lifetime_stack.callback(sha.close)
|
||||
|
||||
return sha
|
||||
|
||||
|
||||
def maybe_open_shm_array(
|
||||
key: str,
|
||||
size: int,
|
||||
dtype: np.dtype | None = None,
|
||||
append_start_index: int | None = None,
|
||||
readonly: bool = False,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tuple[ShmArray, bool]:
|
||||
|
|
@ -641,41 +634,23 @@ def maybe_open_shm_array(
|
|||
try:
|
||||
# see if we already know this key
|
||||
token = _known_tokens[key]
|
||||
return (
|
||||
attach_shm_array(
|
||||
token=token,
|
||||
readonly=readonly,
|
||||
),
|
||||
False,
|
||||
)
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except KeyError:
|
||||
log.debug(f"Could not find {key} in shms cache")
|
||||
log.warning(f"Could not find {key} in shms cache")
|
||||
if dtype:
|
||||
token = _make_token(
|
||||
key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
)
|
||||
token = _make_token(key, dtype)
|
||||
try:
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except FileNotFoundError:
|
||||
log.debug(f"Could not attach to shm with token {token}")
|
||||
log.warning(f"Could not attach to shm with token {token}")
|
||||
|
||||
# This actor does not know about memory
|
||||
# associated with the provided "key".
|
||||
# Attempt to open a block and expect
|
||||
# to fail if a block has been allocated
|
||||
# on the OS by someone else.
|
||||
return (
|
||||
open_shm_array(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
append_start_index=append_start_index,
|
||||
readonly=readonly,
|
||||
),
|
||||
True,
|
||||
)
|
||||
return open_shm_array(key=key, dtype=dtype, **kwargs), True
|
||||
|
||||
|
||||
def try_read(
|
||||
array: np.ndarray
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,47 +18,34 @@
|
|||
numpy data source coversion helpers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any
|
||||
import decimal
|
||||
|
||||
from bidict import bidict
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
# from numba import from_dtype
|
||||
|
||||
|
||||
def_iohlcv_fields: list[tuple[str, type]] = [
|
||||
|
||||
# YES WE KNOW, this isn't needed in polars but we use it for doing
|
||||
# ring-buffer like pre/append ops our our `ShmArray` real-time
|
||||
# numpy-array buffering system such that there is a master index
|
||||
# that can be used for index-arithmetic when write data to the
|
||||
# "middle" of the array. See the ``tractor.ipc.shm`` pkg for more
|
||||
# details.
|
||||
('index', int),
|
||||
|
||||
# presume int for epoch stamps since it's most common
|
||||
# and makes the most sense to avoid float rounding issues.
|
||||
# TODO: if we want higher reso we should use the new
|
||||
# ``time.time_ns()`` in python 3.10+
|
||||
('time', int),
|
||||
ohlc_fields = [
|
||||
('time', float),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
|
||||
# TODO: can we elim this from default field set to save on mem?
|
||||
# i think only kraken really uses this in terms of what we get from
|
||||
# their ohlc history API?
|
||||
# ('bar_wap', float), # shouldn't be default right?
|
||||
('bar_wap', float),
|
||||
]
|
||||
|
||||
# remove index field
|
||||
def_ohlcv_fields: list[tuple[str, type]] = def_iohlcv_fields.copy()
|
||||
def_ohlcv_fields.pop(0)
|
||||
assert (len(def_iohlcv_fields) - len(def_ohlcv_fields)) == 1
|
||||
ohlc_with_index = ohlc_fields.copy()
|
||||
ohlc_with_index.insert(0, ('index', int))
|
||||
|
||||
# our minimum structured array layout for ohlc data
|
||||
base_iohlc_dtype = np.dtype(ohlc_with_index)
|
||||
base_ohlc_dtype = np.dtype(ohlc_fields)
|
||||
|
||||
# TODO: for now need to construct this manually for readonly arrays, see
|
||||
# https://github.com/numba/numba/issues/4511
|
||||
# from numba import from_dtype
|
||||
# base_ohlc_dtype = np.dtype(def_ohlc_fields)
|
||||
# numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
|
||||
|
||||
# map time frame "keys" to seconds values
|
||||
|
|
@ -73,6 +60,28 @@ tf_in_1s = bidict({
|
|||
})
|
||||
|
||||
|
||||
def mk_fqsn(
|
||||
provider: str,
|
||||
symbol: str,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Generate a "fully qualified symbol name" which is
|
||||
a reverse-hierarchical cross broker/provider symbol
|
||||
|
||||
'''
|
||||
return '.'.join([symbol, provider]).lower()
|
||||
|
||||
|
||||
def float_digits(
|
||||
value: float,
|
||||
) -> int:
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(-decimal.Decimal(str(value)).as_tuple().exponent)
|
||||
|
||||
|
||||
def ohlc_zeros(length: int) -> np.ndarray:
|
||||
"""Construct an OHLC field formatted structarray.
|
||||
|
||||
|
|
@ -83,6 +92,168 @@ def ohlc_zeros(length: int) -> np.ndarray:
|
|||
return np.zeros(length, dtype=base_ohlc_dtype)
|
||||
|
||||
|
||||
def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqsn.split('.')
|
||||
if len(tokens) < 3:
|
||||
# probably crypto
|
||||
symbol, broker = tokens
|
||||
return (
|
||||
broker,
|
||||
symbol,
|
||||
'',
|
||||
)
|
||||
|
||||
elif len(tokens) > 3:
|
||||
symbol, venue, suffix, broker = tokens
|
||||
else:
|
||||
symbol, venue, broker = tokens
|
||||
suffix = ''
|
||||
|
||||
# head, _, broker = fqsn.rpartition('.')
|
||||
# symbol, _, suffix = head.rpartition('.')
|
||||
return (
|
||||
broker,
|
||||
'.'.join([symbol, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
class Symbol(BaseModel):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
tick_size: float = 0.01
|
||||
lot_tick_size: float = 0.0 # "volume" precision as min step value
|
||||
tick_size_digits: int = 2
|
||||
lot_size_digits: int = 0
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# specifies a "class" of financial instrument
|
||||
# ex. stock, futer, option, bond etc.
|
||||
|
||||
# @validate_arguments
|
||||
@classmethod
|
||||
def from_broker_info(
|
||||
cls,
|
||||
broker: str,
|
||||
symbol: str,
|
||||
info: dict[str, Any],
|
||||
suffix: str = '',
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_tick_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
key=symbol,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_tick_size,
|
||||
tick_size_digits=float_digits(tick_size),
|
||||
lot_size_digits=float_digits(lot_tick_size),
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_fqsn(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
broker, key, suffix = unpack_fqsn(fqsn)
|
||||
return cls.from_broker_info(
|
||||
broker,
|
||||
key,
|
||||
info=info,
|
||||
suffix=suffix,
|
||||
)
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def brokers(self) -> list[str]:
|
||||
return list(self.broker_info.keys())
|
||||
|
||||
def nearest_tick(self, value: float) -> float:
|
||||
'''
|
||||
Return the nearest tick value based on mininum increment.
|
||||
|
||||
'''
|
||||
mult = 1 / self.tick_size
|
||||
return round(value * mult) / mult
|
||||
|
||||
def front_feed(self) -> tuple[str, str]:
|
||||
'''
|
||||
Return the "current" feed key for this symbol.
|
||||
|
||||
(i.e. the broker + symbol key in a tuple).
|
||||
|
||||
'''
|
||||
return (
|
||||
list(self.broker_info.keys())[0],
|
||||
self.key,
|
||||
)
|
||||
|
||||
def tokens(self) -> tuple[str]:
|
||||
broker, key = self.front_feed()
|
||||
if self.suffix:
|
||||
return (key, self.suffix, broker)
|
||||
else:
|
||||
return (key, broker)
|
||||
|
||||
def front_fqsn(self) -> str:
|
||||
'''
|
||||
fqsn = "fully qualified symbol name"
|
||||
|
||||
Basically the idea here is for all client-ish code (aka programs/actors
|
||||
that ask the provider agnostic layers in the stack for data) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string key of the current form:
|
||||
|
||||
<instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.):
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
'''
|
||||
tokens = self.tokens()
|
||||
fqsn = '.'.join(tokens)
|
||||
return fqsn
|
||||
|
||||
def iterfqsns(self) -> list[str]:
|
||||
keys = []
|
||||
for broker in self.broker_info.keys():
|
||||
fqsn = mk_fqsn(self.key, broker)
|
||||
if self.suffix:
|
||||
fqsn += f'.{self.suffix}'
|
||||
keys.append(fqsn)
|
||||
|
||||
return keys
|
||||
|
||||
|
||||
def _nan_to_closest_num(array: np.ndarray):
|
||||
"""Return interpolated values instead of NaN.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,326 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Financial time series processing utilities usually
|
||||
pertaining to OHLCV style sampled data.
|
||||
|
||||
Routines are generally implemented in either ``numpy`` or ``polars`` B)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from typing import Literal
|
||||
from math import (
|
||||
ceil,
|
||||
floor,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from ._sharedmem import ShmArray
|
||||
from .._profile import (
|
||||
Profiler,
|
||||
pg_profile_enabled,
|
||||
ms_slower_then,
|
||||
)
|
||||
|
||||
|
||||
def slice_from_time(
|
||||
arr: np.ndarray,
|
||||
start_t: float,
|
||||
stop_t: float,
|
||||
step: float, # sampler period step-diff
|
||||
|
||||
) -> slice:
|
||||
'''
|
||||
Calculate array indices mapped from a time range and return them in
|
||||
a slice.
|
||||
|
||||
Given an input array with an epoch `'time'` series entry, calculate
|
||||
the indices which span the time range and return in a slice. Presume
|
||||
each `'time'` step increment is uniform and when the time stamp
|
||||
series contains gaps (the uniform presumption is untrue) use
|
||||
``np.searchsorted()`` binary search to look up the appropriate
|
||||
index.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
msg='slice_from_time()',
|
||||
disabled=not pg_profile_enabled(),
|
||||
ms_threshold=ms_slower_then,
|
||||
)
|
||||
|
||||
times = arr['time']
|
||||
t_first = floor(times[0])
|
||||
t_last = ceil(times[-1])
|
||||
|
||||
# the greatest index we can return which slices to the
|
||||
# end of the input array.
|
||||
read_i_max = arr.shape[0]
|
||||
|
||||
# compute (presumed) uniform-time-step index offsets
|
||||
i_start_t = floor(start_t)
|
||||
read_i_start = floor(((i_start_t - t_first) // step)) - 1
|
||||
|
||||
i_stop_t = ceil(stop_t)
|
||||
|
||||
# XXX: edge case -> always set stop index to last in array whenever
|
||||
# the input stop time is detected to be greater then the equiv time
|
||||
# stamp at that last entry.
|
||||
if i_stop_t >= t_last:
|
||||
read_i_stop = read_i_max
|
||||
else:
|
||||
read_i_stop = ceil((i_stop_t - t_first) // step) + 1
|
||||
|
||||
# always clip outputs to array support
|
||||
# for read start:
|
||||
# - never allow a start < the 0 index
|
||||
# - never allow an end index > the read array len
|
||||
read_i_start = min(
|
||||
max(0, read_i_start),
|
||||
read_i_max - 1,
|
||||
)
|
||||
read_i_stop = max(
|
||||
0,
|
||||
min(read_i_stop, read_i_max),
|
||||
)
|
||||
|
||||
# check for larger-then-latest calculated index for given start
|
||||
# time, in which case we do a binary search for the correct index.
|
||||
# NOTE: this is usually the result of a time series with time gaps
|
||||
# where it is expected that each index step maps to a uniform step
|
||||
# in the time stamp series.
|
||||
t_iv_start = times[read_i_start]
|
||||
if (
|
||||
t_iv_start > i_start_t
|
||||
):
|
||||
# do a binary search for the best index mapping to ``start_t``
|
||||
# given we measured an overshoot using the uniform-time-step
|
||||
# calculation from above.
|
||||
|
||||
# TODO: once we start caching these per source-array,
|
||||
# we can just overwrite ``read_i_start`` directly.
|
||||
new_read_i_start = np.searchsorted(
|
||||
times,
|
||||
i_start_t,
|
||||
side='left',
|
||||
)
|
||||
|
||||
# TODO: minimize binary search work as much as possible:
|
||||
# - cache these remap values which compensate for gaps in the
|
||||
# uniform time step basis where we calc a later start
|
||||
# index for the given input ``start_t``.
|
||||
# - can we shorten the input search sequence by heuristic?
|
||||
# up_to_arith_start = index[:read_i_start]
|
||||
|
||||
if (
|
||||
new_read_i_start <= read_i_start
|
||||
):
|
||||
# t_diff = t_iv_start - start_t
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'start_t:{start_t} -> 0index start_t:{t_iv_start}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# f'REMAPPED START i: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
read_i_start = new_read_i_start
|
||||
|
||||
t_iv_stop = times[read_i_stop - 1]
|
||||
if (
|
||||
t_iv_stop > i_stop_t
|
||||
):
|
||||
# t_diff = stop_t - t_iv_stop
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'calced iv stop:{t_iv_stop} -> stop_t:{stop_t}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# # f'SHOULD REMAP STOP: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
new_read_i_stop = np.searchsorted(
|
||||
times[read_i_start:],
|
||||
# times,
|
||||
i_stop_t,
|
||||
side='right',
|
||||
)
|
||||
|
||||
if (
|
||||
new_read_i_stop <= read_i_stop
|
||||
):
|
||||
read_i_stop = read_i_start + new_read_i_stop + 1
|
||||
|
||||
# sanity checks for range size
|
||||
# samples = (i_stop_t - i_start_t) // step
|
||||
# index_diff = read_i_stop - read_i_start + 1
|
||||
# if index_diff > (samples + 3):
|
||||
# breakpoint()
|
||||
|
||||
# read-relative indexes: gives a slice where `shm.array[read_slc]`
|
||||
# will be the data spanning the input time range `start_t` ->
|
||||
# `stop_t`
|
||||
read_slc = slice(
|
||||
int(read_i_start),
|
||||
int(read_i_stop),
|
||||
)
|
||||
|
||||
profiler(
|
||||
'slicing complete'
|
||||
# f'{start_t} -> {abs_slc.start} | {read_slc.start}\n'
|
||||
# f'{stop_t} -> {abs_slc.stop} | {read_slc.stop}\n'
|
||||
)
|
||||
|
||||
# NOTE: if caller needs absolute buffer indices they can
|
||||
# slice the buffer abs index like so:
|
||||
# index = arr['index']
|
||||
# abs_indx = index[read_slc]
|
||||
# abs_slc = slice(
|
||||
# int(abs_indx[0]),
|
||||
# int(abs_indx[-1]),
|
||||
# )
|
||||
|
||||
return read_slc
|
||||
|
||||
|
||||
def detect_null_time_gap(
|
||||
shm: ShmArray,
|
||||
imargin: int = 1,
|
||||
|
||||
) -> tuple[float, float] | None:
|
||||
'''
|
||||
Detect if there are any zero-epoch stamped rows in
|
||||
the presumed 'time' field-column.
|
||||
|
||||
Filter to the gap and return a surrounding index range.
|
||||
|
||||
NOTE: for now presumes only ONE gap XD
|
||||
|
||||
'''
|
||||
zero_pred: np.ndarray = shm.array['time'] == 0
|
||||
zero_t: np.ndarray = shm.array[zero_pred]
|
||||
if zero_t.size:
|
||||
istart, iend = zero_t['index'][[0, -1]]
|
||||
start, end = shm._array['time'][
|
||||
[istart - imargin, iend + imargin]
|
||||
]
|
||||
return (
|
||||
istart - imargin,
|
||||
start,
|
||||
end,
|
||||
iend + imargin,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
t_unit: Literal[
|
||||
'days',
|
||||
'hours',
|
||||
'minutes',
|
||||
'seconds',
|
||||
'miliseconds',
|
||||
'microseconds',
|
||||
'nanoseconds',
|
||||
]
|
||||
|
||||
|
||||
def with_dts(
|
||||
df: pl.DataFrame,
|
||||
time_col: str = 'time',
|
||||
) -> pl.DataFrame:
|
||||
'''
|
||||
Insert datetime (casted) columns to a (presumably) OHLC sampled
|
||||
time series with an epoch-time column keyed by ``time_col``.
|
||||
|
||||
'''
|
||||
return df.with_columns([
|
||||
pl.col(time_col).shift(1).suffix('_prev'),
|
||||
pl.col(time_col).diff().alias('s_diff'),
|
||||
pl.from_epoch(pl.col(time_col)).alias('dt'),
|
||||
]).with_columns([
|
||||
pl.from_epoch(pl.col(f'{time_col}_prev')).alias('dt_prev'),
|
||||
pl.col('dt').diff().alias('dt_diff'),
|
||||
]) #.with_columns(
|
||||
# pl.col('dt').diff().dt.days().alias('days_dt_diff'),
|
||||
# )
|
||||
|
||||
|
||||
def detect_time_gaps(
|
||||
df: pl.DataFrame,
|
||||
|
||||
time_col: str = 'time',
|
||||
# epoch sampling step diff
|
||||
expect_period: float = 60,
|
||||
|
||||
# datetime diff unit and gap value
|
||||
# crypto mkts
|
||||
# gap_dt_unit: t_unit = 'minutes',
|
||||
# gap_thresh: int = 1,
|
||||
|
||||
# legacy stock mkts
|
||||
gap_dt_unit: t_unit = 'days',
|
||||
gap_thresh: int = 2,
|
||||
|
||||
) -> pl.DataFrame:
|
||||
'''
|
||||
Filter to OHLC datums which contain sample step gaps.
|
||||
|
||||
For eg. legacy markets which have venue close gaps and/or
|
||||
actual missing data segments.
|
||||
|
||||
'''
|
||||
dt_gap_col: str = f'{gap_dt_unit}_diff'
|
||||
return with_dts(
|
||||
df
|
||||
).filter(
|
||||
pl.col('s_diff').abs() > expect_period
|
||||
).with_columns(
|
||||
getattr(
|
||||
pl.col('dt_diff').dt,
|
||||
gap_dt_unit, # NOTE: must be valid ``Expr.dt.<name>``
|
||||
)().alias(dt_gap_col)
|
||||
).filter(
|
||||
pl.col(dt_gap_col).abs() > gap_thresh
|
||||
)
|
||||
|
||||
|
||||
def detect_price_gaps(
|
||||
df: pl.DataFrame,
|
||||
gt_multiplier: float = 2.,
|
||||
price_fields: list[str] = ['high', 'low'],
|
||||
|
||||
) -> pl.DataFrame:
|
||||
'''
|
||||
Detect gaps in clearing price over an OHLC series.
|
||||
|
||||
2 types of gaps generally exist; up gaps and down gaps:
|
||||
|
||||
- UP gap: when any next sample's lo price is strictly greater
|
||||
then the current sample's hi price.
|
||||
|
||||
- DOWN gap: when any next sample's hi price is strictly
|
||||
less then the current samples lo price.
|
||||
|
||||
'''
|
||||
# return df.filter(
|
||||
# pl.col('high') - ) > expect_period,
|
||||
# ).select([
|
||||
# pl.dt.datetime(pl.col(time_col).shift(1)).suffix('_previous'),
|
||||
# pl.all(),
|
||||
# ]).select([
|
||||
# pl.all(),
|
||||
# (pl.col(time_col) - pl.col(f'{time_col}_previous')).alias('diff'),
|
||||
# ])
|
||||
...
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Data layer module commons.
|
||||
|
||||
'''
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.data'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,30 +18,13 @@
|
|||
ToOlS fOr CoPInG wITh "tHE wEB" protocols.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from itertools import count
|
||||
from functools import partial
|
||||
from contextlib import asynccontextmanager, AsyncExitStack
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
AsyncContextManager,
|
||||
AsyncGenerator,
|
||||
Iterable,
|
||||
)
|
||||
from typing import Any, Callable, AsyncGenerator
|
||||
import json
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
from trio_websocket import (
|
||||
WebSocketConnection,
|
||||
open_websocket_url,
|
||||
)
|
||||
from wsproto.utilities import LocalProtocolError
|
||||
import trio_websocket
|
||||
from trio_websocket._impl import (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
|
|
@ -50,71 +33,81 @@ from trio_websocket._impl import (
|
|||
ConnectionTimeout,
|
||||
)
|
||||
|
||||
from ._util import log
|
||||
from .types import Struct
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class NoBsWs:
|
||||
'''
|
||||
Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
"""Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
|
||||
A shim interface that allows client code to stream from some
|
||||
``WebSocketConnection`` but where any connectivy bs is handled
|
||||
automatcially and entirely in the background.
|
||||
|
||||
NOTE: this type should never be created directly but instead is
|
||||
provided via the ``open_autorecon_ws()`` factor below.
|
||||
|
||||
'''
|
||||
# apparently we can QoS for all sorts of reasons..so catch em.
|
||||
"""
|
||||
recon_errors = (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
ConnectionRejected,
|
||||
HandshakeError,
|
||||
ConnectionTimeout,
|
||||
LocalProtocolError,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
rxchan: trio.MemoryReceiveChannel,
|
||||
msg_recv_timeout: float,
|
||||
|
||||
serializer: ModuleType = json
|
||||
token: str,
|
||||
stack: AsyncExitStack,
|
||||
fixture: Callable,
|
||||
serializer: ModuleType = json,
|
||||
):
|
||||
self.url = url
|
||||
self._rx = rxchan
|
||||
self._timeout = msg_recv_timeout
|
||||
self.token = token
|
||||
self.fixture = fixture
|
||||
self._stack = stack
|
||||
self._ws: 'WebSocketConnection' = None # noqa
|
||||
|
||||
# signaling between caller and relay task which determines when
|
||||
# socket is connected (and subscribed).
|
||||
self._connected: trio.Event = trio.Event()
|
||||
async def _connect(
|
||||
self,
|
||||
tries: int = 1000,
|
||||
) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._stack.aclose()
|
||||
except (DisconnectionTimeout, RuntimeError):
|
||||
await trio.sleep(0.5)
|
||||
else:
|
||||
break
|
||||
|
||||
# dynamically reset by the bg relay task
|
||||
self._ws: WebSocketConnection | None = None
|
||||
self._cs: trio.CancelScope | None = None
|
||||
last_err = None
|
||||
for i in range(tries):
|
||||
try:
|
||||
self._ws = await self._stack.enter_async_context(
|
||||
trio_websocket.open_websocket_url(self.url)
|
||||
)
|
||||
# rerun user code fixture
|
||||
if self.token == '':
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self)
|
||||
)
|
||||
else:
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self, self.token)
|
||||
)
|
||||
|
||||
# interchange codec methods
|
||||
# TODO: obviously the method API here may be different
|
||||
# for another interchange format..
|
||||
self._dumps: Callable = serializer.dumps
|
||||
self._loads: Callable = serializer.loads
|
||||
assert ret is None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self._connected.is_set()
|
||||
log.info(f'Connection success: {self.url}')
|
||||
return self._ws
|
||||
|
||||
async def reset(self) -> None:
|
||||
'''
|
||||
Reset the underlying ws connection by cancelling
|
||||
the bg relay task and waiting for it to signal
|
||||
a new connection.
|
||||
|
||||
'''
|
||||
self._connected = trio.Event()
|
||||
self._cs.cancel()
|
||||
await self._connected.wait()
|
||||
except self.recon_errors as err:
|
||||
last_err = err
|
||||
log.error(
|
||||
f'{self} connection bail with '
|
||||
f'{type(err)}...retry attempt {i}'
|
||||
)
|
||||
await trio.sleep(0.5)
|
||||
continue
|
||||
else:
|
||||
log.exception('ws connection fail...')
|
||||
raise last_err
|
||||
|
||||
async def send_msg(
|
||||
self,
|
||||
|
|
@ -122,342 +115,38 @@ class NoBsWs:
|
|||
) -> None:
|
||||
while True:
|
||||
try:
|
||||
msg: Any = self._dumps(data)
|
||||
return await self._ws.send_message(msg)
|
||||
return await self._ws.send_message(json.dumps(data))
|
||||
except self.recon_errors:
|
||||
await self.reset()
|
||||
await self._connect()
|
||||
|
||||
async def recv_msg(self) -> Any:
|
||||
msg: Any = await self._rx.receive()
|
||||
data = self._loads(msg)
|
||||
return data
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
return await self.recv_msg()
|
||||
|
||||
def set_recv_timeout(
|
||||
async def recv_msg(
|
||||
self,
|
||||
timeout: float,
|
||||
) -> None:
|
||||
self._timeout = timeout
|
||||
|
||||
|
||||
async def _reconnect_forever(
|
||||
url: str,
|
||||
snd: trio.MemorySendChannel,
|
||||
nobsws: NoBsWs,
|
||||
reset_after: int, # msg recv timeout before reset attempt
|
||||
|
||||
fixture: AsyncContextManager | None = None,
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
||||
# TODO: can we just report "where" in the call stack
|
||||
# the client code is using the ws stream?
|
||||
# Maybe we can just drop this since it's already in the log msg
|
||||
# orefix?
|
||||
if fixture is not None:
|
||||
src_mod: str = fixture.__module__
|
||||
else:
|
||||
src_mod: str = 'unknown'
|
||||
|
||||
async def proxy_msgs(
|
||||
ws: WebSocketConnection,
|
||||
pcs: trio.CancelScope, # parent cancel scope
|
||||
):
|
||||
'''
|
||||
Receive (under `timeout` deadline) all msgs from from underlying
|
||||
websocket and relay them to (calling) parent task via ``trio``
|
||||
mem chan.
|
||||
|
||||
'''
|
||||
# after so many msg recv timeouts, reset the connection
|
||||
timeouts: int = 0
|
||||
|
||||
) -> Any:
|
||||
while True:
|
||||
with trio.move_on_after(
|
||||
# can be dynamically changed by user code
|
||||
nobsws._timeout,
|
||||
) as cs:
|
||||
try:
|
||||
msg: Any = await ws.get_message()
|
||||
await snd.send(msg)
|
||||
except nobsws.recon_errors:
|
||||
log.exception(
|
||||
f'{src_mod}\n'
|
||||
f'{url} connection bail with:'
|
||||
)
|
||||
await trio.sleep(0.5)
|
||||
pcs.cancel()
|
||||
|
||||
# go back to reonnect loop in parent task
|
||||
return
|
||||
|
||||
if cs.cancelled_caught:
|
||||
timeouts += 1
|
||||
if timeouts > reset_after:
|
||||
log.error(
|
||||
f'{src_mod}\n'
|
||||
'WS feed seems down and slow af.. reconnecting\n'
|
||||
)
|
||||
pcs.cancel()
|
||||
|
||||
# go back to reonnect loop in parent task
|
||||
return
|
||||
|
||||
async def open_fixture(
|
||||
fixture: AsyncContextManager,
|
||||
nobsws: NoBsWs,
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
):
|
||||
'''
|
||||
Open user provided `@acm` and sleep until any connection
|
||||
reset occurs.
|
||||
|
||||
'''
|
||||
async with fixture(nobsws) as ret:
|
||||
assert ret is None
|
||||
task_status.started()
|
||||
await trio.sleep_forever()
|
||||
|
||||
# last_err = None
|
||||
nobsws._connected = trio.Event()
|
||||
task_status.started()
|
||||
|
||||
while not snd._closed:
|
||||
log.info(
|
||||
f'{src_mod}\n'
|
||||
f'{url} trying (RE)CONNECT'
|
||||
)
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
cs = nobsws._cs = n.cancel_scope
|
||||
ws: WebSocketConnection
|
||||
async with open_websocket_url(url) as ws:
|
||||
nobsws._ws = ws
|
||||
log.info(
|
||||
f'{src_mod}\n'
|
||||
f'Connection success: {url}'
|
||||
)
|
||||
|
||||
# begin relay loop to forward msgs
|
||||
n.start_soon(
|
||||
proxy_msgs,
|
||||
ws,
|
||||
cs,
|
||||
)
|
||||
|
||||
if fixture is not None:
|
||||
log.info(
|
||||
f'{src_mod}\n'
|
||||
f'Entering fixture: {fixture}'
|
||||
)
|
||||
|
||||
# TODO: should we return an explicit sub-cs
|
||||
# from this fixture task?
|
||||
await n.start(
|
||||
open_fixture,
|
||||
fixture,
|
||||
nobsws,
|
||||
)
|
||||
|
||||
# indicate to wrapper / opener that we are up and block
|
||||
# to let tasks run **inside** the ws open block above.
|
||||
nobsws._connected.set()
|
||||
await trio.sleep_forever()
|
||||
|
||||
# ws open block end
|
||||
# nursery block end
|
||||
nobsws._connected = trio.Event()
|
||||
if cs.cancelled_caught:
|
||||
log.cancel(
|
||||
f'{url} connection cancelled!'
|
||||
)
|
||||
# if wrapper cancelled us, we expect it to also
|
||||
# have re-assigned a new event
|
||||
assert (
|
||||
nobsws._connected
|
||||
and not nobsws._connected.is_set()
|
||||
)
|
||||
|
||||
# -> from here, move to next reconnect attempt
|
||||
|
||||
else:
|
||||
log.exception(
|
||||
f'{src_mod}\n'
|
||||
'ws connection closed by client...'
|
||||
)
|
||||
try:
|
||||
return json.loads(await self._ws.get_message())
|
||||
except self.recon_errors:
|
||||
await self._connect()
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_autorecon_ws(
|
||||
url: str,
|
||||
|
||||
fixture: AsyncContextManager | None = None,
|
||||
|
||||
# time in sec between msgs received before
|
||||
# we presume connection might need a reset.
|
||||
msg_recv_timeout: float = 16,
|
||||
|
||||
# count of the number of above timeouts before connection reset
|
||||
reset_after: int = 3,
|
||||
|
||||
# TODO: proper type annot smh
|
||||
fixture: Callable,
|
||||
# used for authenticated websockets
|
||||
token: str = '',
|
||||
) -> AsyncGenerator[tuple[...], NoBsWs]:
|
||||
'''
|
||||
An auto-reconnect websocket (wrapper API) around
|
||||
``trio_websocket.open_websocket_url()`` providing automatic
|
||||
re-connection on network errors, msg latency and thus roaming.
|
||||
"""Apparently we can QoS for all sorts of reasons..so catch em.
|
||||
|
||||
Here we implement a re-connect websocket interface where a bg
|
||||
nursery runs ``WebSocketConnection.receive_message()``s in a loop
|
||||
and restarts the full http(s) handshake on catches of certain
|
||||
connetivity errors, or some user defined recv timeout.
|
||||
|
||||
You can provide a ``fixture`` async-context-manager which will be
|
||||
entered/exitted around each connection reset; eg. for (re)requesting
|
||||
subscriptions without requiring streaming setup code to rerun.
|
||||
|
||||
'''
|
||||
snd: trio.MemorySendChannel
|
||||
rcv: trio.MemoryReceiveChannel
|
||||
snd, rcv = trio.open_memory_channel(616)
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
nobsws = NoBsWs(
|
||||
url,
|
||||
rcv,
|
||||
msg_recv_timeout=msg_recv_timeout,
|
||||
)
|
||||
await n.start(
|
||||
partial(
|
||||
_reconnect_forever,
|
||||
url,
|
||||
snd,
|
||||
nobsws,
|
||||
fixture=fixture,
|
||||
reset_after=reset_after,
|
||||
)
|
||||
)
|
||||
await nobsws._connected.wait()
|
||||
assert nobsws._cs
|
||||
assert nobsws.connected()
|
||||
"""
|
||||
async with AsyncExitStack() as stack:
|
||||
ws = NoBsWs(url, token, stack, fixture=fixture)
|
||||
await ws._connect()
|
||||
|
||||
try:
|
||||
yield nobsws
|
||||
yield ws
|
||||
|
||||
finally:
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
'''
|
||||
JSONRPC response-request style machinery for transparent multiplexing of msgs
|
||||
over a NoBsWs.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
id: int
|
||||
jsonrpc: str = '2.0'
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
|
||||
|
||||
@acm
|
||||
async def open_jsonrpc_session(
|
||||
url: str,
|
||||
start_id: int = 0,
|
||||
response_type: type = JSONRPCResult,
|
||||
request_type: Optional[type] = None,
|
||||
request_hook: Optional[Callable] = None,
|
||||
error_hook: Optional[Callable] = None,
|
||||
) -> Callable[[str, dict], dict]:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_autorecon_ws(url) as ws
|
||||
):
|
||||
rpc_id: Iterable = count(start_id)
|
||||
rpc_results: dict[int, dict] = {}
|
||||
|
||||
async def json_rpc(method: str, params: dict) -> dict:
|
||||
'''
|
||||
perform a json rpc call and wait for the result, raise exception in
|
||||
case of error field present on response
|
||||
'''
|
||||
msg = {
|
||||
'jsonrpc': '2.0',
|
||||
'id': next(rpc_id),
|
||||
'method': method,
|
||||
'params': params
|
||||
}
|
||||
_id = msg['id']
|
||||
|
||||
rpc_results[_id] = {
|
||||
'result': None,
|
||||
'event': trio.Event()
|
||||
}
|
||||
|
||||
await ws.send_msg(msg)
|
||||
|
||||
await rpc_results[_id]['event'].wait()
|
||||
|
||||
ret = rpc_results[_id]['result']
|
||||
|
||||
del rpc_results[_id]
|
||||
|
||||
if ret.error is not None:
|
||||
raise Exception(json.dumps(ret.error, indent=4))
|
||||
|
||||
return ret
|
||||
|
||||
async def recv_task():
|
||||
'''
|
||||
receives every ws message and stores it in its corresponding
|
||||
result field, then sets the event to wakeup original sender
|
||||
tasks. also recieves responses to requests originated from
|
||||
the server side.
|
||||
|
||||
'''
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if res_entry := rpc_results.get(mid):
|
||||
|
||||
res_entry['result'] = response_type(**msg)
|
||||
res_entry['event'].set()
|
||||
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if not rpc_results.get(mid):
|
||||
log.warning(
|
||||
f'Unexpected ws msg: {json.dumps(msg, indent=4)}'
|
||||
)
|
||||
|
||||
case {
|
||||
'method': _,
|
||||
'params': _,
|
||||
}:
|
||||
log.debug(f'Recieved\n{msg}')
|
||||
if request_hook:
|
||||
await request_hook(request_type(**msg))
|
||||
|
||||
case {
|
||||
'error': error
|
||||
}:
|
||||
log.warning(f'Recieved\n{error}')
|
||||
if error_hook:
|
||||
await error_hook(response_type(**msg))
|
||||
|
||||
case _:
|
||||
log.warning(f'Unhandled JSON-RPC msg!?\n{msg}')
|
||||
|
||||
n.start_soon(recv_task)
|
||||
yield json_rpc
|
||||
n.cancel_scope.cancel()
|
||||
await stack.aclose()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,196 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
marketstore cli.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
from pprint import pformat
|
||||
|
||||
from anyio_marketstore import open_marketstore_client
|
||||
import trio
|
||||
import tractor
|
||||
import click
|
||||
import numpy as np
|
||||
|
||||
from .marketstore import (
|
||||
get_client,
|
||||
# stream_quotes,
|
||||
ingest_quote_stream,
|
||||
# _url,
|
||||
_tick_tbk_ids,
|
||||
mk_tbk,
|
||||
)
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ..log import get_logger
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
)
|
||||
from ._source import (
|
||||
base_iohlc_dtype,
|
||||
)
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--url',
|
||||
default='ws://localhost:5993/ws',
|
||||
help='HTTP URL of marketstore instance'
|
||||
)
|
||||
@click.argument('names', nargs=-1)
|
||||
@click.pass_obj
|
||||
def ms_stream(
|
||||
config: dict,
|
||||
names: list[str],
|
||||
url: str,
|
||||
) -> None:
|
||||
'''
|
||||
Connect to a marketstore time bucket stream for (a set of) symbols(s)
|
||||
and print to console.
|
||||
|
||||
'''
|
||||
async def main():
|
||||
# async for quote in stream_quotes(symbols=names):
|
||||
# log.info(f"Received quote:\n{quote}")
|
||||
...
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
# @cli.command()
|
||||
# @click.option(
|
||||
# '--url',
|
||||
# default=_url,
|
||||
# help='HTTP URL of marketstore instance'
|
||||
# )
|
||||
# @click.argument('names', nargs=-1)
|
||||
# @click.pass_obj
|
||||
# def ms_destroy(config: dict, names: list[str], url: str) -> None:
|
||||
# """Destroy symbol entries in the local marketstore instance.
|
||||
# """
|
||||
# async def main():
|
||||
# nonlocal names
|
||||
# async with get_client(url) as client:
|
||||
#
|
||||
# if not names:
|
||||
# names = await client.list_symbols()
|
||||
#
|
||||
# # default is to wipe db entirely.
|
||||
# answer = input(
|
||||
# "This will entirely wipe you local marketstore db @ "
|
||||
# f"{url} of the following symbols:\n {pformat(names)}"
|
||||
# "\n\nDelete [N/y]?\n")
|
||||
#
|
||||
# if answer == 'y':
|
||||
# for sym in names:
|
||||
# # tbk = _tick_tbk.format(sym)
|
||||
# tbk = tuple(sym, *_tick_tbk_ids)
|
||||
# print(f"Destroying {tbk}..")
|
||||
# await client.destroy(mk_tbk(tbk))
|
||||
# else:
|
||||
# print("Nothing deleted.")
|
||||
#
|
||||
# tractor.run(main)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--tl',
|
||||
is_flag=True,
|
||||
help='Enable tractor logging')
|
||||
@click.option(
|
||||
'--host',
|
||||
default='localhost'
|
||||
)
|
||||
@click.option(
|
||||
'--port',
|
||||
default=5993
|
||||
)
|
||||
@click.argument('symbols', nargs=-1)
|
||||
@click.pass_obj
|
||||
def storesh(
|
||||
config,
|
||||
tl,
|
||||
host,
|
||||
port,
|
||||
symbols: list[str],
|
||||
):
|
||||
'''
|
||||
Start an IPython shell ready to query the local marketstore db.
|
||||
|
||||
'''
|
||||
from piker.data.marketstore import tsdb_history_update
|
||||
from piker._daemon import open_piker_runtime
|
||||
|
||||
async def main():
|
||||
nonlocal symbols
|
||||
|
||||
async with open_piker_runtime(
|
||||
'storesh',
|
||||
enable_modules=['piker.data._ahab'],
|
||||
):
|
||||
symbol = symbols[0]
|
||||
await tsdb_history_update(symbol)
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--test-file', '-t', help='Test quote stream file')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.argument('name', nargs=1, required=True)
|
||||
@click.pass_obj
|
||||
def ingest(config, name, test_file, tl):
|
||||
'''
|
||||
Ingest real-time broker quotes and ticks to a marketstore instance.
|
||||
|
||||
'''
|
||||
# global opts
|
||||
loglevel = config['loglevel']
|
||||
tractorloglevel = config['tractorloglevel']
|
||||
# log = config['log']
|
||||
|
||||
watchlist_from_file = wl.ensure_watchlists(config['wl_path'])
|
||||
watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins)
|
||||
symbols = watchlists[name]
|
||||
|
||||
grouped_syms = {}
|
||||
for sym in symbols:
|
||||
symbol, _, provider = sym.rpartition('.')
|
||||
if provider not in grouped_syms:
|
||||
grouped_syms[provider] = []
|
||||
|
||||
grouped_syms[provider].append(symbol)
|
||||
|
||||
async def entry_point():
|
||||
async with tractor.open_nursery() as n:
|
||||
for provider, symbols in grouped_syms.items():
|
||||
await n.run_in_actor(
|
||||
ingest_quote_stream,
|
||||
name='ingest_marketstore',
|
||||
symbols=symbols,
|
||||
brokername=provider,
|
||||
tries=1,
|
||||
actorloglevel=loglevel,
|
||||
loglevel=tractorloglevel
|
||||
)
|
||||
|
||||
tractor.run(entry_point)
|
||||
1707
piker/data/feed.py
1707
piker/data/feed.py
File diff suppressed because it is too large
Load Diff
|
|
@ -1,237 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Public abstractions for organizing, managing and generally operating-on
|
||||
real-time data processing data-structures.
|
||||
|
||||
"Streams, flumes, cascades and flows.."
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import pendulum
|
||||
import numpy as np
|
||||
|
||||
from ..accounting import MktPair
|
||||
from ._util import log
|
||||
from .types import Struct
|
||||
from ._sharedmem import (
|
||||
attach_shm_array,
|
||||
ShmArray,
|
||||
_Token,
|
||||
)
|
||||
# from .._profile import (
|
||||
# Profiler,
|
||||
# pg_profile_enabled,
|
||||
# )
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# from pyqtgraph import PlotItem
|
||||
from .feed import Feed
|
||||
|
||||
|
||||
# TODO: ideas for further abstractions as per
|
||||
# https://github.com/pikers/piker/issues/216 and
|
||||
# https://github.com/pikers/piker/issues/270:
|
||||
# - a ``Cascade`` would be the minimal "connection" of 2 ``Flumes``
|
||||
# as per circuit parlance:
|
||||
# https://en.wikipedia.org/wiki/Two-port_network#Cascade_connection
|
||||
# - could cover the combination of our `FspAdmin` and the
|
||||
# backend `.fsp._engine` related machinery to "connect" one flume
|
||||
# to another?
|
||||
# - a (financial signal) ``Flow`` would be the a "collection" of such
|
||||
# minmial cascades. Some engineering based jargon concepts:
|
||||
# - https://en.wikipedia.org/wiki/Signal_chain
|
||||
# - https://en.wikipedia.org/wiki/Daisy_chain_(electrical_engineering)
|
||||
# - https://en.wikipedia.org/wiki/Audio_signal_flow
|
||||
# - https://en.wikipedia.org/wiki/Digital_signal_processing#Implementation
|
||||
# - https://en.wikipedia.org/wiki/Dataflow_programming
|
||||
# - https://en.wikipedia.org/wiki/Signal_programming
|
||||
# - https://en.wikipedia.org/wiki/Incremental_computing
|
||||
|
||||
|
||||
class Flume(Struct):
|
||||
'''
|
||||
Composite reference type which points to all the addressing handles
|
||||
and other meta-data necessary for the read, measure and management
|
||||
of a set of real-time updated data flows.
|
||||
|
||||
Can be thought of as a "flow descriptor" or "flow frame" which
|
||||
describes the high level properties of a set of data flows that can
|
||||
be used seamlessly across process-memory boundaries.
|
||||
|
||||
Each instance's sub-components normally includes:
|
||||
- a msg oriented quote stream provided via an IPC transport
|
||||
- history and real-time shm buffers which are both real-time
|
||||
updated and backfilled.
|
||||
- associated startup indexing information related to both buffer
|
||||
real-time-append and historical prepend addresses.
|
||||
- low level APIs to read and measure the updated data and manage
|
||||
queuing properties.
|
||||
|
||||
'''
|
||||
mkt: MktPair
|
||||
first_quote: dict
|
||||
_rt_shm_token: _Token
|
||||
|
||||
# optional since some data flows won't have a "downsampled" history
|
||||
# buffer/stream (eg. FSPs).
|
||||
_hist_shm_token: _Token | None = None
|
||||
|
||||
# private shm refs loaded dynamically from tokens
|
||||
_hist_shm: ShmArray | None = None
|
||||
_rt_shm: ShmArray | None = None
|
||||
|
||||
stream: tractor.MsgStream | None = None
|
||||
izero_hist: int = 0
|
||||
izero_rt: int = 0
|
||||
throttle_rate: int | None = None
|
||||
|
||||
# TODO: do we need this really if we can pull the `Portal` from
|
||||
# ``tractor``'s internals?
|
||||
feed: Feed | None = None
|
||||
|
||||
@property
|
||||
def rt_shm(self) -> ShmArray:
|
||||
|
||||
if self._rt_shm is None:
|
||||
self._rt_shm = attach_shm_array(
|
||||
token=self._rt_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._rt_shm
|
||||
|
||||
@property
|
||||
def hist_shm(self) -> ShmArray:
|
||||
|
||||
if self._hist_shm_token is None:
|
||||
raise RuntimeError(
|
||||
'No shm token has been set for the history buffer?'
|
||||
)
|
||||
|
||||
if (
|
||||
self._hist_shm is None
|
||||
):
|
||||
self._hist_shm = attach_shm_array(
|
||||
token=self._hist_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._hist_shm
|
||||
|
||||
async def receive(self) -> dict:
|
||||
return await self.stream.receive()
|
||||
|
||||
def get_ds_info(
|
||||
self,
|
||||
) -> tuple[float, float, float]:
|
||||
'''
|
||||
Compute the "downsampling" ratio info between the historical shm
|
||||
buffer and the real-time (HFT) one.
|
||||
|
||||
Return a tuple of the fast sample period, historical sample
|
||||
period and ratio between them.
|
||||
|
||||
'''
|
||||
times = self.hist_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
hist_step_size_s = (end - start).seconds
|
||||
|
||||
times = self.rt_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
rt_step_size_s = (end - start).seconds
|
||||
|
||||
ratio = hist_step_size_s / rt_step_size_s
|
||||
return (
|
||||
rt_step_size_s,
|
||||
hist_step_size_s,
|
||||
ratio,
|
||||
)
|
||||
|
||||
# TODO: get native msgspec decoding for these workinn
|
||||
def to_msg(self) -> dict:
|
||||
|
||||
msg = self.to_dict()
|
||||
msg['mkt'] = self.mkt.to_dict()
|
||||
|
||||
# can't serialize the stream or feed objects, it's expected
|
||||
# you'll have a ref to it since this msg should be rxed on
|
||||
# a stream on whatever far end IPC..
|
||||
msg.pop('stream')
|
||||
msg.pop('feed')
|
||||
return msg
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Load from an IPC msg presumably in either `dict` or
|
||||
`msgspec.Struct` form.
|
||||
|
||||
'''
|
||||
mkt_msg = msg.pop('mkt')
|
||||
mkt = MktPair.from_msg(mkt_msg)
|
||||
return cls(mkt=mkt, **msg)
|
||||
|
||||
def get_index(
|
||||
self,
|
||||
time_s: float,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> int | float:
|
||||
'''
|
||||
Return array shm-buffer index for for epoch time.
|
||||
|
||||
'''
|
||||
times = array['time']
|
||||
first = np.searchsorted(
|
||||
times,
|
||||
time_s,
|
||||
side='left',
|
||||
)
|
||||
imx = times.shape[0] - 1
|
||||
return min(first, imx)
|
||||
|
||||
# only set by external msg or creator, never
|
||||
# manually!
|
||||
_has_vlm: bool = True
|
||||
|
||||
def has_vlm(self) -> bool:
|
||||
|
||||
if not self._has_vlm:
|
||||
return False
|
||||
|
||||
# make sure that the instrument supports volume history
|
||||
# (sometimes this is not the case for some commodities and
|
||||
# derivatives)
|
||||
vlm: np.ndarray = self.rt_shm.array['volume']
|
||||
return not bool(
|
||||
np.all(np.isin(vlm, -1))
|
||||
or np.all(np.isnan(vlm))
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1,967 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Historical data business logic for load, backfill and tsdb storage.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
# from collections import (
|
||||
# Counter,
|
||||
# )
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
# import time
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Callable,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
from pendulum import (
|
||||
Duration,
|
||||
from_timestamp,
|
||||
)
|
||||
import numpy as np
|
||||
|
||||
from ..accounting import (
|
||||
MktPair,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
)
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
ShmArray,
|
||||
)
|
||||
from ._source import def_iohlcv_fields
|
||||
from ._sampling import (
|
||||
open_sample_stream,
|
||||
)
|
||||
from ..brokers._util import (
|
||||
DataUnavailable,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bidict import bidict
|
||||
from ..service.marketstore import StorageClient
|
||||
from .feed import _FeedsBus
|
||||
|
||||
|
||||
# `ShmArray` buffer sizing configuration:
|
||||
_mins_in_day = int(60 * 24)
|
||||
# how much is probably dependent on lifestyle
|
||||
# but we reco a buncha times (but only on a
|
||||
# run-every-other-day kinda week).
|
||||
_secs_in_day = int(60 * _mins_in_day)
|
||||
_days_in_week: int = 7
|
||||
|
||||
_days_worth: int = 3
|
||||
_default_hist_size: int = 6 * 365 * _mins_in_day
|
||||
_hist_buffer_start = int(
|
||||
_default_hist_size - round(7 * _mins_in_day)
|
||||
)
|
||||
|
||||
_default_rt_size: int = _days_worth * _secs_in_day
|
||||
# NOTE: start the append index in rt buffer such that 1 day's worth
|
||||
# can be appenened before overrun.
|
||||
_rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
|
||||
|
||||
|
||||
def diff_history(
|
||||
array: np.ndarray,
|
||||
append_until_dt: datetime | None = None,
|
||||
prepend_until_dt: datetime | None = None,
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
# no diffing with tsdb dt index possible..
|
||||
if (
|
||||
prepend_until_dt is None
|
||||
and append_until_dt is None
|
||||
):
|
||||
return array
|
||||
|
||||
times = array['time']
|
||||
|
||||
if append_until_dt:
|
||||
return array[times < append_until_dt.timestamp()]
|
||||
else:
|
||||
return array[times >= prepend_until_dt.timestamp()]
|
||||
|
||||
|
||||
async def shm_push_in_between(
|
||||
shm: ShmArray,
|
||||
to_push: np.ndarray,
|
||||
prepend_index: int,
|
||||
|
||||
update_start_on_prepend: bool = False,
|
||||
|
||||
) -> int:
|
||||
shm.push(
|
||||
to_push,
|
||||
prepend=True,
|
||||
|
||||
# XXX: only update the ._first index if no tsdb
|
||||
# segment was previously prepended by the
|
||||
# parent task.
|
||||
update_first=update_start_on_prepend,
|
||||
|
||||
# XXX: only prepend from a manually calculated shm
|
||||
# index if there was already a tsdb history
|
||||
# segment prepended (since then the
|
||||
# ._first.value is going to be wayyy in the
|
||||
# past!)
|
||||
start=(
|
||||
prepend_index
|
||||
if not update_start_on_prepend
|
||||
else None
|
||||
),
|
||||
)
|
||||
# XXX: extremely important, there can be no checkpoints
|
||||
# in the block above to avoid entering new ``frames``
|
||||
# values while we're pipelining the current ones to
|
||||
# memory...
|
||||
array = shm.array
|
||||
zeros = array[array['low'] == 0]
|
||||
if (
|
||||
0 < zeros.size < 1000
|
||||
):
|
||||
tractor.breakpoint()
|
||||
|
||||
|
||||
|
||||
async def start_backfill(
|
||||
get_hist,
|
||||
mod: ModuleType,
|
||||
mkt: MktPair,
|
||||
shm: ShmArray,
|
||||
timeframe: float,
|
||||
|
||||
backfill_from_shm_index: int,
|
||||
backfill_from_dt: datetime,
|
||||
|
||||
sampler_stream: tractor.MsgStream,
|
||||
|
||||
backfill_until_dt: datetime | None = None,
|
||||
storage: StorageClient | None = None,
|
||||
|
||||
write_tsdb: bool = True,
|
||||
|
||||
task_status: TaskStatus[tuple] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> int:
|
||||
|
||||
# let caller unblock and deliver latest history frame
|
||||
# and use to signal that backfilling the shm gap until
|
||||
# the tsdb end is complete!
|
||||
bf_done = trio.Event()
|
||||
task_status.started(bf_done)
|
||||
|
||||
# based on the sample step size, maybe load a certain amount history
|
||||
update_start_on_prepend: bool = False
|
||||
if backfill_until_dt is None:
|
||||
|
||||
# TODO: drop this right and just expose the backfill
|
||||
# limits inside a [storage] section in conf.toml?
|
||||
# when no tsdb "last datum" is provided, we just load
|
||||
# some near-term history.
|
||||
# periods = {
|
||||
# 1: {'days': 1},
|
||||
# 60: {'days': 14},
|
||||
# }
|
||||
|
||||
# do a decently sized backfill and load it into storage.
|
||||
periods = {
|
||||
1: {'days': 6},
|
||||
60: {'years': 6},
|
||||
}
|
||||
period_duration: int = periods[timeframe]
|
||||
|
||||
update_start_on_prepend = True
|
||||
|
||||
# NOTE: manually set the "latest" datetime which we intend to
|
||||
# backfill history "until" so as to adhere to the history
|
||||
# settings above when the tsdb is detected as being empty.
|
||||
backfill_until_dt = backfill_from_dt.subtract(**period_duration)
|
||||
|
||||
|
||||
# TODO: can we drop this? without conc i don't think this
|
||||
# is necessary any more?
|
||||
# configure async query throttling
|
||||
# rate = config.get('rate', 1)
|
||||
# XXX: legacy from ``trimeter`` code but unsupported now.
|
||||
# erlangs = config.get('erlangs', 1)
|
||||
# avoid duplicate history frames with a set of datetime frame
|
||||
# starts and associated counts of how many duplicates we see
|
||||
# per time stamp.
|
||||
# starts: Counter[datetime] = Counter()
|
||||
|
||||
# conduct "backward history gap filling" where we push to
|
||||
# the shm buffer until we have history back until the
|
||||
# latest entry loaded from the tsdb's table B)
|
||||
last_start_dt: datetime = backfill_from_dt
|
||||
next_prepend_index: int = backfill_from_shm_index
|
||||
|
||||
while last_start_dt > backfill_until_dt:
|
||||
|
||||
log.debug(
|
||||
f'Requesting {timeframe}s frame ending in {last_start_dt}'
|
||||
)
|
||||
|
||||
try:
|
||||
(
|
||||
array,
|
||||
next_start_dt,
|
||||
next_end_dt,
|
||||
) = await get_hist(
|
||||
timeframe,
|
||||
end_dt=last_start_dt,
|
||||
)
|
||||
|
||||
# broker says there never was or is no more history to pull
|
||||
except DataUnavailable:
|
||||
log.warning(
|
||||
f'NO-MORE-DATA: backend {mod.name} halted history!?'
|
||||
)
|
||||
|
||||
# ugh, what's a better way?
|
||||
# TODO: fwiw, we probably want a way to signal a throttle
|
||||
# condition (eg. with ib) so that we can halt the
|
||||
# request loop until the condition is resolved?
|
||||
return
|
||||
|
||||
# TODO: drop this? see todo above..
|
||||
# if (
|
||||
# next_start_dt in starts
|
||||
# and starts[next_start_dt] <= 6
|
||||
# ):
|
||||
# start_dt = min(starts)
|
||||
# log.warning(
|
||||
# f"{mkt.fqme}: skipping duplicate frame @ {next_start_dt}"
|
||||
# )
|
||||
# starts[start_dt] += 1
|
||||
# await tractor.breakpoint()
|
||||
# continue
|
||||
|
||||
# elif starts[next_start_dt] > 6:
|
||||
# log.warning(
|
||||
# f'NO-MORE-DATA: backend {mod.name} before {next_start_dt}?'
|
||||
# )
|
||||
# return
|
||||
|
||||
# # only update new start point if not-yet-seen
|
||||
# starts[next_start_dt] += 1
|
||||
|
||||
assert array['time'][0] == next_start_dt.timestamp()
|
||||
|
||||
diff = last_start_dt - next_start_dt
|
||||
frame_time_diff_s = diff.seconds
|
||||
|
||||
# frame's worth of sample-period-steps, in seconds
|
||||
frame_size_s = len(array) * timeframe
|
||||
expected_frame_size_s = frame_size_s + timeframe
|
||||
if frame_time_diff_s > expected_frame_size_s:
|
||||
|
||||
# XXX: query result includes a start point prior to our
|
||||
# expected "frame size" and thus is likely some kind of
|
||||
# history gap (eg. market closed period, outage, etc.)
|
||||
# so just report it to console for now.
|
||||
log.warning(
|
||||
f'History frame ending @ {last_start_dt} appears to have a gap:\n'
|
||||
f'{diff} ~= {frame_time_diff_s} seconds'
|
||||
)
|
||||
|
||||
to_push = diff_history(
|
||||
array,
|
||||
prepend_until_dt=backfill_until_dt,
|
||||
)
|
||||
ln = len(to_push)
|
||||
if ln:
|
||||
log.info(f'{ln} bars for {next_start_dt} -> {last_start_dt}')
|
||||
|
||||
else:
|
||||
log.warning(
|
||||
'0 BARS TO PUSH after diff!?\n'
|
||||
f'{next_start_dt} -> {last_start_dt}'
|
||||
)
|
||||
|
||||
# bail gracefully on shm allocation overrun/full
|
||||
# condition
|
||||
try:
|
||||
await shm_push_in_between(
|
||||
shm,
|
||||
to_push,
|
||||
prepend_index=next_prepend_index,
|
||||
update_start_on_prepend=update_start_on_prepend,
|
||||
)
|
||||
await sampler_stream.send({
|
||||
'broadcast_all': {
|
||||
'backfilling': (mkt.fqme, timeframe),
|
||||
},
|
||||
})
|
||||
|
||||
# decrement next prepend point
|
||||
next_prepend_index = next_prepend_index - ln
|
||||
last_start_dt = next_start_dt
|
||||
|
||||
except ValueError as ve:
|
||||
_ve = ve
|
||||
log.error(
|
||||
f'Shm prepend OVERRUN on: {next_start_dt} -> {last_start_dt}?'
|
||||
)
|
||||
|
||||
if next_prepend_index < ln:
|
||||
log.warning(
|
||||
f'Shm buffer can only hold {next_prepend_index} more rows..\n'
|
||||
f'Appending those from recent {ln}-sized frame, no more!'
|
||||
)
|
||||
|
||||
to_push = to_push[-next_prepend_index + 1:]
|
||||
await shm_push_in_between(
|
||||
shm,
|
||||
to_push,
|
||||
prepend_index=next_prepend_index,
|
||||
update_start_on_prepend=update_start_on_prepend,
|
||||
)
|
||||
await sampler_stream.send({
|
||||
'broadcast_all': {
|
||||
'backfilling': (mkt.fqme, timeframe),
|
||||
},
|
||||
})
|
||||
|
||||
# can't push the entire frame? so
|
||||
# push only the amount that can fit..
|
||||
break
|
||||
|
||||
log.info(
|
||||
f'Shm pushed {ln} frame:\n'
|
||||
f'{next_start_dt} -> {last_start_dt}'
|
||||
)
|
||||
|
||||
# FINALLY, maybe write immediately to the tsdb backend for
|
||||
# long-term storage.
|
||||
if (
|
||||
storage is not None
|
||||
and write_tsdb
|
||||
):
|
||||
log.info(
|
||||
f'Writing {ln} frame to storage:\n'
|
||||
f'{next_start_dt} -> {last_start_dt}'
|
||||
)
|
||||
|
||||
if mkt.dst.atype not in {'crypto', 'crypto_currency'}:
|
||||
# for now, our table key schema is not including
|
||||
# the dst[/src] source asset token.
|
||||
col_sym_key: str = mkt.get_fqme(
|
||||
delim_char='',
|
||||
without_src=True,
|
||||
)
|
||||
else:
|
||||
col_sym_key: str = mkt.get_fqme(delim_char='')
|
||||
|
||||
# TODO: implement parquet append!?
|
||||
await storage.write_ohlcv(
|
||||
col_sym_key,
|
||||
shm.array,
|
||||
timeframe,
|
||||
)
|
||||
else:
|
||||
# finally filled gap
|
||||
log.info(
|
||||
f'Finished filling gap to tsdb start @ {backfill_until_dt}!'
|
||||
)
|
||||
# conduct tsdb timestamp gap detection and backfill any
|
||||
# seemingly missing sequence segments..
|
||||
# TODO: ideally these never exist but somehow it seems
|
||||
# sometimes we're writing zero-ed segments on certain
|
||||
# (teardown) cases?
|
||||
from ._timeseries import detect_null_time_gap
|
||||
|
||||
gap_indices: tuple | None = detect_null_time_gap(shm)
|
||||
while gap_indices:
|
||||
(
|
||||
istart,
|
||||
start,
|
||||
end,
|
||||
iend,
|
||||
) = gap_indices
|
||||
|
||||
start_dt = from_timestamp(start)
|
||||
end_dt = from_timestamp(end)
|
||||
(
|
||||
array,
|
||||
next_start_dt,
|
||||
next_end_dt,
|
||||
) = await get_hist(
|
||||
timeframe,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
|
||||
# XXX TODO: pretty sure if i plot tsla, btcusdt.binance
|
||||
# and mnq.cme.ib this causes a Qt crash XXDDD
|
||||
|
||||
# make sure we don't overrun the buffer start
|
||||
len_to_push: int = min(iend, array.size)
|
||||
to_push: np.ndarray = array[-len_to_push:]
|
||||
await shm_push_in_between(
|
||||
shm,
|
||||
to_push,
|
||||
prepend_index=iend,
|
||||
update_start_on_prepend=False,
|
||||
)
|
||||
|
||||
# TODO: UI side needs IPC event to update..
|
||||
# - make sure the UI actually always handles
|
||||
# this update!
|
||||
# - remember that in the display side, only refersh this
|
||||
# if the respective history is actually "in view".
|
||||
# loop
|
||||
await sampler_stream.send({
|
||||
'broadcast_all': {
|
||||
'backfilling': (mkt.fqme, timeframe),
|
||||
},
|
||||
})
|
||||
gap_indices: tuple | None = detect_null_time_gap(shm)
|
||||
|
||||
# XXX: extremely important, there can be no checkpoints
|
||||
# in the block above to avoid entering new ``frames``
|
||||
# values while we're pipelining the current ones to
|
||||
# memory...
|
||||
# await sampler_stream.send('broadcast_all')
|
||||
|
||||
# short-circuit (for now)
|
||||
bf_done.set()
|
||||
|
||||
|
||||
async def back_load_from_tsdb(
|
||||
storemod: ModuleType,
|
||||
storage: StorageClient,
|
||||
|
||||
fqme: str,
|
||||
|
||||
tsdb_history: np.ndarray,
|
||||
|
||||
last_tsdb_dt: datetime,
|
||||
latest_start_dt: datetime,
|
||||
latest_end_dt: datetime,
|
||||
|
||||
bf_done: trio.Event,
|
||||
|
||||
timeframe: int,
|
||||
shm: ShmArray,
|
||||
):
|
||||
assert len(tsdb_history)
|
||||
|
||||
# sync to backend history task's query/load completion
|
||||
# if bf_done:
|
||||
# await bf_done.wait()
|
||||
|
||||
# TODO: eventually it'd be nice to not require a shm array/buffer
|
||||
# to accomplish this.. maybe we can do some kind of tsdb direct to
|
||||
# graphics format eventually in a child-actor?
|
||||
if storemod.name == 'nativedb':
|
||||
return
|
||||
|
||||
await tractor.breakpoint()
|
||||
assert shm._first.value == 0
|
||||
|
||||
array = shm.array
|
||||
|
||||
# if timeframe == 1:
|
||||
# times = shm.array['time']
|
||||
# assert (times[1] - times[0]) == 1
|
||||
|
||||
if len(array):
|
||||
shm_last_dt = from_timestamp(
|
||||
shm.array[0]['time']
|
||||
)
|
||||
else:
|
||||
shm_last_dt = None
|
||||
|
||||
if last_tsdb_dt:
|
||||
assert shm_last_dt >= last_tsdb_dt
|
||||
|
||||
# do diff against start index of last frame of history and only
|
||||
# fill in an amount of datums from tsdb allows for most recent
|
||||
# to be loaded into mem *before* tsdb data.
|
||||
if (
|
||||
last_tsdb_dt
|
||||
and latest_start_dt
|
||||
):
|
||||
backfilled_size_s = (
|
||||
latest_start_dt - last_tsdb_dt
|
||||
).seconds
|
||||
# if the shm buffer len is not large enough to contain
|
||||
# all missing data between the most recent backend-queried frame
|
||||
# and the most recent dt-index in the db we warn that we only
|
||||
# want to load a portion of the next tsdb query to fill that
|
||||
# space.
|
||||
log.info(
|
||||
f'{backfilled_size_s} seconds worth of {timeframe}s loaded'
|
||||
)
|
||||
|
||||
# Load TSDB history into shm buffer (for display) if there is
|
||||
# remaining buffer space.
|
||||
|
||||
time_key: str = 'time'
|
||||
if getattr(storemod, 'ohlc_key_map', False):
|
||||
keymap: bidict = storemod.ohlc_key_map
|
||||
time_key: str = keymap.inverse['time']
|
||||
|
||||
# if (
|
||||
# not len(tsdb_history)
|
||||
# ):
|
||||
# return
|
||||
|
||||
tsdb_last_frame_start: datetime = last_tsdb_dt
|
||||
# load as much from storage into shm possible (depends on
|
||||
# user's shm size settings).
|
||||
while shm._first.value > 0:
|
||||
|
||||
tsdb_history = await storage.read_ohlcv(
|
||||
fqme,
|
||||
timeframe=timeframe,
|
||||
end=tsdb_last_frame_start,
|
||||
)
|
||||
|
||||
# # empty query
|
||||
# if not len(tsdb_history):
|
||||
# break
|
||||
|
||||
next_start = tsdb_history[time_key][0]
|
||||
if next_start >= tsdb_last_frame_start:
|
||||
# no earlier data detected
|
||||
break
|
||||
|
||||
else:
|
||||
tsdb_last_frame_start = next_start
|
||||
|
||||
# TODO: see if there's faster multi-field reads:
|
||||
# https://numpy.org/doc/stable/user/basics.rec.html#accessing-multiple-fields
|
||||
# re-index with a `time` and index field
|
||||
prepend_start = shm._first.value
|
||||
|
||||
to_push = tsdb_history[-prepend_start:]
|
||||
shm.push(
|
||||
to_push,
|
||||
|
||||
# insert the history pre a "days worth" of samples
|
||||
# to leave some real-time buffer space at the end.
|
||||
prepend=True,
|
||||
# update_first=False,
|
||||
# start=prepend_start,
|
||||
field_map=storemod.ohlc_key_map,
|
||||
)
|
||||
|
||||
log.info(f'Loaded {to_push.shape} datums from storage')
|
||||
tsdb_last_frame_start = tsdb_history[time_key][0]
|
||||
|
||||
# manually trigger step update to update charts/fsps
|
||||
# which need an incremental update.
|
||||
# NOTE: the way this works is super duper
|
||||
# un-intuitive right now:
|
||||
# - the broadcaster fires a msg to the fsp subsystem.
|
||||
# - fsp subsys then checks for a sample step diff and
|
||||
# possibly recomputes prepended history.
|
||||
# - the fsp then sends back to the parent actor
|
||||
# (usually a chart showing graphics for said fsp)
|
||||
# which tells the chart to conduct a manual full
|
||||
# graphics loop cycle.
|
||||
# await sampler_stream.send('broadcast_all')
|
||||
|
||||
|
||||
async def tsdb_backfill(
|
||||
mod: ModuleType,
|
||||
storemod: ModuleType,
|
||||
tn: trio.Nursery,
|
||||
|
||||
storage: StorageClient,
|
||||
mkt: MktPair,
|
||||
shm: ShmArray,
|
||||
timeframe: float,
|
||||
|
||||
sampler_stream: tractor.MsgStream,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[ShmArray, ShmArray]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
||||
get_hist: Callable[
|
||||
[int, datetime, datetime],
|
||||
tuple[np.ndarray, str]
|
||||
]
|
||||
config: dict[str, int]
|
||||
async with mod.open_history_client(
|
||||
mkt,
|
||||
) as (get_hist, config):
|
||||
log.info(f'{mod} history client returned backfill config: {config}')
|
||||
|
||||
# get latest query's worth of history all the way
|
||||
# back to what is recorded in the tsdb
|
||||
try:
|
||||
array, mr_start_dt, mr_end_dt = await get_hist(
|
||||
timeframe,
|
||||
end_dt=None,
|
||||
)
|
||||
|
||||
# XXX: timeframe not supported for backend (since
|
||||
# above exception type), terminate immediately since
|
||||
# there's no backfilling possible.
|
||||
except DataUnavailable:
|
||||
task_status.started()
|
||||
return
|
||||
|
||||
times: np.ndarray = array['time']
|
||||
|
||||
# sample period step size in seconds
|
||||
step_size_s = (
|
||||
from_timestamp(times[-1])
|
||||
- from_timestamp(times[-2])
|
||||
).seconds
|
||||
|
||||
if step_size_s not in (1, 60):
|
||||
log.error(f'Last 2 sample period is off!? -> {step_size_s}')
|
||||
step_size_s = (
|
||||
from_timestamp(times[-2])
|
||||
- from_timestamp(times[-3])
|
||||
).seconds
|
||||
|
||||
# NOTE: on the first history, most recent history
|
||||
# frame we PREPEND from the current shm ._last index
|
||||
# and thus a gap between the earliest datum loaded here
|
||||
# and the latest loaded from the tsdb may exist!
|
||||
log.info(f'Pushing {array.size} to shm!')
|
||||
shm.push(
|
||||
array,
|
||||
prepend=True, # append on first frame
|
||||
)
|
||||
backfill_gap_from_shm_index: int = shm._first.value + 1
|
||||
|
||||
# tell parent task to continue
|
||||
task_status.started()
|
||||
|
||||
# loads a (large) frame of data from the tsdb depending
|
||||
# on the db's query size limit; our "nativedb" (using
|
||||
# parquet) generally can load the entire history into mem
|
||||
# but if not then below the remaining history can be lazy
|
||||
# loaded?
|
||||
fqme: str = mkt.fqme
|
||||
tsdb_entry: tuple | None = await storage.load(
|
||||
fqme,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
|
||||
last_tsdb_dt: datetime | None = None
|
||||
if tsdb_entry:
|
||||
(
|
||||
tsdb_history,
|
||||
first_tsdb_dt,
|
||||
last_tsdb_dt,
|
||||
) = tsdb_entry
|
||||
|
||||
# calc the index from which the tsdb data should be
|
||||
# prepended, presuming there is a gap between the
|
||||
# latest frame (loaded/read above) and the latest
|
||||
# sample loaded from the tsdb.
|
||||
backfill_diff: Duration = mr_start_dt - last_tsdb_dt
|
||||
offset_s: float = backfill_diff.in_seconds()
|
||||
offset_samples: int = round(offset_s / timeframe)
|
||||
|
||||
# TODO: see if there's faster multi-field reads:
|
||||
# https://numpy.org/doc/stable/user/basics.rec.html#accessing-multiple-fields
|
||||
# re-index with a `time` and index field
|
||||
prepend_start = shm._first.value - offset_samples + 1
|
||||
|
||||
# tsdb history is so far in the past we can't fit it in
|
||||
# shm buffer space so simply don't load it!
|
||||
if prepend_start > 0:
|
||||
to_push = tsdb_history[-prepend_start:]
|
||||
shm.push(
|
||||
to_push,
|
||||
|
||||
# insert the history pre a "days worth" of samples
|
||||
# to leave some real-time buffer space at the end.
|
||||
prepend=True,
|
||||
# update_first=False,
|
||||
start=prepend_start,
|
||||
field_map=storemod.ohlc_key_map,
|
||||
)
|
||||
|
||||
log.info(f'Loaded {to_push.shape} datums from storage')
|
||||
|
||||
# TODO: maybe start history anal and load missing "history
|
||||
# gaps" via backend..
|
||||
|
||||
if timeframe not in (1, 60):
|
||||
raise ValueError(
|
||||
'`piker` only needs to support 1m and 1s sampling '
|
||||
'but ur api is trying to deliver a longer '
|
||||
f'timeframe of {timeframe} seconds..\n'
|
||||
'So yuh.. dun do dat brudder.'
|
||||
)
|
||||
# if there is a gap to backfill from the first
|
||||
# history frame until the last datum loaded from the tsdb
|
||||
# continue that now in the background
|
||||
bf_done = await tn.start(
|
||||
partial(
|
||||
start_backfill,
|
||||
get_hist,
|
||||
mod,
|
||||
mkt,
|
||||
shm,
|
||||
timeframe,
|
||||
|
||||
backfill_from_shm_index=backfill_gap_from_shm_index,
|
||||
backfill_from_dt=mr_start_dt,
|
||||
|
||||
sampler_stream=sampler_stream,
|
||||
|
||||
backfill_until_dt=last_tsdb_dt,
|
||||
storage=storage,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# if len(hist_shm.array) < 2:
|
||||
# TODO: there's an edge case here to solve where if the last
|
||||
# frame before market close (at least on ib) was pushed and
|
||||
# there was only "1 new" row pushed from the first backfill
|
||||
# query-iteration, then the sample step sizing calcs will
|
||||
# break upstream from here since you can't diff on at least
|
||||
# 2 steps... probably should also add logic to compute from
|
||||
# the tsdb series and stash that somewhere as meta data on
|
||||
# the shm buffer?.. no se.
|
||||
|
||||
# backload any further data from tsdb (concurrently per
|
||||
# timeframe) if not all data was able to be loaded (in memory)
|
||||
# from the ``StorageClient.load()`` call above.
|
||||
try:
|
||||
await trio.sleep_forever()
|
||||
finally:
|
||||
return
|
||||
|
||||
# IF we need to continue backloading incrementall from the
|
||||
# tsdb client..
|
||||
tn.start_soon(
|
||||
back_load_from_tsdb,
|
||||
|
||||
storemod,
|
||||
storage,
|
||||
fqme,
|
||||
|
||||
tsdb_history,
|
||||
last_tsdb_dt,
|
||||
mr_start_dt,
|
||||
mr_end_dt,
|
||||
bf_done,
|
||||
|
||||
timeframe,
|
||||
shm,
|
||||
)
|
||||
|
||||
|
||||
async def manage_history(
|
||||
mod: ModuleType,
|
||||
bus: _FeedsBus,
|
||||
mkt: MktPair,
|
||||
some_data_ready: trio.Event,
|
||||
feed_is_live: trio.Event,
|
||||
timeframe: float = 60, # in seconds
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[ShmArray, ShmArray]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Load and manage historical data including the loading of any
|
||||
available series from any connected tsdb as well as conduct
|
||||
real-time update of both that existing db and the allocated
|
||||
shared memory buffer.
|
||||
|
||||
Init sequence:
|
||||
- allocate shm (numpy array) buffers for 60s & 1s sample rates
|
||||
- configure "zero index" for each buffer: the index where
|
||||
history will prepended *to* and new live data will be
|
||||
appened *from*.
|
||||
- open a ``.storage.StorageClient`` and load any existing tsdb
|
||||
history as well as (async) start a backfill task which loads
|
||||
missing (newer) history from the data provider backend:
|
||||
- tsdb history is loaded first and pushed to shm ASAP.
|
||||
- the backfill task loads the most recent history before
|
||||
unblocking its parent task, so that the `ShmArray._last` is
|
||||
up to date to allow the OHLC sampler to begin writing new
|
||||
samples as the correct buffer index once the provider feed
|
||||
engages.
|
||||
|
||||
'''
|
||||
# TODO: is there a way to make each shm file key
|
||||
# actor-tree-discovery-addr unique so we avoid collisions
|
||||
# when doing tests which also allocate shms for certain instruments
|
||||
# that may be in use on the system by some other running daemons?
|
||||
# from tractor._state import _runtime_vars
|
||||
# port = _runtime_vars['_root_mailbox'][1]
|
||||
|
||||
uid = tractor.current_actor().uid
|
||||
name, uuid = uid
|
||||
service = name.rstrip(f'.{mod.name}')
|
||||
|
||||
fqme: str = mkt.get_fqme(delim_char='')
|
||||
|
||||
# (maybe) allocate shm array for this broker/symbol which will
|
||||
# be used for fast near-term history capture and processing.
|
||||
hist_shm, opened = maybe_open_shm_array(
|
||||
size=_default_hist_size,
|
||||
append_start_index=_hist_buffer_start,
|
||||
|
||||
key=f'piker.{service}[{uuid[:16]}].{fqme}.hist',
|
||||
|
||||
# use any broker defined ohlc dtype:
|
||||
dtype=getattr(mod, '_ohlc_dtype', def_iohlcv_fields),
|
||||
|
||||
# we expect the sub-actor to write
|
||||
readonly=False,
|
||||
)
|
||||
hist_zero_index = hist_shm.index - 1
|
||||
|
||||
# TODO: history validation
|
||||
if not opened:
|
||||
raise RuntimeError(
|
||||
"Persistent shm for sym was already open?!"
|
||||
)
|
||||
|
||||
rt_shm, opened = maybe_open_shm_array(
|
||||
size=_default_rt_size,
|
||||
append_start_index=_rt_buffer_start,
|
||||
key=f'piker.{service}[{uuid[:16]}].{fqme}.rt',
|
||||
|
||||
# use any broker defined ohlc dtype:
|
||||
dtype=getattr(mod, '_ohlc_dtype', def_iohlcv_fields),
|
||||
|
||||
# we expect the sub-actor to write
|
||||
readonly=False,
|
||||
)
|
||||
|
||||
# (for now) set the rt (hft) shm array with space to prepend
|
||||
# only a few days worth of 1s history.
|
||||
days = 2
|
||||
start_index = days*_secs_in_day
|
||||
rt_shm._first.value = start_index
|
||||
rt_shm._last.value = start_index
|
||||
rt_zero_index = rt_shm.index - 1
|
||||
|
||||
if not opened:
|
||||
raise RuntimeError(
|
||||
"Persistent shm for sym was already open?!"
|
||||
)
|
||||
|
||||
open_history_client = getattr(
|
||||
mod,
|
||||
'open_history_client',
|
||||
None,
|
||||
)
|
||||
assert open_history_client
|
||||
|
||||
# TODO: maybe it should be a subpkg of `.data`?
|
||||
from piker import storage
|
||||
|
||||
async with (
|
||||
storage.open_storage_client() as (storemod, client),
|
||||
trio.open_nursery() as tn,
|
||||
):
|
||||
log.info(
|
||||
f'Connecting to storage backend `{storemod.name}`:\n'
|
||||
f'location: {client.address}\n'
|
||||
f'db cardinality: {client.cardinality}\n'
|
||||
# TODO: show backend config, eg:
|
||||
# - network settings
|
||||
# - storage size with compression
|
||||
# - number of loaded time series?
|
||||
)
|
||||
|
||||
# NOTE: this call ONLY UNBLOCKS once the latest-most frame
|
||||
# (i.e. history just before the live feed latest datum) of
|
||||
# history has been loaded and written to the shm buffer:
|
||||
# - the backfiller task can write in reverse chronological
|
||||
# to the shm and tsdb
|
||||
# - the tsdb data can be loaded immediately and the
|
||||
# backfiller can do a single append from it's end datum and
|
||||
# then prepends backward to that from the current time
|
||||
# step.
|
||||
tf2mem: dict = {
|
||||
1: rt_shm,
|
||||
60: hist_shm,
|
||||
}
|
||||
async with open_sample_stream(
|
||||
period_s=1.,
|
||||
shms_by_period={
|
||||
1.: rt_shm.token,
|
||||
60.: hist_shm.token,
|
||||
},
|
||||
|
||||
# NOTE: we want to only open a stream for doing
|
||||
# broadcasts on backfill operations, not receive the
|
||||
# sample index-stream (since there's no code in this
|
||||
# data feed layer that needs to consume it).
|
||||
open_index_stream=True,
|
||||
sub_for_broadcasts=False,
|
||||
|
||||
) as sample_stream:
|
||||
# register 1s and 1m buffers with the global incrementer task
|
||||
log.info(f'Connected to sampler stream: {sample_stream}')
|
||||
|
||||
for timeframe in [60, 1]:
|
||||
await tn.start(
|
||||
tsdb_backfill,
|
||||
mod,
|
||||
storemod,
|
||||
tn,
|
||||
# bus,
|
||||
client,
|
||||
mkt,
|
||||
tf2mem[timeframe],
|
||||
timeframe,
|
||||
|
||||
sample_stream,
|
||||
)
|
||||
|
||||
# indicate to caller that feed can be delivered to
|
||||
# remote requesting client since we've loaded history
|
||||
# data that can be used.
|
||||
some_data_ready.set()
|
||||
|
||||
# wait for a live feed before starting the sampler.
|
||||
await feed_is_live.wait()
|
||||
|
||||
# yield back after client connect with filled shm
|
||||
task_status.started((
|
||||
hist_zero_index,
|
||||
hist_shm,
|
||||
rt_zero_index,
|
||||
rt_shm,
|
||||
))
|
||||
|
||||
# history retreival loop depending on user interaction
|
||||
# and thus a small RPC-prot for remotely controllinlg
|
||||
# what data is loaded for viewing.
|
||||
await trio.sleep_forever()
|
||||
|
|
@ -23,7 +23,7 @@ Api layer likely in here...
|
|||
from types import ModuleType
|
||||
from importlib import import_module
|
||||
|
||||
from ._util import get_logger
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -25,55 +25,50 @@
|
|||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Union,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
import time
|
||||
from math import isnan
|
||||
from pathlib import Path
|
||||
|
||||
from bidict import bidict
|
||||
from msgspec.msgpack import (
|
||||
encode,
|
||||
decode,
|
||||
)
|
||||
# import pyqtgraph as pg
|
||||
import msgpack
|
||||
import pyqtgraph as pg
|
||||
import numpy as np
|
||||
import tractor
|
||||
from trio_websocket import open_websocket_url
|
||||
from anyio_marketstore import ( # noqa
|
||||
from anyio_marketstore import (
|
||||
open_marketstore_client,
|
||||
MarketstoreClient,
|
||||
Params,
|
||||
)
|
||||
import pendulum
|
||||
# TODO: import this for specific error set expected by mkts client
|
||||
# import purerpc
|
||||
|
||||
from ..data.feed import maybe_open_feed
|
||||
from . import Services
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
import purerpc
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import docker
|
||||
from ._ahab import DockerContainer
|
||||
|
||||
from .feed import maybe_open_feed
|
||||
from ..log import get_logger, get_console_log
|
||||
|
||||
|
||||
# ahabd-supervisor and container level config
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# container level config
|
||||
_config = {
|
||||
'grpc_listen_port': 5995,
|
||||
'ws_listen_port': 5993,
|
||||
'log_level': 'debug',
|
||||
'startup_timeout': 2,
|
||||
}
|
||||
|
||||
_yaml_config_str: str = '''
|
||||
_yaml_config = '''
|
||||
# piker's ``marketstore`` config.
|
||||
|
||||
# mount this config using:
|
||||
|
|
@ -92,12 +87,6 @@ stale_threshold: 5
|
|||
enable_add: true
|
||||
enable_remove: false
|
||||
|
||||
# SUPER DUPER CRITICAL to address a super weird issue:
|
||||
# https://github.com/pikers/piker/issues/443
|
||||
# seems like "variable compression" is possibly borked
|
||||
# or snappy compression somehow breaks easily?
|
||||
disable_variable_compression: true
|
||||
|
||||
triggers:
|
||||
- module: ondiskagg.so
|
||||
on: "*/1Sec/OHLCV"
|
||||
|
|
@ -115,18 +104,18 @@ triggers:
|
|||
# config:
|
||||
# filter: "nasdaq"
|
||||
|
||||
'''
|
||||
'''.format(**_config)
|
||||
|
||||
|
||||
def start_marketstore(
|
||||
client: docker.DockerClient,
|
||||
user_config: dict,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tuple[DockerContainer, dict[str, Any]]:
|
||||
'''
|
||||
Start and supervise a marketstore instance with its config
|
||||
bind-mounted in from the piker config directory on the system.
|
||||
Start and supervise a marketstore instance with its config bind-mounted
|
||||
in from the piker config directory on the system.
|
||||
|
||||
The equivalent cli cmd to this code is:
|
||||
|
||||
|
|
@ -142,24 +131,19 @@ def start_marketstore(
|
|||
|
||||
mktsdir = os.path.join(config._config_dir, 'marketstore')
|
||||
|
||||
# create dirs when dne
|
||||
if not os.path.isdir(config._config_dir):
|
||||
Path(config._config_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# create when dne
|
||||
if not os.path.isdir(mktsdir):
|
||||
os.mkdir(mktsdir)
|
||||
|
||||
yml_file = os.path.join(mktsdir, 'mkts.yml')
|
||||
yaml_config = _yaml_config_str.format(**user_config)
|
||||
|
||||
if not os.path.isfile(yml_file):
|
||||
log.warning(
|
||||
f'No `marketstore` config exists?: {yml_file}\n'
|
||||
'Generating new file from template:\n'
|
||||
f'{yaml_config}\n'
|
||||
f'{_yaml_config}\n'
|
||||
)
|
||||
with open(yml_file, 'w') as yf:
|
||||
yf.write(yaml_config)
|
||||
yf.write(_yaml_config)
|
||||
|
||||
# create a mount from user's local piker config dir into container
|
||||
config_dir_mnt = docker.types.Mount(
|
||||
|
|
@ -182,9 +166,6 @@ def start_marketstore(
|
|||
type='bind',
|
||||
)
|
||||
|
||||
grpc_listen_port = int(user_config['grpc_listen_port'])
|
||||
ws_listen_port = int(user_config['ws_listen_port'])
|
||||
|
||||
dcntr: DockerContainer = client.containers.run(
|
||||
'alpacamarkets/marketstore:latest',
|
||||
# do we need this for cmds?
|
||||
|
|
@ -192,88 +173,28 @@ def start_marketstore(
|
|||
|
||||
# '-p 5993:5993',
|
||||
ports={
|
||||
f'{ws_listen_port}/tcp': ws_listen_port,
|
||||
f'{grpc_listen_port}/tcp': grpc_listen_port,
|
||||
'5993/tcp': 5993, # jsonrpc / ws?
|
||||
'5995/tcp': 5995, # grpc
|
||||
},
|
||||
mounts=[
|
||||
config_dir_mnt,
|
||||
data_dir_mnt,
|
||||
],
|
||||
|
||||
# XXX: this must be set to allow backgrounding/non-blocking
|
||||
# usage interaction with the container's process.
|
||||
detach=True,
|
||||
|
||||
# stop_signal='SIGINT',
|
||||
init=True,
|
||||
# remove=True,
|
||||
)
|
||||
|
||||
async def start_matcher(msg: str):
|
||||
return "launching tcp listener for all services..." in msg
|
||||
|
||||
async def stop_matcher(msg: str):
|
||||
return (
|
||||
# not sure when this happens, some kinda stop condition
|
||||
"exiting..." in msg
|
||||
|
||||
# after we send SIGINT..
|
||||
or "initiating graceful shutdown due to 'interrupt' request" in msg
|
||||
)
|
||||
|
||||
return (
|
||||
dcntr,
|
||||
_config,
|
||||
|
||||
# expected startup and stop msgs
|
||||
start_matcher,
|
||||
stop_matcher,
|
||||
"launching tcp listener for all services...",
|
||||
"exiting...",
|
||||
)
|
||||
|
||||
|
||||
@acm
|
||||
async def start_ahab_daemon(
|
||||
service_mngr: Services,
|
||||
user_config: dict | None = None,
|
||||
loglevel: str | None = None,
|
||||
|
||||
) -> tuple[str, dict]:
|
||||
'''
|
||||
Task entrypoint to start the marketstore docker container using the
|
||||
service manager.
|
||||
|
||||
'''
|
||||
from ._ahab import start_ahab_service
|
||||
|
||||
# dict-merge any user settings
|
||||
conf: dict = _config.copy()
|
||||
if user_config:
|
||||
conf: dict = conf | user_config
|
||||
|
||||
dname: str = 'marketstored'
|
||||
log.info(f'Spawning `{dname}` supervisor')
|
||||
async with start_ahab_service(
|
||||
service_mngr,
|
||||
dname,
|
||||
|
||||
# NOTE: docker-py client is passed at runtime
|
||||
start_marketstore,
|
||||
ep_kwargs={'user_config': conf},
|
||||
loglevel=loglevel,
|
||||
) as (
|
||||
_,
|
||||
config,
|
||||
(cid, pid),
|
||||
):
|
||||
log.info(
|
||||
f'`{dname}` up!\n'
|
||||
f'pid: {pid}\n'
|
||||
f'container id: {cid[:12]}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
yield dname, conf
|
||||
|
||||
|
||||
_tick_tbk_ids: tuple[str, str] = ('1Sec', 'TICK')
|
||||
_tick_tbk: str = '{}/' + '/'.join(_tick_tbk_ids)
|
||||
|
||||
|
|
@ -327,6 +248,16 @@ _ohlcv_dt = [
|
|||
]
|
||||
|
||||
|
||||
ohlc_key_map = bidict({
|
||||
'Epoch': 'time',
|
||||
'Open': 'open',
|
||||
'High': 'high',
|
||||
'Low': 'low',
|
||||
'Close': 'close',
|
||||
'Volume': 'volume',
|
||||
})
|
||||
|
||||
|
||||
def mk_tbk(keys: tuple[str, str, str]) -> str:
|
||||
'''
|
||||
Generate a marketstore table key from a tuple.
|
||||
|
|
@ -339,7 +270,7 @@ def mk_tbk(keys: tuple[str, str, str]) -> str:
|
|||
|
||||
def quote_to_marketstore_structarray(
|
||||
quote: dict[str, Any],
|
||||
last_fill: float | None,
|
||||
last_fill: Optional[float]
|
||||
|
||||
) -> np.array:
|
||||
'''
|
||||
|
|
@ -378,6 +309,24 @@ def quote_to_marketstore_structarray(
|
|||
return np.array([tuple(array_input)], dtype=_quote_dt)
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client(
|
||||
host: str = 'localhost',
|
||||
port: int = 5995
|
||||
|
||||
) -> MarketstoreClient:
|
||||
'''
|
||||
Load a ``anyio_marketstore`` grpc client connected
|
||||
to an existing ``marketstore`` server.
|
||||
|
||||
'''
|
||||
async with open_marketstore_client(
|
||||
host,
|
||||
port
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
class MarketStoreError(Exception):
|
||||
"Generic marketstore client error"
|
||||
|
||||
|
|
@ -405,6 +354,340 @@ tf_in_1s = bidict({
|
|||
})
|
||||
|
||||
|
||||
class Storage:
|
||||
'''
|
||||
High level storage api for both real-time and historical ingest.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
client: MarketstoreClient,
|
||||
|
||||
) -> None:
|
||||
# TODO: eventually this should be an api/interface type that
|
||||
# ensures we can support multiple tsdb backends.
|
||||
self.client = client
|
||||
|
||||
# series' cache from tsdb reads
|
||||
self._arrays: dict[str, np.ndarray] = {}
|
||||
|
||||
async def list_keys(self) -> list[str]:
|
||||
return await self.client.list_symbols()
|
||||
|
||||
async def search_keys(self, pattern: str) -> list[str]:
|
||||
'''
|
||||
Search for time series key in the storage backend.
|
||||
|
||||
'''
|
||||
...
|
||||
|
||||
async def write_ticks(self, ticks: list) -> None:
|
||||
...
|
||||
|
||||
async def load(
|
||||
self,
|
||||
fqsn: str,
|
||||
|
||||
) -> tuple[
|
||||
dict[int, np.ndarray], # timeframe (in secs) to series
|
||||
Optional[datetime], # first dt
|
||||
Optional[datetime], # last dt
|
||||
]:
|
||||
|
||||
first_tsdb_dt, last_tsdb_dt = None, None
|
||||
tsdb_arrays = await self.read_ohlcv(
|
||||
fqsn,
|
||||
# on first load we don't need to pull the max
|
||||
# history per request size worth.
|
||||
limit=3000,
|
||||
)
|
||||
log.info(f'Loaded tsdb history {tsdb_arrays}')
|
||||
|
||||
if tsdb_arrays:
|
||||
fastest = list(tsdb_arrays.values())[0]
|
||||
times = fastest['Epoch']
|
||||
first, last = times[0], times[-1]
|
||||
first_tsdb_dt, last_tsdb_dt = map(
|
||||
pendulum.from_timestamp, [first, last]
|
||||
)
|
||||
|
||||
return tsdb_arrays, first_tsdb_dt, last_tsdb_dt
|
||||
|
||||
async def read_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
end: Optional[int] = None,
|
||||
limit: int = int(800e3),
|
||||
|
||||
) -> tuple[
|
||||
MarketstoreClient,
|
||||
Union[dict, np.ndarray]
|
||||
]:
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
|
||||
if fqsn not in syms:
|
||||
return {}
|
||||
|
||||
tfstr = tf_in_1s[1]
|
||||
|
||||
params = Params(
|
||||
symbols=fqsn,
|
||||
timeframe=tfstr,
|
||||
attrgroup='OHLCV',
|
||||
end=end,
|
||||
# limit_from_start=True,
|
||||
|
||||
# TODO: figure the max limit here given the
|
||||
# ``purepc`` msg size limit of purerpc: 33554432
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
if timeframe is None:
|
||||
log.info(f'starting {fqsn} tsdb granularity scan..')
|
||||
# loop through and try to find highest granularity
|
||||
for tfstr in tf_in_1s.values():
|
||||
try:
|
||||
log.info(f'querying for {tfstr}@{fqsn}')
|
||||
params.set('timeframe', tfstr)
|
||||
result = await client.query(params)
|
||||
break
|
||||
|
||||
except purerpc.grpclib.exceptions.UnknownError:
|
||||
# XXX: this is already logged by the container and
|
||||
# thus shows up through `marketstored` logs relay.
|
||||
# log.warning(f'{tfstr}@{fqsn} not found')
|
||||
continue
|
||||
else:
|
||||
return {}
|
||||
|
||||
else:
|
||||
result = await client.query(params)
|
||||
|
||||
# TODO: it turns out column access on recarrays is actually slower:
|
||||
# https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
|
||||
# it might make sense to make these structured arrays?
|
||||
# Fill out a `numpy` array-results map
|
||||
arrays = {}
|
||||
for fqsn, data_set in result.by_symbols().items():
|
||||
arrays.setdefault(fqsn, {})[
|
||||
tf_in_1s.inverse[data_set.timeframe]
|
||||
] = data_set.array
|
||||
|
||||
return arrays[fqsn][timeframe] if timeframe else arrays[fqsn]
|
||||
|
||||
async def delete_ts(
|
||||
self,
|
||||
key: str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
|
||||
) -> bool:
|
||||
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
print(syms)
|
||||
# if key not in syms:
|
||||
# raise KeyError(f'`{fqsn}` table key not found?')
|
||||
|
||||
return await client.destroy(tbk=key)
|
||||
|
||||
async def write_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
ohlcv: np.ndarray,
|
||||
append_and_duplicate: bool = True,
|
||||
limit: int = int(800e3),
|
||||
|
||||
) -> None:
|
||||
# build mkts schema compat array for writing
|
||||
mkts_dt = np.dtype(_ohlcv_dt)
|
||||
mkts_array = np.zeros(
|
||||
len(ohlcv),
|
||||
dtype=mkts_dt,
|
||||
)
|
||||
# copy from shm array (yes it's this easy):
|
||||
# https://numpy.org/doc/stable/user/basics.rec.html#assignment-from-other-structured-arrays
|
||||
mkts_array[:] = ohlcv[[
|
||||
'time',
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
'volume',
|
||||
]]
|
||||
|
||||
m, r = divmod(len(mkts_array), limit)
|
||||
|
||||
for i in range(m, 1):
|
||||
to_push = mkts_array[i-1:i*limit]
|
||||
|
||||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
# TODO: pre deduplicate?
|
||||
isvariablelength=append_and_duplicate,
|
||||
)
|
||||
|
||||
log.info(
|
||||
f'Wrote {mkts_array.size} datums to tsdb\n'
|
||||
)
|
||||
|
||||
for resp in resp.responses:
|
||||
err = resp.error
|
||||
if err:
|
||||
raise MarketStoreError(err)
|
||||
|
||||
if r:
|
||||
to_push = mkts_array[m*limit:]
|
||||
|
||||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
# TODO: pre deduplicate?
|
||||
isvariablelength=append_and_duplicate,
|
||||
)
|
||||
|
||||
log.info(
|
||||
f'Wrote {mkts_array.size} datums to tsdb\n'
|
||||
)
|
||||
|
||||
for resp in resp.responses:
|
||||
err = resp.error
|
||||
if err:
|
||||
raise MarketStoreError(err)
|
||||
|
||||
# XXX: currently the only way to do this is through the CLI:
|
||||
|
||||
# sudo ./marketstore connect --dir ~/.config/piker/data
|
||||
# >> \show mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
|
||||
# and this seems to block and use up mem..
|
||||
# >> \trim mnq.globex.20220617.ib/1Sec/OHLCV 2022-05-15
|
||||
|
||||
# relevant source code for this is here:
|
||||
# https://github.com/alpacahq/marketstore/blob/master/cmd/connect/session/trim.go#L14
|
||||
# def delete_range(self, start_dt, end_dt) -> None:
|
||||
# ...
|
||||
|
||||
@acm
|
||||
async def open_storage_client(
|
||||
fqsn: str,
|
||||
period: Optional[Union[int, str]] = None, # in seconds
|
||||
|
||||
) -> tuple[Storage, dict[str, np.ndarray]]:
|
||||
'''
|
||||
Load a series by key and deliver in ``numpy`` struct array format.
|
||||
|
||||
'''
|
||||
async with (
|
||||
# eventually a storage backend endpoint
|
||||
get_client() as client,
|
||||
):
|
||||
# slap on our wrapper api
|
||||
yield Storage(client)
|
||||
|
||||
|
||||
async def tsdb_history_update(
|
||||
fqsn: Optional[str] = None,
|
||||
|
||||
) -> list[str]:
|
||||
|
||||
# TODO: real-time dedicated task for ensuring
|
||||
# history consistency between the tsdb, shm and real-time feed..
|
||||
|
||||
# update sequence design notes:
|
||||
|
||||
# - load existing highest frequency data from mkts
|
||||
# * how do we want to offer this to the UI?
|
||||
# - lazy loading?
|
||||
# - try to load it all and expect graphics caching/diffing
|
||||
# to hide extra bits that aren't in view?
|
||||
|
||||
# - compute the diff between latest data from broker and shm
|
||||
# * use sql api in mkts to determine where the backend should
|
||||
# start querying for data?
|
||||
# * append any diff with new shm length
|
||||
# * determine missing (gapped) history by scanning
|
||||
# * how far back do we look?
|
||||
|
||||
# - begin rt update ingest and aggregation
|
||||
# * could start by always writing ticks to mkts instead of
|
||||
# worrying about a shm queue for now.
|
||||
# * we have a short list of shm queues worth groking:
|
||||
# - https://github.com/pikers/piker/issues/107
|
||||
# * the original data feed arch blurb:
|
||||
# - https://github.com/pikers/piker/issues/98
|
||||
#
|
||||
profiler = pg.debug.Profiler(
|
||||
disabled=False, # not pg_profile_enabled(),
|
||||
delayed=False,
|
||||
)
|
||||
|
||||
async with (
|
||||
open_storage_client(fqsn) as storage,
|
||||
|
||||
maybe_open_feed(
|
||||
[fqsn],
|
||||
start_stream=False,
|
||||
|
||||
) as (feed, stream),
|
||||
):
|
||||
profiler(f'opened feed for {fqsn}')
|
||||
|
||||
to_append = feed.shm.array
|
||||
to_prepend = None
|
||||
|
||||
if fqsn:
|
||||
symbol = feed.symbols.get(fqsn)
|
||||
if symbol:
|
||||
fqsn = symbol.front_fqsn()
|
||||
|
||||
# diff db history with shm and only write the missing portions
|
||||
ohlcv = feed.shm.array
|
||||
|
||||
# TODO: use pg profiler
|
||||
tsdb_arrays = await storage.read_ohlcv(fqsn)
|
||||
# hist diffing
|
||||
if tsdb_arrays:
|
||||
for secs in (1, 60):
|
||||
ts = tsdb_arrays.get(secs)
|
||||
if ts is not None and len(ts):
|
||||
# these aren't currently used but can be referenced from
|
||||
# within the embedded ipython shell below.
|
||||
to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
|
||||
to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
|
||||
|
||||
profiler('Finished db arrays diffs')
|
||||
|
||||
syms = await storage.client.list_symbols()
|
||||
log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
|
||||
profiler(f'listed symbols {syms}')
|
||||
|
||||
# TODO: ask if user wants to write history for detected
|
||||
# available shm buffers?
|
||||
from tractor.trionics import ipython_embed
|
||||
await ipython_embed()
|
||||
|
||||
# for array in [to_append, to_prepend]:
|
||||
# if array is None:
|
||||
# continue
|
||||
|
||||
# log.info(
|
||||
# f'Writing datums {array.size} -> to tsdb from shm\n'
|
||||
# )
|
||||
# await storage.write_ohlcv(fqsn, array)
|
||||
|
||||
# profiler('Finished db writes')
|
||||
|
||||
|
||||
async def ingest_quote_stream(
|
||||
symbols: list[str],
|
||||
brokername: str,
|
||||
|
|
@ -416,7 +699,6 @@ async def ingest_quote_stream(
|
|||
Ingest a broker quote stream into a ``marketstore`` tsdb.
|
||||
|
||||
'''
|
||||
from piker.storage.marketstore import get_client
|
||||
async with (
|
||||
maybe_open_feed(brokername, symbols, loglevel=loglevel) as feed,
|
||||
get_client() as ms_client,
|
||||
|
|
@ -492,13 +774,12 @@ async def stream_quotes(
|
|||
async with open_websocket_url(f'ws://{host}:{port}/ws') as ws:
|
||||
# send subs topics to server
|
||||
resp = await ws.send_message(
|
||||
|
||||
encode({'streams': list(tbks.values())})
|
||||
msgpack.dumps({'streams': list(tbks.values())})
|
||||
)
|
||||
log.info(resp)
|
||||
|
||||
async def recv() -> dict[str, Any]:
|
||||
return decode((await ws.get_message()), encoding='utf-8')
|
||||
return msgpack.loads((await ws.get_message()), encoding='utf-8')
|
||||
|
||||
streams = (await recv())['streams']
|
||||
log.info(f"Subscribed to {streams}")
|
||||
|
|
@ -1,173 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Tick event stream processing, filter-by-types, format-normalization.
|
||||
|
||||
'''
|
||||
from itertools import chain
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
)
|
||||
|
||||
# tick-type-classes template for all possible "lowest level" events
|
||||
# that can can be emitted by the "top of book" L1 queues and
|
||||
# price-matching (with eventual clearing) in a double auction
|
||||
# market (queuing) system.
|
||||
_tick_groups: dict[str, set[str]] = {
|
||||
'clears': {'trade', 'dark_trade', 'last'},
|
||||
'bids': {'bid', 'bsize'},
|
||||
'asks': {'ask', 'asize'},
|
||||
}
|
||||
|
||||
# XXX alo define the flattened set of all such "fundamental ticks"
|
||||
# so that it can be used as filter, eg. in the graphics display
|
||||
# loop to compute running windowed y-ranges B)
|
||||
_auction_ticks: set[str] = set.union(*_tick_groups.values())
|
||||
|
||||
|
||||
def frame_ticks(
|
||||
quote: dict[str, Any],
|
||||
|
||||
ticks_by_type: dict | None = None,
|
||||
ticks_in_order: list[dict[str, Any]] | None = None
|
||||
|
||||
) -> dict[
|
||||
str,
|
||||
list[dict[str, Any]]
|
||||
]:
|
||||
'''
|
||||
XXX: build a tick-by-type table of lists
|
||||
of tick messages. This allows for less
|
||||
iteration on the receiver side by allowing for
|
||||
a single "latest tick event" look up by
|
||||
indexing the last entry in each sub-list.
|
||||
|
||||
tbt = {
|
||||
'types': ['bid', 'asize', 'last', .. '<type_n>'],
|
||||
|
||||
'bid': [tick0, tick1, tick2, .., tickn],
|
||||
'asize': [tick0, tick1, tick2, .., tickn],
|
||||
'last': [tick0, tick1, tick2, .., tickn],
|
||||
...
|
||||
'<type_n>': [tick0, tick1, tick2, .., tickn],
|
||||
}
|
||||
|
||||
If `ticks_in_order` is provided, append any retrieved ticks
|
||||
since last iteration into this array/buffer/list.
|
||||
|
||||
'''
|
||||
# TODO: once we decide to get fancy really we should
|
||||
# have a shared mem tick buffer that is just
|
||||
# continually filled and the UI just ready from it
|
||||
# at it's display rate.
|
||||
|
||||
tbt = ticks_by_type if ticks_by_type is not None else {}
|
||||
if not (ticks := quote.get('ticks')):
|
||||
return tbt
|
||||
|
||||
# append in reverse FIFO order for in-order iteration on
|
||||
# receiver side.
|
||||
tick: dict[str, Any]
|
||||
for tick in ticks:
|
||||
tbt.setdefault(
|
||||
tick['type'],
|
||||
[],
|
||||
).append(tick)
|
||||
|
||||
# TODO: do we need this any more or can we just
|
||||
# expect the receiver to unwind the below
|
||||
# `ticks_by_type: dict`?
|
||||
# => undwinding would potentially require a
|
||||
# `dict[str, set | list]` instead with an
|
||||
# included `'types' field which is an (ordered)
|
||||
# set of tick type fields in the order which
|
||||
# types arrived?
|
||||
if ticks_in_order:
|
||||
ticks_in_order.extend(ticks)
|
||||
|
||||
return tbt
|
||||
|
||||
|
||||
def iterticks(
|
||||
quote: dict,
|
||||
types: tuple[str] = (
|
||||
'trade',
|
||||
'dark_trade',
|
||||
),
|
||||
deduplicate_darks: bool = False,
|
||||
reverse: bool = False,
|
||||
|
||||
# TODO: should we offer delegating to `frame_ticks()` above
|
||||
# with this?
|
||||
frame_by_type: bool = False,
|
||||
|
||||
) -> AsyncIterator:
|
||||
'''
|
||||
Iterate through ticks delivered per quote cycle, filter and
|
||||
yield any declared in `types`.
|
||||
|
||||
'''
|
||||
if deduplicate_darks:
|
||||
assert 'dark_trade' in types
|
||||
|
||||
# print(f"{quote}\n\n")
|
||||
ticks = quote.get('ticks', ())
|
||||
trades = {}
|
||||
darks = {}
|
||||
|
||||
if ticks:
|
||||
|
||||
# do a first pass and attempt to remove duplicate dark
|
||||
# trades with the same tick signature.
|
||||
if deduplicate_darks:
|
||||
for tick in ticks:
|
||||
ttype = tick.get('type')
|
||||
|
||||
time = tick.get('time', None)
|
||||
if time:
|
||||
sig = (
|
||||
time,
|
||||
tick['price'],
|
||||
tick.get('size')
|
||||
)
|
||||
|
||||
if ttype == 'dark_trade':
|
||||
darks[sig] = tick
|
||||
|
||||
elif ttype == 'trade':
|
||||
trades[sig] = tick
|
||||
|
||||
# filter duplicates
|
||||
for sig, tick in trades.items():
|
||||
tick = darks.pop(sig, None)
|
||||
if tick:
|
||||
ticks.remove(tick)
|
||||
# print(f'DUPLICATE {tick}')
|
||||
|
||||
# re-insert ticks
|
||||
ticks.extend(list(chain(trades.values(), darks.values())))
|
||||
|
||||
# most-recent-first
|
||||
if reverse:
|
||||
ticks = reversed(ticks)
|
||||
|
||||
for tick in ticks:
|
||||
# print(f"{quote['symbol']}: {tick}")
|
||||
ttype = tick.get('type')
|
||||
if ttype in types:
|
||||
yield tick
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) (in stewardship for pikers)
|
||||
# - Tyler Goodlet
|
||||
# - Guillermo Rodriguez
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Extensions to built-in or (heavily used but 3rd party) friend-lib
|
||||
types.
|
||||
|
||||
'''
|
||||
from pprint import pformat
|
||||
|
||||
from msgspec import (
|
||||
msgpack,
|
||||
Struct,
|
||||
structs,
|
||||
)
|
||||
|
||||
|
||||
class Struct(
|
||||
Struct,
|
||||
|
||||
# https://jcristharif.com/msgspec/structs.html#tagged-unions
|
||||
# tag='pikerstruct',
|
||||
# tag=True,
|
||||
):
|
||||
'''
|
||||
A "human friendlier" (aka repl buddy) struct subtype.
|
||||
|
||||
'''
|
||||
def to_dict(self) -> dict:
|
||||
'''
|
||||
Like it sounds.. direct delegation to:
|
||||
https://jcristharif.com/msgspec/api.html#msgspec.structs.asdict
|
||||
|
||||
TODO: probably just drop this method since it's now a built-int method?
|
||||
|
||||
'''
|
||||
return structs.asdict(self)
|
||||
|
||||
def pformat(self) -> str:
|
||||
return f'Struct({pformat(self.to_dict())})'
|
||||
|
||||
def copy(
|
||||
self,
|
||||
update: dict | None = None,
|
||||
|
||||
) -> Struct:
|
||||
'''
|
||||
Validate-typecast all self defined fields, return a copy of
|
||||
us with all such fields.
|
||||
|
||||
NOTE: This is kinda like the default behaviour in
|
||||
`pydantic.BaseModel` except a copy of the object is
|
||||
returned making it compat with `frozen=True`.
|
||||
|
||||
'''
|
||||
if update:
|
||||
for k, v in update.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
# NOTE: roundtrip serialize to validate
|
||||
# - enode to msgpack binary format,
|
||||
# - decode that back to a struct.
|
||||
return msgpack.Decoder(type=type(self)).decode(
|
||||
msgpack.Encoder().encode(self)
|
||||
)
|
||||
|
||||
def typecast(
|
||||
self,
|
||||
|
||||
# TODO: allow only casting a named subset?
|
||||
# fields: set[str] | None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Cast all fields using their declared type annotations
|
||||
(kinda like what `pydantic` does by default).
|
||||
|
||||
NOTE: this of course won't work on frozen types, use
|
||||
``.copy()`` above in such cases.
|
||||
|
||||
'''
|
||||
# https://jcristharif.com/msgspec/api.html#msgspec.structs.fields
|
||||
fi: structs.FieldInfo
|
||||
for fi in structs.fields(self):
|
||||
setattr(
|
||||
self,
|
||||
fi.name,
|
||||
fi.type(getattr(self, fi.name)),
|
||||
)
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
Data feed synchronization protocols, init msgs, and general
|
||||
data-provider-backend-agnostic schema definitions.
|
||||
|
||||
'''
|
||||
from decimal import Decimal
|
||||
from pprint import pformat
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
)
|
||||
|
||||
from msgspec import field
|
||||
|
||||
from .types import Struct
|
||||
from ..accounting import (
|
||||
Asset,
|
||||
MktPair,
|
||||
)
|
||||
from ._util import log
|
||||
|
||||
|
||||
class FeedInitializationError(ValueError):
|
||||
'''
|
||||
Live data feed setup failed due to API / msg incompatiblity!
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class FeedInit(Struct, frozen=True):
|
||||
'''
|
||||
A stringent data provider startup msg schema validator.
|
||||
|
||||
The fields defined here are matched with those absolutely required
|
||||
from each backend broker/data provider.
|
||||
|
||||
'''
|
||||
mkt_info: MktPair
|
||||
|
||||
# NOTE: only field we use rn in ``.data.feed``
|
||||
# TODO: maybe make a SamplerConfig(Struct)?
|
||||
shm_write_opts: dict[str, Any] = field(
|
||||
default_factory=lambda: {
|
||||
'has_vlm': True,
|
||||
'sum_tick_vlm': True,
|
||||
})
|
||||
|
||||
# XXX: we group backend endpoints into 3
|
||||
# groups to determine "degrees" of functionality.
|
||||
_eps: dict[str, list[str]] = {
|
||||
|
||||
# basic API `Client` layer
|
||||
'middleware': [
|
||||
'get_client',
|
||||
],
|
||||
|
||||
# (live) data streaming / loading / search
|
||||
'datad': [
|
||||
'get_mkt_info',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
],
|
||||
|
||||
# live order control and trading
|
||||
'brokerd': [
|
||||
'trades_dialogue',
|
||||
# TODO: ledger normalizer helper?
|
||||
# norm_trades(records: dict[str, Any]) -> TransactionLedger)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def validate_backend(
|
||||
mod: ModuleType,
|
||||
syms: list[str],
|
||||
init_msgs: list[FeedInit] | dict[str, dict[str, Any]],
|
||||
|
||||
# TODO: do a module method scan and report mismatches.
|
||||
check_eps: bool = False,
|
||||
|
||||
api_log_msg_level: str = 'critical'
|
||||
|
||||
) -> FeedInit:
|
||||
'''
|
||||
Fail on malformed live quotes feed config/init or warn on changes
|
||||
that haven't been implemented by this backend yet.
|
||||
|
||||
'''
|
||||
for daemon_name, eps in _eps.items():
|
||||
for name in eps:
|
||||
ep: Callable = getattr(
|
||||
mod,
|
||||
name,
|
||||
None,
|
||||
)
|
||||
if ep is None:
|
||||
log.warning(
|
||||
f'Provider backend {mod.name} is missing '
|
||||
f'{daemon_name} support :(\n'
|
||||
f'The following endpoint is missing: {name}'
|
||||
)
|
||||
|
||||
inits: list[
|
||||
FeedInit | dict[str, Any]
|
||||
] = init_msgs
|
||||
|
||||
# convert to list if from old dict-style
|
||||
if isinstance(init_msgs, dict):
|
||||
inits = list(init_msgs.values())
|
||||
|
||||
init: FeedInit | dict[str, Any]
|
||||
for i, init in enumerate(inits):
|
||||
|
||||
# XXX: eventually this WILL NOT necessarily be true.
|
||||
if i > 0:
|
||||
assert not len(init_msgs) == 1
|
||||
if isinstance(init_msgs, dict):
|
||||
keys: set = set(init_msgs.keys()) - set(syms)
|
||||
raise FeedInitializationError(
|
||||
'TOO MANY INIT MSGS!\n'
|
||||
f'Unexpected keys: {keys}\n'
|
||||
'ALL MSGS:\n'
|
||||
f'{pformat(init_msgs)}\n'
|
||||
)
|
||||
else:
|
||||
raise FeedInitializationError(
|
||||
'TOO MANY INIT MSGS!\n'
|
||||
f'{pformat(init_msgs)}\n'
|
||||
)
|
||||
|
||||
# TODO: once all backends are updated we can remove this branching.
|
||||
rx_msg: bool = False
|
||||
warn_msg: str = ''
|
||||
if not isinstance(init, FeedInit):
|
||||
warn_msg += (
|
||||
'\n'
|
||||
'--------------------------\n'
|
||||
':::DEPRECATED API STYLE:::\n'
|
||||
'--------------------------\n'
|
||||
f'`{mod.name}.stream_quotes()` should deliver '
|
||||
'`.started(FeedInit)`\n'
|
||||
f'|-> CURRENTLY it is using DEPRECATED `.started(dict)` style!\n'
|
||||
f'|-> SEE `FeedInit` in `piker.data.validate`\n'
|
||||
'--------------------------------------------\n'
|
||||
)
|
||||
else:
|
||||
rx_msg = True
|
||||
|
||||
# verify feed init state / schema
|
||||
bs_fqme: str # backend specific fqme
|
||||
mkt: MktPair
|
||||
|
||||
match init:
|
||||
|
||||
# backend is using old dict msg delivery
|
||||
case {
|
||||
'symbol_info': dict(symbol_info),
|
||||
'fqsn': bs_fqme,
|
||||
} | {
|
||||
'mkt_info': dict(symbol_info),
|
||||
'fqsn': bs_fqme,
|
||||
}:
|
||||
symbol_info: dict
|
||||
warn_msg += (
|
||||
'It may also be still using the legacy `Symbol` style API\n'
|
||||
'IT SHOULD BE PORTED TO THE NEW '
|
||||
'`.accounting._mktinfo.MktPair`\n'
|
||||
'STATTTTT!!!\n'
|
||||
)
|
||||
|
||||
# XXX use default legacy (aka discrete precision) mkt
|
||||
# price/size_ticks if none delivered.
|
||||
price_tick = symbol_info.get(
|
||||
'price_tick_size',
|
||||
Decimal('0.01'),
|
||||
)
|
||||
size_tick = symbol_info.get(
|
||||
'lot_tick_size',
|
||||
Decimal('1'),
|
||||
)
|
||||
bs_mktid = init.get('bs_mktid') or bs_fqme
|
||||
mkt = MktPair.from_fqme(
|
||||
fqme=f'{bs_fqme}.{mod.name}',
|
||||
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
|
||||
bs_mktid=str(bs_mktid),
|
||||
_atype=symbol_info['asset_type']
|
||||
)
|
||||
|
||||
# backend is using new `MktPair` but not entirely
|
||||
case {
|
||||
'mkt_info': MktPair(
|
||||
dst=Asset(),
|
||||
) as mkt,
|
||||
'fqsn': bs_fqme,
|
||||
}:
|
||||
warn_msg += (
|
||||
f'{mod.name} in API compat transition?\n'
|
||||
"It's half dict, half man..\n"
|
||||
'-------------------------------------\n'
|
||||
)
|
||||
|
||||
case FeedInit(
|
||||
mkt_info=MktPair(dst=Asset()) as mkt,
|
||||
shm_write_opts=dict(shm_opts),
|
||||
) as init:
|
||||
name: str = mod.name
|
||||
log.info(
|
||||
f"{name}'s `MktPair` info:\n"
|
||||
f'{pformat(mkt.to_dict())}\n'
|
||||
f'shm conf: {pformat(shm_opts)}\n'
|
||||
)
|
||||
|
||||
case _:
|
||||
raise FeedInitializationError(init)
|
||||
|
||||
# build a msg if we received a dict for input.
|
||||
if not rx_msg:
|
||||
assert bs_fqme in mkt.fqme
|
||||
init = FeedInit(
|
||||
mkt_info=mkt,
|
||||
shm_write_opts=init.get('shm_write_opts'),
|
||||
)
|
||||
|
||||
# `MktPair` value audits
|
||||
mkt = init.mkt_info
|
||||
assert mkt.type_key
|
||||
|
||||
# backend is using new `MktPair` but not embedded `Asset` types
|
||||
# for the .src/.dst..
|
||||
if not isinstance(mkt.src, Asset):
|
||||
warn_msg += (
|
||||
f'ALSO, {mod.name.upper()} should try to deliver\n'
|
||||
'the new `MktPair.src: Asset` field!\n'
|
||||
'-----------------------------------------------\n'
|
||||
)
|
||||
|
||||
# complain about any non-idealities
|
||||
if warn_msg:
|
||||
# TODO: would be nice to register an API_COMPAT or something in
|
||||
# maybe cyan for this in general throughput piker no?
|
||||
logmeth = getattr(log, api_log_msg_level)
|
||||
logmeth(warn_msg)
|
||||
|
||||
return init.copy()
|
||||
|
|
@ -78,8 +78,7 @@ class Fsp:
|
|||
# + the consuming fsp *to* the consumers output
|
||||
# shm flow.
|
||||
_flow_registry: dict[
|
||||
tuple[_Token, str],
|
||||
tuple[_Token, Optional[ShmArray]],
|
||||
tuple[_Token, str], _Token,
|
||||
] = {}
|
||||
|
||||
def __init__(
|
||||
|
|
@ -121,6 +120,7 @@ class Fsp:
|
|||
):
|
||||
return self.func(*args, **kwargs)
|
||||
|
||||
# TODO: lru_cache this? prettty sure it'll work?
|
||||
def get_shm(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
|
|
@ -131,27 +131,12 @@ class Fsp:
|
|||
for this "instance" of a signal processor for
|
||||
the given ``key``.
|
||||
|
||||
The destination shm "token" and array are cached if possible to
|
||||
minimize multiple stdlib/system calls.
|
||||
|
||||
'''
|
||||
dst_token, maybe_array = self._flow_registry[
|
||||
dst_token = self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
]
|
||||
if maybe_array is None:
|
||||
self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
] = (
|
||||
dst_token,
|
||||
# "cache" the ``ShmArray`` such that
|
||||
# we call the underlying "attach" code as few
|
||||
# times as possible as per:
|
||||
# - https://github.com/pikers/piker/issues/359
|
||||
# - https://github.com/pikers/piker/issues/332
|
||||
maybe_array := attach_shm_array(dst_token)
|
||||
)
|
||||
|
||||
return maybe_array
|
||||
shm = attach_shm_array(dst_token)
|
||||
return shm
|
||||
|
||||
|
||||
def fsp(
|
||||
|
|
@ -174,10 +159,18 @@ def fsp(
|
|||
return Fsp(wrapped, outputs=(wrapped.__name__,))
|
||||
|
||||
|
||||
def mk_fsp_shm_key(
|
||||
sym: str,
|
||||
target: Fsp
|
||||
|
||||
) -> str:
|
||||
uid = tractor.current_actor().uid
|
||||
return f'{sym}.fsp.{target.name}.{".".join(uid)}'
|
||||
|
||||
|
||||
def maybe_mk_fsp_shm(
|
||||
sym: str,
|
||||
target: Fsp,
|
||||
size: int,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> (str, ShmArray, bool):
|
||||
|
|
@ -186,27 +179,20 @@ def maybe_mk_fsp_shm(
|
|||
exists, otherwise load the shm already existing for that token.
|
||||
|
||||
'''
|
||||
if not isinstance(sym, str):
|
||||
raise ValueError('`sym: str` should be file-name-friendly')
|
||||
assert isinstance(sym, str), '`sym` should be file-name-friendly `str`'
|
||||
|
||||
# TODO: load output types from `Fsp`
|
||||
# - should `index` be a required internal field?
|
||||
fsp_dtype = np.dtype(
|
||||
[('index', int)]
|
||||
+
|
||||
[('time', float)]
|
||||
+
|
||||
[('index', int)] +
|
||||
[(field_name, float) for field_name in target.outputs]
|
||||
)
|
||||
|
||||
# (attempt to) uniquely key the fsp shm buffers
|
||||
actor_name, uuid = tractor.current_actor().uid
|
||||
uuid_snip: str = uuid[:16]
|
||||
key: str = f'piker.{actor_name}[{uuid_snip}].{sym}.{target.name}'
|
||||
key = mk_fsp_shm_key(sym, target)
|
||||
|
||||
shm, opened = maybe_open_shm_array(
|
||||
key,
|
||||
size=size,
|
||||
# TODO: create entry for each time frame
|
||||
dtype=fsp_dtype,
|
||||
readonly=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ core task logic for processing chains
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import (
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Optional,
|
||||
AsyncIterator, Callable, Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
|
@ -36,22 +35,14 @@ from tractor.msg import NamespacePath
|
|||
from ..log import get_logger, get_console_log
|
||||
from .. import data
|
||||
from ..data import attach_shm_array
|
||||
from ..data.feed import (
|
||||
Flume,
|
||||
Feed,
|
||||
)
|
||||
from ..data.feed import Feed
|
||||
from ..data._sharedmem import ShmArray
|
||||
from ..data._sampling import (
|
||||
_default_delay_s,
|
||||
open_sample_stream,
|
||||
)
|
||||
from ..accounting import MktPair
|
||||
from ..data._source import Symbol
|
||||
from ._api import (
|
||||
Fsp,
|
||||
_load_builtins,
|
||||
_Token,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -85,8 +76,8 @@ async def filter_quotes_by_sym(
|
|||
|
||||
async def fsp_compute(
|
||||
|
||||
mkt: MktPair,
|
||||
flume: Flume,
|
||||
symbol: Symbol,
|
||||
feed: Feed,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
|
||||
src: ShmArray,
|
||||
|
|
@ -99,32 +90,31 @@ async def fsp_compute(
|
|||
|
||||
) -> None:
|
||||
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=True
|
||||
)
|
||||
|
||||
fqme = mkt.fqme
|
||||
fqsn = symbol.front_fqsn()
|
||||
out_stream = func(
|
||||
|
||||
# TODO: do we even need this if we do the feed api right?
|
||||
# shouldn't a local stream do this before we get a handle
|
||||
# to the async iterable? it's that or we do some kinda
|
||||
# async itertools style?
|
||||
filter_quotes_by_sym(fqme, quote_stream),
|
||||
filter_quotes_by_sym(fqsn, quote_stream),
|
||||
|
||||
# XXX: currently the ``ohlcv`` arg
|
||||
flume.rt_shm,
|
||||
feed.shm,
|
||||
)
|
||||
|
||||
# HISTORY COMPUTE PHASE
|
||||
# conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output.
|
||||
# Conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output
|
||||
history_output: Union[
|
||||
dict[str, np.ndarray], # multi-output case
|
||||
np.ndarray, # single output case
|
||||
]
|
||||
history_output = await anext(out_stream)
|
||||
history_output = await out_stream.__anext__()
|
||||
|
||||
func_name = func.__name__
|
||||
profiler(f'{func_name} generated history')
|
||||
|
|
@ -136,13 +126,9 @@ async def fsp_compute(
|
|||
# each respective field.
|
||||
fields = getattr(dst.array.dtype, 'fields', None).copy()
|
||||
fields.pop('index')
|
||||
history_by_field: Optional[np.ndarray] = None
|
||||
src_time = src.array['time']
|
||||
history: Optional[np.ndarray] = None # TODO: nptyping here!
|
||||
|
||||
if (
|
||||
fields and
|
||||
len(fields) > 1
|
||||
):
|
||||
if fields and len(fields) > 1 and fields:
|
||||
if not isinstance(history_output, dict):
|
||||
raise ValueError(
|
||||
f'`{func_name}` is a multi-output FSP and should yield a '
|
||||
|
|
@ -153,7 +139,7 @@ async def fsp_compute(
|
|||
if key in history_output:
|
||||
output = history_output[key]
|
||||
|
||||
if history_by_field is None:
|
||||
if history is None:
|
||||
|
||||
if output is None:
|
||||
length = len(src.array)
|
||||
|
|
@ -163,7 +149,7 @@ async def fsp_compute(
|
|||
# using the first output, determine
|
||||
# the length of the struct-array that
|
||||
# will be pushed to shm.
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
length,
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
|
|
@ -171,7 +157,7 @@ async def fsp_compute(
|
|||
if output is None:
|
||||
continue
|
||||
|
||||
history_by_field[key] = output
|
||||
history[key] = output
|
||||
|
||||
# single-key output stream
|
||||
else:
|
||||
|
|
@ -180,15 +166,11 @@ async def fsp_compute(
|
|||
f'`{func_name}` is a single output FSP and should yield an '
|
||||
'`np.ndarray` for history'
|
||||
)
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
len(history_output),
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
history_by_field[func_name] = history_output
|
||||
|
||||
history_by_field['time'] = src_time[-len(history_by_field):]
|
||||
|
||||
history_output['time'] = src.array['time']
|
||||
history[func_name] = history_output
|
||||
|
||||
# TODO: XXX:
|
||||
# THERE'S A BIG BUG HERE WITH THE `index` field since we're
|
||||
|
|
@ -205,10 +187,7 @@ async def fsp_compute(
|
|||
|
||||
# TODO: can we use this `start` flag instead of the manual
|
||||
# setting above?
|
||||
index = dst.push(
|
||||
history_by_field,
|
||||
start=first,
|
||||
)
|
||||
index = dst.push(history, start=first)
|
||||
|
||||
profiler(f'{func_name} pushed history')
|
||||
profiler.finish()
|
||||
|
|
@ -234,14 +213,8 @@ async def fsp_compute(
|
|||
|
||||
log.debug(f"{func_name}: {processed}")
|
||||
key, output = processed
|
||||
# dst.array[-1][key] = output
|
||||
dst.array[[key, 'time']][-1] = (
|
||||
output,
|
||||
# TODO: what about pushing ``time.time_ns()``
|
||||
# in which case we'll need to round at the graphics
|
||||
# processing / sampling layer?
|
||||
src.array[-1]['time']
|
||||
)
|
||||
index = src.index
|
||||
dst.array[-1][key] = output
|
||||
|
||||
# NOTE: for now we aren't streaming this to the consumer
|
||||
# stream latest array index entry which basically just acts
|
||||
|
|
@ -252,7 +225,6 @@ async def fsp_compute(
|
|||
# N-consumers who subscribe for the real-time output,
|
||||
# which we'll likely want to implement using local-mem
|
||||
# chans for the fan out?
|
||||
# index = src.index
|
||||
# if attach_stream:
|
||||
# await client_stream.send(index)
|
||||
|
||||
|
|
@ -271,7 +243,7 @@ async def cascade(
|
|||
ctx: tractor.Context,
|
||||
|
||||
# data feed key
|
||||
fqme: str,
|
||||
fqsn: str,
|
||||
|
||||
src_shm_token: dict,
|
||||
dst_shm_token: tuple[str, np.dtype],
|
||||
|
|
@ -289,7 +261,7 @@ async def cascade(
|
|||
destination shm array buffer.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=False
|
||||
)
|
||||
|
|
@ -312,10 +284,9 @@ async def cascade(
|
|||
# TODO: ugh i hate this wind/unwind to list over the wire
|
||||
# but not sure how else to do it.
|
||||
for (token, fsp_name, dst_token) in shm_registry:
|
||||
Fsp._flow_registry[(
|
||||
_Token.from_msg(token),
|
||||
fsp_name,
|
||||
)] = _Token.from_msg(dst_token), None
|
||||
Fsp._flow_registry[
|
||||
(_Token.from_msg(token), fsp_name)
|
||||
] = _Token.from_msg(dst_token)
|
||||
|
||||
fsp: Fsp = reg.get(
|
||||
NamespacePath(ns_path)
|
||||
|
|
@ -327,9 +298,8 @@ async def cascade(
|
|||
raise ValueError(f'Unknown fsp target: {ns_path}')
|
||||
|
||||
# open a data feed stream with requested broker
|
||||
feed: Feed
|
||||
async with data.feed.maybe_open_feed(
|
||||
[fqme],
|
||||
[fqsn],
|
||||
|
||||
# TODO throttle tick outputs from *this* daemon since
|
||||
# it'll emit tons of ticks due to the throttle only
|
||||
|
|
@ -337,13 +307,14 @@ async def cascade(
|
|||
# needs to get throttled the ticks we generate.
|
||||
# tick_throttle=60,
|
||||
|
||||
) as feed:
|
||||
) as (feed, quote_stream):
|
||||
symbol = feed.symbols[fqsn]
|
||||
|
||||
flume = feed.flumes[fqme]
|
||||
mkt = flume.mkt
|
||||
assert src.token == flume.rt_shm.token
|
||||
profiler(f'{func}: feed up')
|
||||
|
||||
assert src.token == feed.shm.token
|
||||
# last_len = new_len = len(src.array)
|
||||
|
||||
func_name = func.__name__
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
|
|
@ -352,9 +323,9 @@ async def cascade(
|
|||
fsp_target = partial(
|
||||
|
||||
fsp_compute,
|
||||
mkt=mkt,
|
||||
flume=flume,
|
||||
quote_stream=flume.stream,
|
||||
symbol=symbol,
|
||||
feed=feed,
|
||||
quote_stream=quote_stream,
|
||||
|
||||
# shm
|
||||
src=src,
|
||||
|
|
@ -390,7 +361,7 @@ async def cascade(
|
|||
) -> tuple[TaskTracker, int]:
|
||||
# TODO: adopt an incremental update engine/approach
|
||||
# where possible here eventually!
|
||||
log.info(f're-syncing fsp {func_name} to source')
|
||||
log.debug(f're-syncing fsp {func_name} to source')
|
||||
tracker.cs.cancel()
|
||||
await tracker.complete.wait()
|
||||
tracker, index = await n.start(fsp_target)
|
||||
|
|
@ -403,16 +374,14 @@ async def cascade(
|
|||
'key': dst_shm_token,
|
||||
'first': dst._first.value,
|
||||
'last': dst._last.value,
|
||||
}
|
||||
})
|
||||
}})
|
||||
return tracker, index
|
||||
|
||||
def is_synced(
|
||||
src: ShmArray,
|
||||
dst: ShmArray
|
||||
) -> tuple[bool, int, int]:
|
||||
'''
|
||||
Predicate to dertmine if a destination FSP
|
||||
'''Predicate to dertmine if a destination FSP
|
||||
output array is aligned to its source array.
|
||||
|
||||
'''
|
||||
|
|
@ -421,15 +390,16 @@ async def cascade(
|
|||
return not (
|
||||
# the source is likely backfilling and we must
|
||||
# sync history calculations
|
||||
len_diff > 2
|
||||
len_diff > 2 or
|
||||
|
||||
# we aren't step synced to the source and may be
|
||||
# leading/lagging by a step
|
||||
or step_diff > 1
|
||||
or step_diff < 0
|
||||
step_diff > 1 or
|
||||
step_diff < 0
|
||||
), step_diff, len_diff
|
||||
|
||||
async def poll_and_sync_to_step(
|
||||
|
||||
tracker: TaskTracker,
|
||||
src: ShmArray,
|
||||
dst: ShmArray,
|
||||
|
|
@ -448,23 +418,18 @@ async def cascade(
|
|||
# detect sample period step for subscription to increment
|
||||
# signal
|
||||
times = src.array['time']
|
||||
if len(times) > 1:
|
||||
last_ts = times[-1]
|
||||
delay_s = float(last_ts - times[times != last_ts][-1])
|
||||
else:
|
||||
# our default "HFT" sample rate.
|
||||
delay_s = _default_delay_s
|
||||
delay_s = times[-1] - times[times != times[-1]][-1]
|
||||
|
||||
# sub and increment the underlying shared memory buffer
|
||||
# on every step msg received from the global `samplerd`
|
||||
# service.
|
||||
async with open_sample_stream(float(delay_s)) as istream:
|
||||
# Increment the underlying shared memory buffer on every
|
||||
# "increment" msg received from the underlying data feed.
|
||||
async with feed.index_stream(
|
||||
int(delay_s)
|
||||
) as istream:
|
||||
|
||||
profiler(f'{func_name}: sample stream up')
|
||||
profiler.finish()
|
||||
|
||||
async for i in istream:
|
||||
# print(f'FSP incrementing {i}')
|
||||
async for _ in istream:
|
||||
|
||||
# respawn the compute task if the source
|
||||
# array has been updated such that we compute
|
||||
|
|
@ -493,23 +458,3 @@ async def cascade(
|
|||
last = array[-1:].copy()
|
||||
|
||||
dst.push(last)
|
||||
|
||||
# sync with source buffer's time step
|
||||
src_l2 = src.array[-2:]
|
||||
src_li, src_lt = src_l2[-1][['index', 'time']]
|
||||
src_2li, src_2lt = src_l2[-2][['index', 'time']]
|
||||
dst._array['time'][src_li] = src_lt
|
||||
dst._array['time'][src_2li] = src_2lt
|
||||
|
||||
# last2 = dst.array[-2:]
|
||||
# if (
|
||||
# last2[-1]['index'] != src_li
|
||||
# or last2[-2]['index'] != src_2li
|
||||
# ):
|
||||
# dstl2 = list(last2)
|
||||
# srcl2 = list(src_l2)
|
||||
# print(
|
||||
# # f'{dst.token}\n'
|
||||
# f'src: {srcl2}\n'
|
||||
# f'dst: {dstl2}\n'
|
||||
# )
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import numpy as np
|
|||
from numba import jit, float64, optional, int64
|
||||
|
||||
from ._api import fsp
|
||||
from ..data import iterticks
|
||||
from ..data._normalize import iterticks
|
||||
from ..data._sharedmem import ShmArray
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ import numpy as np
|
|||
from tractor.trionics._broadcast import AsyncReceiver
|
||||
|
||||
from ._api import fsp
|
||||
from ..data import iterticks
|
||||
from ..data._normalize import iterticks
|
||||
from ..data._sharedmem import ShmArray
|
||||
from ._momo import _wma
|
||||
from ..log import get_logger
|
||||
|
|
@ -234,7 +234,7 @@ async def flow_rates(
|
|||
# FSPs, user input, and possibly any general event stream in
|
||||
# real-time. Hint: ideally implemented with caching until mutated
|
||||
# ;)
|
||||
period: 'Param[int]' = 1, # noqa
|
||||
period: 'Param[int]' = 6, # noqa
|
||||
|
||||
# TODO: support other means by providing a map
|
||||
# to weights `partial()`-ed with `wma()`?
|
||||
|
|
@ -268,7 +268,8 @@ async def flow_rates(
|
|||
'dark_dvlm_rate': None,
|
||||
}
|
||||
|
||||
quote = await anext(source)
|
||||
# TODO: 3.10 do ``anext()``
|
||||
quote = await source.__anext__()
|
||||
|
||||
# ltr = 0
|
||||
# lvr = 0
|
||||
|
|
|
|||
51
piker/log.py
51
piker/log.py
|
|
@ -21,11 +21,7 @@ import logging
|
|||
import json
|
||||
|
||||
import tractor
|
||||
from pygments import (
|
||||
highlight,
|
||||
lexers,
|
||||
formatters,
|
||||
)
|
||||
from pygments import highlight, lexers, formatters
|
||||
|
||||
# Makes it so we only see the full module name when using ``__name__``
|
||||
# without the extra "piker." prefix.
|
||||
|
|
@ -36,51 +32,26 @@ def get_logger(
|
|||
name: str = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''Return the package log or a sub-log for `name` if provided.
|
||||
'''
|
||||
Return the package log or a sub-log for `name` if provided.
|
||||
|
||||
'''
|
||||
return tractor.log.get_logger(
|
||||
name=name,
|
||||
_root_name=_proj_name,
|
||||
)
|
||||
return tractor.log.get_logger(name=name, _root_name=_proj_name)
|
||||
|
||||
|
||||
def get_console_log(
|
||||
level: str | None = None,
|
||||
name: str | None = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''
|
||||
Get the package logger and enable a handler which writes to stderr.
|
||||
def get_console_log(level: str = None, name: str = None) -> logging.Logger:
|
||||
'''Get the package logger and enable a handler which writes to stderr.
|
||||
|
||||
Yeah yeah, i know we can use ``DictConfig``. You do it...
|
||||
|
||||
'''
|
||||
return tractor.log.get_console_log(
|
||||
level,
|
||||
name=name,
|
||||
_root_name=_proj_name,
|
||||
) # our root logger
|
||||
level, name=name, _root_name=_proj_name) # our root logger
|
||||
|
||||
|
||||
def colorize_json(
|
||||
data: dict,
|
||||
style='algol_nu',
|
||||
):
|
||||
'''
|
||||
Colorize json output using ``pygments``.
|
||||
|
||||
'''
|
||||
formatted_json = json.dumps(
|
||||
data,
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
)
|
||||
def colorize_json(data, style='algol_nu'):
|
||||
"""Colorize json output using ``pygments``.
|
||||
"""
|
||||
formatted_json = json.dumps(data, sort_keys=True, indent=4)
|
||||
return highlight(
|
||||
formatted_json,
|
||||
lexers.JsonLexer(),
|
||||
|
||||
formatted_json, lexers.JsonLexer(),
|
||||
# likeable styles: algol_nu, tango, monokai
|
||||
formatters.TerminalTrueColorFormatter(style=style)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Actor-runtime service orchestration machinery.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from ._mngr import Services
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
open_registry,
|
||||
find_service,
|
||||
check_for_service,
|
||||
)
|
||||
from ._daemon import ( # noqa
|
||||
maybe_spawn_daemon,
|
||||
spawn_emsd,
|
||||
maybe_open_emsd,
|
||||
)
|
||||
from ._actor_runtime import (
|
||||
open_piker_runtime,
|
||||
maybe_open_pikerd,
|
||||
open_pikerd,
|
||||
get_tractor_runtime_kwargs,
|
||||
)
|
||||
from ..brokers._daemon import (
|
||||
spawn_brokerd,
|
||||
maybe_spawn_brokerd,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'check_for_service',
|
||||
'Services',
|
||||
'maybe_spawn_daemon',
|
||||
'spawn_brokerd',
|
||||
'maybe_spawn_brokerd',
|
||||
'spawn_emsd',
|
||||
'maybe_open_emsd',
|
||||
'open_piker_runtime',
|
||||
'maybe_open_pikerd',
|
||||
'open_pikerd',
|
||||
'get_tractor_runtime_kwargs',
|
||||
]
|
||||
|
|
@ -1,291 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
``tractor`` wrapping + default config to bootstrap the `pikerd`.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import os
|
||||
from typing import (
|
||||
Optional,
|
||||
Any,
|
||||
ClassVar,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from ._util import (
|
||||
get_console_log,
|
||||
)
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
open_registry,
|
||||
)
|
||||
|
||||
|
||||
def get_tractor_runtime_kwargs() -> dict[str, Any]:
|
||||
'''
|
||||
Deliver ``tractor`` related runtime variables in a `dict`.
|
||||
|
||||
'''
|
||||
return _tractor_kwargs
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX NOTE XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
# TODO: once we have `rsyscall` support we will read a config
|
||||
# and spawn the service tree distributed per that.
|
||||
start_method: str = 'trio',
|
||||
|
||||
tractor_runtime_overrides: dict | None = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> tuple[
|
||||
tractor.Actor,
|
||||
tuple[str, int],
|
||||
]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically sync with
|
||||
existing piker actors on the local link based on configuration.
|
||||
|
||||
Can be called from a subactor or any program that needs to start
|
||||
a root actor.
|
||||
|
||||
'''
|
||||
try:
|
||||
# check for existing runtime
|
||||
actor = tractor.current_actor().uid
|
||||
|
||||
except tractor._exceptions.NoRuntime:
|
||||
tractor._state._runtime_vars[
|
||||
'piker_vars'] = tractor_runtime_overrides
|
||||
|
||||
registry_addr = registry_addr or _default_reg_addr
|
||||
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=enable_modules,
|
||||
|
||||
**tractor_kwargs,
|
||||
) as _,
|
||||
|
||||
open_registry(registry_addr, ensure_exists=False) as addr,
|
||||
):
|
||||
yield (
|
||||
tractor.current_actor(),
|
||||
addr,
|
||||
)
|
||||
else:
|
||||
async with open_registry(registry_addr) as addr:
|
||||
yield (
|
||||
actor,
|
||||
addr,
|
||||
)
|
||||
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.service._daemon',
|
||||
'piker.brokers._daemon',
|
||||
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
|
||||
'piker.data._sampling',
|
||||
]
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> Services:
|
||||
'''
|
||||
Start a root piker daemon with an indefinite lifetime.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
|
||||
name=_root_dname,
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) as (root_actor, reg_addr),
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
trio.open_nursery() as service_nursery,
|
||||
):
|
||||
if root_actor.accept_addr != reg_addr:
|
||||
raise RuntimeError(
|
||||
f'`pikerd` failed to bind on {reg_addr}!\n'
|
||||
'Maybe you have another daemon already running?'
|
||||
)
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
Services.actor_n = actor_nursery
|
||||
Services.service_n = service_nursery
|
||||
Services.debug_mode = debug_mode
|
||||
|
||||
try:
|
||||
yield Services
|
||||
|
||||
finally:
|
||||
# TODO: is this more clever/efficient?
|
||||
# if 'samplerd' in Services.service_tasks:
|
||||
# await Services.cancel_service('samplerd')
|
||||
service_nursery.cancel_scope.cancel()
|
||||
|
||||
|
||||
# TODO: do we even need this?
|
||||
# @acm
|
||||
# async def maybe_open_runtime(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> None:
|
||||
# '''
|
||||
# Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
# '''
|
||||
# name = kwargs.pop('name')
|
||||
|
||||
# if not tractor.current_actor(err_on_no_runtime=False):
|
||||
# async with open_piker_runtime(
|
||||
# name,
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
# ) as (_, addr):
|
||||
# yield addr,
|
||||
# else:
|
||||
# async with open_registry() as addr:
|
||||
# yield addr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
registry_addr: None | tuple = None,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal | ClassVar[Services]:
|
||||
'''
|
||||
If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
query_name = kwargs.pop(
|
||||
'name',
|
||||
f'piker_query_{os.getpid()}',
|
||||
)
|
||||
|
||||
# TODO: if we need to make the query part faster we could not init
|
||||
# an actor runtime and instead just hit the socket?
|
||||
# from tractor._ipc import _connect_chan, Channel
|
||||
# async with _connect_chan(host, port) as chan:
|
||||
# async with open_portal(chan) as arb_portal:
|
||||
# yield arb_portal
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name=query_name,
|
||||
registry_addr=registry_addr,
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
) as _,
|
||||
|
||||
tractor.find_actor(
|
||||
_root_dname,
|
||||
arbiter_sockaddr=registry_addr,
|
||||
) as portal
|
||||
):
|
||||
# connect to any existing daemon presuming
|
||||
# its registry socket was selected.
|
||||
if (
|
||||
portal is not None
|
||||
):
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
loglevel=loglevel,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
# passthrough to ``tractor`` init
|
||||
**kwargs,
|
||||
|
||||
) as service_manager:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
assert service_manager
|
||||
yield service_manager
|
||||
|
|
@ -1,559 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Supervisor for ``docker`` with included async and SC wrapping
|
||||
to ensure a cancellable container lifetime system.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from collections import ChainMap
|
||||
from functools import partial
|
||||
import os
|
||||
import time
|
||||
from typing import (
|
||||
Optional,
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
from tractor.msg import NamespacePath
|
||||
import docker
|
||||
import json
|
||||
from docker.models.containers import Container as DockerContainer
|
||||
from docker.errors import (
|
||||
DockerException,
|
||||
APIError,
|
||||
# ContainerError,
|
||||
)
|
||||
import requests
|
||||
from requests.exceptions import (
|
||||
ConnectionError,
|
||||
ReadTimeout,
|
||||
)
|
||||
|
||||
from ._mngr import Services
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from .. import config
|
||||
|
||||
|
||||
class DockerNotStarted(Exception):
|
||||
'Prolly you dint start da daemon bruh'
|
||||
|
||||
|
||||
class ApplicationLogError(Exception):
|
||||
'App in container reported an error in logs'
|
||||
|
||||
|
||||
@acm
|
||||
async def open_docker(
|
||||
url: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> docker.DockerClient:
|
||||
|
||||
client: Optional[docker.DockerClient] = None
|
||||
try:
|
||||
client = docker.DockerClient(
|
||||
base_url=url,
|
||||
**kwargs
|
||||
) if url else docker.from_env(**kwargs)
|
||||
|
||||
yield client
|
||||
|
||||
except (
|
||||
DockerException,
|
||||
APIError,
|
||||
) as err:
|
||||
|
||||
def unpack_msg(err: Exception) -> str:
|
||||
args = getattr(err, 'args', None)
|
||||
if args:
|
||||
return args
|
||||
else:
|
||||
return str(err)
|
||||
|
||||
# could be more specific so let's check if it's just perms.
|
||||
if err.args:
|
||||
errs = err.args
|
||||
for err in errs:
|
||||
msg = unpack_msg(err)
|
||||
if 'PermissionError' in msg:
|
||||
raise DockerException('You dint run as root yo!')
|
||||
|
||||
elif 'FileNotFoundError' in msg:
|
||||
raise DockerNotStarted('Did you start da service sister?')
|
||||
|
||||
# not perms?
|
||||
raise
|
||||
|
||||
# finally:
|
||||
# if client:
|
||||
# client.close()
|
||||
|
||||
|
||||
class Container:
|
||||
'''
|
||||
Wrapper around a ``docker.models.containers.Container`` to include
|
||||
log capture and relay through our native logging system and helper
|
||||
method(s) for cancellation/teardown.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
cntr: DockerContainer,
|
||||
) -> None:
|
||||
|
||||
self.cntr = cntr
|
||||
# log msg de-duplication
|
||||
self.seen_so_far = set()
|
||||
|
||||
async def process_logs_until(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
|
||||
# this is a predicate func for matching log msgs emitted by the
|
||||
# underlying containerized app
|
||||
patt_matcher: Callable[[str], bool],
|
||||
|
||||
# XXX WARNING XXX: do not touch this sleep value unless
|
||||
# you know what you are doing! the value is critical to
|
||||
# making sure the caller code inside the startup context
|
||||
# does not timeout BEFORE we receive a match on the
|
||||
# ``patt_matcher()`` predicate above.
|
||||
checkpoint_period: float = 0.001,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Attempt to capture container log messages and relay through our
|
||||
native logging system.
|
||||
|
||||
'''
|
||||
seen_so_far = self.seen_so_far
|
||||
|
||||
while True:
|
||||
logs = self.cntr.logs()
|
||||
try:
|
||||
logs = self.cntr.logs()
|
||||
except (
|
||||
docker.errors.NotFound,
|
||||
docker.errors.APIError
|
||||
):
|
||||
log.exception('Failed to parse logs?')
|
||||
return False
|
||||
|
||||
entries = logs.decode().split('\n')
|
||||
for entry in entries:
|
||||
|
||||
# ignore null lines
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
entry = entry.strip()
|
||||
try:
|
||||
record = json.loads(entry)
|
||||
msg = record[log_msg_key]
|
||||
level = record['level']
|
||||
|
||||
except json.JSONDecodeError:
|
||||
msg = entry
|
||||
level = 'error'
|
||||
|
||||
# TODO: do we need a more general mechanism
|
||||
# for these kinda of "log record entries"?
|
||||
# if 'Error' in entry:
|
||||
# raise RuntimeError(entry)
|
||||
|
||||
if (
|
||||
msg
|
||||
and entry not in seen_so_far
|
||||
):
|
||||
seen_so_far.add(entry)
|
||||
getattr(
|
||||
log,
|
||||
level.lower(),
|
||||
log.error
|
||||
)(f'{msg}')
|
||||
|
||||
if level == 'fatal':
|
||||
raise ApplicationLogError(msg)
|
||||
|
||||
if await patt_matcher(msg):
|
||||
return True
|
||||
|
||||
# do a checkpoint so we don't block if cancelled B)
|
||||
await trio.sleep(checkpoint_period)
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def cuid(self) -> str:
|
||||
fqcn: str = self.cntr.attrs['Config']['Image']
|
||||
return f'{fqcn}[{self.cntr.short_id}]'
|
||||
|
||||
def try_signal(
|
||||
self,
|
||||
signal: str = 'SIGINT',
|
||||
|
||||
) -> bool:
|
||||
try:
|
||||
# XXX: market store doesn't seem to shutdown nicely all the
|
||||
# time with this (maybe because there are still open grpc
|
||||
# connections?) noticably after client connections have been
|
||||
# made or are in use/teardown. It works just fine if you
|
||||
# just start and stop the container tho?..
|
||||
log.cancel(f'SENDING {signal} to {self.cntr.id}')
|
||||
self.cntr.kill(signal)
|
||||
return True
|
||||
|
||||
except docker.errors.APIError as err:
|
||||
if 'is not running' in err.explanation:
|
||||
return False
|
||||
|
||||
def hard_kill(self, start: float) -> None:
|
||||
delay = time.time() - start
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
log.error(
|
||||
f'SIGKILL-ing: {self.cntr.id} after {delay}s\n'
|
||||
)
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
stop_predicate: Callable[[str], bool],
|
||||
|
||||
hard_kill: bool = False,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Attempt to cancel this container gracefully, fail over to
|
||||
a hard kill on timeout.
|
||||
|
||||
'''
|
||||
cid = self.cntr.id
|
||||
|
||||
# first try a graceful cancel
|
||||
log.cancel(
|
||||
f'SIGINT cancelling container: {self.cuid}\n'
|
||||
'waiting on stop predicate...'
|
||||
)
|
||||
self.try_signal('SIGINT')
|
||||
|
||||
start = time.time()
|
||||
for _ in range(6):
|
||||
|
||||
with trio.move_on_after(1) as cs:
|
||||
log.cancel(
|
||||
'polling for CNTR logs for {stop_predicate}..'
|
||||
)
|
||||
|
||||
try:
|
||||
await self.process_logs_until(
|
||||
log_msg_key,
|
||||
stop_predicate,
|
||||
)
|
||||
except ApplicationLogError:
|
||||
hard_kill = True
|
||||
else:
|
||||
# if we aren't cancelled on above checkpoint then we
|
||||
# assume we read the expected stop msg and
|
||||
# terminated.
|
||||
break
|
||||
|
||||
if cs.cancelled_caught:
|
||||
# on timeout just try a hard kill after
|
||||
# a quick container sync-wait.
|
||||
hard_kill = True
|
||||
|
||||
try:
|
||||
log.info(f'Polling for container shutdown:\n{cid}')
|
||||
|
||||
if self.cntr.status not in {'exited', 'not-running'}:
|
||||
self.cntr.wait(
|
||||
timeout=0.1,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
# graceful exit if we didn't time out
|
||||
break
|
||||
|
||||
except (
|
||||
ReadTimeout,
|
||||
):
|
||||
log.info(f'Still waiting on container:\n{cid}')
|
||||
continue
|
||||
|
||||
except (
|
||||
docker.errors.APIError,
|
||||
ConnectionError,
|
||||
requests.exceptions.ConnectionError,
|
||||
trio.Cancelled,
|
||||
):
|
||||
log.exception('Docker connection failure')
|
||||
self.hard_kill(start)
|
||||
raise
|
||||
|
||||
except trio.Cancelled:
|
||||
log.exception('trio cancelled...')
|
||||
self.hard_kill(start)
|
||||
else:
|
||||
hard_kill = True
|
||||
|
||||
if hard_kill:
|
||||
self.hard_kill(start)
|
||||
else:
|
||||
log.cancel(f'Container stopped: {cid}')
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_ahabd(
|
||||
ctx: tractor.Context,
|
||||
endpoint: str, # ns-pointer str-msg-type
|
||||
loglevel: str | None = None,
|
||||
|
||||
**ep_kwargs,
|
||||
|
||||
) -> None:
|
||||
|
||||
log = get_console_log(loglevel or 'cancel')
|
||||
|
||||
async with open_docker() as client:
|
||||
|
||||
# TODO: eventually offer a config-oriented API to do the mounts,
|
||||
# params, etc. passing to ``Containter.run()``?
|
||||
# call into endpoint for container config/init
|
||||
ep_func = NamespacePath(endpoint).load_ref()
|
||||
(
|
||||
dcntr,
|
||||
cntr_config,
|
||||
start_pred,
|
||||
stop_pred,
|
||||
) = ep_func(client, **ep_kwargs)
|
||||
cntr = Container(dcntr)
|
||||
|
||||
conf: ChainMap[str, Any] = ChainMap(
|
||||
|
||||
# container specific
|
||||
cntr_config,
|
||||
|
||||
# defaults
|
||||
{
|
||||
# startup time limit which is the max the supervisor
|
||||
# will wait for the container to be registered in
|
||||
# ``client.containers.list()``
|
||||
'startup_timeout': 1.0,
|
||||
|
||||
# how fast to poll for the starup predicate by sleeping
|
||||
# this amount incrementally thus yielding to the
|
||||
# ``trio`` scheduler on during sync polling execution.
|
||||
'startup_query_period': 0.001,
|
||||
|
||||
# str-key value expected to contain log message body-contents
|
||||
# when read using:
|
||||
# ``json.loads(entry for entry in DockerContainer.logs())``
|
||||
'log_msg_key': 'msg',
|
||||
|
||||
|
||||
# startup sync func, like `Nursery.started()`
|
||||
'started_afunc': None,
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with trio.move_on_after(conf['startup_timeout']) as cs:
|
||||
async with trio.open_nursery() as tn:
|
||||
tn.start_soon(
|
||||
partial(
|
||||
cntr.process_logs_until,
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
patt_matcher=start_pred,
|
||||
checkpoint_period=conf['startup_query_period'],
|
||||
)
|
||||
)
|
||||
|
||||
# optional blocking routine
|
||||
started = conf['started_afunc']
|
||||
if started:
|
||||
await started()
|
||||
|
||||
# poll for container startup or timeout
|
||||
while not cs.cancel_called:
|
||||
if dcntr in client.containers.list():
|
||||
break
|
||||
|
||||
await trio.sleep(conf['startup_query_period'])
|
||||
|
||||
# sync with remote caller actor-task but allow log
|
||||
# processing to continue running in bg.
|
||||
await ctx.started((
|
||||
cntr.cntr.id,
|
||||
os.getpid(),
|
||||
cntr_config,
|
||||
))
|
||||
|
||||
# XXX: if we timeout on finding the "startup msg" we
|
||||
# expect then we want to FOR SURE raise an error
|
||||
# upwards!
|
||||
if cs.cancelled_caught:
|
||||
# if dcntr not in client.containers.list():
|
||||
for entry in cntr.seen_so_far:
|
||||
log.info(entry)
|
||||
|
||||
raise DockerNotStarted(
|
||||
f'Failed to start container: {cntr.cuid}\n'
|
||||
f'due to timeout={conf["startup_timeout"]}s\n\n'
|
||||
"check ur container's logs!"
|
||||
)
|
||||
|
||||
# TODO: we might eventually want a proxy-style msg-prot here
|
||||
# to allow remote control of containers without needing
|
||||
# callers to have root perms?
|
||||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
# TODO: ensure loglevel can be set and teardown logs are
|
||||
# reported if possible on error or cancel..
|
||||
# XXX WARNING: currently shielding here can result in hangs
|
||||
# on ctl-c from user.. ideally we can avoid a cancel getting
|
||||
# consumed and not propagating whilst still doing teardown
|
||||
# logging..
|
||||
with trio.CancelScope(shield=True):
|
||||
await cntr.cancel(
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
stop_predicate=stop_pred,
|
||||
)
|
||||
|
||||
|
||||
@acm
|
||||
async def start_ahab_service(
|
||||
services: Services,
|
||||
service_name: str,
|
||||
|
||||
# endpoint config passed as **kwargs
|
||||
endpoint: Callable[docker.DockerClient, DockerContainer],
|
||||
ep_kwargs: dict,
|
||||
loglevel: str | None = 'cancel',
|
||||
|
||||
# supervisor config
|
||||
drop_root_perms: bool = True,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.Event,
|
||||
dict[str, Any],
|
||||
],
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Start a ``docker`` container supervisor with given service name.
|
||||
|
||||
Currently the actor calling this task should normally be started
|
||||
with root permissions (until we decide to use something that doesn't
|
||||
require this, like docker's rootless mode or some wrapper project) but
|
||||
te root perms are de-escalated after the docker supervisor sub-actor
|
||||
is started.
|
||||
|
||||
'''
|
||||
# global log
|
||||
log = get_console_log(loglevel or 'cancel')
|
||||
|
||||
cn_ready = trio.Event()
|
||||
try:
|
||||
async with tractor.open_nursery() as an:
|
||||
|
||||
portal = await an.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__],
|
||||
loglevel=loglevel,
|
||||
)
|
||||
|
||||
# TODO: we have issues with this on teardown
|
||||
# where ``tractor`` tries to issue ``os.kill()``
|
||||
# and hits perms errors since the root process
|
||||
# doesn't any longer have root perms..
|
||||
|
||||
# de-escalate root perms to the original user
|
||||
# after the docker supervisor actor is spawned.
|
||||
if (
|
||||
drop_root_perms
|
||||
and config._parent_user
|
||||
):
|
||||
import pwd
|
||||
os.setuid(
|
||||
pwd.getpwnam(
|
||||
config._parent_user
|
||||
)[2] # named user's uid
|
||||
)
|
||||
|
||||
cs, first = await services.start_service_task(
|
||||
name=service_name,
|
||||
portal=portal,
|
||||
|
||||
# rest: endpoint inputs
|
||||
target=open_ahabd,
|
||||
endpoint=str(NamespacePath.from_ref(endpoint)),
|
||||
loglevel='cancel',
|
||||
**ep_kwargs,
|
||||
)
|
||||
|
||||
cid, pid, cntr_config = first
|
||||
|
||||
try:
|
||||
yield (
|
||||
cn_ready,
|
||||
cntr_config,
|
||||
(cid, pid),
|
||||
)
|
||||
finally:
|
||||
log.info(f'Cancelling ahab service `{service_name}`')
|
||||
await services.cancel_service(service_name)
|
||||
|
||||
# since we demoted root perms in this parent
|
||||
# we'll get a perms error on proc cleanup in
|
||||
# ``tractor`` nursery exit. just make sure
|
||||
# the child is terminated and don't raise the
|
||||
# error if so.
|
||||
|
||||
# TODO: we could also consider adding
|
||||
# a ``tractor.ZombieDetected`` or something that we could raise
|
||||
# if we find the child didn't terminate.
|
||||
except PermissionError:
|
||||
log.warning('Failed to cancel root permsed container')
|
||||
|
||||
except (
|
||||
trio.MultiError,
|
||||
) as err:
|
||||
for subexc in err.exceptions:
|
||||
if isinstance(subexc, PermissionError):
|
||||
log.warning('Failed to cancel root perms-ed container')
|
||||
return
|
||||
else:
|
||||
raise
|
||||
|
|
@ -1,190 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Daemon-actor spawning "endpoint-hooks".
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._actor_runtime import maybe_open_pikerd
|
||||
from ._registry import find_service
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
|
||||
spawn_args: dict[str, Any],
|
||||
|
||||
loglevel: str | None = None,
|
||||
singleton: bool = False,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Services.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(
|
||||
f"Couldn't find any existing {service_name}\n"
|
||||
'Attempting to spawn new daemon-service..'
|
||||
)
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
started: bool
|
||||
if pikerd_portal is None:
|
||||
started = await service_task_target(
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
else:
|
||||
# request a remote `pikerd` (service manager) to start the
|
||||
# target daemon-task, the target can't return
|
||||
# a non-serializable value since it is expected that service
|
||||
# starting is non-blocking and the target task will persist
|
||||
# running "under" or "within" the `pikerd` actor tree after
|
||||
# the questing client disconnects. in other words this
|
||||
# spawns a persistent daemon actor that continues to live
|
||||
# for the lifespan of whatever the service manager inside
|
||||
# `pikerd` says it should.
|
||||
started = await pikerd_portal.run(
|
||||
service_task_target,
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
if started:
|
||||
log.info(f'Service {service_name} started!')
|
||||
|
||||
# block until we can discover (by IPC connection) to the newly
|
||||
# spawned daemon-actor and then deliver the portal to the
|
||||
# caller.
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from ..clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await Services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
_setup_persistent_emsd,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={},
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
|
@ -1,140 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
daemon-service management API.
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
|
||||
# TODO: we need remote wrapping and a general soln:
|
||||
# - factor this into a ``tractor.highlevel`` extension # pack for the
|
||||
# library.
|
||||
# - wrap a "remote api" wherein you can get a method proxy
|
||||
# to the pikerd actor for starting services remotely!
|
||||
class Services:
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[
|
||||
str,
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
tractor.Portal,
|
||||
trio.Event,
|
||||
]
|
||||
] = {}
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
@classmethod
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
allow_overruns: bool = False,
|
||||
**ctx_kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
trio.Event,
|
||||
Any,
|
||||
]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
|
||||
async with portal.open_context(
|
||||
target,
|
||||
allow_overruns=allow_overruns,
|
||||
**ctx_kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
complete = trio.Event()
|
||||
task_status.started((cs, complete, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
# and any final portal result from the
|
||||
# sub-actor.
|
||||
ctx_res = await ctx.result()
|
||||
|
||||
# NOTE: blocks indefinitely until cancelled
|
||||
# either by error from the target context
|
||||
# function or by being cancelled here by the
|
||||
# surrounding cancel scope.
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
finally:
|
||||
await portal.cancel_actor()
|
||||
complete.set()
|
||||
self.service_tasks.pop(name)
|
||||
|
||||
cs, complete, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal, complete)
|
||||
|
||||
return cs, first
|
||||
|
||||
@classmethod
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
) -> Any:
|
||||
'''
|
||||
Cancel the service task and actor for the given ``name``.
|
||||
|
||||
'''
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal, complete = self.service_tasks[name]
|
||||
cs.cancel()
|
||||
await complete.wait()
|
||||
assert name not in self.service_tasks, \
|
||||
f'Serice task for {name} not terminated?'
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Inter-actor "discovery" (protocol) layer.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
_default_registry_host: str = '127.0.0.1'
|
||||
_default_registry_port: int = 6116
|
||||
_default_reg_addr: tuple[str, int] = (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
|
||||
|
||||
# NOTE: this value is set as an actor-global once the first endpoint
|
||||
# who is capable, spawns a `pikerd` service tree.
|
||||
_registry: Registry | None = None
|
||||
|
||||
|
||||
class Registry:
|
||||
addr: None | tuple[str, int] = None
|
||||
|
||||
# TODO: table of uids to sockaddrs
|
||||
peers: dict[
|
||||
tuple[str, str],
|
||||
tuple[str, int],
|
||||
] = {}
|
||||
|
||||
|
||||
_tractor_kwargs: dict[str, Any] = {}
|
||||
|
||||
|
||||
@acm
|
||||
async def open_registry(
|
||||
addr: None | tuple[str, int] = None,
|
||||
ensure_exists: bool = True,
|
||||
|
||||
) -> tuple[str, int]:
|
||||
|
||||
global _tractor_kwargs
|
||||
actor = tractor.current_actor()
|
||||
uid = actor.uid
|
||||
if (
|
||||
Registry.addr is not None
|
||||
and addr
|
||||
):
|
||||
raise RuntimeError(
|
||||
f'`{uid}` registry addr already bound @ {_registry.sockaddr}'
|
||||
)
|
||||
|
||||
was_set: bool = False
|
||||
|
||||
if (
|
||||
not tractor.is_root_process()
|
||||
and Registry.addr is None
|
||||
):
|
||||
Registry.addr = actor._arb_addr
|
||||
|
||||
if (
|
||||
ensure_exists
|
||||
and Registry.addr is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"`{uid}` registry should already exist bug doesn't?"
|
||||
)
|
||||
|
||||
if (
|
||||
Registry.addr is None
|
||||
):
|
||||
was_set = True
|
||||
Registry.addr = addr or _default_reg_addr
|
||||
|
||||
_tractor_kwargs['arbiter_addr'] = Registry.addr
|
||||
|
||||
try:
|
||||
yield Registry.addr
|
||||
finally:
|
||||
# XXX: always clear the global addr if we set it so that the
|
||||
# next (set of) calls will apply whatever new one is passed
|
||||
# in.
|
||||
if was_set:
|
||||
Registry.addr = None
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> tractor.Portal | None:
|
||||
|
||||
async with open_registry() as reg_addr:
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> None | tuple[str, int]:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with open_registry(ensure_exists=False) as reg_addr:
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue