Compare commits
No commits in common. "pre_overruns_ctxcancelled" and "310_plus" have entirely different histories.
pre_overru
...
310_plus
|
|
@ -3,9 +3,10 @@ name: CI
|
|||
|
||||
on:
|
||||
# Triggers the workflow on push or pull request events but only for the master branch
|
||||
pull_request:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
|
@ -13,46 +14,21 @@ on:
|
|||
|
||||
jobs:
|
||||
|
||||
# test that we can generate a software distribution and install it
|
||||
# thus avoid missing file issues after packaging.
|
||||
sdist-linux:
|
||||
name: 'sdist'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Build sdist
|
||||
run: python setup.py sdist --formats=zip
|
||||
|
||||
- name: Install sdist from .zips
|
||||
run: python -m pip install dist/*.zip
|
||||
|
||||
testing:
|
||||
name: 'install + test-suite'
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build DB container
|
||||
run: docker build -t piker:elastic dockering/elastic
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -U .[es] -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
run: pip install -U . -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
|
||||
- name: Test suite
|
||||
run: pytest tests -rs
|
||||
|
|
|
|||
13
README.rst
13
README.rst
|
|
@ -71,19 +71,6 @@ for a development install::
|
|||
source ./env/bin/activate
|
||||
pip install -r requirements.txt -e .
|
||||
|
||||
install for nixos
|
||||
*****************
|
||||
for users of `NixOS` we offer a development shell envoirment that can be
|
||||
loaded with::
|
||||
|
||||
nix-shell develop.nix
|
||||
|
||||
this will setup the required python environment to run piker, make sure to
|
||||
run::
|
||||
|
||||
pip install -r requirements.txt -e .
|
||||
|
||||
once after loading the shell
|
||||
|
||||
install for tinas
|
||||
*****************
|
||||
|
|
|
|||
|
|
@ -50,8 +50,3 @@ prefer_data_account = [
|
|||
paper = "XX0000000"
|
||||
margin = "X0000000"
|
||||
ira = "X0000000"
|
||||
|
||||
|
||||
[deribit]
|
||||
key_id = 'XXXXXXXX'
|
||||
key_secret = 'Xx_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
|
|
|
|||
32
develop.nix
32
develop.nix
|
|
@ -1,32 +0,0 @@
|
|||
with (import <nixpkgs> {});
|
||||
with python310Packages;
|
||||
stdenv.mkDerivation {
|
||||
name = "pip-env";
|
||||
buildInputs = [
|
||||
# System requirements.
|
||||
readline
|
||||
|
||||
# Python requirements (enough to get a virtualenv going).
|
||||
python310Full
|
||||
virtualenv
|
||||
setuptools
|
||||
pyqt5
|
||||
pip
|
||||
];
|
||||
src = null;
|
||||
shellHook = ''
|
||||
# Allow the use of wheels.
|
||||
SOURCE_DATE_EPOCH=$(date +%s)
|
||||
|
||||
# Augment the dynamic linker path
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${R}/lib/R/lib:${readline}/lib
|
||||
|
||||
export QT_QPA_PLATFORM_PLUGIN_PATH="${qt5.qtbase.bin}/lib/qt-${qt5.qtbase.version}/plugins";
|
||||
|
||||
if [ ! -d "venv" ]; then
|
||||
virtualenv venv
|
||||
fi
|
||||
|
||||
source venv/bin/activate
|
||||
'';
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
FROM elasticsearch:7.17.4
|
||||
|
||||
ENV ES_JAVA_OPTS "-Xms2g -Xmx2g"
|
||||
ENV ELASTIC_USERNAME "elastic"
|
||||
ENV ELASTIC_PASSWORD "password"
|
||||
|
||||
COPY elasticsearch.yml /usr/share/elasticsearch/config/
|
||||
|
||||
RUN printf "password" | ./bin/elasticsearch-keystore add -f -x "bootstrap.password"
|
||||
|
||||
EXPOSE 19200
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
network.host: 0.0.0.0
|
||||
|
||||
http.port: 19200
|
||||
|
||||
discovery.type: single-node
|
||||
|
|
@ -2,26 +2,12 @@
|
|||
# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
|
||||
version: "3.5"
|
||||
|
||||
|
||||
services:
|
||||
|
||||
ib_gw_paper:
|
||||
|
||||
# apparently java is a mega cukc:
|
||||
# https://stackoverflow.com/a/56895801
|
||||
# https://bugs.openjdk.org/browse/JDK-8150460
|
||||
ulimits:
|
||||
# nproc: 65535
|
||||
nproc: 6000
|
||||
nofile:
|
||||
soft: 2000
|
||||
hard: 3000
|
||||
|
||||
ib-gateway:
|
||||
# other image tags available:
|
||||
# https://github.com/waytrade/ib-gateway-docker#supported-tags
|
||||
# image: waytrade/ib-gateway:981.3j
|
||||
image: waytrade/ib-gateway:1012.2i
|
||||
restart: 'no' # restart on boot whenev there's a crash or user clicsk
|
||||
image: waytrade/ib-gateway:981.3j
|
||||
restart: always
|
||||
network_mode: 'host'
|
||||
|
||||
volumes:
|
||||
|
|
@ -53,12 +39,14 @@ services:
|
|||
# this compose file which looks something like:
|
||||
# TWS_USERID='myuser'
|
||||
# TWS_PASSWORD='guest'
|
||||
# TRADING_MODE=paper (or live)
|
||||
# VNC_SERVER_PASSWORD='diggity'
|
||||
|
||||
environment:
|
||||
TWS_USERID: ${TWS_USERID}
|
||||
TWS_PASSWORD: ${TWS_PASSWORD}
|
||||
TRADING_MODE: 'paper'
|
||||
VNC_SERVER_PASSWORD: 'doggy'
|
||||
VNC_SERVER_PORT: '3003'
|
||||
TRADING_MODE: ${TRADING_MODE:-paper}
|
||||
VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
|
||||
|
||||
# ports:
|
||||
# - target: 4002
|
||||
|
|
@ -74,40 +62,3 @@ services:
|
|||
# - "127.0.0.1:4001:4001"
|
||||
# - "127.0.0.1:4002:4002"
|
||||
# - "127.0.0.1:5900:5900"
|
||||
|
||||
# ib_gw_live:
|
||||
# image: waytrade/ib-gateway:1012.2i
|
||||
# restart: no
|
||||
# network_mode: 'host'
|
||||
|
||||
# volumes:
|
||||
# - type: bind
|
||||
# source: ./jts_live.ini
|
||||
# target: /root/jts/jts.ini
|
||||
# # don't let ibc clobber this file for
|
||||
# # the main reason of not having a stupid
|
||||
# # timezone set..
|
||||
# read_only: true
|
||||
|
||||
# # force our own ibc config
|
||||
# - type: bind
|
||||
# source: ./ibc.ini
|
||||
# target: /root/ibc/config.ini
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./fork_ports_delayed.sh
|
||||
# target: /root/scripts/fork_ports_delayed.sh
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./run_x11_vnc.sh
|
||||
# target: /root/scripts/run_x11_vnc.sh
|
||||
# read_only: true
|
||||
|
||||
# # NOTE: to fill these out, define an `.env` file in the same dir as
|
||||
# # this compose file which looks something like:
|
||||
# environment:
|
||||
# TRADING_MODE: 'live'
|
||||
# VNC_SERVER_PASSWORD: 'doggy'
|
||||
# VNC_SERVER_PORT: '3004'
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes
|
|||
#
|
||||
# The default value is 60.
|
||||
|
||||
LoginDialogDisplayTimeout=20
|
||||
LoginDialogDisplayTimeout = 60
|
||||
|
||||
|
||||
|
||||
|
|
@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary
|
|||
# be set dynamically at run-time: most users will never need it,
|
||||
# so don't use it unless you know you need it.
|
||||
|
||||
; OverrideTwsApiPort=4002
|
||||
OverrideTwsApiPort=4002
|
||||
|
||||
|
||||
# Read-only Login
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
[IBGateway]
|
||||
ApiOnly=true
|
||||
LocalServerPort=4001
|
||||
# NOTE: must be set if using IBC's "reject" mode
|
||||
TrustedIPs=127.0.0.1
|
||||
; RemoteHostOrderRouting=ndc1.ibllc.com
|
||||
; WriteDebug=true
|
||||
; RemotePortOrderRouting=4001
|
||||
; useRemoteSettings=false
|
||||
; tradingMode=p
|
||||
; Steps=8
|
||||
; colorPalletName=dark
|
||||
|
||||
# window geo, this may be useful for sending `xdotool` commands?
|
||||
; MainWindow.Width=1986
|
||||
; screenHeight=3960
|
||||
|
||||
|
||||
[Logon]
|
||||
Locale=en
|
||||
# most markets are oriented around this zone
|
||||
# so might as well hard code it.
|
||||
TimeZone=America/New_York
|
||||
UseSSL=true
|
||||
displayedproxymsg=1
|
||||
os_titlebar=true
|
||||
s3store=true
|
||||
useRemoteSettings=false
|
||||
|
||||
[Communication]
|
||||
ctciAutoEncrypt=true
|
||||
Region=usr
|
||||
; Peer=cdc1.ibllc.com:4001
|
||||
|
|
@ -1,35 +1,16 @@
|
|||
#!/bin/sh
|
||||
# start vnc server and listen for connections
|
||||
# on port specced in `$VNC_SERVER_PORT`
|
||||
|
||||
# start VNC server
|
||||
x11vnc \
|
||||
-listen 127.0.0.1 \
|
||||
-allow 127.0.0.1 \
|
||||
-rfbport "${VNC_SERVER_PORT}" \
|
||||
-ncache_cr \
|
||||
-listen localhost \
|
||||
-display :1 \
|
||||
-forever \
|
||||
-shared \
|
||||
-logappend /var/log/x11vnc.log \
|
||||
-bg \
|
||||
-nowf \
|
||||
-noxdamage \
|
||||
-noxfixes \
|
||||
-no6 \
|
||||
-noipv6 \
|
||||
|
||||
|
||||
# -nowcr \
|
||||
# TODO: can't use this because of ``asyncvnc`` issue:
|
||||
-autoport 3003 \
|
||||
# can't use this because of ``asyncvnc`` issue:
|
||||
# https://github.com/barneygale/asyncvnc/issues/1
|
||||
# -passwd 'ibcansmbz'
|
||||
|
||||
# XXX: optional graphics caching flags that seem to rekt the overlay
|
||||
# of the 2 gw windows? When running a single gateway
|
||||
# this seems to maybe optimize some memory usage?
|
||||
# -ncache_cr \
|
||||
# -ncache \
|
||||
|
||||
# NOTE: this will prevent logs from going to the console.
|
||||
# -logappend /var/log/x11vnc.log \
|
||||
|
||||
# where to start allocating ports
|
||||
# -autoport "${VNC_SERVER_PORT}" \
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers.
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,14 +14,7 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"""
|
||||
piker: trading gear for hackers.
|
||||
|
||||
'''
|
||||
from .service import open_piker_runtime
|
||||
from .data.feed import open_feed
|
||||
|
||||
__all__ = [
|
||||
'open_piker_runtime',
|
||||
'open_feed',
|
||||
]
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ Cacheing apis and toolz.
|
|||
|
||||
from collections import OrderedDict
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
asynccontextmanager,
|
||||
)
|
||||
|
||||
from tractor.trionics import maybe_open_context
|
||||
|
|
@ -62,7 +62,7 @@ def async_lifo_cache(maxsize=128):
|
|||
return decorator
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_cached_client(
|
||||
brokername: str,
|
||||
) -> 'Client': # noqa
|
||||
|
|
|
|||
|
|
@ -0,0 +1,561 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Structured, daemon tree service management.
|
||||
|
||||
"""
|
||||
from typing import Optional, Union, Callable, Any
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from collections import defaultdict
|
||||
|
||||
from pydantic import BaseModel
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from .log import get_logger, get_console_log
|
||||
from .brokers import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
|
||||
_registry_addr = ('127.0.0.1', 6116)
|
||||
_tractor_kwargs: dict[str, Any] = {
|
||||
# use a different registry addr then tractor's default
|
||||
'arbiter_addr': _registry_addr
|
||||
}
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
]
|
||||
|
||||
|
||||
class Services(BaseModel):
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[str, tuple[trio.CancelScope, tractor.Portal]] = {}
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
**kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
async with portal.open_context(
|
||||
target,
|
||||
**kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
task_status.started((cs, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
ctx_res = await ctx.result()
|
||||
except tractor.ContextCancelled:
|
||||
return await self.cancel_service(name)
|
||||
else:
|
||||
# wait on any error from the sub-actor
|
||||
# NOTE: this will block indefinitely until
|
||||
# cancelled either by error from the target
|
||||
# context function or by being cancelled here by
|
||||
# the surrounding cancel scope
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
cs, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal)
|
||||
|
||||
return cs, first
|
||||
|
||||
# TODO: per service cancellation by scope, we aren't using this
|
||||
# anywhere right?
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
) -> Any:
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal = self.service_tasks[name]
|
||||
# XXX: not entirely sure why this is required,
|
||||
# and should probably be better fine tuned in
|
||||
# ``tractor``?
|
||||
cs.cancel()
|
||||
return await portal.cancel_actor()
|
||||
|
||||
|
||||
_services: Optional[Services] = None
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a root piker daemon who's lifetime extends indefinitely
|
||||
until cancelled.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=_root_dname,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
):
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
|
||||
# # setup service mngr singleton instance
|
||||
# async with AsyncExitStack() as stack:
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
_services = Services(
|
||||
actor_n=actor_nursery,
|
||||
service_n=service_nursery,
|
||||
debug_mode=debug_mode,
|
||||
)
|
||||
|
||||
yield _services
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically
|
||||
sync with existing piker actors in local network
|
||||
based on configuration.
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
):
|
||||
yield tractor.current_actor()
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_runtime(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
"""
|
||||
Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
"""
|
||||
settings = _tractor_kwargs
|
||||
settings.update(kwargs)
|
||||
|
||||
if not tractor.current_actor(err_on_no_runtime=False):
|
||||
async with tractor.open_root_actor(
|
||||
loglevel=loglevel,
|
||||
**settings,
|
||||
):
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> Union[tractor._portal.Portal, Services]:
|
||||
"""If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
"""
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
async with maybe_open_runtime(loglevel, **kwargs):
|
||||
|
||||
async with tractor.find_actor(_root_dname) as portal:
|
||||
# assert portal is not None
|
||||
if portal is not None:
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
debug_mode=kwargs.get('debug_mode', False),
|
||||
|
||||
) as _:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
yield None
|
||||
|
||||
|
||||
# brokerd enabled modules
|
||||
_data_mods = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
class Brokerd:
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> Optional[tractor.Portal]:
|
||||
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
spawn_args: dict[str, Any],
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Brokerd.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(f"Couldn't find any existing {service_name}")
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
if pikerd_portal is None:
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
await service_task_target(**spawn_args)
|
||||
|
||||
else:
|
||||
# tell the remote `pikerd` to start the target,
|
||||
# the target can't return a non-serializable value
|
||||
# since it is expected that service startingn is
|
||||
# non-blocking and the target task will persist running
|
||||
# on `pikerd` after the client requesting it's start
|
||||
# disconnects.
|
||||
await pikerd_portal.run(
|
||||
service_task_target,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
dname = f'brokerd.{brokername}'
|
||||
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
modpath = brokermod.__name__
|
||||
broker_enable = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
broker_enable.append(subpath)
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + broker_enable,
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from .data import _setup_persistent_brokerd
|
||||
|
||||
await _services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
_setup_persistent_brokerd,
|
||||
brokername=brokername,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client
|
||||
who wishes to use the sub-actor-daemon.
|
||||
|
||||
'''
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={'brokername': brokername, 'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: Optional[str] = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from .clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await _services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
_setup_persistent_emsd,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
# TODO: ideally we can start the tsdb "on demand" but it's
|
||||
# probably going to require "rootless" docker, at least if we don't
|
||||
# want to expect the user to start ``pikerd`` with root perms all the
|
||||
# time.
|
||||
# async def maybe_open_marketstored(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> tractor._portal.Portal: # noqa
|
||||
|
||||
# async with maybe_spawn_daemon(
|
||||
|
||||
# 'marketstored',
|
||||
# service_task_target=spawn_emsd,
|
||||
# spawn_args={'loglevel': loglevel},
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
|
||||
# ) as portal:
|
||||
# yield portal
|
||||
|
|
@ -18,10 +18,7 @@
|
|||
Profiling wrappers for internal libs.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from time import perf_counter
|
||||
from functools import wraps
|
||||
|
||||
# NOTE: you can pass a flag to enable this:
|
||||
|
|
@ -47,193 +44,3 @@ def timeit(fn):
|
|||
return res
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# Modified version of ``pyqtgraph.debug.Profiler`` that
|
||||
# core seems hesitant to land in:
|
||||
# https://github.com/pyqtgraph/pyqtgraph/pull/2281
|
||||
class Profiler(object):
|
||||
'''
|
||||
Simple profiler allowing measurement of multiple time intervals.
|
||||
|
||||
By default, profilers are disabled. To enable profiling, set the
|
||||
environment variable `PYQTGRAPHPROFILE` to a comma-separated list of
|
||||
fully-qualified names of profiled functions.
|
||||
|
||||
Calling a profiler registers a message (defaulting to an increasing
|
||||
counter) that contains the time elapsed since the last call. When the
|
||||
profiler is about to be garbage-collected, the messages are passed to the
|
||||
outer profiler if one is running, or printed to stdout otherwise.
|
||||
|
||||
If `delayed` is set to False, messages are immediately printed instead.
|
||||
|
||||
Example:
|
||||
def function(...):
|
||||
profiler = Profiler()
|
||||
... do stuff ...
|
||||
profiler('did stuff')
|
||||
... do other stuff ...
|
||||
profiler('did other stuff')
|
||||
# profiler is garbage-collected and flushed at function end
|
||||
|
||||
If this function is a method of class C, setting `PYQTGRAPHPROFILE` to
|
||||
"C.function" (without the module name) will enable this profiler.
|
||||
|
||||
For regular functions, use the qualified name of the function, stripping
|
||||
only the initial "pyqtgraph." prefix from the module.
|
||||
'''
|
||||
|
||||
_profilers = os.environ.get("PYQTGRAPHPROFILE", None)
|
||||
_profilers = _profilers.split(",") if _profilers is not None else []
|
||||
|
||||
_depth = 0
|
||||
|
||||
# NOTE: without this defined at the class level
|
||||
# you won't see apprpriately "nested" sub-profiler
|
||||
# instance calls.
|
||||
_msgs = []
|
||||
|
||||
# set this flag to disable all or individual profilers at runtime
|
||||
disable = False
|
||||
|
||||
class DisabledProfiler(object):
|
||||
def __init__(self, *args, **kwds):
|
||||
pass
|
||||
|
||||
def __call__(self, *args):
|
||||
pass
|
||||
|
||||
def finish(self):
|
||||
pass
|
||||
|
||||
def mark(self, msg=None):
|
||||
pass
|
||||
|
||||
_disabledProfiler = DisabledProfiler()
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
msg=None,
|
||||
disabled='env',
|
||||
delayed=True,
|
||||
ms_threshold: float = 0.0,
|
||||
):
|
||||
"""Optionally create a new profiler based on caller's qualname.
|
||||
|
||||
``ms_threshold`` can be set to value in ms for which, if the
|
||||
total measured time of the lifetime of this profiler is **less
|
||||
than** this value, then no profiling messages will be printed.
|
||||
Setting ``delayed=False`` disables this feature since messages
|
||||
are emitted immediately.
|
||||
|
||||
"""
|
||||
if (
|
||||
disabled is True
|
||||
or (
|
||||
disabled == 'env'
|
||||
and len(cls._profilers) == 0
|
||||
)
|
||||
):
|
||||
return cls._disabledProfiler
|
||||
|
||||
# determine the qualified name of the caller function
|
||||
caller_frame = sys._getframe(1)
|
||||
try:
|
||||
caller_object_type = type(caller_frame.f_locals["self"])
|
||||
|
||||
except KeyError: # we are in a regular function
|
||||
qualifier = caller_frame.f_globals["__name__"].split(".", 1)[-1]
|
||||
|
||||
else: # we are in a method
|
||||
qualifier = caller_object_type.__name__
|
||||
func_qualname = qualifier + "." + caller_frame.f_code.co_name
|
||||
|
||||
if disabled == 'env' and func_qualname not in cls._profilers:
|
||||
# don't do anything
|
||||
return cls._disabledProfiler
|
||||
|
||||
cls._depth += 1
|
||||
obj = super(Profiler, cls).__new__(cls)
|
||||
obj._msgs = []
|
||||
|
||||
# create an actual profiling object
|
||||
if cls._depth < 1:
|
||||
cls._msgs = []
|
||||
|
||||
obj._name = msg or func_qualname
|
||||
obj._delayed = delayed
|
||||
obj._markCount = 0
|
||||
obj._finished = False
|
||||
obj._firstTime = obj._lastTime = perf_counter()
|
||||
obj._mt = ms_threshold
|
||||
obj._newMsg("> Entering " + obj._name)
|
||||
return obj
|
||||
|
||||
def __call__(self, msg=None):
|
||||
"""Register or print a new message with timing information.
|
||||
"""
|
||||
if self.disable:
|
||||
return
|
||||
if msg is None:
|
||||
msg = str(self._markCount)
|
||||
|
||||
self._markCount += 1
|
||||
newTime = perf_counter()
|
||||
tot_ms = (newTime - self._firstTime) * 1000
|
||||
ms = (newTime - self._lastTime) * 1000
|
||||
self._newMsg(
|
||||
f" {msg}: {ms:0.4f}, tot:{tot_ms:0.4f}"
|
||||
)
|
||||
|
||||
self._lastTime = newTime
|
||||
|
||||
def mark(self, msg=None):
|
||||
self(msg)
|
||||
|
||||
def _newMsg(self, msg, *args):
|
||||
msg = " " * (self._depth - 1) + msg
|
||||
if self._delayed:
|
||||
self._msgs.append((msg, args))
|
||||
else:
|
||||
print(msg % args)
|
||||
|
||||
def __del__(self):
|
||||
self.finish()
|
||||
|
||||
def finish(self, msg=None):
|
||||
"""Add a final message; flush the message list if no parent profiler.
|
||||
"""
|
||||
if self._finished or self.disable:
|
||||
return
|
||||
|
||||
self._finished = True
|
||||
if msg is not None:
|
||||
self(msg)
|
||||
|
||||
tot_ms = (perf_counter() - self._firstTime) * 1000
|
||||
self._newMsg(
|
||||
"< Exiting %s, total time: %0.4f ms",
|
||||
self._name,
|
||||
tot_ms,
|
||||
)
|
||||
|
||||
if tot_ms < self._mt:
|
||||
# print(f'{tot_ms} < {self._mt}, clearing')
|
||||
# NOTE: this list **must** be an instance var to avoid
|
||||
# deleting common messages during GC I think?
|
||||
self._msgs.clear()
|
||||
# else:
|
||||
# print(f'{tot_ms} > {self._mt}, not clearing')
|
||||
|
||||
# XXX: why is this needed?
|
||||
# don't we **want to show** nested profiler messages?
|
||||
if self._msgs: # and self._depth < 1:
|
||||
|
||||
# if self._msgs:
|
||||
print("\n".join([m[0] % m[1] for m in self._msgs]))
|
||||
|
||||
# clear all entries
|
||||
self._msgs.clear()
|
||||
# type(self)._msgs = []
|
||||
|
||||
type(self)._depth -= 1
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"Accounting for degens": count dem numberz that tracks how much you got
|
||||
for tendiez.
|
||||
|
||||
'''
|
||||
from ..log import get_logger
|
||||
|
||||
from ._ledger import (
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
)
|
||||
from ._pos import (
|
||||
load_pps_from_ledger,
|
||||
open_pps,
|
||||
Position,
|
||||
PpTable,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'Transaction',
|
||||
'TransactionLedger',
|
||||
'open_trade_ledger',
|
||||
'PpTable',
|
||||
'open_pps',
|
||||
'load_pps_from_ledger',
|
||||
'Position',
|
||||
]
|
||||
|
||||
|
||||
def get_likely_pair(
|
||||
src: str,
|
||||
dst: str,
|
||||
bs_mktid: str,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Attempt to get the likely trading pair matching a given destination
|
||||
asset `dst: str`.
|
||||
|
||||
'''
|
||||
try:
|
||||
src_name_start = bs_mktid.rindex(src)
|
||||
except (
|
||||
ValueError, # substr not found
|
||||
):
|
||||
# TODO: handle nested positions..(i.e.
|
||||
# positions where the src fiat was used to
|
||||
# buy some other dst which was furhter used
|
||||
# to buy another dst..)
|
||||
log.warning(
|
||||
f'No src fiat {src} found in {bs_mktid}?'
|
||||
)
|
||||
return
|
||||
|
||||
likely_dst = bs_mktid[:src_name_start]
|
||||
if likely_dst == dst:
|
||||
return bs_mktid
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
from pprint import pformat
|
||||
|
||||
args = sys.argv
|
||||
assert len(args) > 1, 'Specifiy account(s) from `brokers.toml`'
|
||||
args = args[1:]
|
||||
for acctid in args:
|
||||
broker, name = acctid.split('.')
|
||||
trans, updated_pps = load_pps_from_ledger(broker, name)
|
||||
print(
|
||||
f'Processing transactions into pps for {broker}:{acctid}\n'
|
||||
f'{pformat(trans)}\n\n'
|
||||
f'{pformat(updated_pps)}'
|
||||
)
|
||||
|
|
@ -1,254 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Trade and transaction ledger processing.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from collections import UserDict
|
||||
from contextlib import contextmanager as cm
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
Union,
|
||||
Generator
|
||||
)
|
||||
|
||||
from pendulum import (
|
||||
datetime,
|
||||
parse,
|
||||
)
|
||||
import tomli
|
||||
import toml
|
||||
|
||||
from .. import config
|
||||
from ..data.types import Struct
|
||||
from ..log import get_logger
|
||||
from ._mktinfo import (
|
||||
Symbol, # legacy
|
||||
MktPair,
|
||||
Asset,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Transaction(Struct, frozen=True):
|
||||
|
||||
# TODO: unify this with the `MktPair`,
|
||||
# once we have that as a required field,
|
||||
# we don't really need the fqsn any more..
|
||||
fqsn: str
|
||||
|
||||
tid: Union[str, int] # unique transaction id
|
||||
size: float
|
||||
price: float
|
||||
cost: float # commisions or other additional costs
|
||||
dt: datetime
|
||||
|
||||
# TODO: we can drop this right since we
|
||||
# can instead expect the backend to provide this
|
||||
# via the `MktPair`?
|
||||
expiry: datetime | None = None
|
||||
|
||||
# remap for back-compat
|
||||
@property
|
||||
def fqme(self) -> str:
|
||||
return self.fqsn
|
||||
|
||||
# TODO: drop the Symbol type
|
||||
|
||||
# the underlying "transaction system", normally one of a ``MktPair``
|
||||
# (a description of a tradable double auction) or a ledger-recorded
|
||||
# ("ledger" in any sense as long as you can record transfers) of any
|
||||
# sort) ``Asset``.
|
||||
sym: MktPair | Asset | Symbol | None = None
|
||||
|
||||
@property
|
||||
def sys(self) -> Symbol:
|
||||
return self.sym
|
||||
|
||||
# (optional) key-id defined by the broker-service backend which
|
||||
# ensures the instrument-symbol market key for this record is unique
|
||||
# in the "their backend/system" sense; i.e. this uid for the market
|
||||
# as defined (internally) in some namespace defined by the broker
|
||||
# service.
|
||||
bs_mktid: str | int | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
dct = super().to_dict()
|
||||
# ensure we use a pendulum formatted
|
||||
# ISO style str here!@
|
||||
dct['dt'] = str(self.dt)
|
||||
return dct
|
||||
|
||||
|
||||
class TransactionLedger(UserDict):
|
||||
'''
|
||||
Very simple ``dict`` wrapper + ``pathlib.Path`` handle to
|
||||
a TOML formatted transaction file for enabling file writes
|
||||
dynamically whilst still looking exactly like a ``dict`` from the
|
||||
outside.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
ledger_dict: dict,
|
||||
file_path: Path,
|
||||
|
||||
) -> None:
|
||||
self.file_path = file_path
|
||||
super().__init__(ledger_dict)
|
||||
|
||||
def write_config(self) -> None:
|
||||
'''
|
||||
Render the self.data ledger dict to it's TML file form.
|
||||
|
||||
'''
|
||||
with self.file_path.open(mode='w') as fp:
|
||||
toml.dump(self.data, fp)
|
||||
|
||||
def iter_trans(
|
||||
self,
|
||||
broker: str = 'paper',
|
||||
|
||||
) -> Generator[
|
||||
tuple[str, Transaction],
|
||||
None,
|
||||
None,
|
||||
]:
|
||||
'''
|
||||
Deliver trades records in ``(key: str, t: Transaction)``
|
||||
form via generator.
|
||||
|
||||
'''
|
||||
if broker != 'paper':
|
||||
raise NotImplementedError('Per broker support not dun yet!')
|
||||
|
||||
# TODO: lookup some standard normalizer
|
||||
# func in the backend?
|
||||
# from ..brokers import get_brokermod
|
||||
# mod = get_brokermod(broker)
|
||||
# trans_dict = mod.norm_trade_records(self.data)
|
||||
|
||||
# NOTE: instead i propose the normalizer is
|
||||
# a one shot routine (that can be lru cached)
|
||||
# and instead call it for each entry incrementally:
|
||||
# normer = mod.norm_trade_record(txdict)
|
||||
|
||||
for tid, txdict in self.data.items():
|
||||
# special field handling for datetimes
|
||||
# to ensure pendulum is used!
|
||||
fqme = txdict.get('fqme', txdict['fqsn'])
|
||||
dt = parse(txdict['dt'])
|
||||
expiry = txdict.get('expiry')
|
||||
|
||||
yield (
|
||||
tid,
|
||||
Transaction(
|
||||
fqsn=fqme,
|
||||
tid=txdict['tid'],
|
||||
dt=dt,
|
||||
price=txdict['price'],
|
||||
size=txdict['size'],
|
||||
cost=txdict.get('cost', 0),
|
||||
bs_mktid=txdict['bs_mktid'],
|
||||
|
||||
# optional
|
||||
sym=None,
|
||||
expiry=parse(expiry) if expiry else None,
|
||||
)
|
||||
)
|
||||
|
||||
def to_trans(
|
||||
self,
|
||||
broker: str = 'paper',
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Return the entire output from ``.iter_trans()`` in a ``dict``.
|
||||
|
||||
'''
|
||||
return dict(self.iter_trans())
|
||||
|
||||
|
||||
@cm
|
||||
def open_trade_ledger(
|
||||
broker: str,
|
||||
account: str,
|
||||
|
||||
) -> Generator[dict, None, None]:
|
||||
'''
|
||||
Indempotently create and read in a trade log file from the
|
||||
``<configuration_dir>/ledgers/`` directory.
|
||||
|
||||
Files are named per broker account of the form
|
||||
``<brokername>_<accountname>.toml``. The ``accountname`` here is the
|
||||
name as defined in the user's ``brokers.toml`` config.
|
||||
|
||||
'''
|
||||
ldir: Path = config._config_dir / 'ledgers'
|
||||
if not ldir.is_dir():
|
||||
ldir.mkdir()
|
||||
|
||||
fname = f'trades_{broker}_{account}.toml'
|
||||
tradesfile: Path = ldir / fname
|
||||
|
||||
if not tradesfile.is_file():
|
||||
log.info(
|
||||
f'Creating new local trades ledger: {tradesfile}'
|
||||
)
|
||||
tradesfile.touch()
|
||||
|
||||
with tradesfile.open(mode='rb') as cf:
|
||||
start = time.time()
|
||||
ledger_dict = tomli.load(cf)
|
||||
log.info(f'Ledger load took {time.time() - start}s')
|
||||
cpy = ledger_dict.copy()
|
||||
|
||||
ledger = TransactionLedger(
|
||||
ledger_dict=cpy,
|
||||
file_path=tradesfile,
|
||||
)
|
||||
|
||||
try:
|
||||
yield ledger
|
||||
finally:
|
||||
if ledger.data != ledger_dict:
|
||||
|
||||
# TODO: show diff output?
|
||||
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
|
||||
log.info(f'Updating ledger for {tradesfile}:\n')
|
||||
ledger.write_config()
|
||||
|
||||
|
||||
def iter_by_dt(
|
||||
clears: dict[str, Any],
|
||||
|
||||
) -> Iterator[tuple[str, dict]]:
|
||||
'''
|
||||
Iterate entries of a ``clears: dict`` table sorted by entry recorded
|
||||
datetime presumably set at the ``'dt'`` field in each entry.
|
||||
|
||||
'''
|
||||
for tid, data in sorted(
|
||||
list(clears.items()),
|
||||
key=lambda item: item[1]['dt'],
|
||||
):
|
||||
yield tid, data
|
||||
|
|
@ -1,558 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Market (pair) meta-info layer: sane addressing semantics and meta-data
|
||||
for cross-provider marketplaces.
|
||||
|
||||
We intoduce the concept of,
|
||||
|
||||
- a FQMA: fully qualified market address,
|
||||
- a sane schema for FQMAs including derivatives,
|
||||
- a msg-serializeable description of markets for
|
||||
easy sharing with other pikers B)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from decimal import (
|
||||
Decimal,
|
||||
ROUND_HALF_EVEN,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Literal,
|
||||
)
|
||||
|
||||
from ..data.types import Struct
|
||||
|
||||
|
||||
_underlyings: list[str] = [
|
||||
'stock',
|
||||
'bond',
|
||||
'crypto',
|
||||
'fiat',
|
||||
'commodity',
|
||||
]
|
||||
|
||||
|
||||
_derivs: list[str] = [
|
||||
'swap',
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
]
|
||||
|
||||
# NOTE: a tag for other subsystems to try
|
||||
# and do default settings for certain things:
|
||||
# - allocator does unit vs. dolla size limiting.
|
||||
AssetTypeName: Literal[
|
||||
_underlyings
|
||||
+
|
||||
_derivs
|
||||
]
|
||||
|
||||
# egs. stock, futer, option, bond etc.
|
||||
|
||||
|
||||
def dec_digits(
|
||||
value: float | str | Decimal,
|
||||
|
||||
) -> int:
|
||||
'''
|
||||
Return the number of precision digits read from a decimal or float
|
||||
value.
|
||||
|
||||
'''
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(
|
||||
-Decimal(str(value)).as_tuple().exponent
|
||||
)
|
||||
|
||||
|
||||
float_digits = dec_digits
|
||||
|
||||
|
||||
def digits_to_dec(
|
||||
ndigits: int,
|
||||
) -> Decimal:
|
||||
'''
|
||||
Return the minimum float value for an input integer value.
|
||||
|
||||
eg. 3 -> 0.001
|
||||
|
||||
'''
|
||||
if ndigits == 0:
|
||||
return Decimal('0')
|
||||
|
||||
return Decimal('0.' + '0'*(ndigits-1) + '1')
|
||||
|
||||
|
||||
class Asset(Struct, frozen=True):
|
||||
'''
|
||||
Container type describing any transactable asset and its
|
||||
contract-like and/or underlying technology meta-info.
|
||||
|
||||
'''
|
||||
name: str
|
||||
atype: str # AssetTypeName
|
||||
|
||||
# minimum transaction size / precision.
|
||||
# eg. for buttcoin this is a "satoshi".
|
||||
tx_tick: Decimal
|
||||
|
||||
# NOTE: additional info optionally packed in by the backend, but
|
||||
# should not be explicitly required in our generic API.
|
||||
info: dict = {} # make it frozen?
|
||||
|
||||
# TODO?
|
||||
# _to_dict_skip = {'info'}
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
quantized form of the digit precision defined
|
||||
by ``self.lot_tick_size``.
|
||||
|
||||
'''
|
||||
digits = float_digits(self.tx_tick)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
|
||||
def maybe_cons_tokens(
|
||||
tokens: list[Any],
|
||||
delim_char: str = '.',
|
||||
) -> str:
|
||||
'''
|
||||
Construct `str` output from a maybe-concatenation of input
|
||||
sequence of elements in ``tokens``.
|
||||
|
||||
'''
|
||||
return delim_char.join(filter(bool, tokens)).lower()
|
||||
|
||||
|
||||
class MktPair(Struct, frozen=True):
|
||||
'''
|
||||
Market description for a pair of assets which are tradeable:
|
||||
a market which enables transactions of the form,
|
||||
buy: source asset -> destination asset
|
||||
sell: destination asset -> source asset
|
||||
|
||||
The main intention of this type is for a **simple** cross-asset
|
||||
venue/broker normalized descrption type from which all
|
||||
market-auctions can be mapped from FQME identifiers.
|
||||
|
||||
TODO: our eventual target fqme format/schema is:
|
||||
<dst>/<src>.<expiry>.<con_info_1>.<con_info_2>. -> .<venue>.<broker>
|
||||
^ -- optional tokens ------------------------------- ^
|
||||
|
||||
'''
|
||||
dst: str | Asset
|
||||
# "destination asset" (name) used to buy *to*
|
||||
# (or used to sell *from*)
|
||||
|
||||
price_tick: Decimal # minimum price increment
|
||||
size_tick: Decimal # minimum size (aka vlm) increment
|
||||
# the tick size is the number describing the smallest step in value
|
||||
# available in this market between the source and destination
|
||||
# assets.
|
||||
# https://en.wikipedia.org/wiki/Tick_size
|
||||
# https://en.wikipedia.org/wiki/Commodity_tick
|
||||
# https://en.wikipedia.org/wiki/Percentage_in_point
|
||||
|
||||
# unique "broker id" since every market endpoint provider
|
||||
# has their own nomenclature and schema for market maps.
|
||||
bs_mktid: str
|
||||
broker: str # the middle man giving access
|
||||
|
||||
# NOTE: to start this field is optional but should eventually be
|
||||
# required; the reason is for backward compat since more positioning
|
||||
# calculations were not originally stored with a src asset..
|
||||
|
||||
src: str | Asset = ''
|
||||
# "source asset" (name) used to buy *from*
|
||||
# (or used to sell *to*).
|
||||
|
||||
venue: str = '' # market venue provider name
|
||||
expiry: str = '' # for derivs, expiry datetime parseable str
|
||||
|
||||
# destination asset's financial type/classification name
|
||||
# NOTE: this is required for the order size allocator system,
|
||||
# since we use different default settings based on the type
|
||||
# of the destination asset, eg. futes use a units limits vs.
|
||||
# equities a $limit.
|
||||
# dst_type: AssetTypeName | None = None
|
||||
|
||||
# source asset's financial type/classification name
|
||||
# TODO: is a src type required for trading?
|
||||
# there's no reason to need any more then the one-way alloc-limiter
|
||||
# config right?
|
||||
# src_type: AssetTypeName
|
||||
|
||||
# for derivs, info describing contract, egs.
|
||||
# strike price, call or put, swap type, exercise model, etc.
|
||||
contract_info: list[str] | None = None
|
||||
|
||||
_atype: str = ''
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict[str, Any],
|
||||
|
||||
) -> MktPair:
|
||||
'''
|
||||
Constructor for a received msg-dict normally received over IPC.
|
||||
|
||||
'''
|
||||
dst_asset_msg = msg.pop('dst')
|
||||
src_asset_msg = msg.pop('src')
|
||||
|
||||
if isinstance(dst_asset_msg, str):
|
||||
src: str = str(src_asset_msg)
|
||||
assert isinstance(src, str)
|
||||
return cls.from_fqme(
|
||||
dst_asset_msg,
|
||||
src=src,
|
||||
**msg,
|
||||
)
|
||||
|
||||
else:
|
||||
# NOTE: we call `.copy()` here to ensure
|
||||
# type casting!
|
||||
dst = Asset(**dst_asset_msg).copy()
|
||||
if not isinstance(src_asset_msg, str):
|
||||
src = Asset(**src_asset_msg).copy()
|
||||
else:
|
||||
src = str(src_asset_msg)
|
||||
|
||||
return cls(
|
||||
dst=dst,
|
||||
src=src,
|
||||
**msg,
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return isinstance(self.dst, Asset)
|
||||
|
||||
@classmethod
|
||||
def from_fqme(
|
||||
cls,
|
||||
fqme: str,
|
||||
price_tick: float | str,
|
||||
size_tick: float | str,
|
||||
bs_mktid: str,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> MktPair:
|
||||
|
||||
broker, key, suffix = unpack_fqme(fqme)
|
||||
|
||||
# XXX: loading from a fqme string will
|
||||
# leave this pair as "un resolved" meaning
|
||||
# we don't yet have `.dst` set as an `Asset`
|
||||
# which we expect to be filled in by some
|
||||
# backend client with access to that data-info.
|
||||
return cls(
|
||||
dst=key, # not resolved
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
broker=broker,
|
||||
|
||||
**kwargs,
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
'''
|
||||
The "endpoint key" for this market.
|
||||
|
||||
Eg. mnq/usd or btc/usdt or xmr/btc
|
||||
|
||||
In most other tina platforms this is referred to as the
|
||||
"symbol".
|
||||
|
||||
'''
|
||||
return maybe_cons_tokens(
|
||||
[str(self.dst),
|
||||
str(self.src)],
|
||||
delim_char='',
|
||||
)
|
||||
|
||||
@property
|
||||
def suffix(self) -> str:
|
||||
'''
|
||||
The "contract suffix" for this market.
|
||||
|
||||
Eg. mnq/usd.20230616.cme.ib
|
||||
^ ----- ^
|
||||
or tsla/usd.20230324.200c.cboe.ib
|
||||
^ ---------- ^
|
||||
|
||||
In most other tina platforms they only show you these details in
|
||||
some kinda "meta data" format, we have FQMEs so we do this up
|
||||
front and explicit.
|
||||
|
||||
'''
|
||||
field_strs = [self.expiry]
|
||||
con_info = self.contract_info
|
||||
if con_info is not None:
|
||||
field_strs.extend(con_info)
|
||||
|
||||
return maybe_cons_tokens(field_strs)
|
||||
|
||||
# NOTE: the main idea behind an fqme is to map a "market address"
|
||||
# to some endpoint from a transaction provider (eg. a broker) such
|
||||
# that we build a table of `fqme: str -> bs_mktid: Any` where any "piker
|
||||
# market address" maps 1-to-1 to some broker trading endpoint.
|
||||
# @cached_property
|
||||
@property
|
||||
def fqme(self) -> str:
|
||||
'''
|
||||
Return the fully qualified market endpoint-address for the
|
||||
pair of transacting assets.
|
||||
|
||||
fqme = "fully qualified market endpoint"
|
||||
|
||||
And yes, you pronounce it colloquially as read..
|
||||
|
||||
Basically the idea here is for all client code (consumers of piker's
|
||||
APIs which query the data/broker-provider agnostic layer(s)) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string-key of the current form:
|
||||
|
||||
<market-instrument-name>
|
||||
.<venue>
|
||||
.<expiry>
|
||||
.<derivative-suffix-info>
|
||||
.<brokerbackendname>
|
||||
|
||||
eg. for an explicit daq mini futes contract: mnq.cme.20230317.ib
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.)
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
TODO:
|
||||
See community discussion on naming and nomenclature, order
|
||||
of addressing hierarchy, general schema, internal representation:
|
||||
|
||||
https://github.com/pikers/piker/issues/467
|
||||
|
||||
'''
|
||||
return maybe_cons_tokens([
|
||||
self.key, # final "pair name" (eg. qqq[/usd], btcusdt)
|
||||
self.venue,
|
||||
self.suffix, # includes expiry and other con info
|
||||
self.broker,
|
||||
])
|
||||
|
||||
@property
|
||||
def fqsn(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
quantity_type: Literal['price', 'size'] = 'size',
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
and ``.size_tick``'s # of digits.
|
||||
|
||||
'''
|
||||
match quantity_type:
|
||||
case 'price':
|
||||
digits = float_digits(self.price_tick)
|
||||
case 'size':
|
||||
digits = float_digits(self.size_tick)
|
||||
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# TODO: BACKWARD COMPAT, TO REMOVE?
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
if isinstance(self.dst, Asset):
|
||||
return str(self.dst.atype)
|
||||
|
||||
return self._atype
|
||||
|
||||
@property
|
||||
def tick_size_digits(self) -> int:
|
||||
return float_digits(self.price_tick)
|
||||
|
||||
@property
|
||||
def lot_size_digits(self) -> int:
|
||||
return float_digits(self.size_tick)
|
||||
|
||||
|
||||
def unpack_fqme(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[str, str, str]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqme.split('.')
|
||||
|
||||
match tokens:
|
||||
case [mkt_ep, broker]:
|
||||
# probably crypto
|
||||
# mkt_ep, broker = tokens
|
||||
return (
|
||||
broker,
|
||||
mkt_ep,
|
||||
'',
|
||||
)
|
||||
|
||||
# TODO: swap venue and suffix/deriv-info here?
|
||||
case [mkt_ep, venue, suffix, broker]:
|
||||
pass
|
||||
|
||||
case [mkt_ep, venue, broker]:
|
||||
suffix = ''
|
||||
|
||||
case _:
|
||||
raise ValueError(f'Invalid fqme: {fqme}')
|
||||
|
||||
return (
|
||||
broker,
|
||||
'.'.join([mkt_ep, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
unpack_fqsn = unpack_fqme
|
||||
|
||||
|
||||
class Symbol(Struct):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
|
||||
# precision descriptors for price and vlm
|
||||
tick_size: Decimal = Decimal('0.01')
|
||||
lot_tick_size: Decimal = Decimal('0.0')
|
||||
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@classmethod
|
||||
def from_fqsn(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
) -> Symbol:
|
||||
broker, key, suffix = unpack_fqsn(fqsn)
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
key=key,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_size,
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
# compat name mapping
|
||||
from_fqme = from_fqsn
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def tick_size_digits(self) -> int:
|
||||
return float_digits(self.tick_size)
|
||||
|
||||
@property
|
||||
def lot_size_digits(self) -> int:
|
||||
return float_digits(self.lot_tick_size)
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return Decimal(str(self.tick_size))
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return Decimal(str(self.lot_tick_size))
|
||||
|
||||
@property
|
||||
def broker(self) -> str:
|
||||
return list(self.broker_info.keys())[0]
|
||||
|
||||
@property
|
||||
def fqsn(self) -> str:
|
||||
broker = self.broker
|
||||
key = self.key
|
||||
if self.suffix:
|
||||
tokens = (key, self.suffix, broker)
|
||||
else:
|
||||
tokens = (key, broker)
|
||||
|
||||
return '.'.join(tokens).lower()
|
||||
|
||||
fqme = fqsn
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
) -> Decimal:
|
||||
digits = float_digits(self.lot_tick_size)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
|
|
@ -1,908 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Personal/Private position parsing, calculating, summarizing in a way
|
||||
that doesn't try to cuk most humans who prefer to not lose their moneys..
|
||||
|
||||
(looking at you `ib` and dirt-bird friends)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import contextmanager as cm
|
||||
from decimal import Decimal
|
||||
from math import copysign
|
||||
from pprint import pformat
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
Union,
|
||||
Generator
|
||||
)
|
||||
|
||||
import pendulum
|
||||
from pendulum import datetime, now
|
||||
|
||||
from ._toml import (
|
||||
toml,
|
||||
PpsEncoder,
|
||||
)
|
||||
from ._ledger import (
|
||||
Transaction,
|
||||
iter_by_dt,
|
||||
open_trade_ledger,
|
||||
)
|
||||
from ._mktinfo import (
|
||||
Symbol,
|
||||
MktPair,
|
||||
Asset,
|
||||
unpack_fqme,
|
||||
)
|
||||
from .. import config
|
||||
from ..brokers import get_brokermod
|
||||
from ..clearing._messages import BrokerdPosition, Status
|
||||
from ..data.types import Struct
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Position(Struct):
|
||||
'''
|
||||
Basic pp (personal/piker position) model with attached clearing
|
||||
transaction history.
|
||||
|
||||
'''
|
||||
symbol: Symbol | MktPair
|
||||
|
||||
# can be +ve or -ve for long/short
|
||||
size: float
|
||||
|
||||
# "breakeven price" above or below which pnl moves above and below
|
||||
# zero for the entirety of the current "trade state".
|
||||
ppu: float
|
||||
|
||||
# unique "backend system market id"
|
||||
bs_mktid: str
|
||||
|
||||
split_ratio: int | None = None
|
||||
|
||||
# ordered record of known constituent trade messages
|
||||
clears: dict[
|
||||
Union[str, int, Status], # trade id
|
||||
dict[str, Any], # transaction history summaries
|
||||
] = {}
|
||||
first_clear_dt: datetime | None = None
|
||||
|
||||
expiry: datetime | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return pformat(self.to_dict())
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
f: getattr(self, f)
|
||||
for f in self.__struct_fields__
|
||||
}
|
||||
|
||||
def to_pretoml(self) -> tuple[str, dict]:
|
||||
'''
|
||||
Prep this position's data contents for export to toml including
|
||||
re-structuring of the ``.clears`` table to an array of
|
||||
inline-subtables for better ``pps.toml`` compactness.
|
||||
|
||||
'''
|
||||
d = self.to_dict()
|
||||
clears = d.pop('clears')
|
||||
expiry = d.pop('expiry')
|
||||
|
||||
if self.split_ratio is None:
|
||||
d.pop('split_ratio')
|
||||
|
||||
# should be obvious from clears/event table
|
||||
d.pop('first_clear_dt')
|
||||
|
||||
# TODO: we need to figure out how to have one top level
|
||||
# listing venue here even when the backend isn't providing
|
||||
# it via the trades ledger..
|
||||
# drop symbol obj in serialized form
|
||||
s = d.pop('symbol')
|
||||
fqme = s.fqme
|
||||
broker, key, suffix = unpack_fqme(fqme)
|
||||
|
||||
if isinstance(s, Symbol):
|
||||
sym_info = s.broker_info[broker]
|
||||
d['asset_type'] = sym_info['asset_type']
|
||||
d['price_tick'] = (
|
||||
sym_info.get('price_tick_size')
|
||||
or
|
||||
s.tick_size
|
||||
)
|
||||
d['size_tick'] = (
|
||||
sym_info.get('lot_tick_size')
|
||||
or
|
||||
s.lot_tick_size
|
||||
)
|
||||
|
||||
# the newwww wayyy B)
|
||||
else:
|
||||
mkt = s
|
||||
assert isinstance(mkt, MktPair)
|
||||
|
||||
# an asset resolved mkt where we have ``Asset`` info about
|
||||
# each tradeable asset in the market.
|
||||
if mkt.resolved:
|
||||
dst: Asset = mkt.dst
|
||||
d['asset_type'] = dst.atype
|
||||
|
||||
d['price_tick'] = mkt.price_tick
|
||||
d['size_tick'] = mkt.size_tick
|
||||
|
||||
if self.expiry is None:
|
||||
d.pop('expiry', None)
|
||||
elif expiry:
|
||||
d['expiry'] = str(expiry)
|
||||
|
||||
toml_clears_list: list[dict[str, Any]] = []
|
||||
|
||||
# reverse sort so latest clears are at top of section?
|
||||
for tid, data in iter_by_dt(clears):
|
||||
inline_table = toml.TomlDecoder().get_empty_inline_table()
|
||||
|
||||
# serialize datetime to parsable `str`
|
||||
inline_table['dt'] = str(data['dt'])
|
||||
|
||||
# insert optional clear fields in column order
|
||||
for k in ['ppu', 'accum_size']:
|
||||
val = data.get(k)
|
||||
if val:
|
||||
inline_table[k] = val
|
||||
|
||||
# insert required fields
|
||||
for k in ['price', 'size', 'cost']:
|
||||
inline_table[k] = data[k]
|
||||
|
||||
inline_table['tid'] = tid
|
||||
toml_clears_list.append(inline_table)
|
||||
|
||||
d['clears'] = toml_clears_list
|
||||
|
||||
return fqme, d
|
||||
|
||||
def ensure_state(self) -> None:
|
||||
'''
|
||||
Audit either the `.size` and `.ppu` local instance vars against
|
||||
the clears table calculations and return the calc-ed values if
|
||||
they differ and log warnings to console.
|
||||
|
||||
'''
|
||||
clears = list(self.clears.values())
|
||||
self.first_clear_dt = min(list(entry['dt'] for entry in clears))
|
||||
last_clear = clears[-1]
|
||||
|
||||
csize = self.calc_size()
|
||||
accum = last_clear['accum_size']
|
||||
if not self.expired():
|
||||
if (
|
||||
csize != accum
|
||||
and csize != round(accum * self.split_ratio or 1)
|
||||
):
|
||||
raise ValueError(f'Size mismatch: {csize}')
|
||||
else:
|
||||
assert csize == 0, 'Contract is expired but non-zero size?'
|
||||
|
||||
if self.size != csize:
|
||||
log.warning(
|
||||
'Position state mismatch:\n'
|
||||
f'{self.size} => {csize}'
|
||||
)
|
||||
self.size = csize
|
||||
|
||||
cppu = self.calc_ppu()
|
||||
ppu = last_clear['ppu']
|
||||
if (
|
||||
cppu != ppu
|
||||
and self.split_ratio is not None
|
||||
# handle any split info entered (for now) manually by user
|
||||
and cppu != (ppu / self.split_ratio)
|
||||
):
|
||||
raise ValueError(f'PPU mismatch: {cppu}')
|
||||
|
||||
if self.ppu != cppu:
|
||||
log.warning(
|
||||
'Position state mismatch:\n'
|
||||
f'{self.ppu} => {cppu}'
|
||||
)
|
||||
self.ppu = cppu
|
||||
|
||||
def update_from_msg(
|
||||
self,
|
||||
msg: BrokerdPosition,
|
||||
|
||||
) -> None:
|
||||
|
||||
# XXX: better place to do this?
|
||||
symbol = self.symbol
|
||||
|
||||
# TODO: switch to new fields..?
|
||||
# .size_tick_digits, .price_tick_digits
|
||||
size_tick_digits = symbol.lot_size_digits
|
||||
price_tick_digits = symbol.tick_size_digits
|
||||
|
||||
self.ppu = round(
|
||||
# TODO: change this to ppu?
|
||||
msg['avg_price'],
|
||||
ndigits=price_tick_digits,
|
||||
)
|
||||
self.size = round(
|
||||
msg['size'],
|
||||
ndigits=size_tick_digits,
|
||||
)
|
||||
|
||||
@property
|
||||
def dsize(self) -> float:
|
||||
'''
|
||||
The "dollar" size of the pp, normally in trading (fiat) unit
|
||||
terms.
|
||||
|
||||
'''
|
||||
return self.ppu * self.size
|
||||
|
||||
# TODO: idea: "real LIFO" dynamic positioning.
|
||||
# - when a trade takes place where the pnl for
|
||||
# the (set of) trade(s) is below the breakeven price
|
||||
# it may be that the trader took a +ve pnl on a short(er)
|
||||
# term trade in the same account.
|
||||
# - in this case we could recalc the be price to
|
||||
# be reverted back to it's prior value before the nearest term
|
||||
# trade was opened.?
|
||||
# def lifo_price() -> float:
|
||||
# ...
|
||||
|
||||
def iter_clears(self) -> Iterator[tuple[str, dict]]:
|
||||
'''
|
||||
Iterate the internally managed ``.clears: dict`` table in
|
||||
datetime-stamped order.
|
||||
|
||||
'''
|
||||
return iter_by_dt(self.clears)
|
||||
|
||||
def calc_ppu(
|
||||
self,
|
||||
# include transaction cost in breakeven price
|
||||
# and presume the worst case of the same cost
|
||||
# to exit this transaction (even though in reality
|
||||
# it will be dynamic based on exit stratetgy).
|
||||
cost_scalar: float = 2,
|
||||
|
||||
) -> float:
|
||||
'''
|
||||
Compute the "price-per-unit" price for the given non-zero sized
|
||||
rolling position.
|
||||
|
||||
The recurrence relation which computes this (exponential) mean
|
||||
per new clear which **increases** the accumulative postiion size
|
||||
is:
|
||||
|
||||
ppu[-1] = (
|
||||
ppu[-2] * accum_size[-2]
|
||||
+
|
||||
ppu[-1] * size
|
||||
) / accum_size[-1]
|
||||
|
||||
where `cost_basis` for the current step is simply the price
|
||||
* size of the most recent clearing transaction.
|
||||
|
||||
'''
|
||||
asize_h: list[float] = [] # historical accumulative size
|
||||
ppu_h: list[float] = [] # historical price-per-unit
|
||||
|
||||
tid: str
|
||||
entry: dict[str, Any]
|
||||
for (tid, entry) in self.iter_clears():
|
||||
clear_size = entry['size']
|
||||
clear_price = entry['price']
|
||||
|
||||
last_accum_size = asize_h[-1] if asize_h else 0
|
||||
accum_size = last_accum_size + clear_size
|
||||
accum_sign = copysign(1, accum_size)
|
||||
|
||||
sign_change: bool = False
|
||||
|
||||
if accum_size == 0:
|
||||
ppu_h.append(0)
|
||||
asize_h.append(0)
|
||||
continue
|
||||
|
||||
if accum_size == 0:
|
||||
ppu_h.append(0)
|
||||
asize_h.append(0)
|
||||
continue
|
||||
|
||||
# test if the pp somehow went "passed" a net zero size state
|
||||
# resulting in a change of the "sign" of the size (+ve for
|
||||
# long, -ve for short).
|
||||
sign_change = (
|
||||
copysign(1, last_accum_size) + accum_sign == 0
|
||||
and last_accum_size != 0
|
||||
)
|
||||
|
||||
# since we passed the net-zero-size state the new size
|
||||
# after sum should be the remaining size the new
|
||||
# "direction" (aka, long vs. short) for this clear.
|
||||
if sign_change:
|
||||
clear_size = accum_size
|
||||
abs_diff = abs(accum_size)
|
||||
asize_h.append(0)
|
||||
ppu_h.append(0)
|
||||
|
||||
else:
|
||||
# old size minus the new size gives us size diff with
|
||||
# +ve -> increase in pp size
|
||||
# -ve -> decrease in pp size
|
||||
abs_diff = abs(accum_size) - abs(last_accum_size)
|
||||
|
||||
# XXX: LIFO breakeven price update. only an increaze in size
|
||||
# of the position contributes the breakeven price,
|
||||
# a decrease does not (i.e. the position is being made
|
||||
# smaller).
|
||||
# abs_clear_size = abs(clear_size)
|
||||
abs_new_size = abs(accum_size)
|
||||
|
||||
if abs_diff > 0:
|
||||
|
||||
cost_basis = (
|
||||
# cost basis for this clear
|
||||
clear_price * abs(clear_size)
|
||||
+
|
||||
# transaction cost
|
||||
accum_sign * cost_scalar * entry['cost']
|
||||
)
|
||||
|
||||
if asize_h:
|
||||
size_last = abs(asize_h[-1])
|
||||
cb_last = ppu_h[-1] * size_last
|
||||
ppu = (cost_basis + cb_last) / abs_new_size
|
||||
|
||||
else:
|
||||
ppu = cost_basis / abs_new_size
|
||||
|
||||
ppu_h.append(ppu)
|
||||
asize_h.append(accum_size)
|
||||
|
||||
else:
|
||||
# on "exit" clears from a given direction,
|
||||
# only the size changes not the price-per-unit
|
||||
# need to be updated since the ppu remains constant
|
||||
# and gets weighted by the new size.
|
||||
asize_h.append(accum_size)
|
||||
ppu_h.append(ppu_h[-1])
|
||||
|
||||
final_ppu = ppu_h[-1] if ppu_h else 0
|
||||
|
||||
# handle any split info entered (for now) manually by user
|
||||
if self.split_ratio is not None:
|
||||
final_ppu /= self.split_ratio
|
||||
|
||||
return final_ppu
|
||||
|
||||
def expired(self) -> bool:
|
||||
'''
|
||||
Predicate which checks if the contract/instrument is past its expiry.
|
||||
|
||||
'''
|
||||
return bool(self.expiry) and self.expiry < now()
|
||||
|
||||
def calc_size(self) -> float:
|
||||
'''
|
||||
Calculate the unit size of this position in the destination
|
||||
asset using the clears/trade event table; zero if expired.
|
||||
|
||||
'''
|
||||
size: float = 0
|
||||
|
||||
# time-expired pps (normally derivatives) are "closed"
|
||||
# and have a zero size.
|
||||
if self.expired():
|
||||
return 0
|
||||
|
||||
for tid, entry in self.clears.items():
|
||||
size += entry['size']
|
||||
|
||||
if self.split_ratio is not None:
|
||||
size = round(size * self.split_ratio)
|
||||
|
||||
return float(
|
||||
self.symbol.quantize(size),
|
||||
)
|
||||
|
||||
def minimize_clears(
|
||||
self,
|
||||
|
||||
) -> dict[str, dict]:
|
||||
'''
|
||||
Minimize the position's clears entries by removing
|
||||
all transactions before the last net zero size to avoid
|
||||
unecessary history irrelevant to the current pp state.
|
||||
|
||||
'''
|
||||
size: float = 0
|
||||
clears_since_zero: list[tuple(str, dict)] = []
|
||||
|
||||
# TODO: we might just want to always do this when iterating
|
||||
# a ledger? keep a state of the last net-zero and only do the
|
||||
# full iterate when no state was stashed?
|
||||
|
||||
# scan for the last "net zero" position by iterating
|
||||
# transactions until the next net-zero size, rinse, repeat.
|
||||
for tid, clear in self.clears.items():
|
||||
size += clear['size']
|
||||
clears_since_zero.append((tid, clear))
|
||||
|
||||
if size == 0:
|
||||
clears_since_zero.clear()
|
||||
|
||||
self.clears = dict(clears_since_zero)
|
||||
return self.clears
|
||||
|
||||
def add_clear(
|
||||
self,
|
||||
t: Transaction,
|
||||
) -> dict:
|
||||
'''
|
||||
Update clearing table and populate rolling ppu and accumulative
|
||||
size in both the clears entry and local attrs state.
|
||||
|
||||
'''
|
||||
clear = self.clears[t.tid] = {
|
||||
'cost': t.cost,
|
||||
'price': t.price,
|
||||
'size': t.size,
|
||||
'dt': t.dt
|
||||
}
|
||||
|
||||
# TODO: compute these incrementally instead
|
||||
# of re-looping through each time resulting in O(n**2)
|
||||
# behaviour..?
|
||||
|
||||
# NOTE: we compute these **after** adding the entry in order to
|
||||
# make the recurrence relation math work inside
|
||||
# ``.calc_size()``.
|
||||
self.size = clear['accum_size'] = self.calc_size()
|
||||
self.ppu = clear['ppu'] = self.calc_ppu()
|
||||
|
||||
return clear
|
||||
|
||||
# def sugest_split(self) -> float:
|
||||
# ...
|
||||
|
||||
|
||||
class PpTable(Struct):
|
||||
|
||||
brokername: str
|
||||
acctid: str
|
||||
pps: dict[str, Position]
|
||||
conf_path: Path
|
||||
conf: dict | None = {}
|
||||
|
||||
def update_from_trans(
|
||||
self,
|
||||
trans: dict[str, Transaction],
|
||||
cost_scalar: float = 2,
|
||||
|
||||
force_mkt: MktPair | None = None,
|
||||
|
||||
) -> dict[str, Position]:
|
||||
|
||||
pps = self.pps
|
||||
updated: dict[str, Position] = {}
|
||||
|
||||
# lifo update all pps from records, ensuring
|
||||
# we compute the PPU and size sorted in time!
|
||||
for t in sorted(
|
||||
trans.values(),
|
||||
key=lambda t: t.dt,
|
||||
reverse=True,
|
||||
):
|
||||
fqme = t.fqme
|
||||
bs_mktid = t.bs_mktid
|
||||
|
||||
# template the mkt-info presuming a legacy market ticks
|
||||
# if no info exists in the transactions..
|
||||
mkt: MktPair | Symbol | None = force_mkt or t.sys
|
||||
if not mkt:
|
||||
mkt = MktPair.from_fqme(
|
||||
fqme,
|
||||
price_tick='0.01',
|
||||
size_tick='0.0',
|
||||
bs_mktid=bs_mktid,
|
||||
)
|
||||
|
||||
pp = pps.get(bs_mktid)
|
||||
if not pp:
|
||||
# if no existing pp, allocate fresh one.
|
||||
pp = pps[bs_mktid] = Position(
|
||||
mkt,
|
||||
size=0.0,
|
||||
ppu=0.0,
|
||||
bs_mktid=bs_mktid,
|
||||
expiry=t.expiry,
|
||||
)
|
||||
else:
|
||||
# NOTE: if for some reason a "less resolved" mkt pair
|
||||
# info has been set (based on the `.fqme` being
|
||||
# a shorter string), instead use the one from the
|
||||
# transaction since it likely has (more) full
|
||||
# information from the provider.
|
||||
if len(pp.symbol.fqme) < len(fqme):
|
||||
pp.symbol = mkt
|
||||
|
||||
clears = pp.clears
|
||||
if clears:
|
||||
first_clear_dt = pp.first_clear_dt
|
||||
|
||||
# don't do updates for ledger records we already have
|
||||
# included in the current pps state.
|
||||
if (
|
||||
t.tid in clears
|
||||
or (
|
||||
first_clear_dt
|
||||
and t.dt < first_clear_dt
|
||||
)
|
||||
):
|
||||
# NOTE: likely you'll see repeats of the same
|
||||
# ``Transaction`` passed in here if/when you are restarting
|
||||
# a ``brokerd.ib`` where the API will re-report trades from
|
||||
# the current session, so we need to make sure we don't
|
||||
# "double count" these in pp calculations.
|
||||
continue
|
||||
|
||||
# update clearing table
|
||||
pp.add_clear(t)
|
||||
updated[t.bs_mktid] = pp
|
||||
|
||||
# minimize clears tables and update sizing.
|
||||
for bs_mktid, pp in updated.items():
|
||||
pp.ensure_state()
|
||||
|
||||
# deliver only the position entries that were actually updated
|
||||
# (modified the state) from the input transaction set.
|
||||
return updated
|
||||
|
||||
def dump_active(
|
||||
self,
|
||||
) -> tuple[
|
||||
dict[str, Position],
|
||||
dict[str, Position]
|
||||
]:
|
||||
'''
|
||||
Iterate all tabulated positions, render active positions to
|
||||
a ``dict`` format amenable to serialization (via TOML) and drop
|
||||
from state (``.pps``) as well as return in a ``dict`` all
|
||||
``Position``s which have recently closed.
|
||||
|
||||
'''
|
||||
# NOTE: newly closed position are also important to report/return
|
||||
# since a consumer, like an order mode UI ;), might want to react
|
||||
# based on the closure (for example removing the breakeven line
|
||||
# and clearing the entry from any lists/monitors).
|
||||
closed_pp_objs: dict[str, Position] = {}
|
||||
open_pp_objs: dict[str, Position] = {}
|
||||
|
||||
pp_objs = self.pps
|
||||
for bs_mktid in list(pp_objs):
|
||||
pp = pp_objs[bs_mktid]
|
||||
pp.ensure_state()
|
||||
|
||||
if (
|
||||
# "net-zero" is a "closed" position
|
||||
pp.size == 0
|
||||
|
||||
# time-expired pps (normally derivatives) are "closed"
|
||||
or (pp.expiry and pp.expiry < now())
|
||||
):
|
||||
# for expired cases
|
||||
pp.size = 0
|
||||
|
||||
# NOTE: we DO NOT pop the pp here since it can still be
|
||||
# used to check for duplicate clears that may come in as
|
||||
# new transaction from some backend API and need to be
|
||||
# ignored; the closed positions won't be written to the
|
||||
# ``pps.toml`` since ``pp_active_entries`` above is what's
|
||||
# written.
|
||||
closed_pp_objs[bs_mktid] = pp
|
||||
|
||||
else:
|
||||
open_pp_objs[bs_mktid] = pp
|
||||
|
||||
return open_pp_objs, closed_pp_objs
|
||||
|
||||
def to_toml(
|
||||
self,
|
||||
) -> dict[str, Any]:
|
||||
|
||||
active, closed = self.dump_active()
|
||||
|
||||
# ONLY dict-serialize all active positions; those that are
|
||||
# closed we don't store in the ``pps.toml``.
|
||||
to_toml_dict = {}
|
||||
|
||||
for bs_mktid, pos in active.items():
|
||||
|
||||
# keep the minimal amount of clears that make up this
|
||||
# position since the last net-zero state.
|
||||
pos.minimize_clears()
|
||||
pos.ensure_state()
|
||||
|
||||
# serialize to pre-toml form
|
||||
fqme, asdict = pos.to_pretoml()
|
||||
log.info(f'Updating active pp: {fqme}')
|
||||
|
||||
# XXX: ugh, it's cuz we push the section under
|
||||
# the broker name.. maybe we need to rethink this?
|
||||
brokerless_key = fqme.removeprefix(f'{self.brokername}.')
|
||||
to_toml_dict[brokerless_key] = asdict
|
||||
|
||||
return to_toml_dict
|
||||
|
||||
def write_config(self) -> None:
|
||||
'''
|
||||
Write the current position table to the user's ``pps.toml``.
|
||||
|
||||
'''
|
||||
# TODO: show diff output?
|
||||
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
|
||||
# active, closed_pp_objs = table.dump_active()
|
||||
pp_entries = self.to_toml()
|
||||
if pp_entries:
|
||||
log.info(
|
||||
f'Updating positions in ``{self.conf_path}``:\n'
|
||||
f'n{pformat(pp_entries)}'
|
||||
)
|
||||
|
||||
if self.brokername in self.conf:
|
||||
log.warning(
|
||||
f'Rewriting {self.conf_path} keys to drop <broker.acct>!'
|
||||
)
|
||||
# legacy key schema including <brokername.account>, so
|
||||
# rewrite all entries to drop those tables since we now
|
||||
# put that in the filename!
|
||||
accounts = self.conf.pop(self.brokername)
|
||||
assert len(accounts) == 1
|
||||
entries = accounts.pop(self.acctid)
|
||||
self.conf.update(entries)
|
||||
|
||||
self.conf.update(pp_entries)
|
||||
|
||||
# if there are no active position entries according
|
||||
# to the toml dump output above, then clear the config
|
||||
# file of all entries.
|
||||
elif self.conf:
|
||||
for entry in list(self.conf):
|
||||
del self.conf[entry]
|
||||
|
||||
# TODO: why tf haven't they already done this for inline
|
||||
# tables smh..
|
||||
enc = PpsEncoder(preserve=True)
|
||||
# table_bs_type = type(toml.TomlDecoder().get_empty_inline_table())
|
||||
enc.dump_funcs[
|
||||
toml.decoder.InlineTableDict
|
||||
] = enc.dump_inline_table
|
||||
|
||||
config.write(
|
||||
config=self.conf,
|
||||
path=self.conf_path,
|
||||
encoder=enc,
|
||||
fail_empty=False
|
||||
)
|
||||
|
||||
|
||||
def load_pps_from_ledger(
|
||||
|
||||
brokername: str,
|
||||
acctname: str,
|
||||
|
||||
# post normalization filter on ledger entries to be processed
|
||||
filter_by: list[dict] | None = None,
|
||||
|
||||
) -> tuple[
|
||||
dict[str, Transaction],
|
||||
dict[str, Position],
|
||||
]:
|
||||
'''
|
||||
Open a ledger file by broker name and account and read in and
|
||||
process any trade records into our normalized ``Transaction`` form
|
||||
and then update the equivalent ``Pptable`` and deliver the two
|
||||
bs_mktid-mapped dict-sets of the transactions and pps.
|
||||
|
||||
'''
|
||||
with (
|
||||
open_trade_ledger(brokername, acctname) as ledger,
|
||||
open_pps(brokername, acctname) as table,
|
||||
):
|
||||
if not ledger:
|
||||
# null case, no ledger file with content
|
||||
return {}
|
||||
|
||||
mod = get_brokermod(brokername)
|
||||
src_records: dict[str, Transaction] = mod.norm_trade_records(ledger)
|
||||
|
||||
if filter_by:
|
||||
records = {}
|
||||
bs_mktids = set(filter_by)
|
||||
for tid, r in src_records.items():
|
||||
if r.bs_mktid in bs_mktids:
|
||||
records[tid] = r
|
||||
else:
|
||||
records = src_records
|
||||
|
||||
updated = table.update_from_trans(records)
|
||||
|
||||
return records, updated
|
||||
|
||||
|
||||
@cm
|
||||
def open_pps(
|
||||
brokername: str,
|
||||
acctid: str,
|
||||
write_on_exit: bool = False,
|
||||
|
||||
) -> Generator[PpTable, None, None]:
|
||||
'''
|
||||
Read out broker-specific position entries from
|
||||
incremental update file: ``pps.toml``.
|
||||
|
||||
'''
|
||||
conf: dict
|
||||
conf_path: Path
|
||||
conf, conf_path = config.load(
|
||||
f'pps.{brokername}.{acctid}',
|
||||
)
|
||||
|
||||
if brokername in conf:
|
||||
log.warning(
|
||||
f'Rewriting {conf_path} keys to drop <broker.acct>!'
|
||||
)
|
||||
# legacy key schema including <brokername.account>, so
|
||||
# rewrite all entries to drop those tables since we now
|
||||
# put that in the filename!
|
||||
accounts = conf.pop(brokername)
|
||||
for acctid in accounts.copy():
|
||||
entries = accounts.pop(acctid)
|
||||
conf.update(entries)
|
||||
|
||||
# TODO: ideally we can pass in an existing
|
||||
# pps state to this right? such that we
|
||||
# don't have to do a ledger reload all the
|
||||
# time.. a couple ideas I can think of,
|
||||
# - mirror this in some client side actor which
|
||||
# does the actual ledger updates (say the paper
|
||||
# engine proc if we decide to always spawn it?),
|
||||
# - do diffs against updates from the ledger writer
|
||||
# actor and the in-mem state here?
|
||||
|
||||
pp_objs = {}
|
||||
table = PpTable(
|
||||
brokername,
|
||||
acctid,
|
||||
pp_objs,
|
||||
conf_path,
|
||||
conf=conf,
|
||||
)
|
||||
|
||||
# unmarshal/load ``pps.toml`` config entries into object form
|
||||
# and update `PpTable` obj entries.
|
||||
for fqme, entry in conf.items():
|
||||
|
||||
# atype = entry.get('asset_type', '<unknown>')
|
||||
|
||||
# unique broker market id
|
||||
bs_mktid = str(
|
||||
entry.get('bsuid')
|
||||
or entry.get('bs_mktid')
|
||||
)
|
||||
price_tick = Decimal(str(
|
||||
entry.get('price_tick_size')
|
||||
or entry.get('price_tick')
|
||||
or '0.01'
|
||||
))
|
||||
size_tick = Decimal(str(
|
||||
entry.get('lot_tick_size')
|
||||
or entry.get('size_tick')
|
||||
or '0.0'
|
||||
))
|
||||
|
||||
# load the pair using the fqme which
|
||||
# will make the pair "unresolved" until
|
||||
# the backend broker actually loads
|
||||
# the market and position info.
|
||||
mkt = MktPair.from_fqme(
|
||||
fqme,
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
bs_mktid=bs_mktid
|
||||
)
|
||||
|
||||
# TODO: RE: general "events" instead of just "clears":
|
||||
# - make this an `events` field and support more event types
|
||||
# such as 'split', 'name_change', 'mkt_info', etc..
|
||||
# - should be make a ``Struct`` for clear/event entries? convert
|
||||
# "clear events table" from the toml config (list of a dicts)
|
||||
# and load it into object form for use in position processing of
|
||||
# new clear events.
|
||||
|
||||
# convert clears sub-tables (only in this form
|
||||
# for toml re-presentation) back into a master table.
|
||||
toml_clears_list: list[dict[str, Any]] = entry['clears']
|
||||
trans: list[Transaction] = []
|
||||
for clears_table in toml_clears_list:
|
||||
|
||||
tid = clears_table.pop('tid')
|
||||
dtstr = clears_table['dt']
|
||||
dt = pendulum.parse(dtstr)
|
||||
clears_table['dt'] = dt
|
||||
|
||||
trans.append(Transaction(
|
||||
fqsn=bs_mktid,
|
||||
sym=mkt,
|
||||
bs_mktid=bs_mktid,
|
||||
tid=tid,
|
||||
size=clears_table['size'],
|
||||
price=clears_table['price'],
|
||||
cost=clears_table['cost'],
|
||||
dt=dt,
|
||||
))
|
||||
|
||||
size = entry['size']
|
||||
|
||||
# TODO: remove but, handle old field name for now
|
||||
ppu = entry.get(
|
||||
'ppu',
|
||||
entry.get('be_price', 0),
|
||||
)
|
||||
|
||||
split_ratio = entry.get('split_ratio')
|
||||
|
||||
expiry = entry.get('expiry')
|
||||
if expiry:
|
||||
expiry = pendulum.parse(expiry)
|
||||
|
||||
pp = pp_objs[bs_mktid] = Position(
|
||||
mkt,
|
||||
size=size,
|
||||
ppu=ppu,
|
||||
split_ratio=split_ratio,
|
||||
expiry=expiry,
|
||||
bs_mktid=bs_mktid,
|
||||
)
|
||||
|
||||
# XXX: super critical, we need to be sure to include
|
||||
# all pps.toml clears to avoid reusing clears that were
|
||||
# already included in the current incremental update
|
||||
# state, since today's records may have already been
|
||||
# processed!
|
||||
for t in trans:
|
||||
pp.add_clear(t)
|
||||
|
||||
# audit entries loaded from toml
|
||||
pp.ensure_state()
|
||||
|
||||
try:
|
||||
yield table
|
||||
finally:
|
||||
if write_on_exit:
|
||||
table.write_config()
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
TOML codec hacks to make position tables look decent.
|
||||
|
||||
(looking at you "`toml`-lib"..)
|
||||
|
||||
'''
|
||||
import re
|
||||
|
||||
import toml
|
||||
|
||||
|
||||
# TODO: instead see if we can hack tomli and tomli-w to do the same:
|
||||
# - https://github.com/hukkin/tomli
|
||||
# - https://github.com/hukkin/tomli-w
|
||||
class PpsEncoder(toml.TomlEncoder):
|
||||
'''
|
||||
Special "styled" encoder that makes a ``pps.toml`` redable and
|
||||
compact by putting `.clears` tables inline and everything else
|
||||
flat-ish.
|
||||
|
||||
'''
|
||||
separator = ','
|
||||
|
||||
def dump_list(self, v):
|
||||
'''
|
||||
Dump an inline list with a newline after every element and
|
||||
with consideration for denoted inline table types.
|
||||
|
||||
'''
|
||||
retval = "[\n"
|
||||
for u in v:
|
||||
if isinstance(u, toml.decoder.InlineTableDict):
|
||||
out = self.dump_inline_table(u)
|
||||
else:
|
||||
out = str(self.dump_value(u))
|
||||
|
||||
retval += " " + out + "," + "\n"
|
||||
retval += "]"
|
||||
return retval
|
||||
|
||||
def dump_inline_table(self, section):
|
||||
"""Preserve inline table in its compact syntax instead of expanding
|
||||
into subsection.
|
||||
https://github.com/toml-lang/toml#user-content-inline-table
|
||||
"""
|
||||
val_list = []
|
||||
for k, v in section.items():
|
||||
# if isinstance(v, toml.decoder.InlineTableDict):
|
||||
if isinstance(v, dict):
|
||||
val = self.dump_inline_table(v)
|
||||
else:
|
||||
val = str(self.dump_value(v))
|
||||
|
||||
val_list.append(k + " = " + val)
|
||||
|
||||
retval = "{ " + ", ".join(val_list) + " }"
|
||||
return retval
|
||||
|
||||
def dump_sections(self, o, sup):
|
||||
retstr = ""
|
||||
if sup != "" and sup[-1] != ".":
|
||||
sup += '.'
|
||||
retdict = self._dict()
|
||||
arraystr = ""
|
||||
for section in o:
|
||||
qsection = str(section)
|
||||
value = o[section]
|
||||
|
||||
if not re.match(r'^[A-Za-z0-9_-]+$', section):
|
||||
qsection = toml.encoder._dump_str(section)
|
||||
|
||||
# arrayoftables = False
|
||||
if (
|
||||
self.preserve
|
||||
and isinstance(value, toml.decoder.InlineTableDict)
|
||||
):
|
||||
retstr += (
|
||||
qsection
|
||||
+
|
||||
" = "
|
||||
+
|
||||
self.dump_inline_table(o[section])
|
||||
+
|
||||
'\n' # only on the final terminating left brace
|
||||
)
|
||||
|
||||
# XXX: this code i'm pretty sure is just blatantly bad
|
||||
# and/or wrong..
|
||||
# if isinstance(o[section], list):
|
||||
# for a in o[section]:
|
||||
# if isinstance(a, dict):
|
||||
# arrayoftables = True
|
||||
# if arrayoftables:
|
||||
# for a in o[section]:
|
||||
# arraytabstr = "\n"
|
||||
# arraystr += "[[" + sup + qsection + "]]\n"
|
||||
# s, d = self.dump_sections(a, sup + qsection)
|
||||
# if s:
|
||||
# if s[0] == "[":
|
||||
# arraytabstr += s
|
||||
# else:
|
||||
# arraystr += s
|
||||
# while d:
|
||||
# newd = self._dict()
|
||||
# for dsec in d:
|
||||
# s1, d1 = self.dump_sections(d[dsec], sup +
|
||||
# qsection + "." +
|
||||
# dsec)
|
||||
# if s1:
|
||||
# arraytabstr += ("[" + sup + qsection +
|
||||
# "." + dsec + "]\n")
|
||||
# arraytabstr += s1
|
||||
# for s1 in d1:
|
||||
# newd[dsec + "." + s1] = d1[s1]
|
||||
# d = newd
|
||||
# arraystr += arraytabstr
|
||||
|
||||
elif isinstance(value, dict):
|
||||
retdict[qsection] = o[section]
|
||||
|
||||
elif o[section] is not None:
|
||||
retstr += (
|
||||
qsection
|
||||
+
|
||||
" = "
|
||||
+
|
||||
str(self.dump_value(o[section]))
|
||||
)
|
||||
|
||||
# if not isinstance(value, dict):
|
||||
if not isinstance(value, toml.decoder.InlineTableDict):
|
||||
# inline tables should not contain newlines:
|
||||
# https://toml.io/en/v1.0.0#inline-table
|
||||
retstr += '\n'
|
||||
|
||||
else:
|
||||
raise ValueError(value)
|
||||
|
||||
retstr += arraystr
|
||||
return (retstr, retdict)
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
CLI front end for trades ledger and position tracking management.
|
||||
|
||||
'''
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
from rich.console import Console
|
||||
from rich.markdown import Markdown
|
||||
import tractor
|
||||
import trio
|
||||
import typer
|
||||
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
)
|
||||
from ..clearing._messages import BrokerdPosition
|
||||
from ..calc import humanize
|
||||
|
||||
|
||||
ledger = typer.Typer()
|
||||
|
||||
|
||||
def broker_init(
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**start_actor_kwargs,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Given an input broker name, load all named arguments
|
||||
which can be passed to a daemon + context spawn for
|
||||
the relevant `brokerd` service endpoint.
|
||||
|
||||
'''
|
||||
from ..brokers import get_brokermod
|
||||
brokermod = get_brokermod(brokername)
|
||||
modpath = brokermod.__name__
|
||||
|
||||
start_actor_kwargs['name'] = f'brokerd.{brokername}'
|
||||
start_actor_kwargs.update(
|
||||
getattr(
|
||||
brokermod,
|
||||
'_spawn_kwargs',
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
# lookup actor-enabled modules declared by the backend offering the
|
||||
# `brokerd` endpoint(s).
|
||||
enabled = start_actor_kwargs['enable_modules'] = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
enabled.append(subpath)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from ..data import _setup_persistent_brokerd
|
||||
|
||||
return (
|
||||
start_actor_kwargs, # to `ActorNursery.start_actor()`
|
||||
_setup_persistent_brokerd, # service task ep
|
||||
getattr( # trades endpoint
|
||||
brokermod,
|
||||
'trades_dialogue',
|
||||
None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@ledger.command()
|
||||
def sync(
|
||||
fully_qualified_account_name: str,
|
||||
pdb: bool = False,
|
||||
|
||||
loglevel: str = typer.Option(
|
||||
'error',
|
||||
"-l",
|
||||
),
|
||||
):
|
||||
console = Console()
|
||||
|
||||
try:
|
||||
brokername, account = fully_qualified_account_name.split('.')
|
||||
except ValueError:
|
||||
md = Markdown(
|
||||
f'=> `{fully_qualified_account_name}` <=\n\n'
|
||||
'is not a valid '
|
||||
'__fully qualified account name?__\n\n'
|
||||
'Your account name needs to be of the form '
|
||||
'`<brokername>.<account_name>`\n'
|
||||
)
|
||||
console.print(md)
|
||||
return
|
||||
|
||||
start_kwargs, _, trades_ep = broker_init(
|
||||
brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
|
||||
async def main():
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='ledger_cli',
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
|
||||
) as (actor, sockaddr),
|
||||
|
||||
tractor.open_nursery() as an,
|
||||
):
|
||||
portal = await an.start_actor(
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
**start_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
brokername == 'paper'
|
||||
or trades_ep is None
|
||||
):
|
||||
from ..clearing import _paper_engine as paper
|
||||
open_trades_endpoint = paper.open_paperboi(
|
||||
fqme=None, # tell paper to not start clearing loop
|
||||
broker=brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
else:
|
||||
# open live brokerd trades endpoint
|
||||
open_trades_endpoint = portal.open_context(
|
||||
trades_ep,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
|
||||
positions: dict[str, Any]
|
||||
accounts: list[str]
|
||||
async with (
|
||||
open_trades_endpoint as (
|
||||
brokerd_ctx,
|
||||
(positions, accounts,),
|
||||
),
|
||||
):
|
||||
summary: str = (
|
||||
'[dim underline]Piker Position Summary[/] '
|
||||
f'[dim blue underline]{brokername}[/]'
|
||||
'[dim].[/]'
|
||||
f'[blue underline]{account}[/]'
|
||||
f'[dim underline] -> total pps: [/]'
|
||||
f'[green]{len(positions)}[/]\n'
|
||||
)
|
||||
for ppdict in positions:
|
||||
ppmsg = BrokerdPosition(**ppdict)
|
||||
size = ppmsg.size
|
||||
if size:
|
||||
ppu: float = round(
|
||||
ppmsg.avg_price,
|
||||
ndigits=2,
|
||||
)
|
||||
cost_basis: str = humanize(size * ppu)
|
||||
h_size: str = humanize(size)
|
||||
|
||||
if size < 0:
|
||||
pcolor = 'red'
|
||||
else:
|
||||
pcolor = 'green'
|
||||
|
||||
# sematic-highlight of fqme
|
||||
fqme = ppmsg.symbol
|
||||
tokens = fqme.split('.')
|
||||
styled_fqme = f'[blue underline]{tokens[0]}[/]'
|
||||
for tok in tokens[1:]:
|
||||
styled_fqme += '[dim].[/]'
|
||||
styled_fqme += f'[dim blue underline]{tok}[/]'
|
||||
|
||||
# TODO: instead display in a ``rich.Table``?
|
||||
summary += (
|
||||
styled_fqme +
|
||||
'[dim]: [/]'
|
||||
f'[{pcolor}]{h_size}[/]'
|
||||
'[dim blue]u @[/]'
|
||||
f'[{pcolor}]{ppu}[/]'
|
||||
'[dim blue] = [/]'
|
||||
f'[{pcolor}]$ {cost_basis}\n[/]'
|
||||
)
|
||||
|
||||
console.print(summary)
|
||||
await brokerd_ctx.cancel()
|
||||
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ledger()
|
||||
|
|
@ -20,31 +20,22 @@ Broker clients, daemons and general back end machinery.
|
|||
from importlib import import_module
|
||||
from types import ModuleType
|
||||
|
||||
# TODO: move to urllib3/requests once supported
|
||||
import asks
|
||||
asks.init('trio')
|
||||
|
||||
__brokers__ = [
|
||||
'binance',
|
||||
'questrade',
|
||||
'robinhood',
|
||||
'ib',
|
||||
'kraken',
|
||||
|
||||
# broken but used to work
|
||||
# 'questrade',
|
||||
# 'robinhood',
|
||||
|
||||
# TODO: we should get on these stat!
|
||||
# alpaca
|
||||
# wstrade
|
||||
# iex
|
||||
|
||||
# deribit
|
||||
# kucoin
|
||||
# bitso
|
||||
]
|
||||
|
||||
|
||||
def get_brokermod(brokername: str) -> ModuleType:
|
||||
'''
|
||||
Return the imported broker module by name.
|
||||
|
||||
'''
|
||||
"""Return the imported broker module by name.
|
||||
"""
|
||||
module = import_module('.' + brokername, 'piker.brokers')
|
||||
# we only allow monkeying because it's for internal keying
|
||||
module.name = module.__name__.split('.')[-1]
|
||||
|
|
@ -52,9 +43,7 @@ def get_brokermod(brokername: str) -> ModuleType:
|
|||
|
||||
|
||||
def iter_brokermods():
|
||||
'''
|
||||
Iterate all built-in broker modules.
|
||||
|
||||
'''
|
||||
"""Iterate all built-in broker modules.
|
||||
"""
|
||||
for name in __brokers__:
|
||||
yield get_brokermod(name)
|
||||
|
|
|
|||
|
|
@ -15,28 +15,13 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Handy cross-broker utils.
|
||||
|
||||
Handy utils.
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
import json
|
||||
import asks
|
||||
import logging
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
colorize_json,
|
||||
)
|
||||
subsys: str = 'piker.brokers'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
from ..log import colorize_json
|
||||
|
||||
|
||||
class BrokerError(Exception):
|
||||
|
|
@ -84,6 +69,7 @@ class DataThrottle(BrokerError):
|
|||
# TODO: add in throttle metrics/feedback
|
||||
|
||||
|
||||
|
||||
def resproc(
|
||||
resp: asks.response_objects.Response,
|
||||
log: logging.Logger,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -23,14 +20,12 @@ Binance backend
|
|||
"""
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from typing import (
|
||||
Any, Union, Optional,
|
||||
AsyncGenerator, Callable,
|
||||
)
|
||||
import time
|
||||
|
||||
from trio_util import trio_async_generator
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import pendulum
|
||||
|
|
@ -38,28 +33,17 @@ import asks
|
|||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
from pydantic.dataclasses import dataclass
|
||||
from pydantic import BaseModel
|
||||
import wsproto
|
||||
|
||||
from ..accounting._mktinfo import (
|
||||
Asset,
|
||||
MktPair,
|
||||
digits_to_dec,
|
||||
)
|
||||
from .._cacheables import open_cached_client
|
||||
from ._util import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
DataUnavailable,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..data.types import Struct
|
||||
from ..data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
from ._util import resproc, SymbolNotFound
|
||||
from ..log import get_logger, get_console_log
|
||||
from ..data import ShmArray
|
||||
from ..data._web_bs import open_autorecon_ws, NoBsWs
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_url = 'https://api.binance.com'
|
||||
|
|
@ -95,17 +79,12 @@ _show_wap_in_history = False
|
|||
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
|
||||
# TODO: make this frozen again by pre-processing the
|
||||
# filters list to a dict at init time?
|
||||
class Pair(Struct, frozen=True):
|
||||
class Pair(BaseModel):
|
||||
symbol: str
|
||||
status: str
|
||||
|
||||
baseAsset: str
|
||||
baseAssetPrecision: int
|
||||
cancelReplaceAllowed: bool
|
||||
allowTrailingStop: bool
|
||||
quoteAsset: str
|
||||
quotePrecision: int
|
||||
quoteAssetPrecision: int
|
||||
|
|
@ -121,34 +100,18 @@ class Pair(Struct, frozen=True):
|
|||
isSpotTradingAllowed: bool
|
||||
isMarginTradingAllowed: bool
|
||||
|
||||
defaultSelfTradePreventionMode: str
|
||||
allowedSelfTradePreventionModes: list[str]
|
||||
|
||||
filters: dict[
|
||||
str,
|
||||
Union[str, int, float]
|
||||
]
|
||||
filters: list[dict[str, Union[str, int, float]]]
|
||||
permissions: list[str]
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
# XXX: lul, after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
return Decimal(self.filters['PRICE_FILTER']['tickSize'].rstrip('0'))
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return Decimal(self.filters['LOT_SIZE']['stepSize'].rstrip('0'))
|
||||
|
||||
|
||||
class OHLC(Struct):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
@dataclass
|
||||
class OHLC:
|
||||
"""Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-streams
|
||||
|
||||
'''
|
||||
"""
|
||||
time: int
|
||||
|
||||
open: float
|
||||
|
|
@ -171,9 +134,7 @@ class OHLC(Struct):
|
|||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def binance_timestamp(
|
||||
when: datetime
|
||||
) -> int:
|
||||
def binance_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
|
|
@ -182,7 +143,7 @@ class Client:
|
|||
def __init__(self) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._pairs: dict[str, Pair] = {}
|
||||
self._pairs: dict[str, Any] = {}
|
||||
|
||||
async def _api(
|
||||
self,
|
||||
|
|
@ -196,58 +157,48 @@ class Client:
|
|||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def exch_info(
|
||||
async def symbol_info(
|
||||
|
||||
self,
|
||||
sym: str | None = None,
|
||||
sym: Optional[str] = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
'''
|
||||
Fresh exchange-pairs info query for symbol ``sym: str``:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
) -> dict[str, Any]:
|
||||
'''Get symbol info for the exchange.
|
||||
|
||||
'''
|
||||
cached_pair = self._pairs.get(sym)
|
||||
if cached_pair:
|
||||
return cached_pair
|
||||
# TODO: we can load from our self._pairs cache
|
||||
# on repeat calls...
|
||||
|
||||
# retrieve all symbols by default
|
||||
# will retrieve all symbols by default
|
||||
params = {}
|
||||
|
||||
if sym is not None:
|
||||
sym = sym.lower()
|
||||
sym = sym.upper()
|
||||
params = {'symbol': sym}
|
||||
|
||||
resp = await self._api('exchangeInfo', params=params)
|
||||
entries = resp['symbols']
|
||||
if not entries:
|
||||
raise SymbolNotFound(f'{sym} not found:\n{resp}')
|
||||
|
||||
# pre-process .filters field into a table
|
||||
pairs = {}
|
||||
for item in entries:
|
||||
symbol = item['symbol']
|
||||
filters = {}
|
||||
filters_ls: list = item.pop('filters')
|
||||
for entry in filters_ls:
|
||||
ftype = entry['filterType']
|
||||
filters[ftype] = entry
|
||||
|
||||
pairs[symbol] = Pair(
|
||||
filters=filters,
|
||||
**item,
|
||||
resp = await self._api(
|
||||
'exchangeInfo',
|
||||
params=params,
|
||||
)
|
||||
|
||||
# pairs = {
|
||||
# item['symbol']: Pair(**item) for item in entries
|
||||
# }
|
||||
self._pairs.update(pairs)
|
||||
entries = resp['symbols']
|
||||
if not entries:
|
||||
raise SymbolNotFound(f'{sym} not found')
|
||||
|
||||
syms = {item['symbol']: item for item in entries}
|
||||
|
||||
if sym is not None:
|
||||
return pairs[sym]
|
||||
return syms[sym]
|
||||
else:
|
||||
return self._pairs
|
||||
return syms
|
||||
|
||||
symbol_info = exch_info
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
|
|
@ -257,7 +208,7 @@ class Client:
|
|||
if self._pairs is not None:
|
||||
data = self._pairs
|
||||
else:
|
||||
data = await self.exch_info()
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
|
|
@ -279,7 +230,7 @@ class Client:
|
|||
) -> dict:
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC').add(minutes=1)
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
|
|
@ -309,7 +260,6 @@ class Client:
|
|||
for i, bar in enumerate(bars):
|
||||
|
||||
bar = OHLC(*bar)
|
||||
bar.typecast()
|
||||
|
||||
row = []
|
||||
for j, (name, ftype) in enumerate(_ohlc_dtype[1:]):
|
||||
|
|
@ -332,13 +282,12 @@ class Client:
|
|||
@acm
|
||||
async def get_client() -> Client:
|
||||
client = Client()
|
||||
log.info(f'Caching exchange infos..')
|
||||
await client.exch_info()
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
|
||||
|
||||
# validation type
|
||||
class AggTrade(Struct):
|
||||
class AggTrade(BaseModel):
|
||||
e: str # Event type
|
||||
E: int # Event time
|
||||
s: str # Symbol
|
||||
|
|
@ -352,10 +301,7 @@ class AggTrade(Struct):
|
|||
M: bool # Ignore
|
||||
|
||||
|
||||
@trio_async_generator
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
) -> AsyncGenerator[NoBsWs, dict]:
|
||||
async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]:
|
||||
|
||||
timeouts = 0
|
||||
while True:
|
||||
|
|
@ -395,9 +341,7 @@ async def stream_messages(
|
|||
|
||||
elif msg.get('e') == 'aggTrade':
|
||||
|
||||
# NOTE: this is purely for a definition, ``msgspec.Struct``
|
||||
# does not runtime-validate until you decode/encode.
|
||||
# see: https://jcristharif.com/msgspec/structs.html#type-validation
|
||||
# validate
|
||||
msg = AggTrade(**msg)
|
||||
|
||||
# TODO: type out and require this quote format
|
||||
|
|
@ -408,8 +352,8 @@ async def stream_messages(
|
|||
'brokerd_ts': time.time(),
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': float(msg.p),
|
||||
'size': float(msg.q),
|
||||
'price': msg.p,
|
||||
'size': msg.q,
|
||||
'broker_ts': msg.T,
|
||||
}],
|
||||
}
|
||||
|
|
@ -440,65 +384,39 @@ async def open_history_client(
|
|||
async with open_cached_client('binance') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: datetime | None = None,
|
||||
start_dt: datetime | None = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
if timeframe != 60:
|
||||
raise DataUnavailable('Only 1m bars are supported')
|
||||
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
times = array['time']
|
||||
if (
|
||||
end_dt is None
|
||||
):
|
||||
inow = round(time.time())
|
||||
if (inow - times[-1]) > 60:
|
||||
await tractor.breakpoint()
|
||||
|
||||
start_dt = pendulum.from_timestamp(times[0])
|
||||
end_dt = pendulum.from_timestamp(times[-1])
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
|
||||
async def backfill_bars(
|
||||
sym: str,
|
||||
shm: ShmArray, # type: ignore # noqa
|
||||
task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
"""Fill historical bars into shared mem / storage afap.
|
||||
"""
|
||||
with trio.CancelScope() as cs:
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
pair: Pair = await client.exch_info(fqme.upper())
|
||||
|
||||
mkt = MktPair(
|
||||
dst=Asset(
|
||||
name=pair.baseAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.baseAssetPrecision),
|
||||
),
|
||||
src=Asset(
|
||||
name=pair.quoteAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.quoteAssetPrecision),
|
||||
),
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=pair.symbol,
|
||||
broker='binance',
|
||||
)
|
||||
return mkt, pair
|
||||
bars = await client.bars(symbol=sym)
|
||||
shm.push(bars)
|
||||
task_status.started(cs)
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
|
@ -515,15 +433,28 @@ async def stream_quotes(
|
|||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym_infos = {}
|
||||
uid = 0
|
||||
|
||||
async with (
|
||||
open_cached_client('binance') as client,
|
||||
send_chan as send_chan,
|
||||
):
|
||||
mkt_infos: dict[str, MktPair] = {}
|
||||
|
||||
# keep client cached for real-time section
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
for sym in symbols:
|
||||
mkt, pair = await get_mkt_info(sym)
|
||||
mkt_infos[sym] = mkt
|
||||
d = cache[sym.upper()]
|
||||
syminfo = Pair(**d) # validation
|
||||
|
||||
si = sym_infos[sym] = syminfo.dict()
|
||||
|
||||
# XXX: after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
|
||||
si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
|
||||
si['asset_type'] = 'crypto'
|
||||
|
||||
symbol = symbols[0]
|
||||
|
||||
|
|
@ -531,10 +462,9 @@ async def stream_quotes(
|
|||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
symbol: {
|
||||
'fqsn': sym,
|
||||
|
||||
'mkt_info': mkt_infos[sym],
|
||||
'symbol_info': sym_infos[sym],
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -565,7 +495,6 @@ async def stream_quotes(
|
|||
subs.append("{sym}@bookTicker")
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
"method": "UNSUBSCRIBE",
|
||||
"params": subs,
|
||||
|
|
@ -575,23 +504,19 @@ async def stream_quotes(
|
|||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
# XXX: see api docs which show diff addr?
|
||||
# https://developers.binance.com/docs/binance-trading-api/websocket_api#general-api-information
|
||||
# 'wss://ws-api.binance.com:443/ws-api/v3',
|
||||
async with open_autorecon_ws(
|
||||
'wss://stream.binance.com/ws',
|
||||
fixture=subscribe,
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
stream_messages(ws) as msg_gen,
|
||||
):
|
||||
typ, quote = await anext(msg_gen)
|
||||
) as ws:
|
||||
|
||||
# pull a first quote and deliver
|
||||
msg_gen = stream_messages(ws)
|
||||
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
while typ != 'trade':
|
||||
typ, quote = await anext(msg_gen)
|
||||
# TODO: use ``anext()`` when it lands in 3.10!
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
|
|
@ -621,13 +546,13 @@ async def open_symbol_search(
|
|||
async with open_cached_client('binance') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.exch_info()
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# results = await client.exch_info(sym=pattern.upper())
|
||||
# results = await client.symbol_info(sym=pattern.upper())
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
|
|
@ -635,7 +560,7 @@ async def open_symbol_search(
|
|||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
item[0].symbol: item[0]
|
||||
for item in matches
|
||||
})
|
||||
await stream.send(
|
||||
{item[0]['symbol']: item[0]
|
||||
for item in matches}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,170 +28,17 @@ import tractor
|
|||
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..service import (
|
||||
maybe_spawn_brokerd,
|
||||
maybe_open_pikerd,
|
||||
)
|
||||
from ..brokers import (
|
||||
core,
|
||||
get_brokermod,
|
||||
data,
|
||||
)
|
||||
DEFAULT_BROKER = 'binance'
|
||||
from ..log import get_console_log, colorize_json, get_logger
|
||||
from .._daemon import maybe_spawn_brokerd, maybe_open_pikerd
|
||||
from ..brokers import core, get_brokermod, data
|
||||
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
_config_dir = click.get_app_dir('piker')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
|
||||
|
||||
OK = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
FAIL = '\033[91m'
|
||||
ENDC = '\033[0m'
|
||||
|
||||
|
||||
def print_ok(s: str, **kwargs):
|
||||
print(OK + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def print_error(s: str, **kwargs):
|
||||
print(FAIL + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def get_method(client, meth_name: str):
|
||||
print(f'checking client for method \'{meth_name}\'...', end='', flush=True)
|
||||
method = getattr(client, meth_name, None)
|
||||
assert method
|
||||
print_ok('found!.')
|
||||
return method
|
||||
|
||||
|
||||
async def run_method(client, meth_name: str, **kwargs):
|
||||
method = get_method(client, meth_name)
|
||||
print('running...', end='', flush=True)
|
||||
result = await method(**kwargs)
|
||||
print_ok(f'done! result: {type(result)}')
|
||||
return result
|
||||
|
||||
|
||||
async def run_test(broker_name: str):
|
||||
brokermod = get_brokermod(broker_name)
|
||||
total = 0
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
print('getting client...', end='', flush=True)
|
||||
if not hasattr(brokermod, 'get_client'):
|
||||
print_error('fail! no \'get_client\' context manager found.')
|
||||
return
|
||||
|
||||
async with brokermod.get_client(is_brokercheck=True) as client:
|
||||
print_ok('done! inside client context.')
|
||||
|
||||
# check for methods present on brokermod
|
||||
method_list = [
|
||||
'backfill_bars',
|
||||
'get_client',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
|
||||
]
|
||||
|
||||
for method in method_list:
|
||||
print(
|
||||
f'checking brokermod for method \'{method}\'...',
|
||||
end='', flush=True)
|
||||
if not hasattr(brokermod, method):
|
||||
print_error(f'fail! method \'{method}\' not found.')
|
||||
failed += 1
|
||||
else:
|
||||
print_ok('done!')
|
||||
passed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
# for private methods only check is present
|
||||
method_list = [
|
||||
'get_balances',
|
||||
'get_assets',
|
||||
'get_trades',
|
||||
'get_xfers',
|
||||
'submit_limit',
|
||||
'submit_cancel',
|
||||
'search_symbols',
|
||||
]
|
||||
|
||||
for method_name in method_list:
|
||||
try:
|
||||
get_method(client, method_name)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
syms = await run_method(client, 'symbol_info')
|
||||
total += 1
|
||||
|
||||
if len(syms) == 0:
|
||||
raise BaseException('Empty Symbol list?')
|
||||
|
||||
passed += 1
|
||||
|
||||
first_sym = tuple(syms.keys())[0]
|
||||
|
||||
method_list = [
|
||||
('cache_symbols', {}),
|
||||
('search_symbols', {'pattern': first_sym[:-1]}),
|
||||
('bars', {'symbol': first_sym})
|
||||
]
|
||||
|
||||
for method_name, method_kwargs in method_list:
|
||||
try:
|
||||
await run_method(client, method_name, **method_kwargs)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
print(f'total: {total}, passed: {passed}, failed: {failed}')
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('broker', nargs=1, required=True)
|
||||
@click.pass_obj
|
||||
def brokercheck(config, broker):
|
||||
'''
|
||||
Test broker apis for completeness.
|
||||
|
||||
'''
|
||||
async def bcheck_main():
|
||||
async with maybe_spawn_brokerd(broker) as portal:
|
||||
await portal.run(run_test, broker)
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(run_test, broker)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--keys', '-k', multiple=True,
|
||||
help='Return results only for these keys')
|
||||
|
|
|
|||
|
|
@ -26,12 +26,15 @@ from typing import List, Dict, Any, Optional
|
|||
|
||||
import trio
|
||||
|
||||
from ._util import log
|
||||
from ..log import get_logger
|
||||
from . import get_brokermod
|
||||
from ..service import maybe_spawn_brokerd
|
||||
from .._daemon import maybe_spawn_brokerd
|
||||
from .._cacheables import open_cached_client
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def api(brokername: str, methname: str, **kwargs) -> dict:
|
||||
"""Make (proxy through) a broker API call by name and return its result.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@ import tractor
|
|||
from tractor.experimental import msgpub
|
||||
from async_generator import asynccontextmanager
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, get_console_log
|
||||
from . import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def wait_for_network(
|
||||
net_func: Callable,
|
||||
sleep: int = 1
|
||||
|
|
@ -227,28 +227,26 @@ async def get_cached_feed(
|
|||
|
||||
@tractor.stream
|
||||
async def start_quote_stream(
|
||||
stream: tractor.Context, # marks this as a streaming func
|
||||
ctx: tractor.Context, # marks this as a streaming func
|
||||
broker: str,
|
||||
symbols: List[Any],
|
||||
feed_type: str = 'stock',
|
||||
rate: int = 3,
|
||||
) -> None:
|
||||
'''
|
||||
Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
"""Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
pattern.
|
||||
|
||||
Spawns new quoter tasks for each broker backend on-demand.
|
||||
Since most brokers seems to support batch quote requests we
|
||||
limit to one task per process (for now).
|
||||
|
||||
'''
|
||||
"""
|
||||
# XXX: why do we need this again?
|
||||
get_console_log(tractor.current_actor().loglevel)
|
||||
|
||||
# pull global vars from local actor
|
||||
symbols = list(symbols)
|
||||
log.info(
|
||||
f"{stream.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
f"{ctx.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
# another actor task may have already created it
|
||||
async with get_cached_feed(broker) as feed:
|
||||
|
||||
|
|
@ -292,13 +290,13 @@ async def start_quote_stream(
|
|||
assert fquote['displayable']
|
||||
payload[sym] = fquote
|
||||
|
||||
await stream.send_yield(payload)
|
||||
await ctx.send_yield(payload)
|
||||
|
||||
await stream_poll_requests(
|
||||
|
||||
# ``trionics.msgpub`` required kwargs
|
||||
task_name=feed_type,
|
||||
ctx=stream,
|
||||
ctx=ctx,
|
||||
topics=symbols,
|
||||
packetizer=feed.mod.packetizer,
|
||||
|
||||
|
|
@ -321,11 +319,9 @@ async def call_client(
|
|||
|
||||
|
||||
class DataFeed:
|
||||
'''
|
||||
Data feed client for streaming symbol data from and making API
|
||||
client calls to a (remote) ``brokerd`` daemon.
|
||||
|
||||
'''
|
||||
"""Data feed client for streaming symbol data from and making API client calls
|
||||
to a (remote) ``brokerd`` daemon.
|
||||
"""
|
||||
_allowed = ('stock', 'option')
|
||||
|
||||
def __init__(self, portal, brokermod):
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
``deribit`` backend
|
||||
------------------
|
||||
pretty good liquidity crypto derivatives, uses custom json rpc over ws for
|
||||
client methods, then `cryptofeed` for data streams.
|
||||
|
||||
status
|
||||
******
|
||||
- supports option charts
|
||||
- no order support yet
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[deribit]
|
||||
key_id = 'XXXXXXXX'
|
||||
key_secret = 'Xx_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
|
||||
To obtain an api id and secret you need to create an account, which can be a
|
||||
real market account over at:
|
||||
|
||||
- deribit.com (requires KYC for deposit address)
|
||||
|
||||
Or a testnet account over at:
|
||||
|
||||
- test.deribit.com
|
||||
|
||||
For testnet once the account is created here is how you deposit fake crypto to
|
||||
try it out:
|
||||
|
||||
1) Go to Wallet:
|
||||
|
||||
.. figure:: assets/0_wallet.png
|
||||
:align: center
|
||||
:target: assets/0_wallet.png
|
||||
:alt: wallet page
|
||||
|
||||
2) Then click on the elipsis menu and select deposit
|
||||
|
||||
.. figure:: assets/1_wallet_select_deposit.png
|
||||
:align: center
|
||||
:target: assets/1_wallet_select_deposit.png
|
||||
:alt: wallet deposit page
|
||||
|
||||
3) This will take you to the deposit address page
|
||||
|
||||
.. figure:: assets/2_gen_deposit_addr.png
|
||||
:align: center
|
||||
:target: assets/2_gen_deposit_addr.png
|
||||
:alt: generate deposit address page
|
||||
|
||||
4) After clicking generate you should see the address, copy it and go to the
|
||||
`coin faucet <https://test.deribit.com/dericoin/BTC/deposit>`_ and send fake
|
||||
coins to that address.
|
||||
|
||||
.. figure:: assets/3_deposit_address.png
|
||||
:align: center
|
||||
:target: assets/3_deposit_address.png
|
||||
:alt: generated address
|
||||
|
||||
5) Back in the deposit address page you should see the deposit in your history
|
||||
|
||||
.. figure:: assets/4_wallet_deposit_history.png
|
||||
:align: center
|
||||
:target: assets/4_wallet_deposit_history.png
|
||||
:alt: wallet deposit history
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
|
||||
from piker.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
backfill_bars
|
||||
)
|
||||
# from .broker import (
|
||||
# trades_dialogue,
|
||||
# norm_trade_records,
|
||||
# )
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
# 'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
# 'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
# 'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
# annotation to let backend agnostic code
|
||||
# know if ``brokerd`` should be spawned with
|
||||
# ``tractor``'s aio mode.
|
||||
_infect_asyncio: bool = True
|
||||
|
|
@ -1,672 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
|
||||
from contextlib import asynccontextmanager as acm, AsyncExitStack
|
||||
from functools import partial
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, Iterable, Callable
|
||||
|
||||
import pendulum
|
||||
import asks
|
||||
import trio
|
||||
from trio_typing import Nursery, TaskStatus
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
|
||||
from piker.data.types import Struct
|
||||
from piker.data._web_bs import (
|
||||
NoBsWs,
|
||||
open_autorecon_ws,
|
||||
open_jsonrpc_session
|
||||
)
|
||||
|
||||
from .._util import resproc
|
||||
|
||||
from piker import config
|
||||
from piker.log import get_logger
|
||||
|
||||
from tractor.trionics import (
|
||||
broadcast_receiver,
|
||||
BroadcastReceiver,
|
||||
maybe_open_context
|
||||
)
|
||||
from tractor import to_asyncio
|
||||
|
||||
from cryptofeed import FeedHandler
|
||||
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT,
|
||||
L1_BOOK, TRADES,
|
||||
OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
_url = 'https://www.deribit.com'
|
||||
_ws_url = 'wss://www.deribit.com/ws/api/v2'
|
||||
_testnet_ws_url = 'wss://test.deribit.com/ws/api/v2'
|
||||
|
||||
|
||||
# Broker specific ohlc schema (rest)
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('bar_wap', float), # will be zeroed by sampler if not filled
|
||||
]
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
id: int
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
usIn: int
|
||||
usOut: int
|
||||
usDiff: int
|
||||
testnet: bool
|
||||
|
||||
class JSONRPCChannel(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
method: str
|
||||
params: dict
|
||||
|
||||
|
||||
class KLinesResult(Struct):
|
||||
close: list[float]
|
||||
cost: list[float]
|
||||
high: list[float]
|
||||
low: list[float]
|
||||
open: list[float]
|
||||
status: str
|
||||
ticks: list[int]
|
||||
volume: list[float]
|
||||
|
||||
class Trade(Struct):
|
||||
trade_seq: int
|
||||
trade_id: str
|
||||
timestamp: int
|
||||
tick_direction: int
|
||||
price: float
|
||||
mark_price: float
|
||||
iv: float
|
||||
instrument_name: str
|
||||
index_price: float
|
||||
direction: str
|
||||
combo_trade_id: Optional[int] = 0,
|
||||
combo_id: Optional[str] = '',
|
||||
amount: float
|
||||
|
||||
class LastTradesResult(Struct):
|
||||
trades: list[Trade]
|
||||
has_more: bool
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def deribit_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
def str_to_cb_sym(name: str) -> Symbol:
|
||||
base, strike_price, expiry_date, option_type = name.split('-')
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'put':
|
||||
option_type = PUT
|
||||
elif option_type == 'call':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date,
|
||||
expiry_normalize=False)
|
||||
|
||||
|
||||
def piker_sym_to_cb_sym(name: str) -> Symbol:
|
||||
base, expiry_date, strike_price, option_type = tuple(
|
||||
name.upper().split('-'))
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'P':
|
||||
option_type = PUT
|
||||
elif option_type == 'C':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date.upper())
|
||||
|
||||
|
||||
def cb_sym_to_deribit_inst(sym: Symbol):
|
||||
# cryptofeed normalized
|
||||
cb_norm = ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z']
|
||||
|
||||
# deribit specific
|
||||
months = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
|
||||
|
||||
exp = sym.expiry_date
|
||||
|
||||
# YYMDD
|
||||
# 01234
|
||||
year, month, day = (
|
||||
exp[:2], months[cb_norm.index(exp[2:3])], exp[3:])
|
||||
|
||||
otype = 'C' if sym.option_type == CALL else 'P'
|
||||
|
||||
return f'{sym.base}-{day}{month}{year}-{sym.strike_price}-{otype}'
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
|
||||
section = conf.get('deribit')
|
||||
|
||||
# TODO: document why we send this, basically because logging params for cryptofeed
|
||||
conf['log'] = {}
|
||||
conf['log']['disabled'] = True
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for deribit in {path}')
|
||||
|
||||
return conf
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
def __init__(self, json_rpc: Callable) -> None:
|
||||
self._pairs: dict[str, Any] = None
|
||||
|
||||
config = get_config().get('deribit', {})
|
||||
|
||||
if ('key_id' in config) and ('key_secret' in config):
|
||||
self._key_id = config['key_id']
|
||||
self._key_secret = config['key_secret']
|
||||
|
||||
else:
|
||||
self._key_id = None
|
||||
self._key_secret = None
|
||||
|
||||
self.json_rpc = json_rpc
|
||||
|
||||
@property
|
||||
def currencies(self):
|
||||
return ['btc', 'eth', 'sol', 'usd']
|
||||
|
||||
async def get_balances(self, kind: str = 'option') -> dict[str, float]:
|
||||
"""Return the set of positions for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_positions', params={
|
||||
'currency': currency.upper(),
|
||||
'kind': kind})
|
||||
|
||||
balances[currency] = resp.result
|
||||
|
||||
return balances
|
||||
|
||||
async def get_assets(self) -> dict[str, float]:
|
||||
"""Return the set of asset balances for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_account_summary', params={
|
||||
'currency': currency.upper()})
|
||||
|
||||
balances[currency] = resp.result['balance']
|
||||
|
||||
return balances
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float
|
||||
) -> dict:
|
||||
"""Place an order
|
||||
"""
|
||||
params = {
|
||||
'instrument_name': symbol.upper(),
|
||||
'amount': size,
|
||||
'type': 'limit',
|
||||
'price': price,
|
||||
}
|
||||
resp = await self.json_rpc(
|
||||
f'private/{action}', params)
|
||||
|
||||
return resp.result
|
||||
|
||||
async def submit_cancel(self, oid: str):
|
||||
"""Send cancel request for order id
|
||||
"""
|
||||
resp = await self.json_rpc(
|
||||
'private/cancel', {'order_id': oid})
|
||||
return resp.result
|
||||
|
||||
async def symbol_info(
|
||||
self,
|
||||
instrument: Optional[str] = None,
|
||||
currency: str = 'btc', # BTC, ETH, SOL, USDC
|
||||
kind: str = 'option',
|
||||
expired: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""Get symbol info for the exchange.
|
||||
|
||||
"""
|
||||
if self._pairs:
|
||||
return self._pairs
|
||||
|
||||
# will retrieve all symbols by default
|
||||
params = {
|
||||
'currency': currency.upper(),
|
||||
'kind': kind,
|
||||
'expired': str(expired).lower()
|
||||
}
|
||||
|
||||
resp = await self.json_rpc('public/get_instruments', params)
|
||||
results = resp.result
|
||||
|
||||
instruments = {
|
||||
item['instrument_name'].lower(): item
|
||||
for item in results
|
||||
}
|
||||
|
||||
if instrument is not None:
|
||||
return instruments[instrument]
|
||||
else:
|
||||
return instruments
|
||||
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
data,
|
||||
score_cutoff=35,
|
||||
limit=limit
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0]['instrument_name'].lower(): item[0]
|
||||
for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str,
|
||||
start_dt: Optional[datetime] = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
limit: int = 1000,
|
||||
as_np: bool = True,
|
||||
) -> dict:
|
||||
instrument = symbol
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_time = deribit_timestamp(start_dt)
|
||||
end_time = deribit_timestamp(end_dt)
|
||||
|
||||
# https://docs.deribit.com/#public-get_tradingview_chart_data
|
||||
resp = await self.json_rpc(
|
||||
'public/get_tradingview_chart_data',
|
||||
params={
|
||||
'instrument_name': instrument.upper(),
|
||||
'start_timestamp': start_time,
|
||||
'end_timestamp': end_time,
|
||||
'resolution': '1'
|
||||
})
|
||||
|
||||
result = KLinesResult(**resp.result)
|
||||
new_bars = []
|
||||
for i in range(len(result.close)):
|
||||
|
||||
_open = result.open[i]
|
||||
high = result.high[i]
|
||||
low = result.low[i]
|
||||
close = result.close[i]
|
||||
volume = result.volume[i]
|
||||
|
||||
row = [
|
||||
(start_time + (i * (60 * 1000))) / 1000.0, # time
|
||||
result.open[i],
|
||||
result.high[i],
|
||||
result.low[i],
|
||||
result.close[i],
|
||||
result.volume[i],
|
||||
0
|
||||
]
|
||||
|
||||
new_bars.append((i,) + tuple(row))
|
||||
|
||||
array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else klines
|
||||
return array
|
||||
|
||||
async def last_trades(
|
||||
self,
|
||||
instrument: str,
|
||||
count: int = 10
|
||||
):
|
||||
resp = await self.json_rpc(
|
||||
'public/get_last_trades_by_instrument',
|
||||
params={
|
||||
'instrument_name': instrument,
|
||||
'count': count
|
||||
})
|
||||
|
||||
return LastTradesResult(**resp.result)
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client(
|
||||
is_brokercheck: bool = False
|
||||
) -> Client:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_jsonrpc_session(
|
||||
_testnet_ws_url, dtype=JSONRPCResult) as json_rpc
|
||||
):
|
||||
client = Client(json_rpc)
|
||||
|
||||
_refresh_token: Optional[str] = None
|
||||
_access_token: Optional[str] = None
|
||||
|
||||
async def _auth_loop(
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED
|
||||
):
|
||||
"""Background task that adquires a first access token and then will
|
||||
refresh the access token while the nursery isn't cancelled.
|
||||
|
||||
https://docs.deribit.com/?python#authentication-2
|
||||
"""
|
||||
renew_time = 10
|
||||
access_scope = 'trade:read_write'
|
||||
_expiry_time = time.time()
|
||||
got_access = False
|
||||
nonlocal _refresh_token
|
||||
nonlocal _access_token
|
||||
|
||||
while True:
|
||||
if time.time() - _expiry_time < renew_time:
|
||||
# if we are close to token expiry time
|
||||
|
||||
if _refresh_token != None:
|
||||
# if we have a refresh token already dont need to send
|
||||
# secret
|
||||
params = {
|
||||
'grant_type': 'refresh_token',
|
||||
'refresh_token': _refresh_token,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
else:
|
||||
# we don't have refresh token, send secret to initialize
|
||||
params = {
|
||||
'grant_type': 'client_credentials',
|
||||
'client_id': client._key_id,
|
||||
'client_secret': client._key_secret,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
resp = await json_rpc('public/auth', params)
|
||||
result = resp.result
|
||||
|
||||
_expiry_time = time.time() + result['expires_in']
|
||||
_refresh_token = result['refresh_token']
|
||||
|
||||
if 'access_token' in result:
|
||||
_access_token = result['access_token']
|
||||
|
||||
if not got_access:
|
||||
# first time this loop runs we must indicate task is
|
||||
# started, we have auth
|
||||
got_access = True
|
||||
task_status.started()
|
||||
|
||||
else:
|
||||
await trio.sleep(renew_time / 2)
|
||||
|
||||
# if we have client creds launch auth loop
|
||||
if client._key_id is not None:
|
||||
await n.start(_auth_loop)
|
||||
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
@acm
|
||||
async def open_feed_handler():
|
||||
fh = FeedHandler(config=get_config())
|
||||
yield fh
|
||||
await to_asyncio.run_task(fh.stop_async)
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_feed_handler() -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_context(
|
||||
acm_func=open_feed_handler,
|
||||
key='feedhandler',
|
||||
) as (cache_hit, fh):
|
||||
yield fh
|
||||
|
||||
|
||||
async def aio_price_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _trade(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('trade', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'last': data,
|
||||
'broker_ts': time.time(),
|
||||
'data': data.to_dict(),
|
||||
'receipt': receipt_timestamp
|
||||
}))
|
||||
|
||||
async def _l1(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('l1', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'ticks': [
|
||||
{'type': 'bid',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'bsize',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'ask',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)},
|
||||
{'type': 'asize',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)}
|
||||
]
|
||||
}))
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[TRADES, L1_BOOK],
|
||||
symbols=[piker_sym_to_cb_sym(instrument)],
|
||||
callbacks={
|
||||
TRADES: _trade,
|
||||
L1_BOOK: _l1
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_price_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_price_feed,
|
||||
kwargs={
|
||||
'instrument': instrument
|
||||
},
|
||||
key=f'{instrument}-price',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
|
||||
|
||||
|
||||
async def aio_order_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _fill(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
async def _order_info(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[FILLS, ORDER_INFO],
|
||||
symbols=[instrument.upper()],
|
||||
callbacks={
|
||||
FILLS: _fill,
|
||||
ORDER_INFO: _order_info,
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_order_feed(
|
||||
instrument: list[str]
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_order_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_order_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_order_feed,
|
||||
kwargs={
|
||||
'instrument': instrument,
|
||||
'fh': fh
|
||||
},
|
||||
key=f'{instrument}-order',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 169 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 132 KiB |
|
|
@ -1,185 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, Callable
|
||||
import time
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import pendulum
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
|
||||
from piker._cacheables import open_cached_client
|
||||
from piker.log import get_logger, get_console_log
|
||||
from piker.data import ShmArray
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataUnavailable,
|
||||
)
|
||||
|
||||
from cryptofeed import FeedHandler
|
||||
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT, L1_BOOK, TRADES, OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
from .api import (
|
||||
Client, Trade,
|
||||
get_config,
|
||||
str_to_cb_sym, piker_sym_to_cb_sym, cb_sym_to_deribit_inst,
|
||||
maybe_open_price_feed
|
||||
)
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
instrument: str,
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
array = await client.bars(
|
||||
instrument,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
if len(array) == 0:
|
||||
raise DataUnavailable
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym = symbols[0]
|
||||
|
||||
async with (
|
||||
open_cached_client('deribit') as client,
|
||||
send_chan as send_chan
|
||||
):
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
sym: {
|
||||
'symbol_info': {
|
||||
'asset_type': 'option',
|
||||
'price_tick_size': 0.0005
|
||||
},
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
nsym = piker_sym_to_cb_sym(sym)
|
||||
|
||||
async with maybe_open_price_feed(sym) as stream:
|
||||
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
last_trades = (await client.last_trades(
|
||||
cb_sym_to_deribit_inst(nsym), count=1)).trades
|
||||
|
||||
if len(last_trades) == 0:
|
||||
last_trade = None
|
||||
async for typ, quote in stream:
|
||||
if typ == 'trade':
|
||||
last_trade = Trade(**(quote['data']))
|
||||
break
|
||||
|
||||
else:
|
||||
last_trade = Trade(**(last_trades[0]))
|
||||
|
||||
first_quote = {
|
||||
'symbol': sym,
|
||||
'last': last_trade.price,
|
||||
'brokerd_ts': last_trade.timestamp,
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': last_trade.price,
|
||||
'size': last_trade.amount,
|
||||
'broker_ts': last_trade.timestamp
|
||||
}]
|
||||
}
|
||||
task_status.started((init_msgs, first_quote))
|
||||
|
||||
feed_is_live.set()
|
||||
|
||||
async for typ, quote in stream:
|
||||
topic = quote['symbol']
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> Client:
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# repack in dict form
|
||||
await stream.send(
|
||||
await client.search_symbols(pattern))
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
``ib`` backend
|
||||
--------------
|
||||
more or less the "everything broker" for traditional and international
|
||||
markets. they are the "go to" provider for automatic retail trading
|
||||
and we interface to their APIs using the `ib_insync` project.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
currently there is not yet full support for:
|
||||
- options charting and trading
|
||||
- paxos based crypto rt feeds and trading
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib]
|
||||
hosts = [
|
||||
"127.0.0.1",
|
||||
]
|
||||
# TODO: when we eventually spawn gateways in our
|
||||
# container, we can just dynamically allocate these
|
||||
# using IBC.
|
||||
ports = [
|
||||
4002,
|
||||
4003,
|
||||
4006,
|
||||
4001,
|
||||
7497,
|
||||
]
|
||||
|
||||
# XXX: for a paper account the flex web query service
|
||||
# is not supported so you have to manually download
|
||||
# and XML report and put it in a location that can be
|
||||
# accessed by the ``brokerd.ib`` backend code for parsing.
|
||||
flex_token = '1111111111111111'
|
||||
flex_trades_query_id = '6969696' # live accounts only?
|
||||
|
||||
# 3rd party web-api token
|
||||
# (XXX: not sure if this works yet)
|
||||
trade_log_token = '111111111111111'
|
||||
|
||||
# when clients are being scanned this determines
|
||||
# which clients are preferred to be used for data feeds
|
||||
# based on account names which are detected as active
|
||||
# on each client.
|
||||
prefer_data_account = [
|
||||
# this has to be first in order to make data work with dual paper + live
|
||||
'main',
|
||||
'algopaper',
|
||||
]
|
||||
|
||||
[ib.accounts]
|
||||
main = 'U69696969'
|
||||
algopaper = 'DU9696969'
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_ib_algopaper.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
["0000e1a7.630f5e5a.01.01"]
|
||||
secType = "FUT"
|
||||
conId = 515416577
|
||||
symbol = "MNQ"
|
||||
lastTradeDateOrContractMonth = "20221216"
|
||||
strike = 0.0
|
||||
right = ""
|
||||
multiplier = "2"
|
||||
exchange = "GLOBEX"
|
||||
primaryExchange = ""
|
||||
currency = "USD"
|
||||
localSymbol = "MNQZ2"
|
||||
tradingClass = "MNQ"
|
||||
includeExpired = false
|
||||
secIdType = ""
|
||||
secId = ""
|
||||
comboLegsDescrip = ""
|
||||
comboLegs = []
|
||||
execId = "0000e1a7.630f5e5a.01.01"
|
||||
time = 1661972086.0
|
||||
acctNumber = "DU69696969"
|
||||
side = "BOT"
|
||||
shares = 1.0
|
||||
price = 12372.75
|
||||
permId = 441472655
|
||||
clientId = 6116
|
||||
orderId = 985
|
||||
liquidation = 0
|
||||
cumQty = 1.0
|
||||
avgPrice = 12372.75
|
||||
orderRef = ""
|
||||
evRule = ""
|
||||
evMultiplier = 0.0
|
||||
modelCode = ""
|
||||
lastLiquidity = 1
|
||||
broker_time = 1661972086.0
|
||||
name = "ib"
|
||||
commission = 0.57
|
||||
realizedPNL = 243.41
|
||||
yield_ = 0.0
|
||||
yieldRedemptionDate = 0
|
||||
listingExchange = "GLOBEX"
|
||||
date = "2022-08-31T18:54:46+00:00"
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib.algopaper."mnq.globex.20221216"]
|
||||
size = -1.0
|
||||
ppu = 12423.630576923071
|
||||
bs_mktid = 515416577
|
||||
expiry = "2022-12-16T00:00:00+00:00"
|
||||
clears = [
|
||||
{ dt = "2022-08-31T18:54:46+00:00", ppu = 12423.630576923071, accum_size = -19.0, price = 12372.75, size = 1.0, cost = 0.57, tid = "0000e1a7.630f5e5a.01.01" },
|
||||
]
|
||||
|
|
@ -20,10 +20,15 @@ Interactive Brokers API backend.
|
|||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
- ``data.py`` for real-time data feed endpoints
|
||||
|
||||
- ``client.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
- ``report.py`` for the hackery to build manual pp calcs
|
||||
to avoid ib's absolute bullshit FIFO style position
|
||||
tracking..
|
||||
|
||||
"""
|
||||
from .api import (
|
||||
get_client,
|
||||
|
|
@ -33,9 +38,7 @@ from .feed import (
|
|||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
trades_dialogue,
|
||||
)
|
||||
from .broker import trades_dialogue
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
|
|
@ -45,22 +48,13 @@ __all__ = [
|
|||
'stream_quotes',
|
||||
]
|
||||
|
||||
_brokerd_mods: list[str] = [
|
||||
'api',
|
||||
'broker',
|
||||
]
|
||||
|
||||
_datad_mods: list[str] = [
|
||||
'feed',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = (
|
||||
_brokerd_mods
|
||||
+
|
||||
_datad_mods
|
||||
)
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
|
|
|
|||
|
|
@ -1,187 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
"FLEX" report processing utils.
|
||||
|
||||
"""
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
from .api import (
|
||||
get_config,
|
||||
log,
|
||||
)
|
||||
from piker.accounting import (
|
||||
open_trade_ledger,
|
||||
)
|
||||
|
||||
|
||||
def parse_flex_dt(
|
||||
record: str,
|
||||
) -> pendulum.datetime:
|
||||
date, ts = record.split(';')
|
||||
dt = pendulum.parse(date)
|
||||
ts = f'{ts[:2]}:{ts[2:4]}:{ts[4:]}'
|
||||
tsdt = pendulum.parse(ts)
|
||||
return dt.set(hour=tsdt.hour, minute=tsdt.minute, second=tsdt.second)
|
||||
|
||||
|
||||
def flex_records_to_ledger_entries(
|
||||
accounts: bidict,
|
||||
trade_entries: list[object],
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Convert flex report entry objects into ``dict`` form, pretty much
|
||||
straight up without modification except add a `pydatetime` field
|
||||
from the parsed timestamp.
|
||||
|
||||
'''
|
||||
trades_by_account = {}
|
||||
for t in trade_entries:
|
||||
entry = t.__dict__
|
||||
|
||||
# XXX: LOL apparently ``toml`` has a bug
|
||||
# where a section key error will show up in the write
|
||||
# if you leave a table key as an `int`? So i guess
|
||||
# cast to strs for all keys..
|
||||
|
||||
# oddly for some so-called "BookTrade" entries
|
||||
# this field seems to be blank, no cuckin clue.
|
||||
# trade['ibExecID']
|
||||
tid = str(entry.get('ibExecID') or entry['tradeID'])
|
||||
# date = str(entry['tradeDate'])
|
||||
|
||||
# XXX: is it going to cause problems if a account name
|
||||
# get's lost? The user should be able to find it based
|
||||
# on the actual exec history right?
|
||||
acctid = accounts[str(entry['accountId'])]
|
||||
|
||||
# probably a flex record with a wonky non-std timestamp..
|
||||
dt = entry['pydatetime'] = parse_flex_dt(entry['dateTime'])
|
||||
entry['datetime'] = str(dt)
|
||||
|
||||
if not tid:
|
||||
# this is likely some kind of internal adjustment
|
||||
# transaction, likely one of the following:
|
||||
# - an expiry event that will show a "book trade" indicating
|
||||
# some adjustment to cash balances: zeroing or itm settle.
|
||||
# - a manual cash balance position adjustment likely done by
|
||||
# the user from the accounts window in TWS where they can
|
||||
# manually set the avg price and size:
|
||||
# https://api.ibkr.com/lib/cstools/faq/web1/index.html#/tag/DTWS_ADJ_AVG_COST
|
||||
log.warning(f'Skipping ID-less ledger entry:\n{pformat(entry)}')
|
||||
continue
|
||||
|
||||
trades_by_account.setdefault(
|
||||
acctid, {}
|
||||
)[tid] = entry
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_account[acctid] = dict(sorted(
|
||||
trades_by_account[acctid].items(),
|
||||
key=lambda entry: entry[1]['pydatetime'],
|
||||
))
|
||||
|
||||
return trades_by_account
|
||||
|
||||
|
||||
def load_flex_trades(
|
||||
path: str | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
|
||||
from ib_insync import flexreport, util
|
||||
|
||||
conf = get_config()
|
||||
|
||||
if not path:
|
||||
# load ``brokers.toml`` and try to get the flex
|
||||
# token and query id that must be previously defined
|
||||
# by the user.
|
||||
token = conf.get('flex_token')
|
||||
if not token:
|
||||
raise ValueError(
|
||||
'You must specify a ``flex_token`` field in your'
|
||||
'`brokers.toml` in order load your trade log, see our'
|
||||
'intructions for how to set this up here:\n'
|
||||
'PUT LINK HERE!'
|
||||
)
|
||||
|
||||
qid = conf['flex_trades_query_id']
|
||||
|
||||
# TODO: hack this into our logging
|
||||
# system like we do with the API client..
|
||||
util.logToConsole()
|
||||
|
||||
# TODO: rewrite the query part of this with async..httpx?
|
||||
report = flexreport.FlexReport(
|
||||
token=token,
|
||||
queryId=qid,
|
||||
)
|
||||
|
||||
else:
|
||||
# XXX: another project we could potentially look at,
|
||||
# https://pypi.org/project/ibflex/
|
||||
report = flexreport.FlexReport(path=path)
|
||||
|
||||
trade_entries = report.extract('Trade')
|
||||
ln = len(trade_entries)
|
||||
log.info(f'Loaded {ln} trades from flex query')
|
||||
|
||||
trades_by_account = flex_records_to_ledger_entries(
|
||||
conf['accounts'].inverse, # reverse map to user account names
|
||||
trade_entries,
|
||||
)
|
||||
|
||||
ledger_dict: dict | None = None
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_id = trades_by_account[acctid]
|
||||
|
||||
with open_trade_ledger('ib', acctid) as ledger_dict:
|
||||
tid_delta = set(trades_by_id) - set(ledger_dict)
|
||||
log.info(
|
||||
'New trades detected\n'
|
||||
f'{pformat(tid_delta)}'
|
||||
)
|
||||
if tid_delta:
|
||||
sorted_delta = dict(sorted(
|
||||
{tid: trades_by_id[tid] for tid in tid_delta}.items(),
|
||||
key=lambda entry: entry[1].pop('pydatetime'),
|
||||
))
|
||||
ledger_dict.update(sorted_delta)
|
||||
|
||||
return ledger_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
import os
|
||||
|
||||
args = sys.argv
|
||||
if len(args) > 1:
|
||||
args = args[1:]
|
||||
for arg in args:
|
||||
path = os.path.abspath(arg)
|
||||
load_flex_trades(path=path)
|
||||
else:
|
||||
# expect brokers.toml to have an entry and
|
||||
# pull from the web service.
|
||||
load_flex_trades()
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
``ib`` utilities and hacks suitable for use in the backend and/or as
|
||||
runnable script-programs.
|
||||
|
||||
'''
|
||||
from typing import Literal
|
||||
import subprocess
|
||||
|
||||
import tractor
|
||||
|
||||
from .._util import log
|
||||
|
||||
|
||||
_reset_tech: Literal[
|
||||
'vnc',
|
||||
'i3ipc_xdotool',
|
||||
|
||||
# TODO: in theory we can use a different linux DE API or
|
||||
# some other type of similar window scanning/mgmt client
|
||||
# (on other OSs) to do the same.
|
||||
|
||||
] = 'vnc'
|
||||
|
||||
|
||||
async def data_reset_hack(
|
||||
reset_type: str = 'data',
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Run key combos for resetting data feeds and yield back to caller
|
||||
when complete.
|
||||
|
||||
NOTE: this is a linux-only hack around!
|
||||
|
||||
There are multiple "techs" you can use depending on your infra setup:
|
||||
|
||||
- if running ib-gw in a container with a VNC server running the most
|
||||
performant method is the `'vnc'` option.
|
||||
|
||||
- if running ib-gw/tws locally, and you are using `i3` you can use
|
||||
the ``i3ipc`` lib and ``xdotool`` to send the appropriate click
|
||||
and key-combos automatically to your local desktop's java X-apps.
|
||||
|
||||
https://interactivebrokers.github.io/tws-api/historical_limitations.html#pacing_violations
|
||||
|
||||
TODOs:
|
||||
- a return type that hopefully determines if the hack was
|
||||
successful.
|
||||
- other OS support?
|
||||
- integration with ``ib-gw`` run in docker + Xorg?
|
||||
- is it possible to offer a local server that can be accessed by
|
||||
a client? Would be sure be handy for running native java blobs
|
||||
that need to be wrangle.
|
||||
|
||||
'''
|
||||
global _reset_tech
|
||||
|
||||
match _reset_tech:
|
||||
case 'vnc':
|
||||
try:
|
||||
await tractor.to_asyncio.run_task(vnc_click_hack)
|
||||
except OSError:
|
||||
_reset_tech = 'i3ipc_xdotool'
|
||||
try:
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
case 'i3ipc_xdotool':
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
|
||||
case _ as tech:
|
||||
raise RuntimeError(f'{tech} is not supported for reset tech!?')
|
||||
|
||||
# we don't really need the ``xdotool`` approach any more B)
|
||||
return True
|
||||
|
||||
|
||||
async def vnc_click_hack(
|
||||
reset_type: str = 'data'
|
||||
) -> None:
|
||||
'''
|
||||
Reset the data or netowork connection for the VNC attached
|
||||
ib gateway using magic combos.
|
||||
|
||||
'''
|
||||
key = {'data': 'f', 'connection': 'r'}[reset_type]
|
||||
|
||||
import asyncvnc
|
||||
|
||||
async with asyncvnc.connect(
|
||||
'localhost',
|
||||
port=3003,
|
||||
# password='ibcansmbz',
|
||||
) as client:
|
||||
|
||||
# move to middle of screen
|
||||
# 640x1800
|
||||
client.mouse.move(
|
||||
x=500,
|
||||
y=500,
|
||||
)
|
||||
client.mouse.click()
|
||||
client.keyboard.press('Ctrl', 'Alt', key) # keys are stacked
|
||||
|
||||
|
||||
def i3ipc_xdotool_manual_click_hack() -> None:
|
||||
import i3ipc
|
||||
|
||||
i3 = i3ipc.Connection()
|
||||
t = i3.get_tree()
|
||||
|
||||
orig_win_id = t.find_focused().window
|
||||
|
||||
# for tws
|
||||
win_names: list[str] = [
|
||||
'Interactive Brokers', # tws running in i3
|
||||
'IB Gateway', # gw running in i3
|
||||
# 'IB', # gw running in i3 (newer version?)
|
||||
]
|
||||
|
||||
try:
|
||||
for name in win_names:
|
||||
results = t.find_titled(name)
|
||||
print(f'results for {name}: {results}')
|
||||
if results:
|
||||
con = results[0]
|
||||
print(f'Resetting data feed for {name}')
|
||||
win_id = str(con.window)
|
||||
w, h = con.rect.width, con.rect.height
|
||||
|
||||
# TODO: seems to be a few libs for python but not sure
|
||||
# if they support all the sub commands we need, order of
|
||||
# most recent commit history:
|
||||
# https://github.com/rr-/pyxdotool
|
||||
# https://github.com/ShaneHutter/pyxdotool
|
||||
# https://github.com/cphyc/pyxdotool
|
||||
|
||||
# TODO: only run the reconnect (2nd) kc on a detected
|
||||
# disconnect?
|
||||
for key_combo, timeout in [
|
||||
# only required if we need a connection reset.
|
||||
# ('ctrl+alt+r', 12),
|
||||
# data feed reset.
|
||||
('ctrl+alt+f', 6)
|
||||
]:
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', win_id,
|
||||
|
||||
# move mouse to bottom left of window (where
|
||||
# there should be nothing to click).
|
||||
'mousemove_relative', '--sync', str(w-4), str(h-4),
|
||||
|
||||
# NOTE: we may need to stick a `--retry 3` in here..
|
||||
'click', '--window', win_id,
|
||||
'--repeat', '3', '1',
|
||||
|
||||
# hackzorzes
|
||||
'key', key_combo,
|
||||
],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# re-activate and focus original window
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', str(orig_win_id),
|
||||
'click', '--window', str(orig_win_id), '1',
|
||||
])
|
||||
except subprocess.TimeoutExpired:
|
||||
log.exception('xdotool timed out?')
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,64 +0,0 @@
|
|||
``kraken`` backend
|
||||
------------------
|
||||
though they don't have the most liquidity of all the cexes they sure are
|
||||
accommodating to those of us who appreciate a little ``xmr``.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken]
|
||||
accounts.spot = 'spot'
|
||||
key_descr = "spot"
|
||||
api_key = "69696969696969696696969696969696969696969696969696969696"
|
||||
secret = "BOOBSBOOBSBOOBSBOOBSBOOBSSMBZ69696969696969669969696969696"
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_kraken_spot.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[TFJBKK-SMBZS-VJ4UWS]
|
||||
ordertxid = "SMBZSA-7CNQU-3HWLNJ"
|
||||
postxid = "SMBZSE-M7IF5-CFI7LT"
|
||||
pair = "XXMRZEUR"
|
||||
time = 1655691993.4133966
|
||||
type = "buy"
|
||||
ordertype = "limit"
|
||||
price = "103.97000000"
|
||||
cost = "499.99999977"
|
||||
fee = "0.80000000"
|
||||
vol = "4.80907954"
|
||||
margin = "0.00000000"
|
||||
misc = ""
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken.spot."xmreur.kraken"]
|
||||
size = 4.80907954
|
||||
ppu = 103.97000000
|
||||
bs_mktid = "XXMRZEUR"
|
||||
clears = [
|
||||
{ tid = "TFJBKK-SMBZS-VJ4UWS", cost = 0.8, price = 103.97, size = 4.80907954, dt = "2022-05-20T02:26:33.413397+00:00" },
|
||||
]
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken backend.
|
||||
|
||||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
'''
|
||||
|
||||
from piker.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
get_mkt_info,
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
trades_dialogue,
|
||||
norm_trade_records,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
|
@ -1,699 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken web API wrapping.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import itertools
|
||||
from typing import (
|
||||
Any,
|
||||
Union,
|
||||
)
|
||||
import time
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import asks
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import urllib.parse
|
||||
import hashlib
|
||||
import hmac
|
||||
import base64
|
||||
import trio
|
||||
|
||||
from piker import config
|
||||
from piker.data.types import Struct
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
MktPair,
|
||||
digits_to_dec,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
)
|
||||
from piker.accounting import Transaction
|
||||
from . import log
|
||||
|
||||
# <uri>/<version>/
|
||||
_url = 'https://api.kraken.com/0'
|
||||
|
||||
|
||||
# Broker specific ohlc schema which includes a vwap field
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('count', int),
|
||||
('bar_wap', float),
|
||||
]
|
||||
|
||||
# UI components allow this to be declared such that additional
|
||||
# (historical) fields can be exposed.
|
||||
ohlc_dtype = np.dtype(_ohlc_dtype)
|
||||
|
||||
_show_wap_in_history = True
|
||||
_symbol_info_translation: dict[str, str] = {
|
||||
'tick_decimals': 'pair_decimals',
|
||||
}
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
section = conf.get('kraken')
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for kraken in {path}')
|
||||
return {}
|
||||
|
||||
return section
|
||||
|
||||
|
||||
def get_kraken_signature(
|
||||
urlpath: str,
|
||||
data: dict[str, Any],
|
||||
secret: str
|
||||
) -> str:
|
||||
postdata = urllib.parse.urlencode(data)
|
||||
encoded = (str(data['nonce']) + postdata).encode()
|
||||
message = urlpath.encode() + hashlib.sha256(encoded).digest()
|
||||
|
||||
mac = hmac.new(base64.b64decode(secret), message, hashlib.sha512)
|
||||
sigdigest = base64.b64encode(mac.digest())
|
||||
return sigdigest.decode()
|
||||
|
||||
|
||||
class InvalidKey(ValueError):
|
||||
'''
|
||||
EAPI:Invalid key
|
||||
This error is returned when the API key used for the call is
|
||||
either expired or disabled, please review the API key in your
|
||||
Settings -> API tab of account management or generate a new one
|
||||
and update your application.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
# https://www.kraken.com/features/api#get-tradable-pairs
|
||||
class Pair(Struct):
|
||||
altname: str # alternate pair name
|
||||
wsname: str # WebSocket pair name (if available)
|
||||
aclass_base: str # asset class of base component
|
||||
base: str # asset id of base component
|
||||
aclass_quote: str # asset class of quote component
|
||||
quote: str # asset id of quote component
|
||||
lot: str # volume lot size
|
||||
|
||||
cost_decimals: int
|
||||
costmin: float
|
||||
pair_decimals: int # scaling decimal places for pair
|
||||
lot_decimals: int # scaling decimal places for volume
|
||||
|
||||
# amount to multiply lot volume by to get currency volume
|
||||
lot_multiplier: float
|
||||
|
||||
# array of leverage amounts available when buying
|
||||
leverage_buy: list[int]
|
||||
# array of leverage amounts available when selling
|
||||
leverage_sell: list[int]
|
||||
|
||||
# fee schedule array in [volume, percent fee] tuples
|
||||
fees: list[tuple[int, float]]
|
||||
|
||||
# maker fee schedule array in [volume, percent fee] tuples (if on
|
||||
# maker/taker)
|
||||
fees_maker: list[tuple[int, float]]
|
||||
|
||||
fee_volume_currency: str # volume discount currency
|
||||
margin_call: str # margin call level
|
||||
margin_stop: str # stop-out/liquidation margin level
|
||||
ordermin: float # minimum order volume for pair
|
||||
tick_size: float # min price step size
|
||||
status: str
|
||||
|
||||
short_position_limit: float = 0
|
||||
long_position_limit: float = float('inf')
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.pair_decimals)
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.lot_decimals)
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
# symbol mapping from all names to the altname
|
||||
_ntable: dict[str, str] = {}
|
||||
|
||||
# 2-way map of symbol names to their "alt names" ffs XD
|
||||
_altnames: bidict[str, str] = bidict()
|
||||
|
||||
_pairs: dict[str, Pair] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: dict[str, str],
|
||||
name: str = '',
|
||||
api_key: str = '',
|
||||
secret: str = ''
|
||||
) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._sesh.headers.update({
|
||||
'User-Agent':
|
||||
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
|
||||
})
|
||||
self._name = name
|
||||
self._api_key = api_key
|
||||
self._secret = secret
|
||||
|
||||
self.conf: dict[str, str] = config
|
||||
self.assets: dict[str, Asset] = {}
|
||||
|
||||
@property
|
||||
def pairs(self) -> dict[str, Pair]:
|
||||
if self._pairs is None:
|
||||
raise RuntimeError(
|
||||
"Make sure to run `cache_symbols()` on startup!"
|
||||
)
|
||||
# retreive and cache all symbols
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def _public(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
) -> dict[str, Any]:
|
||||
resp = await self._sesh.post(
|
||||
path=f'/public/{method}',
|
||||
json=data,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def _private(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
uri_path: str
|
||||
) -> dict[str, Any]:
|
||||
headers = {
|
||||
'Content-Type':
|
||||
'application/x-www-form-urlencoded',
|
||||
'API-Key':
|
||||
self._api_key,
|
||||
'API-Sign':
|
||||
get_kraken_signature(uri_path, data, self._secret)
|
||||
}
|
||||
resp = await self._sesh.post(
|
||||
path=f'/private/{method}',
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def endpoint(
|
||||
self,
|
||||
method: str,
|
||||
data: dict[str, Any]
|
||||
|
||||
) -> dict[str, Any]:
|
||||
uri_path = f'/0/private/{method}'
|
||||
data['nonce'] = str(int(1000*time.time()))
|
||||
return await self._private(method, data, uri_path)
|
||||
|
||||
async def get_balances(
|
||||
self,
|
||||
) -> dict[str, float]:
|
||||
'''
|
||||
Return the set of asset balances for this account
|
||||
by symbol.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'Balance',
|
||||
{},
|
||||
)
|
||||
by_bsmktid = resp['result']
|
||||
|
||||
# TODO: we need to pull out the "asset" decimals
|
||||
# data and return a `decimal.Decimal` instead here!
|
||||
# using the underlying Asset
|
||||
return {
|
||||
self._altnames[sym].lower(): float(bal)
|
||||
for sym, bal in by_bsmktid.items()
|
||||
}
|
||||
|
||||
async def get_assets(self) -> dict[str, dict]:
|
||||
'''
|
||||
Get all assets available for trading and xfer.
|
||||
|
||||
https://docs.kraken.com/rest/#tag/Market-Data/operation/getAssetInfo
|
||||
|
||||
return msg:
|
||||
"asset1": {
|
||||
"aclass": "string",
|
||||
"altname": "string",
|
||||
"decimals": 0,
|
||||
"display_decimals": 0,
|
||||
"collateral_value": 0,
|
||||
"status": "string"
|
||||
}
|
||||
|
||||
'''
|
||||
resp = await self._public('Assets', {})
|
||||
return resp['result']
|
||||
|
||||
async def cache_assets(self) -> None:
|
||||
'''
|
||||
Load and cache all asset infos and pack into
|
||||
our native ``Asset`` struct.
|
||||
|
||||
'''
|
||||
assets = await self.get_assets()
|
||||
for bs_mktid, info in assets.items():
|
||||
|
||||
aname = self._altnames[bs_mktid] = info['altname']
|
||||
aclass = info['aclass']
|
||||
|
||||
self.assets[bs_mktid] = Asset(
|
||||
name=aname.lower(),
|
||||
atype=f'crypto_{aclass}',
|
||||
tx_tick=digits_to_dec(info['decimals']),
|
||||
info=info,
|
||||
)
|
||||
|
||||
async def get_trades(
|
||||
self,
|
||||
fetch_limit: int | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Get the trades (aka cleared orders) history from the rest endpoint:
|
||||
https://docs.kraken.com/rest/#operation/getTradeHistory
|
||||
|
||||
'''
|
||||
ofs = 0
|
||||
trades_by_id: dict[str, Any] = {}
|
||||
|
||||
for i in itertools.count():
|
||||
if (
|
||||
fetch_limit
|
||||
and i >= fetch_limit
|
||||
):
|
||||
break
|
||||
|
||||
# increment 'ofs' pagination offset
|
||||
ofs = i*50
|
||||
|
||||
resp = await self.endpoint(
|
||||
'TradesHistory',
|
||||
{'ofs': ofs},
|
||||
)
|
||||
by_id = resp['result']['trades']
|
||||
trades_by_id.update(by_id)
|
||||
|
||||
# can get up to 50 results per query, see:
|
||||
# https://docs.kraken.com/rest/#tag/User-Data/operation/getTradeHistory
|
||||
if (
|
||||
len(by_id) < 50
|
||||
):
|
||||
err = resp.get('error')
|
||||
if err:
|
||||
raise BrokerError(err)
|
||||
|
||||
# we know we received the max amount of
|
||||
# trade results so there may be more history.
|
||||
# catch the end of the trades
|
||||
count = resp['result']['count']
|
||||
break
|
||||
|
||||
# santity check on update
|
||||
assert count == len(trades_by_id.values())
|
||||
return trades_by_id
|
||||
|
||||
async def get_xfers(
|
||||
self,
|
||||
asset: str,
|
||||
src_asset: str = '',
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Get asset balance transfer transactions.
|
||||
|
||||
Currently only withdrawals are supported.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'WithdrawStatus',
|
||||
{'asset': asset},
|
||||
)
|
||||
try:
|
||||
xfers: list[dict] = resp['result']
|
||||
except KeyError:
|
||||
log.exception(f'Kraken suxxx: {resp}')
|
||||
return []
|
||||
|
||||
# eg. resp schema:
|
||||
# 'result': [{'method': 'Bitcoin', 'aclass': 'currency', 'asset':
|
||||
# 'XXBT', 'refid': 'AGBJRMB-JHD2M4-NDI3NR', 'txid':
|
||||
# 'b95d66d3bb6fd76cbccb93f7639f99a505cb20752c62ea0acc093a0e46547c44',
|
||||
# 'info': 'bc1qc8enqjekwppmw3g80p56z5ns7ze3wraqk5rl9z',
|
||||
# 'amount': '0.00300726', 'fee': '0.00001000', 'time':
|
||||
# 1658347714, 'status': 'Success'}]}
|
||||
|
||||
trans: dict[str, Transaction] = {}
|
||||
for entry in xfers:
|
||||
|
||||
# look up the normalized name and asset info
|
||||
asset_key = entry['asset']
|
||||
asset = self.assets[asset_key]
|
||||
asset_key = self._altnames[asset_key].lower()
|
||||
|
||||
# XXX: this is in the asset units (likely) so it isn't
|
||||
# quite the same as a commisions cost necessarily..)
|
||||
cost = float(entry['fee'])
|
||||
|
||||
fqme = asset_key + '.kraken'
|
||||
|
||||
tx = Transaction(
|
||||
fqsn=fqme,
|
||||
sym=asset,
|
||||
tid=entry['txid'],
|
||||
dt=pendulum.from_timestamp(entry['time']),
|
||||
bs_mktid=f'{asset_key}{src_asset}',
|
||||
size=-1*(
|
||||
float(entry['amount'])
|
||||
+
|
||||
cost
|
||||
),
|
||||
# since this will be treated as a "sell" it
|
||||
# shouldn't be needed to compute the be price.
|
||||
price='NaN',
|
||||
|
||||
# XXX: see note above
|
||||
cost=cost,
|
||||
)
|
||||
trans[tx.tid] = tx
|
||||
|
||||
return trans
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str = None,
|
||||
validate: bool = False # set True test call without a real submission
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
|
||||
'''
|
||||
# Build common data dict for common keys from both endpoints
|
||||
data = {
|
||||
"pair": symbol,
|
||||
"price": str(price),
|
||||
"validate": validate
|
||||
}
|
||||
if reqid is None:
|
||||
# Build order data for kraken api
|
||||
data |= {
|
||||
"ordertype": "limit",
|
||||
"type": action,
|
||||
"volume": str(size),
|
||||
}
|
||||
return await self.endpoint('AddOrder', data)
|
||||
|
||||
else:
|
||||
# Edit order data for kraken api
|
||||
data["txid"] = reqid
|
||||
return await self.endpoint('EditOrder', data)
|
||||
|
||||
async def submit_cancel(
|
||||
self,
|
||||
reqid: str,
|
||||
) -> dict:
|
||||
'''
|
||||
Send cancel request for order id ``reqid``.
|
||||
|
||||
'''
|
||||
# txid is a transaction id given by kraken
|
||||
return await self.endpoint('CancelOrder', {"txid": reqid})
|
||||
|
||||
async def pair_info(
|
||||
self,
|
||||
pair: str | None = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
|
||||
if pair is not None:
|
||||
pairs = {'pair': pair}
|
||||
else:
|
||||
pairs = None # get all pairs
|
||||
|
||||
resp = await self._public('AssetPairs', pairs)
|
||||
err = resp['error']
|
||||
if err:
|
||||
symbolname = pairs['pair'] if pair else None
|
||||
raise SymbolNotFound(f'{symbolname}.kraken')
|
||||
|
||||
pairs = resp['result']
|
||||
|
||||
if pair is not None:
|
||||
_, data = next(iter(pairs.items()))
|
||||
return Pair(**data)
|
||||
else:
|
||||
return {
|
||||
key: Pair(**data)
|
||||
for key, data in pairs.items()
|
||||
}
|
||||
|
||||
async def mkt_info(
|
||||
self,
|
||||
pair_str: str,
|
||||
|
||||
) -> MktPair:
|
||||
|
||||
(
|
||||
bs_mktid, # str
|
||||
pair_info, # Pair
|
||||
) = Client.normalize_symbol(pair_str)
|
||||
|
||||
dst_asset = self.assets[pair_info.base]
|
||||
|
||||
# NOTE XXX parse out the src asset name until we figure out
|
||||
# how to get the src asset's `Pair` info from kraken..
|
||||
src_key = pair_str.lstrip(dst_asset.name.upper()).lower()
|
||||
|
||||
return MktPair(
|
||||
dst=dst_asset,
|
||||
price_tick=pair_info.price_tick,
|
||||
size_tick=pair_info.size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
src=src_key,
|
||||
broker='kraken',
|
||||
)
|
||||
|
||||
async def cache_symbols(self) -> dict:
|
||||
'''
|
||||
Load all market pair info build and cache it for downstream use.
|
||||
|
||||
A ``._ntable: dict[str, str]`` is available for mapping the
|
||||
websocket pair name-keys and their http endpoint API (smh)
|
||||
equivalents to the "alternative name" which is generally the one
|
||||
we actually want to use XD
|
||||
|
||||
'''
|
||||
if not self._pairs:
|
||||
self._pairs.update(await self.pair_info())
|
||||
|
||||
# table of all ws and rest keys to their alt-name values.
|
||||
ntable: dict[str, str] = {}
|
||||
|
||||
for rest_key in list(self._pairs.keys()):
|
||||
|
||||
pair: Pair = self._pairs[rest_key]
|
||||
altname = pair.altname
|
||||
wsname = pair.wsname
|
||||
ntable[altname] = ntable[rest_key] = ntable[wsname] = altname
|
||||
|
||||
# register the pair under all monikers, a giant flat
|
||||
# surjection of all possible names to each info obj.
|
||||
self._pairs[altname] = self._pairs[wsname] = pair
|
||||
|
||||
self._ntable.update(ntable)
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Search for a symbol by "alt name"..
|
||||
|
||||
It is expected that the ``Client._pairs`` table
|
||||
gets populated before conducting the underlying fuzzy-search
|
||||
over the pair-key set.
|
||||
|
||||
'''
|
||||
if not len(self._pairs):
|
||||
await self.cache_symbols()
|
||||
assert self._pairs, '`Client.cache_symbols()` was never called!?'
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
self._pairs,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0].altname: item[0] for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str = 'XBTUSD',
|
||||
|
||||
# UTC 2017-07-02 12:53:20
|
||||
since: Union[int, datetime] | None = None,
|
||||
count: int = 720, # <- max allowed per query
|
||||
as_np: bool = True,
|
||||
|
||||
) -> dict:
|
||||
|
||||
if since is None:
|
||||
since = pendulum.now('UTC').start_of('minute').subtract(
|
||||
minutes=count).timestamp()
|
||||
|
||||
elif isinstance(since, int):
|
||||
since = pendulum.from_timestamp(since).timestamp()
|
||||
|
||||
else: # presumably a pendulum datetime
|
||||
since = since.timestamp()
|
||||
|
||||
# UTC 2017-07-02 12:53:20 is oldest seconds value
|
||||
since = str(max(1499000000, int(since)))
|
||||
json = await self._public(
|
||||
'OHLC',
|
||||
data={
|
||||
'pair': symbol,
|
||||
'since': since,
|
||||
},
|
||||
)
|
||||
try:
|
||||
res = json['result']
|
||||
res.pop('last')
|
||||
bars = next(iter(res.values()))
|
||||
|
||||
new_bars = []
|
||||
|
||||
first = bars[0]
|
||||
last_nz_vwap = first[-3]
|
||||
if last_nz_vwap == 0:
|
||||
# use close if vwap is zero
|
||||
last_nz_vwap = first[-4]
|
||||
|
||||
# convert all fields to native types
|
||||
for i, bar in enumerate(bars):
|
||||
# normalize weird zero-ed vwap values..cmon kraken..
|
||||
# indicates vwap didn't change since last bar
|
||||
vwap = float(bar.pop(-3))
|
||||
if vwap != 0:
|
||||
last_nz_vwap = vwap
|
||||
if vwap == 0:
|
||||
vwap = last_nz_vwap
|
||||
|
||||
# re-insert vwap as the last of the fields
|
||||
bar.append(vwap)
|
||||
|
||||
new_bars.append(
|
||||
(i,) + tuple(
|
||||
ftype(bar[j]) for j, (name, ftype) in enumerate(
|
||||
_ohlc_dtype[1:]
|
||||
)
|
||||
)
|
||||
)
|
||||
array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars
|
||||
return array
|
||||
except KeyError:
|
||||
errmsg = json['error'][0]
|
||||
|
||||
if 'not found' in errmsg:
|
||||
raise SymbolNotFound(errmsg + f': {symbol}')
|
||||
|
||||
elif 'Too many requests' in errmsg:
|
||||
raise DataThrottle(f'{symbol}')
|
||||
|
||||
else:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
@classmethod
|
||||
def normalize_symbol(
|
||||
cls,
|
||||
ticker: str
|
||||
) -> tuple[str, Pair]:
|
||||
'''
|
||||
Normalize symbol names to to a 3x3 pair from the global
|
||||
definition map which we build out from the data retreived from
|
||||
the 'AssetPairs' endpoint, see methods above.
|
||||
|
||||
'''
|
||||
ticker = cls._ntable[ticker]
|
||||
return ticker.lower(), cls._pairs[ticker]
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> Client:
|
||||
|
||||
conf = get_config()
|
||||
if conf:
|
||||
client = Client(
|
||||
conf,
|
||||
name=conf['key_descr'],
|
||||
api_key=conf['api_key'],
|
||||
secret=conf['secret']
|
||||
)
|
||||
else:
|
||||
client = Client({})
|
||||
|
||||
# at startup, load all symbols, and asset info in
|
||||
# batch requests.
|
||||
async with trio.open_nursery() as nurse:
|
||||
nurse.start_soon(client.cache_assets)
|
||||
await client.cache_symbols()
|
||||
|
||||
yield client
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,487 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Real-time and historical data feed endpoints.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
)
|
||||
import time
|
||||
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import pendulum
|
||||
from trio_typing import TaskStatus
|
||||
from trio_util import trio_async_generator
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from piker.accounting._mktinfo import (
|
||||
MktPair,
|
||||
)
|
||||
from piker._cacheables import open_cached_client
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
DataUnavailable,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
from piker.data._web_bs import open_autorecon_ws, NoBsWs
|
||||
from . import log
|
||||
from .api import (
|
||||
Client,
|
||||
Pair,
|
||||
)
|
||||
|
||||
|
||||
class OHLC(Struct):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://docs.kraken.com/websockets/#message-ohlc
|
||||
|
||||
'''
|
||||
chan_id: int # internal kraken id
|
||||
chan_name: str # eg. ohlc-1 (name-interval)
|
||||
pair: str # fx pair
|
||||
time: float # Begin time of interval, in seconds since epoch
|
||||
etime: float # End time of interval, in seconds since epoch
|
||||
open: float # Open price of interval
|
||||
high: float # High price within interval
|
||||
low: float # Low price within interval
|
||||
close: float # Close price of interval
|
||||
vwap: float # Volume weighted average price within interval
|
||||
volume: float # Accumulated volume **within interval**
|
||||
count: int # Number of trades within interval
|
||||
# (sampled) generated tick data
|
||||
ticks: list[Any] = []
|
||||
|
||||
|
||||
@trio_async_generator
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Message stream parser and heartbeat handler.
|
||||
|
||||
Deliver ws subscription messages as well as handle heartbeat logic
|
||||
though a single async generator.
|
||||
|
||||
'''
|
||||
too_slow_count = last_hb = 0
|
||||
|
||||
while True:
|
||||
|
||||
with trio.move_on_after(5) as cs:
|
||||
msg = await ws.recv_msg()
|
||||
|
||||
# trigger reconnection if heartbeat is laggy
|
||||
if cs.cancelled_caught:
|
||||
|
||||
too_slow_count += 1
|
||||
|
||||
if too_slow_count > 20:
|
||||
log.warning(
|
||||
"Heartbeat is too slow, resetting ws connection")
|
||||
|
||||
await ws._connect()
|
||||
too_slow_count = 0
|
||||
continue
|
||||
|
||||
match msg:
|
||||
case {'event': 'heartbeat'}:
|
||||
now = time.time()
|
||||
delay = now - last_hb
|
||||
last_hb = now
|
||||
|
||||
# XXX: why tf is this not printing without --tl flag?
|
||||
log.debug(f"Heartbeat after {delay}")
|
||||
# print(f"Heartbeat after {delay}")
|
||||
|
||||
continue
|
||||
|
||||
case _:
|
||||
# passthrough sub msgs
|
||||
yield msg
|
||||
|
||||
|
||||
@trio_async_generator
|
||||
async def process_data_feed_msgs(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Parse and pack data feed messages.
|
||||
|
||||
'''
|
||||
async with stream_messages(ws) as ws_stream:
|
||||
async for msg in ws_stream:
|
||||
match msg:
|
||||
case {
|
||||
'errorMessage': errmsg
|
||||
}:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
case {
|
||||
'event': 'subscriptionStatus',
|
||||
} as sub:
|
||||
log.info(
|
||||
'WS subscription is active:\n'
|
||||
f'{sub}'
|
||||
)
|
||||
continue
|
||||
|
||||
case [
|
||||
chan_id,
|
||||
*payload_array,
|
||||
chan_name,
|
||||
pair
|
||||
]:
|
||||
if 'ohlc' in chan_name:
|
||||
ohlc = OHLC(
|
||||
chan_id,
|
||||
chan_name,
|
||||
pair,
|
||||
*payload_array[0]
|
||||
)
|
||||
ohlc.typecast()
|
||||
yield 'ohlc', ohlc
|
||||
|
||||
elif 'spread' in chan_name:
|
||||
|
||||
bid, ask, ts, bsize, asize = map(
|
||||
float, payload_array[0])
|
||||
|
||||
# TODO: really makes you think IB has a horrible API...
|
||||
quote = {
|
||||
'symbol': pair.replace('/', ''),
|
||||
'ticks': [
|
||||
{'type': 'bid', 'price': bid, 'size': bsize},
|
||||
{'type': 'bsize', 'price': bid, 'size': bsize},
|
||||
|
||||
{'type': 'ask', 'price': ask, 'size': asize},
|
||||
{'type': 'asize', 'price': ask, 'size': asize},
|
||||
],
|
||||
}
|
||||
yield 'l1', quote
|
||||
|
||||
# elif 'book' in msg[-2]:
|
||||
# chan_id, *payload_array, chan_name, pair = msg
|
||||
# print(msg)
|
||||
|
||||
case {
|
||||
'connectionID': conid,
|
||||
'event': 'systemStatus',
|
||||
'status': 'online',
|
||||
'version': ver,
|
||||
}:
|
||||
log.info(
|
||||
f'Established {ver} ws connection with id: {conid}'
|
||||
)
|
||||
continue
|
||||
|
||||
case _:
|
||||
print(f'UNHANDLED MSG: {msg}')
|
||||
# yield msg
|
||||
|
||||
|
||||
def normalize(
|
||||
ohlc: OHLC,
|
||||
|
||||
) -> dict:
|
||||
quote = ohlc.to_dict()
|
||||
quote['broker_ts'] = quote['time']
|
||||
quote['brokerd_ts'] = time.time()
|
||||
quote['symbol'] = quote['pair'] = quote['pair'].replace('/', '')
|
||||
quote['last'] = quote['close']
|
||||
quote['bar_wap'] = ohlc.vwap
|
||||
|
||||
# seriously eh? what's with this non-symmetry everywhere
|
||||
# in subscription systems...
|
||||
# XXX: piker style is always lowercases symbols.
|
||||
topic = quote['pair'].replace('/', '').lower()
|
||||
|
||||
# print(quote)
|
||||
return topic, quote
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
symbol: str,
|
||||
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# lol, kraken won't send any more then the "last"
|
||||
# 720 1m bars.. so we have to just ignore further
|
||||
# requests of this type..
|
||||
queries: int = 0
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
nonlocal queries
|
||||
if (
|
||||
queries > 0
|
||||
or timeframe != 60
|
||||
):
|
||||
raise DataUnavailable(
|
||||
'Only a single query for 1m bars supported')
|
||||
|
||||
count = 0
|
||||
while count <= 3:
|
||||
try:
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
since=end_dt,
|
||||
)
|
||||
count += 1
|
||||
queries += 1
|
||||
break
|
||||
except DataThrottle:
|
||||
log.warning(f'kraken OHLC throttle for {symbol}')
|
||||
await trio.sleep(1)
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 1, 'rate': 1}
|
||||
|
||||
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
'''
|
||||
Query for and return a `MktPair` and backend-native `Pair` (or
|
||||
wtv else) info.
|
||||
|
||||
If more then one fqme is provided return a ``dict`` of native
|
||||
key-strs to `MktPair`s.
|
||||
|
||||
'''
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# uppercase since kraken bs_mktid is always upper
|
||||
sym_str = fqme.upper()
|
||||
pair: Pair = await client.pair_info(sym_str)
|
||||
mkt: MktPair = await client.mkt_info(sym_str)
|
||||
return mkt, pair
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# backend specific
|
||||
sub_type: str = 'ohlc',
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Subscribe for ohlc stream of quotes for ``pairs``.
|
||||
|
||||
``pairs`` must be formatted <crypto_symbol>/<fiat_symbol>.
|
||||
|
||||
'''
|
||||
|
||||
ws_pairs: list[str] = []
|
||||
mkt_infos: dict[str, MktPair] = {}
|
||||
|
||||
async with (
|
||||
send_chan as send_chan,
|
||||
):
|
||||
for sym_str in symbols:
|
||||
mkt, pair = await get_mkt_info(sym_str)
|
||||
mkt_infos[sym_str] = mkt
|
||||
ws_pairs.append(pair.wsname)
|
||||
|
||||
symbol = symbols[0].lower()
|
||||
|
||||
# sync with `.data.feed` caller
|
||||
# TODO: should we make this init msg a `Struct`?
|
||||
init_msgs = {
|
||||
symbol: {
|
||||
'fqsn': sym_str,
|
||||
'mkt_info': mkt_infos[sym_str],
|
||||
'shm_write_opts': {
|
||||
'sum_tick_vml': False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: NoBsWs):
|
||||
|
||||
# XXX: setup subs
|
||||
# https://docs.kraken.com/websockets/#message-subscribe
|
||||
# specific logic for this in kraken's sync client:
|
||||
# https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
|
||||
ohlc_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'ohlc',
|
||||
'interval': 1,
|
||||
},
|
||||
}
|
||||
|
||||
# TODO: we want to eventually allow unsubs which should
|
||||
# be completely fine to request from a separate task
|
||||
# since internally the ws methods appear to be FIFO
|
||||
# locked.
|
||||
await ws.send_msg(ohlc_sub)
|
||||
|
||||
# trade data (aka L1)
|
||||
l1_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'spread',
|
||||
# 'depth': 10}
|
||||
},
|
||||
}
|
||||
|
||||
# pull a first quote and deliver
|
||||
await ws.send_msg(l1_sub)
|
||||
|
||||
yield
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
'pair': ws_pairs,
|
||||
'event': 'unsubscribe',
|
||||
'subscription': ['ohlc', 'spread'],
|
||||
})
|
||||
|
||||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
# see the tips on reconnection logic:
|
||||
# https://support.kraken.com/hc/en-us/articles/360044504011-WebSocket-API-unexpected-disconnections-from-market-data-feeds
|
||||
ws: NoBsWs
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
'wss://ws.kraken.com/',
|
||||
fixture=subscribe,
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
# NOTE: not sure this actually works XD particularly
|
||||
# if we call `ws._connect()` manally in the streaming
|
||||
# async gen..
|
||||
process_data_feed_msgs(ws) as msg_gen,
|
||||
):
|
||||
# pull a first quote and deliver
|
||||
typ, ohlc_last = await anext(msg_gen)
|
||||
topic, quote = normalize(ohlc_last)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
# lol, only "closes" when they're margin squeezing clients ;P
|
||||
feed_is_live.set()
|
||||
|
||||
# keep start of last interval for volume tracking
|
||||
last_interval_start = ohlc_last.etime
|
||||
|
||||
# start streaming
|
||||
async for typ, ohlc in msg_gen:
|
||||
|
||||
if typ == 'ohlc':
|
||||
|
||||
# TODO: can get rid of all this by using
|
||||
# ``trades`` subscription...
|
||||
|
||||
# generate tick values to match time & sales pane:
|
||||
# https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m
|
||||
volume = ohlc.volume
|
||||
|
||||
# new OHLC sample interval
|
||||
if ohlc.etime > last_interval_start:
|
||||
last_interval_start = ohlc.etime
|
||||
tick_volume = volume
|
||||
|
||||
else:
|
||||
# this is the tick volume *within the interval*
|
||||
tick_volume = volume - ohlc_last.volume
|
||||
|
||||
ohlc_last = ohlc
|
||||
last = ohlc.close
|
||||
|
||||
if tick_volume:
|
||||
ohlc.ticks.append({
|
||||
'type': 'trade',
|
||||
'price': last,
|
||||
'size': tick_volume,
|
||||
})
|
||||
|
||||
topic, quote = normalize(ohlc)
|
||||
|
||||
elif typ == 'l1':
|
||||
quote = ohlc
|
||||
topic = quote['symbol'].lower()
|
||||
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
|
||||
) -> Client:
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started(cache)
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
cache,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
pair[0].altname: pair[0]
|
||||
for pair in matches
|
||||
})
|
||||
|
|
@ -43,13 +43,10 @@ from ..calc import humanize, percent_change
|
|||
from .._cacheables import open_cached_client, async_lifo_cache
|
||||
from .. import config
|
||||
from ._util import resproc, BrokerError, SymbolNotFound
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from .util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, colorize_json, get_console_log
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_use_practice_account = False
|
||||
_refresh_token_ep = 'https://{}login.questrade.com/oauth2/'
|
||||
|
|
|
|||
|
|
@ -27,13 +27,12 @@ from typing import List
|
|||
from async_generator import asynccontextmanager
|
||||
import asks
|
||||
|
||||
from ._util import (
|
||||
resproc,
|
||||
BrokerError,
|
||||
log,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._util import resproc, BrokerError
|
||||
from ..calc import percent_change
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_service_ep = 'https://api.robinhood.com'
|
||||
|
||||
|
||||
|
|
@ -66,10 +65,8 @@ class Client:
|
|||
self.api = _API(self._sess)
|
||||
|
||||
def _zip_in_order(self, symbols: [str], quotes: List[dict]):
|
||||
return {
|
||||
quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, quotes)
|
||||
}
|
||||
return {quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, results_dict)}
|
||||
|
||||
async def quote(self, symbols: [str]):
|
||||
"""Retrieve quotes for a list of ``symbols``.
|
||||
|
|
|
|||
|
|
@ -18,17 +18,3 @@
|
|||
Market machinery for order executions, book, management.
|
||||
|
||||
"""
|
||||
from ..log import get_logger
|
||||
from ._client import (
|
||||
open_ems,
|
||||
OrderClient,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'open_ems',
|
||||
'OrderClient',
|
||||
|
||||
]
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
|
|||
|
|
@ -22,10 +22,54 @@ from enum import Enum
|
|||
from typing import Optional
|
||||
|
||||
from bidict import bidict
|
||||
from pydantic import BaseModel, validator
|
||||
|
||||
from ._pos import Position
|
||||
from ._mktinfo import Symbol
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
from ._messages import BrokerdPosition, Status
|
||||
|
||||
|
||||
class Position(BaseModel):
|
||||
'''
|
||||
Basic pp (personal position) model with attached fills history.
|
||||
|
||||
This type should be IPC wire ready?
|
||||
|
||||
'''
|
||||
symbol: Symbol
|
||||
|
||||
# last size and avg entry price
|
||||
size: float
|
||||
avg_price: float # TODO: contextual pricing
|
||||
|
||||
# ordered record of known constituent trade messages
|
||||
fills: list[Status] = []
|
||||
|
||||
def update_from_msg(
|
||||
self,
|
||||
msg: BrokerdPosition,
|
||||
|
||||
) -> None:
|
||||
|
||||
# XXX: better place to do this?
|
||||
symbol = self.symbol
|
||||
|
||||
lot_size_digits = symbol.lot_size_digits
|
||||
avg_price, size = (
|
||||
round(msg['avg_price'], ndigits=symbol.tick_size_digits),
|
||||
round(msg['size'], ndigits=lot_size_digits),
|
||||
)
|
||||
|
||||
self.avg_price = avg_price
|
||||
self.size = size
|
||||
|
||||
@property
|
||||
def dsize(self) -> float:
|
||||
'''
|
||||
The "dollar" size of the pp, normally in trading (fiat) unit
|
||||
terms.
|
||||
|
||||
'''
|
||||
return self.avg_price * self.size
|
||||
|
||||
|
||||
_size_units = bidict({
|
||||
|
|
@ -40,9 +84,34 @@ SizeUnit = Enum(
|
|||
)
|
||||
|
||||
|
||||
class Allocator(Struct):
|
||||
class Allocator(BaseModel):
|
||||
|
||||
class Config:
|
||||
validate_assignment = True
|
||||
copy_on_model_validation = False
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
# required to get the account validator lookup working?
|
||||
extra = 'allow'
|
||||
underscore_attrs_are_private = False
|
||||
|
||||
symbol: Symbol
|
||||
account: Optional[str] = 'paper'
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
size_unit: str = 'currency'
|
||||
_size_units: dict[str, Optional[str]] = _size_units
|
||||
|
||||
@validator('size_unit', pre=True)
|
||||
def maybe_lookup_key(cls, v):
|
||||
# apply the corresponding enum key for the text "description" value
|
||||
if v not in _size_units:
|
||||
return _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
return v
|
||||
|
||||
# TODO: if we ever want ot support non-uniform entry-slot-proportion
|
||||
# "sizes"
|
||||
|
|
@ -51,28 +120,6 @@ class Allocator(Struct):
|
|||
units_limit: float
|
||||
currency_limit: float
|
||||
slots: int
|
||||
account: Optional[str] = 'paper'
|
||||
|
||||
_size_units: bidict[str, Optional[str]] = _size_units
|
||||
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
_size_unit: str = 'currency'
|
||||
|
||||
@property
|
||||
def size_unit(self) -> str:
|
||||
return self._size_unit
|
||||
|
||||
@size_unit.setter
|
||||
def size_unit(self, v: str) -> Optional[str]:
|
||||
if v not in _size_units:
|
||||
v = _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
self._size_unit = v
|
||||
return v
|
||||
|
||||
def step_sizes(
|
||||
self,
|
||||
|
|
@ -93,13 +140,10 @@ class Allocator(Struct):
|
|||
else:
|
||||
return self.units_limit
|
||||
|
||||
def limit_info(self) -> tuple[str, float]:
|
||||
return self.size_unit, self.limit()
|
||||
|
||||
def next_order_info(
|
||||
self,
|
||||
|
||||
# we only need a startup size for exit calcs, we can then
|
||||
# we only need a startup size for exit calcs, we can the
|
||||
# determine how large slots should be if the initial pp size was
|
||||
# larger then the current live one, and the live one is smaller
|
||||
# then the initial config settings.
|
||||
|
|
@ -129,7 +173,7 @@ class Allocator(Struct):
|
|||
l_sub_pp = self.units_limit - abs_live_size
|
||||
|
||||
elif size_unit == 'currency':
|
||||
live_cost_basis = abs_live_size * live_pp.ppu
|
||||
live_cost_basis = abs_live_size * live_pp.avg_price
|
||||
slot_size = currency_per_slot / price
|
||||
l_sub_pp = (self.currency_limit - live_cost_basis) / price
|
||||
|
||||
|
|
@ -140,14 +184,12 @@ class Allocator(Struct):
|
|||
|
||||
# an entry (adding-to or starting a pp)
|
||||
if (
|
||||
action == 'buy' and live_size > 0 or
|
||||
action == 'sell' and live_size < 0 or
|
||||
live_size == 0
|
||||
or (action == 'buy' and live_size > 0)
|
||||
or action == 'sell' and live_size < 0
|
||||
):
|
||||
order_size = min(
|
||||
slot_size,
|
||||
max(l_sub_pp, 0),
|
||||
)
|
||||
|
||||
order_size = min(slot_size, l_sub_pp)
|
||||
|
||||
# an exit (removing-from or going to net-zero pp)
|
||||
else:
|
||||
|
|
@ -163,7 +205,7 @@ class Allocator(Struct):
|
|||
if size_unit == 'currency':
|
||||
# compute the "projected" limit's worth of units at the
|
||||
# current pp (weighted) price:
|
||||
slot_size = currency_per_slot / live_pp.ppu
|
||||
slot_size = currency_per_slot / live_pp.avg_price
|
||||
|
||||
else:
|
||||
slot_size = u_per_slot
|
||||
|
|
@ -202,12 +244,7 @@ class Allocator(Struct):
|
|||
if order_size < slot_size:
|
||||
# compute a fractional slots size to display
|
||||
slots_used = self.slots_used(
|
||||
Position(
|
||||
symbol=sym,
|
||||
size=order_size,
|
||||
ppu=price,
|
||||
bs_mktid=sym,
|
||||
)
|
||||
Position(symbol=sym, size=order_size, avg_price=price)
|
||||
)
|
||||
|
||||
return {
|
||||
|
|
@ -234,8 +271,8 @@ class Allocator(Struct):
|
|||
abs_pp_size = abs(pp.size)
|
||||
|
||||
if self.size_unit == 'currency':
|
||||
# live_currency_size = size or (abs_pp_size * pp.ppu)
|
||||
live_currency_size = abs_pp_size * pp.ppu
|
||||
# live_currency_size = size or (abs_pp_size * pp.avg_price)
|
||||
live_currency_size = abs_pp_size * pp.avg_price
|
||||
prop = live_currency_size / self.currency_limit
|
||||
|
||||
else:
|
||||
|
|
@ -247,6 +284,14 @@ class Allocator(Struct):
|
|||
return round(prop * self.slots)
|
||||
|
||||
|
||||
_derivs = (
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
)
|
||||
|
||||
|
||||
def mk_allocator(
|
||||
|
||||
symbol: Symbol,
|
||||
|
|
@ -255,7 +300,7 @@ def mk_allocator(
|
|||
# default allocation settings
|
||||
defaults: dict[str, float] = {
|
||||
'account': None, # select paper by default
|
||||
# 'size_unit': 'currency',
|
||||
'size_unit': 'currency',
|
||||
'units_limit': 400,
|
||||
'currency_limit': 5e3,
|
||||
'slots': 4,
|
||||
|
|
@ -273,9 +318,42 @@ def mk_allocator(
|
|||
'currency_limit': 6e3,
|
||||
'slots': 6,
|
||||
}
|
||||
|
||||
defaults.update(user_def)
|
||||
|
||||
return Allocator(
|
||||
alloc = Allocator(
|
||||
symbol=symbol,
|
||||
**defaults,
|
||||
)
|
||||
|
||||
asset_type = symbol.type_key
|
||||
|
||||
# specific configs by asset class / type
|
||||
|
||||
if asset_type in _derivs:
|
||||
# since it's harder to know how currency "applies" in this case
|
||||
# given leverage properties
|
||||
alloc.size_unit = '# units'
|
||||
|
||||
# set units limit to slots size thus making make the next
|
||||
# entry step 1.0
|
||||
alloc.units_limit = alloc.slots
|
||||
|
||||
# if the current position is already greater then the limit
|
||||
# settings, increase the limit to the current position
|
||||
if alloc.size_unit == 'currency':
|
||||
startup_size = startup_pp.size * startup_pp.avg_price
|
||||
|
||||
if startup_size > alloc.currency_limit:
|
||||
alloc.currency_limit = round(startup_size, ndigits=2)
|
||||
|
||||
else:
|
||||
startup_size = abs(startup_pp.size)
|
||||
|
||||
if startup_size > alloc.units_limit:
|
||||
alloc.units_limit = startup_size
|
||||
|
||||
if asset_type in _derivs:
|
||||
alloc.slots = alloc.units_limit
|
||||
|
||||
return alloc
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,294 +18,211 @@
|
|||
Orders and execution client API.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from typing import Dict
|
||||
from pprint import pformat
|
||||
from typing import TYPE_CHECKING
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import trio
|
||||
import tractor
|
||||
from tractor.trionics import broadcast_receiver
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..accounting._mktinfo import unpack_fqme
|
||||
from ..data.types import Struct
|
||||
from ..service import maybe_open_emsd
|
||||
from ._messages import (
|
||||
Order,
|
||||
Cancel,
|
||||
BrokerdPosition,
|
||||
)
|
||||
from ..brokers import get_brokermod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._messages import (
|
||||
Status,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._ems import _emsd_main
|
||||
from .._daemon import maybe_open_emsd
|
||||
from ._messages import Order, Cancel
|
||||
|
||||
|
||||
class OrderClient(Struct):
|
||||
'''
|
||||
EMS-client-side order book ctl and tracking.
|
||||
log = get_logger(__name__)
|
||||
|
||||
(A)sync API for submitting orders and alerts to the `emsd` service;
|
||||
this is the main control for execution management from client code.
|
||||
|
||||
@dataclass
|
||||
class OrderBook:
|
||||
'''EMS-client-side order book ctl and tracking.
|
||||
|
||||
A style similar to "model-view" is used here where this api is
|
||||
provided as a supervised control for an EMS actor which does all the
|
||||
hard/fast work of talking to brokers/exchanges to conduct
|
||||
executions.
|
||||
|
||||
Currently, this is mostly for keeping local state to match the EMS
|
||||
and use received events to trigger graphics updates.
|
||||
|
||||
'''
|
||||
# IPC stream to `emsd` actor
|
||||
_ems_stream: tractor.MsgStream
|
||||
|
||||
# mem channels used to relay order requests to the EMS daemon
|
||||
_to_relay_task: trio.abc.SendChannel
|
||||
_from_sync_order_client: trio.abc.ReceiveChannel
|
||||
_to_ems: trio.abc.SendChannel
|
||||
_from_order_book: trio.abc.ReceiveChannel
|
||||
|
||||
# history table
|
||||
_sent_orders: dict[str, Order] = {}
|
||||
_sent_orders: Dict[str, Order] = field(default_factory=dict)
|
||||
_ready_to_receive: trio.Event = trio.Event()
|
||||
|
||||
def send_nowait(
|
||||
def send(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
msg: Order,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Sync version of ``.send()``.
|
||||
|
||||
'''
|
||||
) -> dict:
|
||||
self._sent_orders[msg.oid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
return msg
|
||||
|
||||
async def send(
|
||||
def update(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Send a new order msg async to the `emsd` service.
|
||||
|
||||
'''
|
||||
self._sent_orders[msg.oid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def update_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Sync version of ``.update()``.
|
||||
|
||||
'''
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
return msg
|
||||
msg = cmd.dict()
|
||||
msg.update(data)
|
||||
self._sent_orders[uuid] = Order(**msg)
|
||||
self._to_ems.send_nowait(msg)
|
||||
return cmd
|
||||
|
||||
async def update(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
) -> dict:
|
||||
'''
|
||||
Update an existing order dialog with a msg updated from
|
||||
``update`` kwargs.
|
||||
def cancel(self, uuid: str) -> bool:
|
||||
"""Cancel an order (or alert) in the EMS.
|
||||
|
||||
'''
|
||||
"""
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def _mk_cancel_msg(
|
||||
self,
|
||||
uuid: str,
|
||||
) -> Cancel:
|
||||
cmd = self._sent_orders.get(uuid)
|
||||
if not cmd:
|
||||
log.error(
|
||||
f'Unknown order {uuid}!?\n'
|
||||
f'Maybe there is a stale entry or line?\n'
|
||||
f'You should report this as a bug!'
|
||||
)
|
||||
fqme = str(cmd.symbol)
|
||||
return Cancel(
|
||||
msg = Cancel(
|
||||
oid=uuid,
|
||||
symbol=fqme,
|
||||
)
|
||||
|
||||
def cancel_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Sync version of ``.cancel()``.
|
||||
|
||||
'''
|
||||
self._to_relay_task.send_nowait(
|
||||
self._mk_cancel_msg(uuid)
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Cancel an already existintg order (or alert) dialog.
|
||||
|
||||
'''
|
||||
await self._ems_stream.send(
|
||||
self._mk_cancel_msg(uuid)
|
||||
symbol=cmd.symbol,
|
||||
)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
|
||||
|
||||
_client: OrderClient = None
|
||||
_orders: OrderBook = None
|
||||
|
||||
|
||||
async def relay_orders_from_sync_code(
|
||||
def get_orders(
|
||||
emsd_uid: tuple[str, str] = None
|
||||
) -> OrderBook:
|
||||
""""
|
||||
OrderBook singleton factory per actor.
|
||||
|
||||
client: OrderClient,
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
"""
|
||||
if emsd_uid is not None:
|
||||
# TODO: read in target emsd's active book on startup
|
||||
pass
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Order submission relay task: deliver orders sent from synchronous (UI)
|
||||
code to the EMS via ``OrderClient._from_sync_order_client``.
|
||||
global _orders
|
||||
|
||||
This is run in the UI actor (usually the one running Qt but could be
|
||||
any other client service code). This process simply delivers order
|
||||
messages to the above ``_to_relay_task`` send channel (from sync code using
|
||||
``.send_nowait()``), these values are pulled from the channel here
|
||||
and relayed to any consumer(s) that called this function using
|
||||
a ``tractor`` portal.
|
||||
|
||||
This effectively makes order messages look like they're being
|
||||
"pushed" from the parent to the EMS where local sync code is likely
|
||||
doing the pushing from some non-async UI handler.
|
||||
|
||||
'''
|
||||
async with (
|
||||
client._from_sync_order_client.subscribe() as sync_order_cmds
|
||||
):
|
||||
async for cmd in sync_order_cmds:
|
||||
sym = cmd.symbol
|
||||
msg = pformat(cmd.to_dict())
|
||||
|
||||
if sym == symbol_key:
|
||||
log.info(f'Send order cmd:\n{msg}')
|
||||
# send msg over IPC / wire
|
||||
await to_ems_stream.send(cmd)
|
||||
|
||||
else:
|
||||
log.warning(
|
||||
f'Ignoring unmatched order cmd for {sym} != {symbol_key}:'
|
||||
f'\n{msg}'
|
||||
)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ems(
|
||||
fqme: str,
|
||||
mode: str = 'live',
|
||||
loglevel: str = 'error',
|
||||
|
||||
) -> tuple[
|
||||
OrderClient,
|
||||
tractor.MsgStream,
|
||||
dict[
|
||||
# brokername, acctid
|
||||
tuple[str, str],
|
||||
dict[str, BrokerdPosition],
|
||||
],
|
||||
list[str],
|
||||
dict[str, Status],
|
||||
]:
|
||||
'''
|
||||
(Maybe) spawn an EMS-daemon (emsd), deliver an `OrderClient` for
|
||||
requesting orders/alerts and a `trades_stream` which delivers all
|
||||
response-msgs.
|
||||
|
||||
This is a "client side" entrypoint which may spawn the `emsd` service
|
||||
if it can't be discovered and generally speaking is the lowest level
|
||||
broker control client-API.
|
||||
|
||||
'''
|
||||
broker, symbol, suffix = unpack_fqme(fqme)
|
||||
|
||||
async with maybe_open_emsd(
|
||||
broker,
|
||||
loglevel=loglevel,
|
||||
) as portal:
|
||||
|
||||
mod = get_brokermod(broker)
|
||||
if (
|
||||
not getattr(mod, 'trades_dialogue', None)
|
||||
or mode == 'paper'
|
||||
):
|
||||
mode = 'paper'
|
||||
|
||||
from ._ems import _emsd_main
|
||||
async with (
|
||||
# connect to emsd
|
||||
portal.open_context(
|
||||
_emsd_main,
|
||||
fqme=fqme,
|
||||
exec_mode=mode,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (
|
||||
ctx,
|
||||
(
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
),
|
||||
|
||||
# open 2-way trade command stream
|
||||
ctx.open_stream() as trades_stream,
|
||||
):
|
||||
# use any pre-existing actor singleton client.
|
||||
global _client
|
||||
if _client is None:
|
||||
if _orders is None:
|
||||
size = 100
|
||||
tx, rx = trio.open_memory_channel(size)
|
||||
brx = broadcast_receiver(rx, size)
|
||||
|
||||
# setup local ui event streaming channels for request/resp
|
||||
# streamging with EMS daemon
|
||||
_client = OrderClient(
|
||||
_ems_stream=trades_stream,
|
||||
_to_relay_task=tx,
|
||||
_from_sync_order_client=brx,
|
||||
_orders = OrderBook(
|
||||
_to_ems=tx,
|
||||
_from_order_book=brx,
|
||||
)
|
||||
|
||||
_client._ems_stream = trades_stream
|
||||
return _orders
|
||||
|
||||
# start sync code order msg delivery task
|
||||
|
||||
# TODO: we can get rid of this relay loop once we move
|
||||
# order_mode inputs to async code!
|
||||
async def relay_order_cmds_from_sync_code(
|
||||
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
|
||||
) -> None:
|
||||
"""
|
||||
Order streaming task: deliver orders transmitted from UI
|
||||
to downstream consumers.
|
||||
|
||||
This is run in the UI actor (usually the one running Qt but could be
|
||||
any other client service code). This process simply delivers order
|
||||
messages to the above ``_to_ems`` send channel (from sync code using
|
||||
``.send_nowait()``), these values are pulled from the channel here
|
||||
and relayed to any consumer(s) that called this function using
|
||||
a ``tractor`` portal.
|
||||
|
||||
This effectively makes order messages look like they're being
|
||||
"pushed" from the parent to the EMS where local sync code is likely
|
||||
doing the pushing from some UI.
|
||||
|
||||
"""
|
||||
book = get_orders()
|
||||
async with book._from_order_book.subscribe() as orders_stream:
|
||||
async for cmd in orders_stream:
|
||||
if cmd['symbol'] == symbol_key:
|
||||
log.info(f'Send order cmd:\n{pformat(cmd)}')
|
||||
# send msg over IPC / wire
|
||||
await to_ems_stream.send(cmd)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ems(
|
||||
fqsn: str,
|
||||
|
||||
) -> (
|
||||
OrderBook,
|
||||
tractor.MsgStream,
|
||||
dict,
|
||||
):
|
||||
'''
|
||||
Spawn an EMS daemon and begin sending orders and receiving
|
||||
alerts.
|
||||
|
||||
This EMS tries to reduce most broker's terrible order entry apis to
|
||||
a very simple protocol built on a few easy to grok and/or
|
||||
"rantsy" premises:
|
||||
|
||||
- most users will prefer "dark mode" where orders are not submitted
|
||||
to a broker until and execution condition is triggered
|
||||
(aka client-side "hidden orders")
|
||||
|
||||
- Brokers over-complicate their apis and generally speaking hire
|
||||
poor designers to create them. We're better off using creating a super
|
||||
minimal, schema-simple, request-event-stream protocol to unify all the
|
||||
existing piles of shit (and shocker, it'll probably just end up
|
||||
looking like a decent crypto exchange's api)
|
||||
|
||||
- all order types can be implemented with client-side limit orders
|
||||
|
||||
- we aren't reinventing a wheel in this case since none of these
|
||||
brokers are exposing FIX protocol; it is they doing the re-invention.
|
||||
|
||||
|
||||
TODO: make some fancy diagrams using mermaid.io
|
||||
|
||||
the possible set of responses from the stream is currently:
|
||||
- 'dark_submitted', 'broker_submitted'
|
||||
- 'dark_cancelled', 'broker_cancelled'
|
||||
- 'dark_executed', 'broker_executed'
|
||||
- 'broker_filled'
|
||||
|
||||
'''
|
||||
# wait for service to connect back to us signalling
|
||||
# ready for order commands
|
||||
book = get_orders()
|
||||
|
||||
from ..data._source import unpack_fqsn
|
||||
broker, symbol, suffix = unpack_fqsn(fqsn)
|
||||
|
||||
async with maybe_open_emsd(broker) as portal:
|
||||
|
||||
async with (
|
||||
# connect to emsd
|
||||
portal.open_context(
|
||||
|
||||
_emsd_main,
|
||||
fqsn=fqsn,
|
||||
|
||||
) as (ctx, (positions, accounts)),
|
||||
|
||||
# open 2-way trade command stream
|
||||
ctx.open_stream() as trades_stream,
|
||||
):
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(
|
||||
relay_orders_from_sync_code,
|
||||
_client,
|
||||
fqme,
|
||||
relay_order_cmds_from_sync_code,
|
||||
fqsn,
|
||||
trades_stream
|
||||
)
|
||||
|
||||
yield (
|
||||
_client,
|
||||
trades_stream,
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
|
||||
# stop the sync-msg-relay task on exit.
|
||||
n.cancel_scope.cancel()
|
||||
yield book, trades_stream, positions, accounts
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -15,162 +15,108 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Clearing sub-system message and protocols.
|
||||
Clearing system messagingn types and protocols.
|
||||
|
||||
"""
|
||||
# from collections import (
|
||||
# ChainMap,
|
||||
# deque,
|
||||
# )
|
||||
from typing import (
|
||||
Optional,
|
||||
Literal,
|
||||
)
|
||||
from typing import Optional, Union
|
||||
|
||||
from msgspec import field
|
||||
# TODO: try out just encoding/send direction for now?
|
||||
# import msgspec
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
|
||||
|
||||
# TODO: a composite for tracking msg flow on 2-legged
|
||||
# dialogs.
|
||||
# class Dialog(ChainMap):
|
||||
# '''
|
||||
# Msg collection abstraction to easily track the state changes of
|
||||
# a msg flow in one high level, query-able and immutable construct.
|
||||
|
||||
# The main use case is to query data from a (long-running)
|
||||
# msg-transaction-sequence
|
||||
|
||||
|
||||
# '''
|
||||
# def update(
|
||||
# self,
|
||||
# msg,
|
||||
# ) -> None:
|
||||
# self.maps.insert(0, msg.to_dict())
|
||||
|
||||
# def flatten(self) -> dict:
|
||||
# return dict(self)
|
||||
|
||||
|
||||
# TODO: ``msgspec`` stuff worth paying attention to:
|
||||
# - schema evolution:
|
||||
# https://jcristharif.com/msgspec/usage.html#schema-evolution
|
||||
# - for eg. ``BrokerdStatus``, instead just have separate messages?
|
||||
# - use literals for a common msg determined by diff keys?
|
||||
# - https://jcristharif.com/msgspec/usage.html#literal
|
||||
|
||||
# --------------
|
||||
# Client -> emsd
|
||||
# --------------
|
||||
|
||||
class Order(Struct):
|
||||
|
||||
# TODO: ideally we can combine these 2 fields into
|
||||
# 1 and just use the size polarity to determine a buy/sell.
|
||||
# i would like to see this become more like
|
||||
# https://jcristharif.com/msgspec/usage.html#literal
|
||||
# action: Literal[
|
||||
# 'live',
|
||||
# 'dark',
|
||||
# 'alert',
|
||||
# ]
|
||||
|
||||
action: Literal[
|
||||
'buy',
|
||||
'sell',
|
||||
'alert',
|
||||
]
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: Literal[
|
||||
'dark',
|
||||
'live',
|
||||
# 'paper', no right?
|
||||
]
|
||||
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
# TODO: figure out how to optionally typecast this to `MktPair`?
|
||||
symbol: str # | MktPair
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float # -ve is "sell", +ve is "buy"
|
||||
|
||||
brokers: list[str] = []
|
||||
|
||||
|
||||
class Cancel(Struct):
|
||||
'''
|
||||
Cancel msg for removing a dark (ems triggered) or
|
||||
class Cancel(BaseModel):
|
||||
'''Cancel msg for removing a dark (ems triggered) or
|
||||
broker-submitted (live) trigger/order.
|
||||
|
||||
'''
|
||||
action: str = 'cancel'
|
||||
oid: str # uuid4
|
||||
symbol: str
|
||||
action: str = 'cancel'
|
||||
|
||||
|
||||
# --------------
|
||||
class Order(BaseModel):
|
||||
|
||||
action: str # {'buy', 'sell', 'alert'}
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
symbol: Union[str, Symbol]
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float
|
||||
brokers: list[str]
|
||||
|
||||
# Assigned once initial ack is received
|
||||
# ack_time_ns: Optional[int] = None
|
||||
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: str # {'dark', 'live', 'paper'}
|
||||
|
||||
class Config:
|
||||
# just for pre-loading a ``Symbol`` when used
|
||||
# in the order mode staging process
|
||||
arbitrary_types_allowed = True
|
||||
# don't copy this model instance when used in
|
||||
# a recursive model
|
||||
copy_on_model_validation = False
|
||||
|
||||
# Client <- emsd
|
||||
# --------------
|
||||
# update msgs from ems which relay state change info
|
||||
# from the active clearing engine.
|
||||
|
||||
class Status(Struct):
|
||||
|
||||
time_ns: int
|
||||
oid: str # uuid4 ems-order dialog id
|
||||
|
||||
resp: Literal[
|
||||
'pending', # acked by broker but not yet open
|
||||
'open',
|
||||
'dark_open', # dark/algo triggered order is open in ems clearing loop
|
||||
'triggered', # above triggered order sent to brokerd, or an alert closed
|
||||
'closed', # fully cleared all size/units
|
||||
'fill', # partial execution
|
||||
'canceled',
|
||||
'error',
|
||||
]
|
||||
class Status(BaseModel):
|
||||
|
||||
name: str = 'status'
|
||||
oid: str # uuid4
|
||||
time_ns: int
|
||||
|
||||
# {
|
||||
# 'dark_submitted',
|
||||
# 'dark_cancelled',
|
||||
# 'dark_triggered',
|
||||
|
||||
# 'broker_submitted',
|
||||
# 'broker_cancelled',
|
||||
# 'broker_executed',
|
||||
# 'broker_filled',
|
||||
# 'broker_errored',
|
||||
|
||||
# 'alert_submitted',
|
||||
# 'alert_triggered',
|
||||
|
||||
# }
|
||||
resp: str # "response", see above
|
||||
|
||||
# symbol: str
|
||||
|
||||
# trigger info
|
||||
trigger_price: Optional[float] = None
|
||||
# price: float
|
||||
|
||||
# broker: Optional[str] = None
|
||||
|
||||
# this maps normally to the ``BrokerdOrder.reqid`` below, an id
|
||||
# normally allocated internally by the backend broker routing system
|
||||
reqid: Optional[int | str] = None
|
||||
broker_reqid: Optional[Union[int, str]] = None
|
||||
|
||||
# the (last) source order/request msg if provided
|
||||
# (eg. the Order/Cancel which causes this msg) and
|
||||
# acts as a back-reference to the corresponding
|
||||
# request message which was the source of this msg.
|
||||
req: Order | None = None
|
||||
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
src: Optional[str] = None
|
||||
|
||||
# set when a cancel request msg was set for this order flow dialog
|
||||
# but the brokerd dialog isn't yet in a cancelled state.
|
||||
cancel_called: bool = False
|
||||
|
||||
# for relaying a boxed brokerd-dialog-side msg data "through" the
|
||||
# ems layer to clients.
|
||||
# for relaying backend msg data "through" the ems layer
|
||||
brokerd_msg: dict = {}
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd -> brokerd
|
||||
# ---------------
|
||||
# requests *sent* from ems to respective backend broker daemon
|
||||
|
||||
class BrokerdCancel(Struct):
|
||||
class BrokerdCancel(BaseModel):
|
||||
|
||||
action: str = 'cancel'
|
||||
oid: str # piker emsd order id
|
||||
time_ns: int
|
||||
|
||||
|
|
@ -181,39 +127,34 @@ class BrokerdCancel(Struct):
|
|||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: Optional[int | str] = None
|
||||
action: str = 'cancel'
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
|
||||
class BrokerdOrder(Struct):
|
||||
class BrokerdOrder(BaseModel):
|
||||
|
||||
action: str # {buy, sell}
|
||||
oid: str
|
||||
account: str
|
||||
time_ns: int
|
||||
|
||||
symbol: str # fqsn
|
||||
price: float
|
||||
size: float
|
||||
|
||||
# TODO: if we instead rely on a +ve/-ve size to determine
|
||||
# the action we more or less don't need this field right?
|
||||
action: str = '' # {buy, sell}
|
||||
|
||||
# "broker request id": broker specific/internal order id if this is
|
||||
# None, creates a new order otherwise if the id is valid the backend
|
||||
# api must modify the existing matching order. If the broker allows
|
||||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: Optional[int | str] = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
symbol: str # symbol.<providername> ?
|
||||
price: float
|
||||
size: float
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd <- brokerd
|
||||
# ---------------
|
||||
# requests *received* to ems from broker backend
|
||||
|
||||
class BrokerdOrderAck(Struct):
|
||||
|
||||
class BrokerdOrderAck(BaseModel):
|
||||
'''
|
||||
Immediate reponse to a brokerd order request providing the broker
|
||||
specific unique order id so that the EMS can associate this
|
||||
|
|
@ -221,93 +162,102 @@ class BrokerdOrderAck(Struct):
|
|||
``.oid`` (which is a uuid4).
|
||||
|
||||
'''
|
||||
name: str = 'ack'
|
||||
|
||||
# defined and provided by backend
|
||||
reqid: int | str
|
||||
reqid: Union[int, str]
|
||||
|
||||
# emsd id originally sent in matching request msg
|
||||
oid: str
|
||||
account: str = ''
|
||||
name: str = 'ack'
|
||||
|
||||
|
||||
class BrokerdStatus(Struct):
|
||||
class BrokerdStatus(BaseModel):
|
||||
|
||||
reqid: int | str
|
||||
time_ns: int
|
||||
status: Literal[
|
||||
'open',
|
||||
'canceled',
|
||||
'fill',
|
||||
'pending',
|
||||
'error',
|
||||
]
|
||||
|
||||
account: str
|
||||
name: str = 'status'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
|
||||
# XXX: should be best effort set for every update
|
||||
account: str = ''
|
||||
|
||||
# {
|
||||
# 'submitted',
|
||||
# 'cancelled',
|
||||
# 'filled',
|
||||
# }
|
||||
status: str
|
||||
|
||||
filled: float = 0.0
|
||||
reason: str = ''
|
||||
remaining: float = 0.0
|
||||
|
||||
# external: bool = False
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
external: bool = False
|
||||
|
||||
# XXX: not required schema as of yet
|
||||
broker_details: dict = field(default_factory=lambda: {
|
||||
broker_details: dict = {
|
||||
'name': '',
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
class BrokerdFill(Struct):
|
||||
class BrokerdFill(BaseModel):
|
||||
'''
|
||||
A single message indicating a "fill-details" event from the broker
|
||||
if avaiable.
|
||||
|
||||
'''
|
||||
name: str = 'fill'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
|
||||
# order exeuction related
|
||||
action: str
|
||||
size: float
|
||||
price: float
|
||||
|
||||
broker_details: dict = {} # meta-data (eg. commisions etc.)
|
||||
|
||||
# brokerd timestamp required for order mode arrow placement on x-axis
|
||||
|
||||
# TODO: maybe int if we force ns?
|
||||
# we need to normalize this somehow since backends will use their
|
||||
# own format and likely across many disparate epoch clocks...
|
||||
broker_time: float
|
||||
reqid: int | str
|
||||
time_ns: int
|
||||
|
||||
# order exeuction related
|
||||
size: float
|
||||
price: float
|
||||
|
||||
name: str = 'fill'
|
||||
action: Optional[str] = None
|
||||
broker_details: dict = {} # meta-data (eg. commisions etc.)
|
||||
|
||||
|
||||
class BrokerdError(Struct):
|
||||
class BrokerdError(BaseModel):
|
||||
'''
|
||||
Optional error type that can be relayed to emsd for error handling.
|
||||
|
||||
This is still a TODO thing since we're not sure how to employ it yet.
|
||||
|
||||
'''
|
||||
name: str = 'error'
|
||||
oid: str
|
||||
symbol: str
|
||||
reason: str
|
||||
|
||||
# if no brokerd order request was actually submitted (eg. we errored
|
||||
# at the ``pikerd`` layer) then there will be ``reqid`` allocated.
|
||||
reqid: Optional[int | str] = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
name: str = 'error'
|
||||
symbol: str
|
||||
reason: str
|
||||
broker_details: dict = {}
|
||||
|
||||
|
||||
class BrokerdPosition(Struct):
|
||||
'''
|
||||
Position update event from brokerd.
|
||||
class BrokerdPosition(BaseModel):
|
||||
'''Position update event from brokerd.
|
||||
|
||||
'''
|
||||
name: str = 'position'
|
||||
|
||||
broker: str
|
||||
account: str
|
||||
symbol: str
|
||||
currency: str
|
||||
size: float
|
||||
avg_price: float
|
||||
currency: str = ''
|
||||
name: str = 'position'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,78 +18,54 @@
|
|||
Fake trading for forward testing.
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from operator import itemgetter
|
||||
import itertools
|
||||
import time
|
||||
from typing import (
|
||||
Callable,
|
||||
)
|
||||
from typing import Tuple, Optional, Callable
|
||||
import uuid
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import trio
|
||||
import tractor
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..brokers import get_brokermod
|
||||
from .. import data
|
||||
from ..data.types import Struct
|
||||
from ..accounting._mktinfo import (
|
||||
Symbol,
|
||||
MktPair,
|
||||
)
|
||||
from ..accounting import (
|
||||
Position,
|
||||
PpTable,
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
open_pps,
|
||||
)
|
||||
from ..data._normalize import iterticks
|
||||
from ..accounting._mktinfo import unpack_fqme
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..data._source import unpack_fqsn
|
||||
from ..log import get_logger
|
||||
from ._messages import (
|
||||
BrokerdCancel,
|
||||
BrokerdOrder,
|
||||
BrokerdOrderAck,
|
||||
BrokerdStatus,
|
||||
BrokerdFill,
|
||||
BrokerdPosition,
|
||||
BrokerdError,
|
||||
BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
|
||||
BrokerdFill, BrokerdPosition, BrokerdError
|
||||
)
|
||||
|
||||
|
||||
class PaperBoi(Struct):
|
||||
'''
|
||||
Emulates a broker order client providing approximately the same API
|
||||
and delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements (eg open, closed, fill msgs).
|
||||
log = get_logger(__name__)
|
||||
|
||||
'''
|
||||
|
||||
@dataclass
|
||||
class PaperBoi:
|
||||
"""
|
||||
Emulates a broker order client providing the same API and
|
||||
delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements.
|
||||
|
||||
"""
|
||||
broker: str
|
||||
|
||||
ems_trades_stream: tractor.MsgStream
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
|
||||
# map of paper "live" orders which be used
|
||||
# to simulate fills based on paper engine settings
|
||||
_buys: defaultdict[str, bidict]
|
||||
_sells: defaultdict[str, bidict]
|
||||
_buys: bidict
|
||||
_sells: bidict
|
||||
_reqids: bidict
|
||||
_syms: dict[str, Symbol] = {}
|
||||
_positions: dict[str, BrokerdPosition]
|
||||
|
||||
# init edge case L1 spread
|
||||
last_ask: tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: tuple[float, float] = (0, 0)
|
||||
last_ask: Tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: Tuple[float, float] = (0, 0)
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
|
|
@ -98,25 +74,28 @@ class PaperBoi(Struct):
|
|||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str | None,
|
||||
|
||||
reqid: Optional[str],
|
||||
) -> int:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
"""Place an order and return integer request id provided by client.
|
||||
|
||||
"""
|
||||
is_modify: bool = False
|
||||
if reqid is None:
|
||||
reqid = str(uuid.uuid4())
|
||||
|
||||
else:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = self._reqids[reqid]
|
||||
assert old_price != price
|
||||
is_modify = True
|
||||
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
'''
|
||||
if action == 'alert':
|
||||
# bypass all fill simulation
|
||||
return reqid
|
||||
|
||||
entry = self._reqids.get(reqid)
|
||||
if entry:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = entry
|
||||
else:
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
# TODO: net latency model
|
||||
# we checkpoint here quickly particulalry
|
||||
# for dark orders since we want the dark_executed
|
||||
|
|
@ -124,25 +103,19 @@ class PaperBoi(Struct):
|
|||
# in the broker trades event processing loop
|
||||
await trio.sleep(0.05)
|
||||
|
||||
if (
|
||||
action == 'sell'
|
||||
and size > 0
|
||||
):
|
||||
if action == 'sell':
|
||||
size = -size
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='open',
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='submitted',
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
filled=0.0,
|
||||
reason='paper_trigger',
|
||||
remaining=size,
|
||||
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
# if we're already a clearing price simulate an immediate fill
|
||||
if (
|
||||
|
|
@ -150,28 +123,28 @@ class PaperBoi(Struct):
|
|||
) or (
|
||||
action == 'sell' and (clear_price := self.last_bid[0]) >= price
|
||||
):
|
||||
await self.fake_fill(
|
||||
symbol,
|
||||
clear_price,
|
||||
size,
|
||||
action,
|
||||
reqid,
|
||||
oid,
|
||||
)
|
||||
await self.fake_fill(symbol, clear_price, size, action, reqid, oid)
|
||||
|
||||
# register this submissions as a paper live order
|
||||
else:
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
# register this submissions as a paper live order
|
||||
|
||||
# submit order to book simulation fill loop
|
||||
if action == 'buy':
|
||||
orders = self._buys
|
||||
|
||||
elif action == 'sell':
|
||||
orders = self._sells
|
||||
|
||||
# {symbol -> bidict[oid, (<price data>)]}
|
||||
orders[symbol][oid] = (price, size, reqid, action)
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
|
||||
if is_modify:
|
||||
# remove any existing order for the old price
|
||||
orders[symbol].pop((oid, old_price))
|
||||
|
||||
# buys/sells: (symbol -> (price -> order))
|
||||
orders.setdefault(symbol, {})[(oid, price)] = (size, reqid, action)
|
||||
|
||||
return reqid
|
||||
|
||||
|
|
@ -184,26 +157,26 @@ class PaperBoi(Struct):
|
|||
oid, symbol, action, price = self._reqids[reqid]
|
||||
|
||||
if action == 'buy':
|
||||
self._buys[symbol].pop(oid, None)
|
||||
self._buys[symbol].pop((oid, price))
|
||||
elif action == 'sell':
|
||||
self._sells[symbol].pop(oid, None)
|
||||
self._sells[symbol].pop((oid, price))
|
||||
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.05)
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='canceled',
|
||||
account='paper',
|
||||
status='cancelled',
|
||||
oid=oid,
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
async def fake_fill(
|
||||
self,
|
||||
|
||||
fqme: str,
|
||||
symbol: str,
|
||||
price: float,
|
||||
size: float,
|
||||
action: str, # one of {'buy', 'sell'}
|
||||
|
|
@ -217,21 +190,21 @@ class PaperBoi(Struct):
|
|||
remaining: float = 0,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Pretend to fill a broker order @ price and size.
|
||||
"""Pretend to fill a broker order @ price and size.
|
||||
|
||||
'''
|
||||
"""
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.05)
|
||||
fill_time_ns = time.time_ns()
|
||||
fill_time_s = time.time()
|
||||
|
||||
fill_msg = BrokerdFill(
|
||||
msg = BrokerdFill(
|
||||
|
||||
reqid=reqid,
|
||||
time_ns=fill_time_ns,
|
||||
time_ns=time.time_ns(),
|
||||
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
|
||||
broker_time=datetime.now().timestamp(),
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
|
|
@ -241,69 +214,79 @@ class PaperBoi(Struct):
|
|||
'name': self.broker + '_paper',
|
||||
},
|
||||
)
|
||||
log.info(f'Fake filling order:\n{fill_msg}')
|
||||
await self.ems_trades_stream.send(fill_msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
if order_complete:
|
||||
|
||||
msg = BrokerdStatus(
|
||||
|
||||
reqid=reqid,
|
||||
time_ns=time.time_ns(),
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='closed',
|
||||
|
||||
status='filled',
|
||||
filled=size,
|
||||
remaining=0 if order_complete else remaining,
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
|
||||
# NOTE: for paper we set the "bs_mktid" as just the fqme since
|
||||
# we don't actually have any unique backend symbol ourselves
|
||||
# other then this thing, our fqme address.
|
||||
bs_mktid: str = fqme
|
||||
t = Transaction(
|
||||
fqsn=fqme,
|
||||
sym=self._syms[fqme],
|
||||
tid=oid,
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
cost=0, # TODO: cost model
|
||||
dt=pendulum.from_timestamp(fill_time_s),
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
'oid': oid,
|
||||
},
|
||||
'name': self.broker,
|
||||
},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
tx = t.to_dict()
|
||||
tx.pop('sym')
|
||||
|
||||
# update in-mem ledger and pos table
|
||||
self.ledger.update({oid: tx})
|
||||
self.ppt.update_from_trans({oid: t})
|
||||
|
||||
# transmit pp msg to ems
|
||||
pp = self.ppt.pps[bs_mktid]
|
||||
pp_msg = BrokerdPosition(
|
||||
# lookup any existing position
|
||||
token = f'{symbol}.{self.broker}'
|
||||
pp_msg = self._positions.setdefault(
|
||||
token,
|
||||
BrokerdPosition(
|
||||
broker=self.broker,
|
||||
account='paper',
|
||||
symbol=fqme,
|
||||
|
||||
size=pp.size,
|
||||
avg_price=pp.ppu,
|
||||
|
||||
symbol=symbol,
|
||||
# TODO: we need to look up the asset currency from
|
||||
# broker info. i guess for crypto this can be
|
||||
# inferred from the pair?
|
||||
# currency=bs_mktid,
|
||||
currency='',
|
||||
size=0.0,
|
||||
avg_price=0,
|
||||
)
|
||||
)
|
||||
await self.ems_trades_stream.send(pp_msg)
|
||||
|
||||
# write all updates to filesys
|
||||
self.ledger.write_config()
|
||||
self.ppt.write_config()
|
||||
# "avg position price" calcs
|
||||
# TODO: eventually it'd be nice to have a small set of routines
|
||||
# to do this stuff from a sequence of cleared orders to enable
|
||||
# so called "contextual positions".
|
||||
new_size = size + pp_msg.size
|
||||
|
||||
# old size minus the new size gives us size differential with
|
||||
# +ve -> increase in pp size
|
||||
# -ve -> decrease in pp size
|
||||
size_diff = abs(new_size) - abs(pp_msg.size)
|
||||
|
||||
if new_size == 0:
|
||||
pp_msg.avg_price = 0
|
||||
|
||||
elif size_diff > 0:
|
||||
# only update the "average position price" when the position
|
||||
# size increases not when it decreases (i.e. the position is
|
||||
# being made smaller)
|
||||
pp_msg.avg_price = (
|
||||
abs(size) * price + pp_msg.avg_price * abs(pp_msg.size)
|
||||
) / abs(new_size)
|
||||
|
||||
pp_msg.size = new_size
|
||||
|
||||
await self.ems_trades_stream.send(pp_msg.dict())
|
||||
|
||||
|
||||
async def simulate_fills(
|
||||
quote_stream: tractor.MsgStream, # noqa
|
||||
quote_stream: 'tractor.ReceiveStream', # noqa
|
||||
client: PaperBoi,
|
||||
|
||||
) -> None:
|
||||
|
||||
# TODO: more machinery to better simulate real-world market things:
|
||||
|
|
@ -323,116 +306,61 @@ async def simulate_fills(
|
|||
|
||||
# this stream may eventually contain multiple symbols
|
||||
async for quotes in quote_stream:
|
||||
|
||||
for sym, quote in quotes.items():
|
||||
|
||||
for tick in iterticks(
|
||||
quote,
|
||||
# dark order price filter(s)
|
||||
types=('ask', 'bid', 'trade', 'last')
|
||||
):
|
||||
tick_price = tick['price']
|
||||
# print(tick)
|
||||
tick_price = tick.get('price')
|
||||
ttype = tick['type']
|
||||
|
||||
buys: bidict[str, tuple] = client._buys[sym]
|
||||
iter_buys = reversed(sorted(
|
||||
buys.values(),
|
||||
key=itemgetter(0),
|
||||
))
|
||||
if ttype in ('ask',):
|
||||
|
||||
def buy_on_ask(our_price):
|
||||
return tick_price <= our_price
|
||||
|
||||
sells: bidict[str, tuple] = client._sells[sym]
|
||||
iter_sells = sorted(
|
||||
sells.values(),
|
||||
key=itemgetter(0)
|
||||
)
|
||||
|
||||
def sell_on_bid(our_price):
|
||||
return tick_price >= our_price
|
||||
|
||||
match tick:
|
||||
|
||||
# on an ask queue tick, only clear buy entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'ask',
|
||||
}:
|
||||
client.last_ask = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_ask[1]),
|
||||
)
|
||||
|
||||
iter_entries = zip(
|
||||
iter_buys,
|
||||
itertools.repeat(buy_on_ask)
|
||||
)
|
||||
orders = client._buys.get(sym, {})
|
||||
|
||||
book_sequence = reversed(
|
||||
sorted(orders.keys(), key=itemgetter(1)))
|
||||
|
||||
def pred(our_price):
|
||||
return tick_price < our_price
|
||||
|
||||
elif ttype in ('bid',):
|
||||
|
||||
# on a bid queue tick, only clear sell entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'bid',
|
||||
}:
|
||||
client.last_bid = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_bid[1]),
|
||||
)
|
||||
|
||||
iter_entries = zip(
|
||||
iter_sells,
|
||||
itertools.repeat(sell_on_bid)
|
||||
)
|
||||
orders = client._sells.get(sym, {})
|
||||
book_sequence = sorted(orders.keys(), key=itemgetter(1))
|
||||
|
||||
# TODO: fix this block, though it definitely
|
||||
# costs a lot more CPU-wise
|
||||
# - doesn't seem like clears are happening still on
|
||||
# "resting" limit orders?
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': ('trade' | 'last'),
|
||||
}:
|
||||
# in the clearing price / last price case we
|
||||
# want to iterate both sides of our book for
|
||||
# clears since we don't know which direction the
|
||||
# price is going to move (especially with HFT)
|
||||
# and thus we simply interleave both sides (buys
|
||||
# and sells) until one side clears and then
|
||||
# break until the next tick?
|
||||
def interleave():
|
||||
for pair in zip(
|
||||
iter_buys,
|
||||
iter_sells,
|
||||
):
|
||||
for order_info, pred in zip(
|
||||
pair,
|
||||
itertools.cycle([buy_on_ask, sell_on_bid]),
|
||||
):
|
||||
yield order_info, pred
|
||||
def pred(our_price):
|
||||
return tick_price > our_price
|
||||
|
||||
iter_entries = interleave()
|
||||
|
||||
# NOTE: all other (non-clearable) tick event types
|
||||
# - we don't want to sping the simulated clear loop
|
||||
# below unecessarily and further don't want to pop
|
||||
# simulated live orders prematurely.
|
||||
case _:
|
||||
elif ttype in ('trade', 'last'):
|
||||
# TODO: simulate actual book queues and our orders
|
||||
# place in it, might require full L2 data?
|
||||
continue
|
||||
|
||||
# iterate all potentially clearable book prices
|
||||
# in FIFO order per side.
|
||||
for order_info, pred in iter_entries:
|
||||
(our_price, size, reqid, action) = order_info
|
||||
# iterate book prices descending
|
||||
for oid, our_price in book_sequence:
|
||||
if pred(our_price):
|
||||
|
||||
# print(order_info)
|
||||
clearable = pred(our_price)
|
||||
if clearable:
|
||||
# pop and retreive order info
|
||||
oid = {
|
||||
'buy': buys,
|
||||
'sell': sells
|
||||
}[action].inverse.pop(order_info)
|
||||
# retreive order info
|
||||
(size, reqid, action) = orders.pop((oid, our_price))
|
||||
|
||||
# clearing price would have filled entirely
|
||||
await client.fake_fill(
|
||||
fqme=sym,
|
||||
symbol=sym,
|
||||
# todo slippage to determine fill price
|
||||
price=tick_price,
|
||||
size=size,
|
||||
|
|
@ -440,6 +368,9 @@ async def simulate_fills(
|
|||
reqid=reqid,
|
||||
oid=oid,
|
||||
)
|
||||
else:
|
||||
# prices are iterated in sorted order so we're done
|
||||
break
|
||||
|
||||
|
||||
async def handle_order_requests(
|
||||
|
|
@ -449,214 +380,119 @@ async def handle_order_requests(
|
|||
|
||||
) -> None:
|
||||
|
||||
request_msg: dict
|
||||
# order_request: dict
|
||||
async for request_msg in ems_order_stream:
|
||||
match request_msg:
|
||||
case {'action': ('buy' | 'sell')}:
|
||||
order = BrokerdOrder(**request_msg)
|
||||
account = order.account
|
||||
|
||||
# error on bad inputs
|
||||
reason = None
|
||||
action = request_msg['action']
|
||||
|
||||
if action in {'buy', 'sell'}:
|
||||
|
||||
account = request_msg['account']
|
||||
if account != 'paper':
|
||||
reason = f'No account found:`{account}` (paper only)?'
|
||||
|
||||
elif order.size == 0:
|
||||
reason = 'Invalid size: 0'
|
||||
|
||||
if reason:
|
||||
log.error(reason)
|
||||
log.error(
|
||||
'This is a paper account, only a `paper` selection is valid'
|
||||
)
|
||||
await ems_order_stream.send(BrokerdError(
|
||||
oid=order.oid,
|
||||
symbol=order.symbol,
|
||||
reason=reason,
|
||||
))
|
||||
oid=request_msg['oid'],
|
||||
symbol=request_msg['symbol'],
|
||||
reason=f'Paper only. No account found: `{account}` ?',
|
||||
).dict())
|
||||
continue
|
||||
|
||||
reqid = order.reqid or str(uuid.uuid4())
|
||||
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
oid=order.oid,
|
||||
reqid=reqid,
|
||||
account='paper'
|
||||
)
|
||||
)
|
||||
# validate
|
||||
order = BrokerdOrder(**request_msg)
|
||||
|
||||
# call our client api to submit the order
|
||||
reqid = await client.submit_limit(
|
||||
|
||||
oid=order.oid,
|
||||
symbol=f'{order.symbol}.{client.broker}',
|
||||
symbol=order.symbol,
|
||||
price=order.price,
|
||||
action=order.action,
|
||||
size=order.size,
|
||||
|
||||
# XXX: by default 0 tells ``ib_insync`` methods that
|
||||
# there is no existing order so ask the client to create
|
||||
# a new one (which it seems to do by allocating an int
|
||||
# counter - collision prone..)
|
||||
reqid=reqid,
|
||||
reqid=order.reqid,
|
||||
)
|
||||
log.info(f'Submitted paper LIMIT {reqid}:\n{order}')
|
||||
|
||||
case {'action': 'cancel'}:
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
|
||||
# ems order request id
|
||||
oid=order.oid,
|
||||
|
||||
# broker specific request id
|
||||
reqid=reqid,
|
||||
|
||||
).dict()
|
||||
)
|
||||
|
||||
elif action == 'cancel':
|
||||
msg = BrokerdCancel(**request_msg)
|
||||
|
||||
await client.submit_cancel(
|
||||
reqid=msg.reqid
|
||||
)
|
||||
|
||||
case _:
|
||||
else:
|
||||
log.error(f'Unknown order command: {request_msg}')
|
||||
|
||||
|
||||
_reqids: bidict[str, tuple] = {}
|
||||
_buys: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
_sells: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def trades_dialogue(
|
||||
|
||||
ctx: tractor.Context,
|
||||
broker: str,
|
||||
fqme: str | None = None, # if empty, we only boot broker mode
|
||||
loglevel: str = 'warning',
|
||||
fqsn: str,
|
||||
loglevel: str = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
tractor.log.get_console_log(loglevel)
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
with (
|
||||
open_pps(
|
||||
broker,
|
||||
'paper',
|
||||
write_on_exit=True,
|
||||
) as ppt,
|
||||
|
||||
open_trade_ledger(
|
||||
broker,
|
||||
'paper',
|
||||
) as ledger
|
||||
):
|
||||
# attempt to get market info from the backend instead of presuming
|
||||
# the ledger entries have everything correct.
|
||||
# TODO: how to process ledger info from backends?
|
||||
# - should we be rolling our own actor-cached version of these
|
||||
# client API refs or using portal IPC to send requests to the
|
||||
# existing brokerd daemon?
|
||||
# - alternatively we can possibly expect and use
|
||||
# a `.broker.norm_trade_records()` ep?
|
||||
fqmes: list[str] = [fqme]
|
||||
if fqme is None:
|
||||
fqmes = list(ppt.pps)
|
||||
|
||||
for fqme in fqmes:
|
||||
mkt: MktPair | None = None
|
||||
brokermod = get_brokermod(broker)
|
||||
gmi = getattr(brokermod, 'get_mkt_info', None)
|
||||
if gmi:
|
||||
mkt, pair = await brokermod.get_mkt_info(
|
||||
fqme.rstrip(f'.{broker}'),
|
||||
)
|
||||
|
||||
# update pos table from ledger history
|
||||
ppt.update_from_trans(
|
||||
ledger.to_trans(),
|
||||
|
||||
# NOTE: here we pass in any `MktPair` provided by the
|
||||
# backend broker instead of assuming the pps.toml contains
|
||||
# the correct contents!
|
||||
force_mkt=mkt
|
||||
)
|
||||
|
||||
pp_msgs: list[BrokerdPosition] = []
|
||||
pos: Position
|
||||
token: str # f'{symbol}.{self.broker}'
|
||||
for token, pos in ppt.pps.items():
|
||||
pp_msgs.append(BrokerdPosition(
|
||||
broker=broker,
|
||||
account='paper',
|
||||
symbol=pos.symbol.fqme,
|
||||
size=pos.size,
|
||||
avg_price=pos.ppu,
|
||||
))
|
||||
|
||||
await ctx.started((
|
||||
pp_msgs,
|
||||
['paper'],
|
||||
))
|
||||
|
||||
# write new positions state in case ledger was
|
||||
# newer then that tracked in pps.toml
|
||||
ppt.write_config()
|
||||
|
||||
# exit early since no fqme was passed,
|
||||
# normally this case is just to load
|
||||
# positions "offline".
|
||||
if fqme is None:
|
||||
log.warning(
|
||||
'Paper engine only running in position delivery mode!\n'
|
||||
'NO SIMULATED CLEARING LOOP IS ACTIVE!'
|
||||
)
|
||||
await trio.sleep_forever()
|
||||
return
|
||||
|
||||
async with (
|
||||
|
||||
data.open_feed(
|
||||
[fqme],
|
||||
[fqsn],
|
||||
loglevel=loglevel,
|
||||
) as feed,
|
||||
|
||||
):
|
||||
# TODO: load paper positions per broker from .toml config file
|
||||
# and pass as symbol to position data mapping: ``dict[str, dict]``
|
||||
# await ctx.started(all_positions)
|
||||
await ctx.started(({}, {'paper',}))
|
||||
|
||||
async with (
|
||||
ctx.open_stream() as ems_stream,
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
|
||||
client = PaperBoi(
|
||||
broker=broker,
|
||||
ems_trades_stream=ems_stream,
|
||||
ppt=ppt,
|
||||
ledger=ledger,
|
||||
|
||||
_buys=_buys,
|
||||
_sells=_sells,
|
||||
_reqids=_reqids,
|
||||
|
||||
# TODO: load postions from ledger file
|
||||
_syms={
|
||||
fqme: flume.symbol
|
||||
for fqme, flume in feed.flumes.items()
|
||||
}
|
||||
)
|
||||
|
||||
n.start_soon(
|
||||
handle_order_requests,
|
||||
client,
|
||||
broker,
|
||||
ems_stream,
|
||||
_buys={},
|
||||
_sells={},
|
||||
|
||||
_reqids={},
|
||||
|
||||
# TODO: load paper positions from ``positions.toml``
|
||||
_positions={},
|
||||
)
|
||||
|
||||
n.start_soon(handle_order_requests, client, ems_stream)
|
||||
|
||||
# paper engine simulator clearing task
|
||||
await simulate_fills(feed.streams[broker], client)
|
||||
await simulate_fills(feed.stream, client)
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_paperboi(
|
||||
fqme: str | None = None,
|
||||
broker: str | None = None,
|
||||
loglevel: str | None = None,
|
||||
fqsn: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> Callable:
|
||||
'''
|
||||
|
|
@ -664,39 +500,28 @@ async def open_paperboi(
|
|||
its context.
|
||||
|
||||
'''
|
||||
if not fqme:
|
||||
assert broker, 'One of `broker` or `fqme` is required siss..!'
|
||||
else:
|
||||
broker, symbol, expiry = unpack_fqme(fqme)
|
||||
|
||||
we_spawned: bool = False
|
||||
broker, symbol, expiry = unpack_fqsn(fqsn)
|
||||
service_name = f'paperboi.{broker}'
|
||||
|
||||
async with (
|
||||
tractor.find_actor(service_name) as portal,
|
||||
tractor.open_nursery() as tn,
|
||||
):
|
||||
# NOTE: only spawn if no paperboi already is up since we likely
|
||||
# don't need more then one actor for simulated order clearing
|
||||
# per broker-backend.
|
||||
# only spawn if no paperboi already is up
|
||||
# (we likely don't need more then one proc for basic
|
||||
# simulated order clearing)
|
||||
if portal is None:
|
||||
log.info('Starting new paper-engine actor')
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
we_spawned = True
|
||||
|
||||
async with portal.open_context(
|
||||
trades_dialogue,
|
||||
broker=broker,
|
||||
fqme=fqme,
|
||||
fqsn=fqsn,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (ctx, first):
|
||||
yield ctx, first
|
||||
|
||||
# tear down connection and any spawned actor on exit
|
||||
await ctx.cancel()
|
||||
if we_spawned:
|
||||
await portal.cancel_actor()
|
||||
yield ctx, first
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Sub-sys module commons.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.clearing'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -19,58 +19,38 @@ CLI commons.
|
|||
|
||||
'''
|
||||
import os
|
||||
from pprint import pformat
|
||||
|
||||
import click
|
||||
import trio
|
||||
import tractor
|
||||
|
||||
from ..log import (
|
||||
get_console_log,
|
||||
get_logger,
|
||||
colorize_json,
|
||||
)
|
||||
from ..log import get_console_log, get_logger, colorize_json
|
||||
from ..brokers import get_brokermod
|
||||
from ..service import (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
from .._daemon import _tractor_kwargs
|
||||
from .. import config
|
||||
|
||||
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--pdb', is_flag=True, help='Enable tractor debug mode')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.option('--host', '-h', default='127.0.0.1', help='Host address to bind')
|
||||
@click.option(
|
||||
'--tsdb',
|
||||
is_flag=True,
|
||||
help='Enable local ``marketstore`` instance'
|
||||
)
|
||||
@click.option(
|
||||
'--es',
|
||||
is_flag=True,
|
||||
help='Enable local ``elasticsearch`` instance'
|
||||
)
|
||||
def pikerd(
|
||||
loglevel: str,
|
||||
host: str,
|
||||
port: int,
|
||||
tl: bool,
|
||||
pdb: bool,
|
||||
tsdb: bool,
|
||||
es: bool,
|
||||
):
|
||||
def pikerd(loglevel, host, tl, pdb, tsdb):
|
||||
'''
|
||||
Spawn the piker broker-daemon.
|
||||
|
||||
'''
|
||||
|
||||
from ..service import open_pikerd
|
||||
from .._daemon import open_pikerd
|
||||
log = get_console_log(loglevel)
|
||||
|
||||
if pdb:
|
||||
|
|
@ -82,25 +62,32 @@ def pikerd(
|
|||
"\n"
|
||||
))
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
|
||||
async def main():
|
||||
|
||||
async with (
|
||||
open_pikerd(
|
||||
tsdb=tsdb,
|
||||
es=es,
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
registry_addr=reg_addr,
|
||||
|
||||
), # normally delivers a ``Services`` handle
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
if tsdb:
|
||||
from piker.data._ahab import start_ahab
|
||||
from piker.data.marketstore import start_marketstore
|
||||
|
||||
log.info('Spawning `marketstore` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await n.start(
|
||||
start_ahab,
|
||||
'marketstored',
|
||||
start_marketstore,
|
||||
|
||||
)
|
||||
log.info(
|
||||
f'`marketstore` up!\n'
|
||||
f'`marketstored` pid: {pid}\n'
|
||||
f'docker container id: {cid}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
|
@ -110,46 +97,25 @@ def pikerd(
|
|||
@click.group(context_settings=config._context_defaults)
|
||||
@click.option(
|
||||
'--brokers', '-b',
|
||||
default=None,
|
||||
default=[DEFAULT_BROKER],
|
||||
multiple=True,
|
||||
help='Broker backend to use'
|
||||
)
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--configdir', '-c', help='Configuration directory')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.pass_context
|
||||
def cli(
|
||||
ctx: click.Context,
|
||||
brokers: list[str],
|
||||
loglevel: str,
|
||||
tl: bool,
|
||||
configdir: str,
|
||||
host: str,
|
||||
port: int,
|
||||
|
||||
) -> None:
|
||||
def cli(ctx, brokers, loglevel, tl, configdir):
|
||||
if configdir is not None:
|
||||
assert os.path.isdir(configdir), f"`{configdir}` is not a valid path"
|
||||
config._override_config_dir(configdir)
|
||||
|
||||
ctx.ensure_object(dict)
|
||||
|
||||
if not brokers:
|
||||
# (try to) load all (supposedly) supported data/broker backends
|
||||
from piker.brokers import __brokers__
|
||||
brokers = __brokers__
|
||||
|
||||
if len(brokers) == 1:
|
||||
brokermods = [get_brokermod(brokers[0])]
|
||||
else:
|
||||
brokermods = [get_brokermod(broker) for broker in brokers]
|
||||
assert brokermods
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
|
||||
ctx.obj.update({
|
||||
'brokers': brokers,
|
||||
|
|
@ -159,7 +125,6 @@ def cli(
|
|||
'log': get_console_log(loglevel),
|
||||
'confdir': config._config_dir,
|
||||
'wl_path': config._watchlists_data_path,
|
||||
'registry_addr': reg_addr,
|
||||
})
|
||||
|
||||
# allow enabling same loglevel in ``tractor`` machinery
|
||||
|
|
@ -169,45 +134,33 @@ def cli(
|
|||
|
||||
@cli.command()
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.argument('ports', nargs=-1, required=False)
|
||||
@click.argument('names', nargs=-1, required=False)
|
||||
@click.pass_obj
|
||||
def services(config, tl, ports):
|
||||
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
_default_registry_port,
|
||||
_default_registry_host,
|
||||
)
|
||||
|
||||
host = _default_registry_host
|
||||
if not ports:
|
||||
ports = [_default_registry_port]
|
||||
def services(config, tl, names):
|
||||
|
||||
async def list_services():
|
||||
nonlocal host
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
),
|
||||
tractor.get_arbiter(
|
||||
host=host,
|
||||
port=ports[0]
|
||||
) as portal
|
||||
):
|
||||
|
||||
async with tractor.get_arbiter(
|
||||
*_tractor_kwargs['arbiter_addr']
|
||||
) as portal:
|
||||
registry = await portal.run_from_ns('self', 'get_registry')
|
||||
json_d = {}
|
||||
for key, socket in registry.items():
|
||||
# name, uuid = uid
|
||||
host, port = socket
|
||||
json_d[key] = f'{host}:{port}'
|
||||
click.echo(f"{colorize_json(json_d)}")
|
||||
|
||||
trio.run(list_services)
|
||||
tractor.run(
|
||||
list_services,
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
arbiter_addr=_tractor_kwargs['arbiter_addr'],
|
||||
)
|
||||
|
||||
|
||||
def _load_clis() -> None:
|
||||
from ..service import marketstore # noqa
|
||||
from ..service import elastic
|
||||
from ..data import marketstore # noqa
|
||||
from ..data import cli # noqa
|
||||
from ..brokers import cli # noqa
|
||||
from ..ui import cli # noqa
|
||||
|
|
|
|||
170
piker/config.py
170
piker/config.py
|
|
@ -15,15 +15,15 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Platform configuration (files) mgmt.
|
||||
Broker configuration mgmt.
|
||||
|
||||
"""
|
||||
import platform
|
||||
import sys
|
||||
import os
|
||||
from os.path import dirname
|
||||
import shutil
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
from bidict import bidict
|
||||
import toml
|
||||
|
|
@ -33,16 +33,9 @@ from .log import get_logger
|
|||
log = get_logger('broker-config')
|
||||
|
||||
|
||||
# XXX NOTE: taken from ``click`` since apparently they have some
|
||||
# taken from ``click`` since apparently they have some
|
||||
# super weirdness with sigint and sudo..no clue
|
||||
# we're probably going to slowly just modify it to our own version over
|
||||
# time..
|
||||
def get_app_dir(
|
||||
app_name: str,
|
||||
roaming: bool = True,
|
||||
force_posix: bool = False,
|
||||
|
||||
) -> str:
|
||||
def get_app_dir(app_name, roaming=True, force_posix=False):
|
||||
r"""Returns the config folder for the application. The default behavior
|
||||
is to return whatever is most appropriate for the operating system.
|
||||
|
||||
|
|
@ -81,30 +74,7 @@ def get_app_dir(
|
|||
def _posixify(name):
|
||||
return "-".join(name.split()).lower()
|
||||
|
||||
# NOTE: for testing with `pytest` we leverage the `tmp_dir`
|
||||
# fixture to generate (and clean up) a test-request-specific
|
||||
# directory for isolated configuration files such that,
|
||||
# - multiple tests can run (possibly in parallel) without data races
|
||||
# on the config state,
|
||||
# - we don't need to ever worry about leaking configs into the
|
||||
# system thus avoiding needing to manage config cleaup fixtures or
|
||||
# other bothers (since obviously `tmp_dir` cleans up after itself).
|
||||
#
|
||||
# In order to "pass down" the test dir path to all (sub-)actors in
|
||||
# the actor tree we preload the root actor's runtime vars state (an
|
||||
# internal mechanism for inheriting state down an actor tree in
|
||||
# `tractor`) with the testing dir and check for it whenever we
|
||||
# detect `pytest` is being used (which it isn't under normal
|
||||
# operation).
|
||||
if "pytest" in sys.modules:
|
||||
import tractor
|
||||
actor = tractor.current_actor(err_on_no_runtime=False)
|
||||
if actor: # runtime is up
|
||||
rvs = tractor._state._runtime_vars
|
||||
testdirpath = Path(rvs['piker_vars']['piker_test_dir'])
|
||||
assert testdirpath.exists(), 'piker test harness might be borked!?'
|
||||
app_name = str(testdirpath)
|
||||
|
||||
# if WIN:
|
||||
if platform.system() == 'Windows':
|
||||
key = "APPDATA" if roaming else "LOCALAPPDATA"
|
||||
folder = os.environ.get(key)
|
||||
|
|
@ -124,35 +94,28 @@ def get_app_dir(
|
|||
)
|
||||
|
||||
|
||||
_click_config_dir: Path = Path(get_app_dir('piker'))
|
||||
_config_dir: Path = _click_config_dir
|
||||
_parent_user: str = os.environ.get('SUDO_USER')
|
||||
_config_dir = _click_config_dir = get_app_dir('piker')
|
||||
_parent_user = os.environ.get('SUDO_USER')
|
||||
|
||||
if _parent_user:
|
||||
non_root_user_dir = Path(
|
||||
os.path.expanduser(f'~{_parent_user}')
|
||||
non_root_user_dir = os.path.expanduser(
|
||||
f'~{_parent_user}'
|
||||
)
|
||||
root: str = 'root'
|
||||
_ccds: str = str(_click_config_dir) # click config dir string
|
||||
i_tail: int = int(_ccds.rfind(root) + len(root))
|
||||
root = 'root'
|
||||
_config_dir = (
|
||||
non_root_user_dir
|
||||
/
|
||||
Path(_ccds[i_tail+1:]) # +1 to capture trailing '/'
|
||||
non_root_user_dir +
|
||||
_click_config_dir[
|
||||
_click_config_dir.rfind(root) + len(root):
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_conf_names: set[str] = {
|
||||
'brokers',
|
||||
# 'pps',
|
||||
'trades',
|
||||
'watchlists',
|
||||
'paper_trades'
|
||||
}
|
||||
|
||||
# TODO: probably drop all this super legacy, questrade specific,
|
||||
# config stuff XD ?
|
||||
_watchlists_data_path: Path = _config_dir / Path('watchlists.json')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
_context_defaults = dict(
|
||||
default_map={
|
||||
# Questrade specific quote poll rates
|
||||
|
|
@ -183,85 +146,72 @@ def _conf_fn_w_ext(
|
|||
def get_conf_path(
|
||||
conf_name: str = 'brokers',
|
||||
|
||||
) -> Path:
|
||||
'''
|
||||
Return the top-level default config path normally under
|
||||
``~/.config/piker`` on linux for a given ``conf_name``, the config
|
||||
name.
|
||||
) -> str:
|
||||
"""Return the default config path normally under
|
||||
``~/.config/piker`` on linux.
|
||||
|
||||
Contains files such as:
|
||||
- brokers.toml
|
||||
- pp.toml
|
||||
- watchlists.toml
|
||||
- trades.toml
|
||||
|
||||
# maybe coming soon ;)
|
||||
- signals.toml
|
||||
- strats.toml
|
||||
|
||||
'''
|
||||
if 'pps.' not in conf_name:
|
||||
assert str(conf_name) in _conf_names
|
||||
|
||||
"""
|
||||
assert conf_name in _conf_names
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
return _config_dir / Path(fn)
|
||||
return os.path.join(
|
||||
_config_dir,
|
||||
fn,
|
||||
)
|
||||
|
||||
|
||||
def repodir() -> Path:
|
||||
def repodir():
|
||||
'''
|
||||
Return the abspath as ``Path`` to the git repo's root dir.
|
||||
Return the abspath to the repo directory.
|
||||
|
||||
'''
|
||||
return Path(__file__).absolute().parent.parent
|
||||
dirpath = os.path.abspath(
|
||||
# we're 3 levels down in **this** module file
|
||||
dirname(dirname(os.path.realpath(__file__)))
|
||||
)
|
||||
return dirpath
|
||||
|
||||
|
||||
def load(
|
||||
conf_name: str = 'brokers',
|
||||
path: Path | None = None,
|
||||
path: str = None
|
||||
|
||||
**tomlkws,
|
||||
|
||||
) -> tuple[dict, str]:
|
||||
) -> (dict, str):
|
||||
'''
|
||||
Load config file by name.
|
||||
|
||||
'''
|
||||
path: Path = path or get_conf_path(conf_name)
|
||||
path = path or get_conf_path(conf_name)
|
||||
if not os.path.isfile(path):
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
|
||||
if not _config_dir.is_dir():
|
||||
_config_dir.mkdir(
|
||||
parents=True,
|
||||
exist_ok=True,
|
||||
template = os.path.join(
|
||||
repodir(),
|
||||
'config',
|
||||
fn
|
||||
)
|
||||
|
||||
if not path.is_file():
|
||||
fn: str = _conf_fn_w_ext(conf_name)
|
||||
|
||||
# try to copy in a template config to the user's directory if
|
||||
# one exists.
|
||||
template: Path = repodir() / 'config' / fn
|
||||
if template.is_file():
|
||||
# try to copy in a template config to the user's directory
|
||||
# if one exists.
|
||||
if os.path.isfile(template):
|
||||
shutil.copyfile(template, path)
|
||||
else:
|
||||
# create empty file
|
||||
with path.open(mode='x'):
|
||||
pass
|
||||
else:
|
||||
with path.open(mode='r'):
|
||||
pass # touch it
|
||||
|
||||
config: dict = toml.load(str(path), **tomlkws)
|
||||
config = toml.load(path)
|
||||
log.debug(f"Read config file {path}")
|
||||
return config, path
|
||||
|
||||
|
||||
def write(
|
||||
config: dict, # toml config as dict
|
||||
|
||||
name: str | None = None,
|
||||
path: Path | None = None,
|
||||
fail_empty: bool = True,
|
||||
|
||||
**toml_kwargs,
|
||||
name: str = 'brokers',
|
||||
path: str = None,
|
||||
|
||||
) -> None:
|
||||
''''
|
||||
|
|
@ -270,34 +220,26 @@ def write(
|
|||
Create a ``brokers.ini`` file if one does not exist.
|
||||
|
||||
'''
|
||||
if name:
|
||||
path: Path = path or get_conf_path(name)
|
||||
dirname: Path = path.parent
|
||||
if not dirname.is_dir():
|
||||
path = path or get_conf_path(name)
|
||||
dirname = os.path.dirname(path)
|
||||
if not os.path.isdir(dirname):
|
||||
log.debug(f"Creating config dir {_config_dir}")
|
||||
dirname.mkdir()
|
||||
os.makedirs(dirname)
|
||||
|
||||
if (
|
||||
not config
|
||||
and fail_empty
|
||||
):
|
||||
if not config:
|
||||
raise ValueError(
|
||||
"Watch out you're trying to write a blank config!"
|
||||
)
|
||||
"Watch out you're trying to write a blank config!")
|
||||
|
||||
log.debug(
|
||||
f"Writing config `{name}` file to:\n"
|
||||
f"{path}"
|
||||
)
|
||||
with path.open(mode='w') as cf:
|
||||
return toml.dump(
|
||||
config,
|
||||
cf,
|
||||
**toml_kwargs,
|
||||
)
|
||||
with open(path, 'w') as cf:
|
||||
return toml.dump(config, cf)
|
||||
|
||||
|
||||
def load_accounts(
|
||||
|
||||
providers: Optional[list[str]] = None
|
||||
|
||||
) -> bidict[str, Optional[str]]:
|
||||
|
|
|
|||
|
|
@ -22,12 +22,6 @@ and storing data from your brokers as well as
|
|||
sharing live streams over a network.
|
||||
|
||||
"""
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from ..log import (
|
||||
get_console_log,
|
||||
)
|
||||
from ._normalize import iterticks
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
|
|
@ -38,6 +32,7 @@ from ._sharedmem import (
|
|||
)
|
||||
from .feed import (
|
||||
open_feed,
|
||||
_setup_persistent_brokerd,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -49,41 +44,5 @@ __all__ = [
|
|||
'attach_shm_array',
|
||||
'open_shm_array',
|
||||
'get_shm_token',
|
||||
'_setup_persistent_brokerd',
|
||||
]
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def _setup_persistent_brokerd(
|
||||
ctx: tractor.Context,
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Allocate a actor-wide service nursery in ``brokerd``
|
||||
such that feeds can be run in the background persistently by
|
||||
the broker backend as needed.
|
||||
|
||||
'''
|
||||
get_console_log(
|
||||
loglevel or tractor.current_actor().loglevel,
|
||||
)
|
||||
|
||||
from .feed import (
|
||||
_bus,
|
||||
get_feed_bus,
|
||||
)
|
||||
global _bus
|
||||
assert not _bus
|
||||
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
# assign a nursery to the feeds bus for spawning
|
||||
# background tasks from clients
|
||||
get_feed_bus(brokername, service_nursery)
|
||||
|
||||
# unblock caller
|
||||
await ctx.started()
|
||||
|
||||
# we pin this task to keep the feeds manager active until the
|
||||
# parent actor decides to tear it down
|
||||
await trio.sleep_forever()
|
||||
|
|
|
|||
|
|
@ -15,12 +15,9 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Supervisor for ``docker`` with included async and SC wrapping
|
||||
to ensure a cancellable container lifetime system.
|
||||
Supervisor for docker with included specific-image service helpers.
|
||||
|
||||
'''
|
||||
from collections import ChainMap
|
||||
from functools import partial
|
||||
import os
|
||||
import time
|
||||
from typing import (
|
||||
|
|
@ -40,27 +37,21 @@ from docker.models.containers import Container as DockerContainer
|
|||
from docker.errors import (
|
||||
DockerException,
|
||||
APIError,
|
||||
# ContainerError,
|
||||
)
|
||||
import requests
|
||||
from requests.exceptions import (
|
||||
ConnectionError,
|
||||
ReadTimeout,
|
||||
)
|
||||
from requests.exceptions import ConnectionError, ReadTimeout
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, get_console_log
|
||||
from .. import config
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DockerNotStarted(Exception):
|
||||
'Prolly you dint start da daemon bruh'
|
||||
|
||||
|
||||
class ApplicationLogError(Exception):
|
||||
'App in container reported an error in logs'
|
||||
class ContainerError(RuntimeError):
|
||||
'Error reported via app-container logging level'
|
||||
|
||||
|
||||
@acm
|
||||
|
|
@ -105,9 +96,9 @@ async def open_docker(
|
|||
# not perms?
|
||||
raise
|
||||
|
||||
# finally:
|
||||
# if client:
|
||||
# client.close()
|
||||
finally:
|
||||
if client:
|
||||
client.close()
|
||||
|
||||
|
||||
class Container:
|
||||
|
|
@ -128,19 +119,8 @@ class Container:
|
|||
|
||||
async def process_logs_until(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
|
||||
# this is a predicate func for matching log msgs emitted by the
|
||||
# underlying containerized app
|
||||
patt_matcher: Callable[[str], bool],
|
||||
|
||||
# XXX WARNING XXX: do not touch this sleep value unless
|
||||
# you know what you are doing! the value is critical to
|
||||
# making sure the caller code inside the startup context
|
||||
# does not timeout BEFORE we receive a match on the
|
||||
# ``patt_matcher()`` predicate above.
|
||||
checkpoint_period: float = 0.001,
|
||||
|
||||
patt: str,
|
||||
bp_on_msg: bool = False,
|
||||
) -> bool:
|
||||
'''
|
||||
Attempt to capture container log messages and relay through our
|
||||
|
|
@ -151,15 +131,6 @@ class Container:
|
|||
|
||||
while True:
|
||||
logs = self.cntr.logs()
|
||||
try:
|
||||
logs = self.cntr.logs()
|
||||
except (
|
||||
docker.errors.NotFound,
|
||||
docker.errors.APIError
|
||||
):
|
||||
log.exception('Failed to parse logs?')
|
||||
return False
|
||||
|
||||
entries = logs.decode().split('\n')
|
||||
for entry in entries:
|
||||
|
||||
|
|
@ -167,44 +138,34 @@ class Container:
|
|||
if not entry:
|
||||
continue
|
||||
|
||||
entry = entry.strip()
|
||||
try:
|
||||
record = json.loads(entry)
|
||||
msg = record[log_msg_key]
|
||||
level = record['level']
|
||||
|
||||
record = json.loads(entry.strip())
|
||||
except json.JSONDecodeError:
|
||||
msg = entry
|
||||
level = 'error'
|
||||
if 'Error' in entry:
|
||||
raise RuntimeError(entry)
|
||||
raise
|
||||
|
||||
# TODO: do we need a more general mechanism
|
||||
# for these kinda of "log record entries"?
|
||||
# if 'Error' in entry:
|
||||
# raise RuntimeError(entry)
|
||||
|
||||
if (
|
||||
msg
|
||||
and entry not in seen_so_far
|
||||
):
|
||||
msg = record['msg']
|
||||
level = record['level']
|
||||
if msg and entry not in seen_so_far:
|
||||
seen_so_far.add(entry)
|
||||
getattr(log, level.lower(), log.error)(f'{msg}')
|
||||
if bp_on_msg:
|
||||
await tractor.breakpoint()
|
||||
|
||||
if level == 'fatal':
|
||||
raise ApplicationLogError(msg)
|
||||
getattr(log, level, log.error)(f'{msg}')
|
||||
|
||||
if await patt_matcher(msg):
|
||||
# print(f'level: {level}')
|
||||
if level in ('error', 'fatal'):
|
||||
raise ContainerError(msg)
|
||||
|
||||
if patt in msg:
|
||||
return True
|
||||
|
||||
# do a checkpoint so we don't block if cancelled B)
|
||||
await trio.sleep(checkpoint_period)
|
||||
await trio.sleep(0.01)
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def cuid(self) -> str:
|
||||
fqcn: str = self.cntr.attrs['Config']['Image']
|
||||
return f'{fqcn}[{self.cntr.short_id}]'
|
||||
|
||||
def try_signal(
|
||||
self,
|
||||
signal: str = 'SIGINT',
|
||||
|
|
@ -224,66 +185,30 @@ class Container:
|
|||
if 'is not running' in err.explanation:
|
||||
return False
|
||||
|
||||
def hard_kill(self, start: float) -> None:
|
||||
delay = time.time() - start
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
log.error(
|
||||
f'SIGKILL-ing: {self.cntr.id} after {delay}s\n'
|
||||
)
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
stop_predicate: Callable[[str], bool],
|
||||
|
||||
hard_kill: bool = False,
|
||||
|
||||
stop_msg: str,
|
||||
) -> None:
|
||||
'''
|
||||
Attempt to cancel this container gracefully, fail over to
|
||||
a hard kill on timeout.
|
||||
|
||||
'''
|
||||
cid = self.cntr.id
|
||||
|
||||
# first try a graceful cancel
|
||||
log.cancel(
|
||||
f'SIGINT cancelling container: {self.cuid}\n'
|
||||
'waiting on stop predicate...'
|
||||
f'SIGINT cancelling container: {cid}\n'
|
||||
f'waiting on stop msg: "{stop_msg}"'
|
||||
)
|
||||
self.try_signal('SIGINT')
|
||||
|
||||
start = time.time()
|
||||
for _ in range(6):
|
||||
for _ in range(30):
|
||||
|
||||
with trio.move_on_after(0.5) as cs:
|
||||
log.cancel('polling for CNTR logs...')
|
||||
cs.shield = True
|
||||
await self.process_logs_until(stop_msg)
|
||||
|
||||
try:
|
||||
await self.process_logs_until(
|
||||
log_msg_key,
|
||||
stop_predicate,
|
||||
)
|
||||
except ApplicationLogError:
|
||||
hard_kill = True
|
||||
else:
|
||||
# if we aren't cancelled on above checkpoint then we
|
||||
# assume we read the expected stop msg and
|
||||
# terminated.
|
||||
# assume we read the expected stop msg and terminated.
|
||||
break
|
||||
|
||||
if cs.cancelled_caught:
|
||||
# on timeout just try a hard kill after
|
||||
# a quick container sync-wait.
|
||||
hard_kill = True
|
||||
|
||||
try:
|
||||
log.info(f'Polling for container shutdown:\n{cid}')
|
||||
|
||||
|
|
@ -293,7 +218,6 @@ class Container:
|
|||
condition='not-running',
|
||||
)
|
||||
|
||||
# graceful exit if we didn't time out
|
||||
break
|
||||
|
||||
except (
|
||||
|
|
@ -305,22 +229,24 @@ class Container:
|
|||
except (
|
||||
docker.errors.APIError,
|
||||
ConnectionError,
|
||||
requests.exceptions.ConnectionError,
|
||||
trio.Cancelled,
|
||||
):
|
||||
log.exception('Docker connection failure')
|
||||
self.hard_kill(start)
|
||||
raise
|
||||
|
||||
except trio.Cancelled:
|
||||
log.exception('trio cancelled...')
|
||||
self.hard_kill(start)
|
||||
break
|
||||
else:
|
||||
hard_kill = True
|
||||
delay = time.time() - start
|
||||
log.error(
|
||||
f'Failed to kill container {cid} after {delay}s\n'
|
||||
'sending SIGKILL..'
|
||||
)
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
if hard_kill:
|
||||
self.hard_kill(start)
|
||||
else:
|
||||
log.cancel(f'Container stopped: {cid}')
|
||||
|
||||
|
||||
|
|
@ -328,16 +254,11 @@ class Container:
|
|||
async def open_ahabd(
|
||||
ctx: tractor.Context,
|
||||
endpoint: str, # ns-pointer str-msg-type
|
||||
loglevel: str | None = 'cancel',
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
|
||||
log = get_console_log(
|
||||
loglevel,
|
||||
name=__name__,
|
||||
)
|
||||
get_console_log('info', name=__name__)
|
||||
|
||||
async with open_docker() as client:
|
||||
|
||||
|
|
@ -348,84 +269,26 @@ async def open_ahabd(
|
|||
(
|
||||
dcntr,
|
||||
cntr_config,
|
||||
start_pred,
|
||||
stop_pred,
|
||||
start_msg,
|
||||
stop_msg,
|
||||
) = ep_func(client)
|
||||
cntr = Container(dcntr)
|
||||
|
||||
conf: ChainMap[str, Any] = ChainMap(
|
||||
with trio.move_on_after(1):
|
||||
found = await cntr.process_logs_until(start_msg)
|
||||
|
||||
# container specific
|
||||
cntr_config,
|
||||
|
||||
# defaults
|
||||
{
|
||||
# startup time limit which is the max the supervisor
|
||||
# will wait for the container to be registered in
|
||||
# ``client.containers.list()``
|
||||
'startup_timeout': 1.0,
|
||||
|
||||
# how fast to poll for the starup predicate by sleeping
|
||||
# this amount incrementally thus yielding to the
|
||||
# ``trio`` scheduler on during sync polling execution.
|
||||
'startup_query_period': 0.001,
|
||||
|
||||
# str-key value expected to contain log message body-contents
|
||||
# when read using:
|
||||
# ``json.loads(entry for entry in DockerContainer.logs())``
|
||||
'log_msg_key': 'msg',
|
||||
|
||||
|
||||
# startup sync func, like `Nursery.started()`
|
||||
'started_afunc': None,
|
||||
},
|
||||
if not found and cntr not in client.containers.list():
|
||||
raise RuntimeError(
|
||||
'Failed to start `marketstore` check logs deats'
|
||||
)
|
||||
|
||||
try:
|
||||
with trio.move_on_after(conf['startup_timeout']) as cs:
|
||||
async with trio.open_nursery() as tn:
|
||||
tn.start_soon(
|
||||
partial(
|
||||
cntr.process_logs_until,
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
patt_matcher=start_pred,
|
||||
checkpoint_period=conf['startup_query_period'],
|
||||
)
|
||||
)
|
||||
|
||||
# optional blocking routine
|
||||
started = conf['started_afunc']
|
||||
if started:
|
||||
await started()
|
||||
|
||||
# poll for container startup or timeout
|
||||
while not cs.cancel_called:
|
||||
if dcntr in client.containers.list():
|
||||
break
|
||||
|
||||
await trio.sleep(conf['startup_query_period'])
|
||||
|
||||
# sync with remote caller actor-task but allow log
|
||||
# processing to continue running in bg.
|
||||
await ctx.started((
|
||||
cntr.cntr.id,
|
||||
os.getpid(),
|
||||
cntr_config,
|
||||
))
|
||||
|
||||
# XXX: if we timeout on finding the "startup msg" we
|
||||
# expect then we want to FOR SURE raise an error
|
||||
# upwards!
|
||||
if cs.cancelled_caught:
|
||||
# if dcntr not in client.containers.list():
|
||||
for entry in cntr.seen_so_far:
|
||||
log.info(entry)
|
||||
|
||||
raise DockerNotStarted(
|
||||
f'Failed to start container: {cntr.cuid}\n'
|
||||
f'due to timeout={conf["startup_timeout"]}s\n\n'
|
||||
"check ur container's logs!"
|
||||
)
|
||||
try:
|
||||
|
||||
# TODO: we might eventually want a proxy-style msg-prot here
|
||||
# to allow remote control of containers without needing
|
||||
|
|
@ -433,25 +296,13 @@ async def open_ahabd(
|
|||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
# TODO: ensure loglevel can be set and teardown logs are
|
||||
# reported if possible on error or cancel..
|
||||
# XXX WARNING: currently shielding here can result in hangs
|
||||
# on ctl-c from user.. ideally we can avoid a cancel getting
|
||||
# consumed and not propagating whilst still doing teardown
|
||||
# logging..
|
||||
with trio.CancelScope(shield=True):
|
||||
await cntr.cancel(
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
stop_predicate=stop_pred,
|
||||
)
|
||||
await cntr.cancel(stop_msg)
|
||||
|
||||
|
||||
async def start_ahab(
|
||||
service_name: str,
|
||||
endpoint: Callable[docker.DockerClient, DockerContainer],
|
||||
loglevel: str | None = 'cancel',
|
||||
drop_root_perms: bool = True,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.Event,
|
||||
|
|
@ -472,12 +323,13 @@ async def start_ahab(
|
|||
'''
|
||||
cn_ready = trio.Event()
|
||||
try:
|
||||
async with tractor.open_nursery() as an:
|
||||
async with tractor.open_nursery(
|
||||
loglevel='runtime',
|
||||
) as tn:
|
||||
|
||||
portal = await an.start_actor(
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__],
|
||||
loglevel=loglevel,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
|
||||
# TODO: we have issues with this on teardown
|
||||
|
|
@ -487,10 +339,7 @@ async def start_ahab(
|
|||
|
||||
# de-escalate root perms to the original user
|
||||
# after the docker supervisor actor is spawned.
|
||||
if (
|
||||
drop_root_perms
|
||||
and config._parent_user
|
||||
):
|
||||
if config._parent_user:
|
||||
import pwd
|
||||
os.setuid(
|
||||
pwd.getpwnam(
|
||||
|
|
@ -501,7 +350,6 @@ async def start_ahab(
|
|||
async with portal.open_context(
|
||||
open_ahabd,
|
||||
endpoint=str(NamespacePath.from_ref(endpoint)),
|
||||
loglevel='cancel',
|
||||
) as (ctx, first):
|
||||
|
||||
cid, pid, cntr_config = first
|
||||
|
|
@ -1,827 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Pre-(path)-graphics formatted x/y nd/1d rendering subsystem.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
from msgspec import field
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ._sharedmem import (
|
||||
ShmArray,
|
||||
)
|
||||
from ._pathops import (
|
||||
path_arrays_from_ohlc,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._dataviz import (
|
||||
Viz,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
|
||||
class IncrementalFormatter(msgspec.Struct):
|
||||
'''
|
||||
Incrementally updating, pre-path-graphics tracking, formatter.
|
||||
|
||||
Allows tracking source data state in an updateable pre-graphics
|
||||
``np.ndarray`` format (in local process memory) as well as
|
||||
incrementally rendering from that format **to** 1d x/y for path
|
||||
generation using ``pg.functions.arrayToQPath()``.
|
||||
|
||||
'''
|
||||
shm: ShmArray
|
||||
viz: Viz
|
||||
|
||||
# the value to be multiplied any any index into the x/y_1d arrays
|
||||
# given the input index is based on the original source data array.
|
||||
flat_index_ratio: float = 1
|
||||
|
||||
@property
|
||||
def index_field(self) -> 'str':
|
||||
'''
|
||||
Value (``str``) used to look up the "index series" from the
|
||||
underlying source ``numpy`` struct-array; delegate directly to
|
||||
the managing ``Viz``.
|
||||
|
||||
'''
|
||||
return self.viz.index_field
|
||||
|
||||
# Incrementally updated xy ndarray formatted data, a pre-1d
|
||||
# format which is updated and cached independently of the final
|
||||
# pre-graphics-path 1d format.
|
||||
x_nd: Optional[np.ndarray] = None
|
||||
y_nd: Optional[np.ndarray] = None
|
||||
|
||||
@property
|
||||
def xy_nd(self) -> tuple[np.ndarray, np.ndarray]:
|
||||
return (
|
||||
self.x_nd[self.xy_slice],
|
||||
self.y_nd[self.xy_slice],
|
||||
)
|
||||
|
||||
@property
|
||||
def xy_slice(self) -> slice:
|
||||
return slice(
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
)
|
||||
|
||||
# indexes which slice into the above arrays (which are allocated
|
||||
# based on source data shm input size) and allow retrieving
|
||||
# incrementally updated data.
|
||||
xy_nd_start: int | None = None
|
||||
xy_nd_stop: int | None = None
|
||||
|
||||
# TODO: eventually incrementally update 1d-pre-graphics path data?
|
||||
x_1d: np.ndarray | None = None
|
||||
y_1d: np.ndarray | None = None
|
||||
|
||||
# incremental view-change state(s) tracking
|
||||
_last_vr: tuple[float, float] | None = None
|
||||
_last_ivdr: tuple[float, float] | None = None
|
||||
|
||||
@property
|
||||
def index_step_size(self) -> float:
|
||||
'''
|
||||
Readonly value computed on first ``.diff()`` call.
|
||||
|
||||
'''
|
||||
return self.viz.index_step()
|
||||
|
||||
def diff(
|
||||
self,
|
||||
new_read: tuple[np.ndarray],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
# TODO:
|
||||
# - can the renderer just call ``Viz.read()`` directly? unpack
|
||||
# latest source data read
|
||||
# - eventually maybe we can implement some kind of
|
||||
# transform on the ``QPainterPath`` that will more or less
|
||||
# detect the diff in "elements" terms? update diff state since
|
||||
# we've now rendered paths.
|
||||
(
|
||||
xfirst,
|
||||
xlast,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
) = new_read
|
||||
|
||||
index = array['index']
|
||||
|
||||
# if the first index in the read array is 0 then
|
||||
# it means the source buffer has bee completely backfilled to
|
||||
# available space.
|
||||
src_start = index[0]
|
||||
src_stop = index[-1] + 1
|
||||
|
||||
# these are the "formatted output data" indices
|
||||
# for the pre-graphics arrays.
|
||||
nd_start = self.xy_nd_start
|
||||
nd_stop = self.xy_nd_stop
|
||||
|
||||
if (
|
||||
nd_start is None
|
||||
):
|
||||
assert nd_stop is None
|
||||
|
||||
# setup to do a prepend of all existing src history
|
||||
nd_start = self.xy_nd_start = src_stop
|
||||
# set us in a zero-to-append state
|
||||
nd_stop = self.xy_nd_stop = src_stop
|
||||
|
||||
# compute the length diffs between the first/last index entry in
|
||||
# the input data and the last indexes we have on record from the
|
||||
# last time we updated the curve index.
|
||||
prepend_length = int(nd_start - src_start)
|
||||
append_length = int(src_stop - nd_stop)
|
||||
|
||||
# blah blah blah
|
||||
# do diffing for prepend, append and last entry
|
||||
return (
|
||||
slice(src_start, nd_start),
|
||||
prepend_length,
|
||||
append_length,
|
||||
slice(nd_stop, src_stop),
|
||||
)
|
||||
|
||||
def _track_inview_range(
|
||||
self,
|
||||
view_range: tuple[int, int],
|
||||
|
||||
) -> bool:
|
||||
# if a view range is passed, plan to draw the
|
||||
# source ouput that's "in view" of the chart.
|
||||
vl, vr = view_range
|
||||
zoom_or_append = False
|
||||
last_vr = self._last_vr
|
||||
|
||||
# incremental in-view data update.
|
||||
if last_vr:
|
||||
lvl, lvr = last_vr # relative slice indices
|
||||
|
||||
# TODO: detecting more specifically the interaction changes
|
||||
# last_ivr = self._last_ivdr or (vl, vr)
|
||||
# al, ar = last_ivr # abs slice indices
|
||||
# left_change = abs(x_iv[0] - al) >= 1
|
||||
# right_change = abs(x_iv[-1] - ar) >= 1
|
||||
|
||||
# likely a zoom/pan view change or data append update
|
||||
if (
|
||||
(vr - lvr) > 2
|
||||
or vl < lvl
|
||||
|
||||
# append / prepend update
|
||||
# we had an append update where the view range
|
||||
# didn't change but the data-viewed (shifted)
|
||||
# underneath, so we need to redraw.
|
||||
# or left_change and right_change and last_vr == view_range
|
||||
|
||||
# not (left_change and right_change) and ivr
|
||||
# (
|
||||
# or abs(x_iv[ivr] - livr) > 1
|
||||
):
|
||||
zoom_or_append = True
|
||||
|
||||
self._last_vr = view_range
|
||||
|
||||
return zoom_or_append
|
||||
|
||||
def format_to_1d(
|
||||
self,
|
||||
new_read: tuple,
|
||||
array_key: str,
|
||||
profiler: Profiler,
|
||||
|
||||
slice_to_inview: bool = True,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
shm = self.shm
|
||||
|
||||
(
|
||||
_,
|
||||
_,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
|
||||
) = new_read
|
||||
|
||||
(
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
append_len,
|
||||
post_slice,
|
||||
) = self.diff(new_read)
|
||||
|
||||
# we first need to allocate xy data arrays
|
||||
# from the source data.
|
||||
if self.y_nd is None:
|
||||
self.xy_nd_start = shm._first.value
|
||||
self.xy_nd_stop = shm._last.value
|
||||
self.x_nd, self.y_nd = self.allocate_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
)
|
||||
profiler('allocated xy history')
|
||||
|
||||
# once allocated we do incremental pre/append
|
||||
# updates from the diff with the source buffer.
|
||||
else:
|
||||
if prepend_len:
|
||||
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
# this is the pre-sliced, "normally expected"
|
||||
# new data that an updater would normally be
|
||||
# expected to process, however in some cases (like
|
||||
# step curves) the updater routine may want to do
|
||||
# the source history-data reading itself, so we pass
|
||||
# both here.
|
||||
shm._array[pre_slice],
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=False,
|
||||
)
|
||||
|
||||
self.xy_nd_start -= prepend_len
|
||||
profiler('prepended xy history: {prepend_length}')
|
||||
|
||||
if append_len:
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
shm._array[post_slice],
|
||||
post_slice,
|
||||
append_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=True,
|
||||
)
|
||||
self.xy_nd_stop += append_len
|
||||
profiler('appened xy history: {append_length}')
|
||||
# sanity
|
||||
# slice_ln = post_slice.stop - post_slice.start
|
||||
# assert append_len == slice_ln
|
||||
|
||||
view_changed: bool = False
|
||||
view_range: tuple[int, int] = (ivl, ivr)
|
||||
if slice_to_inview:
|
||||
view_changed = self._track_inview_range(view_range)
|
||||
array = in_view
|
||||
profiler(f'{self.viz.name} view range slice {view_range}')
|
||||
|
||||
# TODO: we need to check if the last-datum-in-view is true and
|
||||
# if so only slice to the 2nd last datumonly slice to the 2nd
|
||||
# last datum.
|
||||
# hist = array[:slice_to_head]
|
||||
|
||||
# XXX: WOA WTF TRACTOR DEBUGGING BUGGG
|
||||
# assert 0
|
||||
|
||||
# xy-path data transform: convert source data to a format
|
||||
# able to be passed to a `QPainterPath` rendering routine.
|
||||
if not len(array):
|
||||
# XXX: this might be why the profiler only has exits?
|
||||
return
|
||||
|
||||
# TODO: hist here should be the pre-sliced
|
||||
# x/y_data in the case where allocate_xy is
|
||||
# defined?
|
||||
x_1d, y_1d, connect = self.format_xy_nd_to_1d(
|
||||
array,
|
||||
array_key,
|
||||
view_range,
|
||||
)
|
||||
# cache/save last 1d outputs for use by other
|
||||
# readers (eg. `Viz.draw_last_datum()` in the
|
||||
# only-draw-last-uppx case).
|
||||
self.x_1d = x_1d
|
||||
self.y_1d = y_1d
|
||||
|
||||
# app_tres = None
|
||||
# if append_len:
|
||||
# appended = array[-append_len-1:slice_to_head]
|
||||
# app_tres = self.format_xy_nd_to_1d(
|
||||
# appended,
|
||||
# array_key,
|
||||
# (
|
||||
# view_range[1] - append_len + slice_to_head,
|
||||
# view_range[1]
|
||||
# ),
|
||||
# )
|
||||
# # assert (len(appended) - 1) == append_len
|
||||
# # assert len(appended) == append_len
|
||||
# print(
|
||||
# f'{self.viz.name} APPEND LEN: {append_len}\n'
|
||||
# f'{self.viz.name} APPENDED: {appended}\n'
|
||||
# f'{self.viz.name} app_tres: {app_tres}\n'
|
||||
# )
|
||||
|
||||
# update the last "in view data range"
|
||||
if len(x_1d):
|
||||
self._last_ivdr = x_1d[0], x_1d[-1]
|
||||
|
||||
profiler('.format_to_1d()')
|
||||
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
connect,
|
||||
prepend_len,
|
||||
append_len,
|
||||
view_changed,
|
||||
# app_tres,
|
||||
)
|
||||
|
||||
###############################
|
||||
# Sub-type override interface #
|
||||
###############################
|
||||
|
||||
x_offset: np.ndarray = np.array([0])
|
||||
|
||||
# optional pre-graphics xy formatted data which
|
||||
# is incrementally updated in sync with the source data.
|
||||
# XXX: was ``.allocate_xy()``
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert the structured-array ``src_shm`` format to
|
||||
a equivalently shaped (and field-less) ``np.ndarray``.
|
||||
|
||||
Eg. a 4 field x N struct-array => (N, 4)
|
||||
|
||||
'''
|
||||
y_nd = src_shm._array[data_field].copy()
|
||||
x_nd = (
|
||||
src_shm._array[self.index_field].copy()
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
return x_nd, y_nd
|
||||
|
||||
# XXX: was ``.update_xy()``
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write pushed data to flattened copy
|
||||
y_nd_new = new_from_src[data_field]
|
||||
self.y_nd[read_slc] = y_nd_new
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name.upper()}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{list(x_nd[-s:])}\n'
|
||||
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{list(y_nd[-s:])}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new[0]}\n'
|
||||
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
# XXX: was ``.format_xy()``
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # 1d x
|
||||
np.ndarray, # 1d y
|
||||
np.ndarray | str, # connection array/style
|
||||
]:
|
||||
'''
|
||||
Default xy-nd array to 1d pre-graphics-path render routine.
|
||||
|
||||
Return single field column data verbatim
|
||||
|
||||
'''
|
||||
# NOTE: we don't include the very last datum which is filled in
|
||||
# normally by another graphics object.
|
||||
x_1d = array[self.index_field][:-1]
|
||||
y_1d = array[array_key][:-1]
|
||||
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# x_nd = list(self.x_nd[self.xy_slice][-s:-1])
|
||||
# y_nd = list(self.y_nd[self.xy_slice][-s:-1])
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# f'XY data:\n'
|
||||
# f'x: {x_nd}\n'
|
||||
# f'y: {y_nd}\n\n'
|
||||
# f'x_1d: {list(x_1d[-s:])}\n'
|
||||
# f'y_1d: {list(y_1d[-s:])}\n\n'
|
||||
|
||||
# )
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
|
||||
# 1d connection array or style-key to
|
||||
# ``pg.functions.arrayToQPath()``
|
||||
'all',
|
||||
)
|
||||
|
||||
|
||||
class OHLCBarsFmtr(IncrementalFormatter):
|
||||
x_offset: np.ndarray = np.array([
|
||||
-0.5,
|
||||
0,
|
||||
0,
|
||||
0.5,
|
||||
])
|
||||
|
||||
fields: list[str] = field(
|
||||
default_factory=lambda: ['open', 'high', 'low', 'close']
|
||||
)
|
||||
flat_index_ratio: float = 4
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
ohlc_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input struct-array holding OHLC samples into a pair of
|
||||
flattened x, y arrays with the same size (datums wise) as the source
|
||||
data.
|
||||
|
||||
'''
|
||||
y_nd = ohlc_shm.ustruct(self.fields)
|
||||
|
||||
# generate an flat-interpolated x-domain
|
||||
x_nd = (
|
||||
np.broadcast_to(
|
||||
ohlc_shm._array[self.index_field][:, None],
|
||||
(
|
||||
ohlc_shm._array.size,
|
||||
# 4, # only ohlc
|
||||
y_nd.shape[1],
|
||||
),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
assert y_nd.any()
|
||||
|
||||
# write pushed data to flattened copy
|
||||
return (
|
||||
x_nd,
|
||||
y_nd,
|
||||
)
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write newly pushed data to flattened copy
|
||||
# a struct-arr is always passed in.
|
||||
new_y_nd = rfn.structured_to_unstructured(
|
||||
new_from_src[self.fields]
|
||||
)
|
||||
self.y_nd[read_slc] = new_y_nd
|
||||
|
||||
# generate same-valued-per-row x support based on y shape
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = np.broadcast_to(
|
||||
new_from_src[self.index_field][:, None],
|
||||
new_y_nd.shape,
|
||||
) + self.x_offset
|
||||
|
||||
# TODO: can we drop this frame and just use the above?
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
start: int = 0, # XXX: do we need this?
|
||||
# 0.5 is no overlap between arms, 1.0 is full overlap
|
||||
w: float = 0.16,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
More or less direct proxy to the ``numba``-fied
|
||||
``path_arrays_from_ohlc()`` (above) but with closed in kwargs
|
||||
for line spacing.
|
||||
|
||||
'''
|
||||
x, y, c = path_arrays_from_ohlc(
|
||||
array[:-1],
|
||||
start,
|
||||
bar_w=self.index_step_size,
|
||||
bar_gap=w * self.index_step_size,
|
||||
|
||||
# XXX: don't ask, due to a ``numba`` bug..
|
||||
use_time_index=(self.index_field == 'time'),
|
||||
)
|
||||
return x, y, c
|
||||
|
||||
|
||||
class OHLCBarsAsCurveFmtr(OHLCBarsFmtr):
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
# TODO: in the case of an existing ``.update_xy()``
|
||||
# should we be passing in array as an xy arrays tuple?
|
||||
|
||||
# 2 more datum-indexes to capture zero at end
|
||||
x_flat = self.x_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
y_flat = self.y_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
|
||||
# slice to view
|
||||
ivl, ivr = vr
|
||||
x_iv_flat = x_flat[ivl:ivr]
|
||||
y_iv_flat = y_flat[ivl:ivr]
|
||||
|
||||
# reshape to 1d for graphics rendering
|
||||
y_iv = y_iv_flat.reshape(-1)
|
||||
x_iv = x_iv_flat.reshape(-1)
|
||||
|
||||
return x_iv, y_iv, 'all'
|
||||
|
||||
|
||||
class StepCurveFmtr(IncrementalFormatter):
|
||||
|
||||
x_offset: np.ndarray = np.array([
|
||||
0,
|
||||
1,
|
||||
])
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input 1d shm array to a "step array" format
|
||||
for use by path graphics generation.
|
||||
|
||||
'''
|
||||
i = shm._array[self.index_field].copy()
|
||||
out = shm._array[data_field].copy()
|
||||
|
||||
x_out = (
|
||||
np.broadcast_to(
|
||||
i[:, None],
|
||||
(i.size, 2),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# fill out Nx2 array to hold each step's left + right vertices.
|
||||
y_out = np.empty(
|
||||
x_out.shape,
|
||||
dtype=out.dtype,
|
||||
)
|
||||
# fill in (current) values from source shm buffer
|
||||
y_out[:] = out[:, np.newaxis]
|
||||
|
||||
# TODO: pretty sure we can drop this?
|
||||
# start y at origin level
|
||||
# y_out[0, 0] = 0
|
||||
# y_out[self.xy_nd_start] = 0
|
||||
return x_out, y_out
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
array_key: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
slice,
|
||||
]:
|
||||
# NOTE: for a step curve we slice from one datum prior
|
||||
# to the current "update slice" to get the previous
|
||||
# "level".
|
||||
#
|
||||
# why this is needed,
|
||||
# - the current new append slice will often have a zero
|
||||
# value in the latest datum-step (at least for zero-on-new
|
||||
# cases like vlm in the) as per configuration of the FSP
|
||||
# engine.
|
||||
# - we need to look back a datum to get the last level which
|
||||
# will be used to terminate/complete the last step x-width
|
||||
# which will be set to pair with the last x-index THIS MEANS
|
||||
#
|
||||
# XXX: this means WE CAN'T USE the append slice since we need to
|
||||
# "look backward" one step to get the needed back-to-zero level
|
||||
# and the update data in ``new_from_src`` will only contain the
|
||||
# latest new data.
|
||||
back_1 = slice(
|
||||
read_slc.start - 1,
|
||||
read_slc.stop,
|
||||
)
|
||||
|
||||
to_write = src_shm._array[back_1]
|
||||
y_nd_new = self.y_nd[back_1]
|
||||
y_nd_new[:] = to_write[array_key][:, None]
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field][:, None]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# XXX: uncomment for debugging
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'dolla_vlm' in name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{x_nd[-s:]}\n'
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{y_nd[-s:]}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new}\n'
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
last_t, last = array[-1][[self.index_field, array_key]]
|
||||
|
||||
start = self.xy_nd_start
|
||||
stop = self.xy_nd_stop
|
||||
|
||||
x_step = self.x_nd[start:stop]
|
||||
y_step = self.y_nd[start:stop]
|
||||
|
||||
# slice out in-view data
|
||||
ivl, ivr = vr
|
||||
|
||||
# NOTE: add an extra step to get the vertical-line-down-to-zero
|
||||
# adjacent to the last-datum graphic (filled rect).
|
||||
x_step_iv = x_step[ivl:ivr+1]
|
||||
y_step_iv = y_step[ivl:ivr+1]
|
||||
|
||||
# flatten to 1d
|
||||
x_1d = x_step_iv.reshape(x_step_iv.size)
|
||||
y_1d = y_step_iv.reshape(y_step_iv.size)
|
||||
|
||||
# debugging
|
||||
# if y_1d.any():
|
||||
# s = 6
|
||||
# print(
|
||||
# f'x_step_iv:\n{x_step_iv[-s:]}\n'
|
||||
# f'y_step_iv:\n{y_step_iv[-s:]}\n\n'
|
||||
# f'x_1d:\n{x_1d[-s:]}\n'
|
||||
# f'y_1d:\n{y_1d[-s:]}\n'
|
||||
# )
|
||||
|
||||
return x_1d, y_1d, 'all'
|
||||
|
|
@ -56,7 +56,7 @@ def iterticks(
|
|||
sig = (
|
||||
time,
|
||||
tick['price'],
|
||||
tick.get('size')
|
||||
tick['size']
|
||||
)
|
||||
|
||||
if ttype == 'dark_trade':
|
||||
|
|
|
|||
|
|
@ -1,446 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Super fast ``QPainterPath`` generation related operator routines.
|
||||
|
||||
"""
|
||||
from math import (
|
||||
ceil,
|
||||
floor,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
from numba import (
|
||||
# types,
|
||||
njit,
|
||||
float64,
|
||||
int64,
|
||||
# optional,
|
||||
)
|
||||
|
||||
# TODO: for ``numba`` typing..
|
||||
# from ._source import numba_ohlc_dtype
|
||||
from ._m4 import ds_m4
|
||||
from .._profile import (
|
||||
Profiler,
|
||||
pg_profile_enabled,
|
||||
ms_slower_then,
|
||||
)
|
||||
|
||||
|
||||
def xy_downsample(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
|
||||
x_spacer: float = 0.5,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
float,
|
||||
float,
|
||||
]:
|
||||
'''
|
||||
Downsample 1D (flat ``numpy.ndarray``) arrays using M4 given an input
|
||||
``uppx`` (units-per-pixel) and add space between discreet datums.
|
||||
|
||||
'''
|
||||
# downsample whenever more then 1 pixels per datum can be shown.
|
||||
# always refresh data bounds until we get diffing
|
||||
# working properly, see above..
|
||||
m4_out = ds_m4(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
)
|
||||
|
||||
if m4_out is not None:
|
||||
bins, x, y, ymn, ymx = m4_out
|
||||
# flatten output to 1d arrays suitable for path-graphics generation.
|
||||
x = np.broadcast_to(x[:, None], y.shape)
|
||||
x = (x + np.array(
|
||||
[-x_spacer, 0, 0, x_spacer]
|
||||
)).flatten()
|
||||
y = y.flatten()
|
||||
|
||||
return x, y, ymn, ymx
|
||||
|
||||
# XXX: we accept a None output for the case where the input range
|
||||
# to ``ds_m4()`` is bad (-ve) and we want to catch and debug
|
||||
# that (seemingly super rare) circumstance..
|
||||
return None
|
||||
|
||||
|
||||
@njit(
|
||||
# NOTE: need to construct this manually for readonly
|
||||
# arrays, see https://github.com/numba/numba/issues/4511
|
||||
# (
|
||||
# types.Array(
|
||||
# numba_ohlc_dtype,
|
||||
# 1,
|
||||
# 'C',
|
||||
# readonly=True,
|
||||
# ),
|
||||
# int64,
|
||||
# types.unicode_type,
|
||||
# optional(float64),
|
||||
# ),
|
||||
nogil=True
|
||||
)
|
||||
def path_arrays_from_ohlc(
|
||||
data: np.ndarray,
|
||||
start: int64,
|
||||
bar_w: float64,
|
||||
bar_gap: float64 = 0.16,
|
||||
use_time_index: bool = True,
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
Generate an array of lines objects from input ohlc data.
|
||||
|
||||
'''
|
||||
size = int(data.shape[0] * 6)
|
||||
|
||||
# XXX: see this for why the dtype might have to be defined outside
|
||||
# the routine.
|
||||
# https://github.com/numba/numba/issues/4098#issuecomment-493914533
|
||||
x = np.zeros(
|
||||
shape=size,
|
||||
dtype=float64,
|
||||
)
|
||||
y, c = x.copy(), x.copy()
|
||||
|
||||
half_w: float = bar_w/2
|
||||
|
||||
# TODO: report bug for assert @
|
||||
# /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
|
||||
for i, q in enumerate(data[start:], start):
|
||||
|
||||
open = q['open']
|
||||
high = q['high']
|
||||
low = q['low']
|
||||
close = q['close']
|
||||
|
||||
if use_time_index:
|
||||
index = float64(q['time'])
|
||||
else:
|
||||
index = float64(q['index'])
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index = float64(q[index_field])
|
||||
# AND this (probably)
|
||||
# open, high, low, close, index = q[
|
||||
# ['open', 'high', 'low', 'close', 'index']]
|
||||
|
||||
istart = i * 6
|
||||
istop = istart + 6
|
||||
|
||||
# x,y detail the 6 points which connect all vertexes of a ohlc bar
|
||||
mid: float = index + half_w
|
||||
x[istart:istop] = (
|
||||
index + bar_gap,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
index + bar_w - bar_gap,
|
||||
)
|
||||
y[istart:istop] = (
|
||||
open,
|
||||
open,
|
||||
low,
|
||||
high,
|
||||
close,
|
||||
close,
|
||||
)
|
||||
|
||||
# specifies that the first edge is never connected to the
|
||||
# prior bars last edge thus providing a small "gap"/"space"
|
||||
# between bars determined by ``bar_gap``.
|
||||
c[istart:istop] = (1, 1, 1, 1, 1, 0)
|
||||
|
||||
return x, y, c
|
||||
|
||||
|
||||
def hl2mxmn(
|
||||
ohlc: np.ndarray,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Convert a OHLC struct-array containing 'high'/'low' columns
|
||||
to a "joined" max/min 1-d array.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
hls = ohlc[[
|
||||
'low',
|
||||
'high',
|
||||
]]
|
||||
|
||||
mxmn = np.empty(2*hls.size, dtype=np.float64)
|
||||
x = np.empty(2*hls.size, dtype=np.float64)
|
||||
trace_hl(hls, mxmn, x, index[0])
|
||||
x = x + index[0]
|
||||
|
||||
return mxmn, x
|
||||
|
||||
|
||||
@njit(
|
||||
# TODO: the type annots..
|
||||
# float64[:](float64[:],),
|
||||
)
|
||||
def trace_hl(
|
||||
hl: 'np.ndarray',
|
||||
out: np.ndarray,
|
||||
x: np.ndarray,
|
||||
start: int,
|
||||
|
||||
# the "offset" values in the x-domain which
|
||||
# place the 2 output points around each ``int``
|
||||
# master index.
|
||||
margin: float = 0.43,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
"Trace" the outline of the high-low values of an ohlc sequence
|
||||
as a line such that the maximum deviation (aka disperaion) between
|
||||
bars if preserved.
|
||||
|
||||
This routine is expected to modify input arrays in-place.
|
||||
|
||||
'''
|
||||
last_l = hl['low'][0]
|
||||
last_h = hl['high'][0]
|
||||
|
||||
for i in range(hl.size):
|
||||
row = hl[i]
|
||||
l, h = row['low'], row['high']
|
||||
|
||||
up_diff = h - last_l
|
||||
down_diff = last_h - l
|
||||
|
||||
if up_diff > down_diff:
|
||||
out[2*i + 1] = h
|
||||
out[2*i] = last_l
|
||||
else:
|
||||
out[2*i + 1] = l
|
||||
out[2*i] = last_h
|
||||
|
||||
last_l = l
|
||||
last_h = h
|
||||
|
||||
x[2*i] = int(i) - margin
|
||||
x[2*i + 1] = int(i) + margin
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def ohlc_flatten(
|
||||
ohlc: np.ndarray,
|
||||
use_mxmn: bool = True,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
'''
|
||||
Convert an OHLCV struct-array into a flat ready-for-line-plotting
|
||||
1-d array that is 4 times the size with x-domain values distributed
|
||||
evenly (by 0.5 steps) over each index.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
|
||||
if use_mxmn:
|
||||
# traces a line optimally over highs to lows
|
||||
# using numba. NOTE: pretty sure this is faster
|
||||
# and looks about the same as the below output.
|
||||
flat, x = hl2mxmn(ohlc)
|
||||
|
||||
else:
|
||||
flat = rfn.structured_to_unstructured(
|
||||
ohlc[['open', 'high', 'low', 'close']]
|
||||
).flatten()
|
||||
|
||||
x = np.linspace(
|
||||
start=index[0] - 0.5,
|
||||
stop=index[-1] + 0.5,
|
||||
num=len(flat),
|
||||
)
|
||||
return x, flat
|
||||
|
||||
|
||||
def slice_from_time(
|
||||
arr: np.ndarray,
|
||||
start_t: float,
|
||||
stop_t: float,
|
||||
step: float, # sampler period step-diff
|
||||
|
||||
) -> slice:
|
||||
'''
|
||||
Calculate array indices mapped from a time range and return them in
|
||||
a slice.
|
||||
|
||||
Given an input array with an epoch `'time'` series entry, calculate
|
||||
the indices which span the time range and return in a slice. Presume
|
||||
each `'time'` step increment is uniform and when the time stamp
|
||||
series contains gaps (the uniform presumption is untrue) use
|
||||
``np.searchsorted()`` binary search to look up the appropriate
|
||||
index.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
msg='slice_from_time()',
|
||||
disabled=not pg_profile_enabled(),
|
||||
ms_threshold=ms_slower_then,
|
||||
)
|
||||
|
||||
times = arr['time']
|
||||
t_first = floor(times[0])
|
||||
t_last = ceil(times[-1])
|
||||
|
||||
# the greatest index we can return which slices to the
|
||||
# end of the input array.
|
||||
read_i_max = arr.shape[0]
|
||||
|
||||
# compute (presumed) uniform-time-step index offsets
|
||||
i_start_t = floor(start_t)
|
||||
read_i_start = floor(((i_start_t - t_first) // step)) - 1
|
||||
|
||||
i_stop_t = ceil(stop_t)
|
||||
|
||||
# XXX: edge case -> always set stop index to last in array whenever
|
||||
# the input stop time is detected to be greater then the equiv time
|
||||
# stamp at that last entry.
|
||||
if i_stop_t >= t_last:
|
||||
read_i_stop = read_i_max
|
||||
else:
|
||||
read_i_stop = ceil((i_stop_t - t_first) // step) + 1
|
||||
|
||||
# always clip outputs to array support
|
||||
# for read start:
|
||||
# - never allow a start < the 0 index
|
||||
# - never allow an end index > the read array len
|
||||
read_i_start = min(
|
||||
max(0, read_i_start),
|
||||
read_i_max - 1,
|
||||
)
|
||||
read_i_stop = max(
|
||||
0,
|
||||
min(read_i_stop, read_i_max),
|
||||
)
|
||||
|
||||
# check for larger-then-latest calculated index for given start
|
||||
# time, in which case we do a binary search for the correct index.
|
||||
# NOTE: this is usually the result of a time series with time gaps
|
||||
# where it is expected that each index step maps to a uniform step
|
||||
# in the time stamp series.
|
||||
t_iv_start = times[read_i_start]
|
||||
if (
|
||||
t_iv_start > i_start_t
|
||||
):
|
||||
# do a binary search for the best index mapping to ``start_t``
|
||||
# given we measured an overshoot using the uniform-time-step
|
||||
# calculation from above.
|
||||
|
||||
# TODO: once we start caching these per source-array,
|
||||
# we can just overwrite ``read_i_start`` directly.
|
||||
new_read_i_start = np.searchsorted(
|
||||
times,
|
||||
i_start_t,
|
||||
side='left',
|
||||
)
|
||||
|
||||
# TODO: minimize binary search work as much as possible:
|
||||
# - cache these remap values which compensate for gaps in the
|
||||
# uniform time step basis where we calc a later start
|
||||
# index for the given input ``start_t``.
|
||||
# - can we shorten the input search sequence by heuristic?
|
||||
# up_to_arith_start = index[:read_i_start]
|
||||
|
||||
if (
|
||||
new_read_i_start <= read_i_start
|
||||
):
|
||||
# t_diff = t_iv_start - start_t
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'start_t:{start_t} -> 0index start_t:{t_iv_start}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# f'REMAPPED START i: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
read_i_start = new_read_i_start
|
||||
|
||||
t_iv_stop = times[read_i_stop - 1]
|
||||
if (
|
||||
t_iv_stop > i_stop_t
|
||||
):
|
||||
# t_diff = stop_t - t_iv_stop
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'calced iv stop:{t_iv_stop} -> stop_t:{stop_t}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# # f'SHOULD REMAP STOP: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
new_read_i_stop = np.searchsorted(
|
||||
times[read_i_start:],
|
||||
# times,
|
||||
i_stop_t,
|
||||
side='right',
|
||||
)
|
||||
|
||||
if (
|
||||
new_read_i_stop <= read_i_stop
|
||||
):
|
||||
read_i_stop = read_i_start + new_read_i_stop + 1
|
||||
|
||||
# sanity checks for range size
|
||||
# samples = (i_stop_t - i_start_t) // step
|
||||
# index_diff = read_i_stop - read_i_start + 1
|
||||
# if index_diff > (samples + 3):
|
||||
# breakpoint()
|
||||
|
||||
# read-relative indexes: gives a slice where `shm.array[read_slc]`
|
||||
# will be the data spanning the input time range `start_t` ->
|
||||
# `stop_t`
|
||||
read_slc = slice(
|
||||
int(read_i_start),
|
||||
int(read_i_stop),
|
||||
)
|
||||
|
||||
profiler(
|
||||
'slicing complete'
|
||||
# f'{start_t} -> {abs_slc.start} | {read_slc.start}\n'
|
||||
# f'{stop_t} -> {abs_slc.stop} | {read_slc.stop}\n'
|
||||
)
|
||||
|
||||
# NOTE: if caller needs absolute buffer indices they can
|
||||
# slice the buffer abs index like so:
|
||||
# index = arr['index']
|
||||
# abs_indx = index[read_slc]
|
||||
# abs_slc = slice(
|
||||
# int(abs_indx[0]),
|
||||
# int(abs_indx[-1]),
|
||||
# )
|
||||
|
||||
return read_slc
|
||||
|
|
@ -20,95 +20,53 @@ financial data flows.
|
|||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import (
|
||||
Counter,
|
||||
defaultdict,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from collections import Counter
|
||||
import time
|
||||
from typing import (
|
||||
AsyncIterator,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
import tractor
|
||||
from tractor.trionics import (
|
||||
maybe_open_nursery,
|
||||
)
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
from ..service import maybe_spawn_daemon
|
||||
from ..log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._sharedmem import (
|
||||
ShmArray,
|
||||
)
|
||||
from ._sharedmem import ShmArray
|
||||
from .feed import _FeedsBus
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# highest frequency sample step is 1 second by default, though in
|
||||
# the future we may want to support shorter periods or a dynamic style
|
||||
# tick-event stream.
|
||||
_default_delay_s: float = 1.0
|
||||
|
||||
|
||||
class Sampler:
|
||||
class sampler:
|
||||
'''
|
||||
Global sampling engine registry.
|
||||
|
||||
Manages state for sampling events, shm incrementing and
|
||||
sample period logic.
|
||||
|
||||
This non-instantiated type is meant to be a singleton within
|
||||
a `samplerd` actor-service spawned once by the user wishing to
|
||||
time-step-sample (real-time) quote feeds, see
|
||||
``.service.maybe_open_samplerd()`` and the below
|
||||
``register_with_sampler()``.
|
||||
|
||||
'''
|
||||
service_nursery: None | trio.Nursery = None
|
||||
|
||||
# TODO: we could stick these in a composed type to avoid
|
||||
# angering the "i hate module scoped variables crowd" (yawn).
|
||||
ohlcv_shms: dict[float, list[ShmArray]] = {}
|
||||
ohlcv_shms: dict[int, list[ShmArray]] = {}
|
||||
|
||||
# holds one-task-per-sample-period tasks which are spawned as-needed by
|
||||
# data feed requests with a given detected time step usually from
|
||||
# history loading.
|
||||
incr_task_cs: trio.CancelScope | None = None
|
||||
incrementers: dict[int, trio.CancelScope] = {}
|
||||
|
||||
# holds all the ``tractor.Context`` remote subscriptions for
|
||||
# a particular sample period increment event: all subscribers are
|
||||
# notified on a step.
|
||||
subscribers: defaultdict[
|
||||
float,
|
||||
list[
|
||||
float,
|
||||
set[tractor.MsgStream]
|
||||
],
|
||||
] = defaultdict(
|
||||
lambda: [
|
||||
round(time.time()),
|
||||
set(),
|
||||
]
|
||||
)
|
||||
subscribers: dict[int, tractor.Context] = {}
|
||||
|
||||
@classmethod
|
||||
async def increment_ohlc_buffer(
|
||||
self,
|
||||
period_s: float,
|
||||
|
||||
async def increment_ohlc_buffer(
|
||||
delay_s: int,
|
||||
task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
):
|
||||
):
|
||||
'''
|
||||
Task which inserts new bars into the provide shared memory array
|
||||
every ``period_s`` seconds.
|
||||
every ``delay_s`` seconds.
|
||||
|
||||
This task fulfills 2 purposes:
|
||||
- it takes the subscribed set of shm arrays and increments them
|
||||
|
|
@ -120,152 +78,101 @@ class Sampler:
|
|||
the underlying buffers will actually be incremented.
|
||||
|
||||
'''
|
||||
# # wait for brokerd to signal we should start sampling
|
||||
# await shm_incrementing(shm_token['shm_name']).wait()
|
||||
|
||||
# TODO: right now we'll spin printing bars if the last time stamp is
|
||||
# before a large period of no market activity. Likely the best way
|
||||
# to solve this is to make this task aware of the instrument's
|
||||
# tradable hours?
|
||||
|
||||
total_s: float = 0 # total seconds counted
|
||||
ad = period_s - 0.001 # compensate for trio processing time
|
||||
# adjust delay to compensate for trio processing time
|
||||
ad = min(sampler.ohlcv_shms.keys()) - 0.001
|
||||
|
||||
total_s = 0 # total seconds counted
|
||||
lowest = min(sampler.ohlcv_shms.keys())
|
||||
lowest_shm = sampler.ohlcv_shms[lowest][0]
|
||||
ad = lowest - 0.001
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
|
||||
# register this time period step as active
|
||||
sampler.incrementers[delay_s] = cs
|
||||
task_status.started(cs)
|
||||
|
||||
# sample step loop:
|
||||
# includes broadcasting to all connected consumers on every
|
||||
# new sample step as well incrementing any registered
|
||||
# buffers by registered sample period.
|
||||
while True:
|
||||
# TODO: do we want to support dynamically
|
||||
# adding a "lower" lowest increment period?
|
||||
await trio.sleep(ad)
|
||||
total_s += period_s
|
||||
total_s += lowest
|
||||
|
||||
# increment all subscribed shm arrays
|
||||
# TODO:
|
||||
# - this in ``numba``
|
||||
# - just lookup shms for this step instead of iterating?
|
||||
|
||||
i_epoch = round(time.time())
|
||||
broadcasted: set[float] = set()
|
||||
|
||||
# print(f'epoch: {i_epoch} -> REGISTRY {self.ohlcv_shms}')
|
||||
for shm_period_s, shms in self.ohlcv_shms.items():
|
||||
|
||||
# short-circuit on any not-ready because slower sample
|
||||
# rate consuming shm buffers.
|
||||
if total_s % shm_period_s != 0:
|
||||
# print(f'skipping `{shm_period_s}s` sample update')
|
||||
for delay_s, shms in sampler.ohlcv_shms.items():
|
||||
if total_s % delay_s != 0:
|
||||
continue
|
||||
|
||||
# update last epoch stamp for this period group
|
||||
if shm_period_s not in broadcasted:
|
||||
sub_pair = self.subscribers[shm_period_s]
|
||||
sub_pair[0] = i_epoch
|
||||
broadcasted.add(shm_period_s)
|
||||
|
||||
# TODO: ``numba`` this!
|
||||
for shm in shms:
|
||||
# print(f'UPDATE {shm_period_s}s STEP for {shm.token}')
|
||||
# TODO: in theory we could make this faster by copying the
|
||||
# "last" readable value into the underlying larger buffer's
|
||||
# next value and then incrementing the counter instead of
|
||||
# using ``.push()``?
|
||||
|
||||
# append new entry to buffer thus "incrementing"
|
||||
# the bar
|
||||
# append new entry to buffer thus "incrementing" the bar
|
||||
array = shm.array
|
||||
last = array[-1:][shm._write_fields].copy()
|
||||
# (index, t, close) = last[0][['index', 'time', 'close']]
|
||||
(t, close) = last[0][['time', 'close']]
|
||||
|
||||
# guard against startup backfilling races where
|
||||
# the buffer has not yet been filled.
|
||||
if not last.size:
|
||||
continue
|
||||
|
||||
(t, close) = last[0][[
|
||||
'time',
|
||||
'close',
|
||||
]]
|
||||
|
||||
next_t = t + shm_period_s
|
||||
|
||||
if shm_period_s <= 1:
|
||||
next_t = i_epoch
|
||||
|
||||
# this copies non-std fields (eg. vwap) from the
|
||||
# last datum
|
||||
last[[
|
||||
'time',
|
||||
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
|
||||
'volume',
|
||||
]][0] = (
|
||||
# epoch timestamp
|
||||
next_t,
|
||||
|
||||
# OHLC
|
||||
close,
|
||||
close,
|
||||
close,
|
||||
close,
|
||||
|
||||
0, # vlm
|
||||
)
|
||||
|
||||
# TODO: in theory we could make this faster by
|
||||
# copying the "last" readable value into the
|
||||
# underlying larger buffer's next value and then
|
||||
# incrementing the counter instead of using
|
||||
# ``.push()``?
|
||||
# this copies non-std fields (eg. vwap) from the last datum
|
||||
last[
|
||||
['time', 'volume', 'open', 'high', 'low', 'close']
|
||||
][0] = (t + delay_s, 0, close, close, close, close)
|
||||
|
||||
# write to the buffer
|
||||
shm.push(last)
|
||||
|
||||
# broadcast increment msg to all updated subs per period
|
||||
for shm_period_s in broadcasted:
|
||||
await self.broadcast(
|
||||
period_s=shm_period_s,
|
||||
time_stamp=i_epoch,
|
||||
)
|
||||
await broadcast(delay_s, shm=lowest_shm)
|
||||
|
||||
@classmethod
|
||||
async def broadcast(
|
||||
self,
|
||||
period_s: float,
|
||||
time_stamp: float | None = None,
|
||||
|
||||
) -> None:
|
||||
async def broadcast(
|
||||
delay_s: int,
|
||||
shm: Optional[ShmArray] = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Broadcast the period size and last index step value to all
|
||||
Broadcast the given ``shm: ShmArray``'s buffer index step to any
|
||||
subscribers for a given sample period.
|
||||
|
||||
The sent msg will include the first and last index which slice into
|
||||
the buffer's non-empty data.
|
||||
|
||||
'''
|
||||
pair: list[float, set]
|
||||
pair = self.subscribers[period_s]
|
||||
subs = sampler.subscribers.get(delay_s, ())
|
||||
|
||||
last_ts: float
|
||||
subs: set
|
||||
last_ts, subs = pair
|
||||
first = last = -1
|
||||
|
||||
task = trio.lowlevel.current_task()
|
||||
log.debug(
|
||||
f'SUBS {self.subscribers}\n'
|
||||
f'PAIR {pair}\n'
|
||||
f'TASK: {task}: {id(task)}\n'
|
||||
f'broadcasting {period_s} -> {last_ts}\n'
|
||||
# f'consumers: {subs}'
|
||||
)
|
||||
borked: set[tractor.MsgStream] = set()
|
||||
sent: set[tractor.MsgStream] = set()
|
||||
while True:
|
||||
try:
|
||||
for stream in (subs - sent):
|
||||
if shm is None:
|
||||
periods = sampler.ohlcv_shms.keys()
|
||||
# if this is an update triggered by a history update there
|
||||
# might not actually be any sampling bus setup since there's
|
||||
# no "live feed" active yet.
|
||||
if periods:
|
||||
lowest = min(periods)
|
||||
shm = sampler.ohlcv_shms[lowest][0]
|
||||
first = shm._first.value
|
||||
last = shm._last.value
|
||||
|
||||
for stream in subs:
|
||||
try:
|
||||
await stream.send({
|
||||
'index': time_stamp or last_ts,
|
||||
'period': period_s,
|
||||
'first': first,
|
||||
'last': last,
|
||||
'index': last,
|
||||
})
|
||||
sent.add(stream)
|
||||
|
||||
except (
|
||||
trio.BrokenResourceError,
|
||||
trio.ClosedResourceError
|
||||
|
|
@ -273,270 +180,60 @@ class Sampler:
|
|||
log.error(
|
||||
f'{stream._ctx.chan.uid} dropped connection'
|
||||
)
|
||||
borked.add(stream)
|
||||
else:
|
||||
break
|
||||
except RuntimeError:
|
||||
log.warning(f'Client subs {subs} changed while broadcasting')
|
||||
continue
|
||||
|
||||
for stream in borked:
|
||||
try:
|
||||
subs.remove(stream)
|
||||
except KeyError:
|
||||
except ValueError:
|
||||
log.warning(
|
||||
f'{stream._ctx.chan.uid} sub already removed!?'
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def broadcast_all(self) -> None:
|
||||
for period_s in self.subscribers:
|
||||
await self.broadcast(period_s)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def register_with_sampler(
|
||||
async def iter_ohlc_periods(
|
||||
ctx: tractor.Context,
|
||||
period_s: float,
|
||||
shms_by_period: dict[float, dict] | None = None,
|
||||
|
||||
open_index_stream: bool = True, # open a 2way stream for sample step msgs?
|
||||
sub_for_broadcasts: bool = True, # sampler side to send step updates?
|
||||
delay_s: int,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Subscribe to OHLC sampling "step" events: when the time
|
||||
aggregation period increments, this event stream emits an index
|
||||
event.
|
||||
|
||||
get_console_log(tractor.current_actor().loglevel)
|
||||
incr_was_started: bool = False
|
||||
|
||||
try:
|
||||
async with maybe_open_nursery(
|
||||
Sampler.service_nursery
|
||||
) as service_nursery:
|
||||
|
||||
# init startup, create (actor-)local service nursery and start
|
||||
# increment task
|
||||
Sampler.service_nursery = service_nursery
|
||||
|
||||
# always ensure a period subs entry exists
|
||||
last_ts, subs = Sampler.subscribers[float(period_s)]
|
||||
|
||||
async with trio.Lock():
|
||||
if Sampler.incr_task_cs is None:
|
||||
Sampler.incr_task_cs = await service_nursery.start(
|
||||
Sampler.increment_ohlc_buffer,
|
||||
1.,
|
||||
)
|
||||
incr_was_started = True
|
||||
|
||||
# insert the base 1s period (for OHLC style sampling) into
|
||||
# the increment buffer set to update and shift every second.
|
||||
if shms_by_period is not None:
|
||||
from ._sharedmem import (
|
||||
attach_shm_array,
|
||||
_Token,
|
||||
)
|
||||
for period in shms_by_period:
|
||||
|
||||
# load and register shm handles
|
||||
shm_token_msg = shms_by_period[period]
|
||||
shm = attach_shm_array(
|
||||
_Token.from_msg(shm_token_msg),
|
||||
readonly=False,
|
||||
)
|
||||
shms_by_period[period] = shm
|
||||
Sampler.ohlcv_shms.setdefault(period, []).append(shm)
|
||||
|
||||
assert Sampler.ohlcv_shms
|
||||
|
||||
# unblock caller
|
||||
await ctx.started(set(Sampler.ohlcv_shms.keys()))
|
||||
|
||||
if open_index_stream:
|
||||
try:
|
||||
'''
|
||||
# add our subscription
|
||||
subs = sampler.subscribers.setdefault(delay_s, [])
|
||||
await ctx.started()
|
||||
async with ctx.open_stream() as stream:
|
||||
if sub_for_broadcasts:
|
||||
subs.add(stream)
|
||||
subs.append(stream)
|
||||
|
||||
# except broadcast requests from the subscriber
|
||||
async for msg in stream:
|
||||
if msg == 'broadcast_all':
|
||||
await Sampler.broadcast_all()
|
||||
finally:
|
||||
if sub_for_broadcasts:
|
||||
subs.remove(stream)
|
||||
else:
|
||||
# if no shms are passed in we just wait until cancelled
|
||||
# by caller.
|
||||
try:
|
||||
# stream and block until cancelled
|
||||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
# TODO: why tf isn't this working?
|
||||
if shms_by_period is not None:
|
||||
for period, shm in shms_by_period.items():
|
||||
Sampler.ohlcv_shms[period].remove(shm)
|
||||
|
||||
if incr_was_started:
|
||||
Sampler.incr_task_cs.cancel()
|
||||
Sampler.incr_task_cs = None
|
||||
|
||||
|
||||
async def spawn_samplerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Daemon-side service task: start a sampling daemon for common step
|
||||
update and increment count write and stream broadcasting.
|
||||
|
||||
'''
|
||||
from piker.service import Services
|
||||
|
||||
dname = 'samplerd'
|
||||
log.info(f'Spawning `{dname}`')
|
||||
|
||||
# singleton lock creation of ``samplerd`` since we only ever want
|
||||
# one daemon per ``pikerd`` proc tree.
|
||||
# TODO: make this built-into the service api?
|
||||
async with Services.locks[dname + '_singleton']:
|
||||
|
||||
if dname not in Services.service_tasks:
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=[
|
||||
'piker.data._sampling',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
try:
|
||||
subs.remove(stream)
|
||||
except ValueError:
|
||||
log.error(
|
||||
f'iOHLC step stream was already dropped {ctx.chan.uid}?'
|
||||
)
|
||||
|
||||
await Services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
register_with_sampler,
|
||||
period_s=1,
|
||||
sub_for_broadcasts=False,
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_samplerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal: # noqa
|
||||
'''
|
||||
Client-side helper to maybe startup the ``samplerd`` service
|
||||
under the ``pikerd`` tree.
|
||||
|
||||
'''
|
||||
dname = 'samplerd'
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
dname,
|
||||
service_task_target=spawn_samplerd,
|
||||
spawn_args={},
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
@acm
|
||||
async def open_sample_stream(
|
||||
period_s: float,
|
||||
shms_by_period: dict[float, dict] | None = None,
|
||||
open_index_stream: bool = True,
|
||||
sub_for_broadcasts: bool = True,
|
||||
|
||||
cache_key: str | None = None,
|
||||
allow_new_sampler: bool = True,
|
||||
|
||||
) -> AsyncIterator[dict[str, float]]:
|
||||
'''
|
||||
Subscribe to OHLC sampling "step" events: when the time aggregation
|
||||
period increments, this event stream emits an index event.
|
||||
|
||||
This is a client-side endpoint that does all the work of ensuring
|
||||
the `samplerd` actor is up and that mult-consumer-tasks are given
|
||||
a broadcast stream when possible.
|
||||
|
||||
'''
|
||||
# TODO: wrap this manager with the following to make it cached
|
||||
# per client-multitasks entry.
|
||||
# maybe_open_context(
|
||||
# acm_func=partial(
|
||||
# portal.open_context,
|
||||
# register_with_sampler,
|
||||
# ),
|
||||
# key=cache_key or period_s,
|
||||
# )
|
||||
# if cache_hit:
|
||||
# # add a new broadcast subscription for the quote stream
|
||||
# # if this feed is likely already in use
|
||||
# async with istream.subscribe() as bistream:
|
||||
# yield bistream
|
||||
# else:
|
||||
|
||||
async with (
|
||||
# XXX: this should be singleton on a host,
|
||||
# a lone broker-daemon per provider should be
|
||||
# created for all practical purposes
|
||||
maybe_open_samplerd() as portal,
|
||||
|
||||
portal.open_context(
|
||||
register_with_sampler,
|
||||
**{
|
||||
'period_s': period_s,
|
||||
'shms_by_period': shms_by_period,
|
||||
'open_index_stream': open_index_stream,
|
||||
'sub_for_broadcasts': sub_for_broadcasts,
|
||||
},
|
||||
) as (ctx, first)
|
||||
):
|
||||
async with (
|
||||
ctx.open_stream() as istream,
|
||||
|
||||
# TODO: we don't need this task-bcasting right?
|
||||
# istream.subscribe() as istream,
|
||||
):
|
||||
yield istream
|
||||
|
||||
|
||||
async def sample_and_broadcast(
|
||||
|
||||
bus: _FeedsBus, # noqa
|
||||
rt_shm: ShmArray,
|
||||
hist_shm: ShmArray,
|
||||
shm: ShmArray,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
brokername: str,
|
||||
sum_tick_vlm: bool = True,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
`brokerd`-side task which writes latest datum sampled data.
|
||||
|
||||
This task is meant to run in the same actor (mem space) as the
|
||||
`brokerd` real-time quote feed which is being sampled to
|
||||
a ``ShmArray`` buffer.
|
||||
|
||||
'''
|
||||
log.info("Started shared mem bar writer")
|
||||
|
||||
overruns = Counter()
|
||||
|
||||
# iterate stream delivered by broker
|
||||
async for quotes in quote_stream:
|
||||
# print(quotes)
|
||||
|
||||
# TODO: ``numba`` this!
|
||||
for broker_symbol, quote in quotes.items():
|
||||
# TODO: in theory you can send the IPC msg *before* writing
|
||||
|
|
@ -560,9 +257,6 @@ async def sample_and_broadcast(
|
|||
|
||||
last = tick['price']
|
||||
|
||||
# more compact inline-way to do this assignment
|
||||
# to both buffers?
|
||||
for shm in [rt_shm, hist_shm]:
|
||||
# update last entry
|
||||
# benchmarked in the 4-5 us range
|
||||
o, high, low, v = shm.array[-1][
|
||||
|
|
@ -599,37 +293,29 @@ async def sample_and_broadcast(
|
|||
volume,
|
||||
)
|
||||
|
||||
# TODO: PUT THIS IN A ``_FeedsBus.broadcast()`` method!
|
||||
# XXX: we need to be very cautious here that no
|
||||
# context-channel is left lingering which doesn't have
|
||||
# a far end receiver actor-task. In such a case you can
|
||||
# end up triggering backpressure which which will
|
||||
# eventually block this producer end of the feed and
|
||||
# thus other consumers still attached.
|
||||
sub_key: str = broker_symbol.lower()
|
||||
subs: list[
|
||||
tuple[
|
||||
tractor.MsgStream | trio.MemorySendChannel,
|
||||
float | None, # tick throttle in Hz
|
||||
Union[tractor.MsgStream, trio.MemorySendChannel],
|
||||
tractor.Context,
|
||||
Optional[float], # tick throttle in Hz
|
||||
]
|
||||
] = bus.get_subs(sub_key)
|
||||
] = bus._subscribers[broker_symbol.lower()]
|
||||
|
||||
# NOTE: by default the broker backend doesn't append
|
||||
# it's own "name" into the fqsn schema (but maybe it
|
||||
# should?) so we have to manually generate the correct
|
||||
# key here.
|
||||
fqsn = f'{broker_symbol}.{brokername}'
|
||||
bsym = f'{broker_symbol}.{brokername}'
|
||||
lags: int = 0
|
||||
|
||||
# TODO: speed up this loop in an AOT compiled lang (like
|
||||
# rust or nim or zig) and/or instead of doing a fan out to
|
||||
# TCP sockets here, we add a shm-style tick queue which
|
||||
# readers can pull from instead of placing the burden of
|
||||
# broadcast on solely on this `brokerd` actor. see issues:
|
||||
# - https://github.com/pikers/piker/issues/98
|
||||
# - https://github.com/pikers/piker/issues/107
|
||||
for (stream, ctx, tick_throttle) in subs:
|
||||
|
||||
for (stream, tick_throttle) in subs.copy():
|
||||
try:
|
||||
with trio.move_on_after(0.2) as cs:
|
||||
if tick_throttle:
|
||||
|
|
@ -637,39 +323,47 @@ async def sample_and_broadcast(
|
|||
# pushes to the ``uniform_rate_send()`` below.
|
||||
try:
|
||||
stream.send_nowait(
|
||||
(fqsn, quote)
|
||||
(bsym, quote)
|
||||
)
|
||||
except trio.WouldBlock:
|
||||
overruns[sub_key] += 1
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
|
||||
if ctx:
|
||||
log.warning(
|
||||
f'Feed OVERRUN {sub_key}'
|
||||
'@{bus.brokername} -> \n'
|
||||
f'feed @ {chan.uid}\n'
|
||||
f'throttle = {tick_throttle} Hz'
|
||||
f'Feed overrun {bus.brokername} ->'
|
||||
f'{chan.uid} !!!'
|
||||
)
|
||||
|
||||
if overruns[sub_key] > 6:
|
||||
else:
|
||||
key = id(stream)
|
||||
overruns[key] += 1
|
||||
log.warning(
|
||||
f'Feed overrun {broker_symbol}'
|
||||
'@{bus.brokername} -> '
|
||||
f'feed @ {tick_throttle} Hz'
|
||||
)
|
||||
if overruns[key] > 6:
|
||||
# TODO: should we check for the
|
||||
# context being cancelled? this
|
||||
# could happen but the
|
||||
# channel-ipc-pipe is still up.
|
||||
if (
|
||||
not chan.connected()
|
||||
or ctx._cancel_called
|
||||
):
|
||||
if not chan.connected():
|
||||
log.warning(
|
||||
'Dropping broken consumer:\n'
|
||||
f'{sub_key}:'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
await stream.aclose()
|
||||
raise trio.BrokenResourceError
|
||||
else:
|
||||
log.warning(
|
||||
'Feed getting overrun bro!\n'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
continue
|
||||
|
||||
else:
|
||||
await stream.send(
|
||||
{fqsn: quote}
|
||||
{bsym: quote}
|
||||
)
|
||||
|
||||
if cs.cancelled_caught:
|
||||
|
|
@ -682,7 +376,6 @@ async def sample_and_broadcast(
|
|||
trio.ClosedResourceError,
|
||||
trio.EndOfChannel,
|
||||
):
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
if ctx:
|
||||
log.warning(
|
||||
|
|
@ -698,65 +391,19 @@ async def sample_and_broadcast(
|
|||
# so far seems like no since this should all
|
||||
# be single-threaded. Doing it anyway though
|
||||
# since there seems to be some kinda race..
|
||||
bus.remove_subs(
|
||||
sub_key,
|
||||
{(stream, tick_throttle)},
|
||||
try:
|
||||
subs.remove((stream, tick_throttle))
|
||||
except ValueError:
|
||||
log.error(
|
||||
f'Stream was already removed from subs!?\n'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
|
||||
|
||||
# a working tick-type-classes template
|
||||
_tick_groups = {
|
||||
'clears': {'trade', 'dark_trade', 'last'},
|
||||
'bids': {'bid', 'bsize'},
|
||||
'asks': {'ask', 'asize'},
|
||||
}
|
||||
|
||||
|
||||
def frame_ticks(
|
||||
first_quote: dict,
|
||||
last_quote: dict,
|
||||
ticks_by_type: dict,
|
||||
) -> None:
|
||||
# append quotes since last iteration into the last quote's
|
||||
# tick array/buffer.
|
||||
ticks = last_quote.get('ticks')
|
||||
|
||||
# TODO: once we decide to get fancy really we should
|
||||
# have a shared mem tick buffer that is just
|
||||
# continually filled and the UI just ready from it
|
||||
# at it's display rate.
|
||||
if ticks:
|
||||
# TODO: do we need this any more or can we just
|
||||
# expect the receiver to unwind the below
|
||||
# `ticks_by_type: dict`?
|
||||
# => undwinding would potentially require a
|
||||
# `dict[str, set | list]` instead with an
|
||||
# included `'types' field which is an (ordered)
|
||||
# set of tick type fields in the order which
|
||||
# types arrived?
|
||||
first_quote['ticks'].extend(ticks)
|
||||
|
||||
# XXX: build a tick-by-type table of lists
|
||||
# of tick messages. This allows for less
|
||||
# iteration on the receiver side by allowing for
|
||||
# a single "latest tick event" look up by
|
||||
# indexing the last entry in each sub-list.
|
||||
# tbt = {
|
||||
# 'types': ['bid', 'asize', 'last', .. '<type_n>'],
|
||||
|
||||
# 'bid': [tick0, tick1, tick2, .., tickn],
|
||||
# 'asize': [tick0, tick1, tick2, .., tickn],
|
||||
# 'last': [tick0, tick1, tick2, .., tickn],
|
||||
# ...
|
||||
# '<type_n>': [tick0, tick1, tick2, .., tickn],
|
||||
# }
|
||||
|
||||
# append in reverse FIFO order for in-order iteration on
|
||||
# receiver side.
|
||||
for tick in ticks:
|
||||
ttype = tick['type']
|
||||
ticks_by_type[ttype].append(tick)
|
||||
|
||||
# TODO: a less naive throttler, here's some snippets:
|
||||
# token bucket by njs:
|
||||
# https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||
|
||||
async def uniform_rate_send(
|
||||
|
||||
|
|
@ -767,23 +414,6 @@ async def uniform_rate_send(
|
|||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Throttle a real-time (presumably tick event) stream to a uniform
|
||||
transmissiom rate, normally for the purposes of throttling a data
|
||||
flow being consumed by a graphics rendering actor which itself is limited
|
||||
by a fixed maximum display rate.
|
||||
|
||||
Though this function isn't documented (nor was intentially written
|
||||
to be) a token-bucket style algo, it effectively operates as one (we
|
||||
think?).
|
||||
|
||||
TODO: a less naive throttler, here's some snippets:
|
||||
token bucket by njs:
|
||||
https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||
|
||||
'''
|
||||
# try not to error-out on overruns of the subscribed client
|
||||
stream._ctx._backpressure = True
|
||||
|
||||
# TODO: compute the approx overhead latency per cycle
|
||||
left_to_sleep = throttle_period = 1/rate - 0.000616
|
||||
|
|
@ -794,12 +424,6 @@ async def uniform_rate_send(
|
|||
diff = 0
|
||||
|
||||
task_status.started()
|
||||
ticks_by_type: defaultdict[
|
||||
str,
|
||||
list[dict],
|
||||
] = defaultdict(list)
|
||||
|
||||
clear_types = _tick_groups['clears']
|
||||
|
||||
while True:
|
||||
|
||||
|
|
@ -818,17 +442,34 @@ async def uniform_rate_send(
|
|||
|
||||
if not first_quote:
|
||||
first_quote = last_quote
|
||||
# first_quote['tbt'] = ticks_by_type
|
||||
|
||||
if (throttle_period - diff) > 0:
|
||||
# received a quote but the send cycle period hasn't yet
|
||||
# expired we aren't supposed to send yet so append
|
||||
# to the tick frame.
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
last_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# append quotes since last iteration into the last quote's
|
||||
# tick array/buffer.
|
||||
ticks = last_quote.get('ticks')
|
||||
|
||||
# XXX: idea for frame type data structure we could
|
||||
# use on the wire instead of a simple list?
|
||||
# frames = {
|
||||
# 'index': ['type_a', 'type_c', 'type_n', 'type_n'],
|
||||
|
||||
# 'type_a': [tick0, tick1, tick2, .., tickn],
|
||||
# 'type_b': [tick0, tick1, tick2, .., tickn],
|
||||
# 'type_c': [tick0, tick1, tick2, .., tickn],
|
||||
# ...
|
||||
# 'type_n': [tick0, tick1, tick2, .., tickn],
|
||||
# }
|
||||
|
||||
# TODO: once we decide to get fancy really we should
|
||||
# have a shared mem tick buffer that is just
|
||||
# continually filled and the UI just ready from it
|
||||
# at it's display rate.
|
||||
if ticks:
|
||||
first_quote['ticks'].extend(ticks)
|
||||
|
||||
# send cycle isn't due yet so continue waiting
|
||||
continue
|
||||
|
|
@ -845,50 +486,17 @@ async def uniform_rate_send(
|
|||
# received quote ASAP.
|
||||
sym, first_quote = await quote_stream.receive()
|
||||
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
first_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# we have a quote already so send it now.
|
||||
|
||||
with trio.move_on_after(throttle_period) as cs:
|
||||
while (
|
||||
not set(ticks_by_type).intersection(clear_types)
|
||||
):
|
||||
try:
|
||||
sym, last_quote = await quote_stream.receive()
|
||||
except trio.EndOfChannel:
|
||||
log.exception(f"feed for {stream} ended?")
|
||||
break
|
||||
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
last_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# measured_rate = 1 / (time.time() - last_send)
|
||||
# log.info(
|
||||
# f'`{sym}` throttled send hz: {round(measured_rate, ndigits=1)}'
|
||||
# )
|
||||
first_quote['tbt'] = ticks_by_type
|
||||
|
||||
# TODO: now if only we could sync this to the display
|
||||
# rate timing exactly lul
|
||||
try:
|
||||
await stream.send({sym: first_quote})
|
||||
except tractor.RemoteActorError as rme:
|
||||
if rme.type is not tractor._exceptions.StreamOverrun:
|
||||
raise
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
log.warning(
|
||||
'Throttled quote-stream overrun!\n'
|
||||
f'{sym}:{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
|
||||
except (
|
||||
# NOTE: any of these can be raised by ``tractor``'s IPC
|
||||
# transport-layer and we want to be highly resilient
|
||||
|
|
@ -909,4 +517,3 @@ async def uniform_rate_send(
|
|||
first_quote = last_quote = None
|
||||
diff = 0
|
||||
last_send = time.time()
|
||||
ticks_by_type.clear()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -27,14 +27,13 @@ from multiprocessing.shared_memory import SharedMemory, _USE_POSIX
|
|||
if _USE_POSIX:
|
||||
from _posixshmem import shm_unlink
|
||||
|
||||
# import msgspec
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
import tractor
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ..log import get_logger
|
||||
from ._source import base_iohlc_dtype
|
||||
from .types import Struct
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
@ -50,11 +49,7 @@ _rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
|
|||
|
||||
|
||||
def cuckoff_mantracker():
|
||||
'''
|
||||
Disable all ``multiprocessing``` "resource tracking" machinery since
|
||||
it's an absolute multi-threaded mess of non-SC madness.
|
||||
|
||||
'''
|
||||
from multiprocessing import resource_tracker as mantracker
|
||||
|
||||
# Tell the "resource tracker" thing to fuck off.
|
||||
|
|
@ -112,39 +107,36 @@ class SharedInt:
|
|||
log.warning(f'Shm for {name} already unlinked?')
|
||||
|
||||
|
||||
class _Token(Struct, frozen=True):
|
||||
class _Token(BaseModel):
|
||||
'''
|
||||
Internal represenation of a shared memory "token"
|
||||
which can be used to key a system wide post shm entry.
|
||||
|
||||
'''
|
||||
class Config:
|
||||
frozen = True
|
||||
|
||||
shm_name: str # this servers as a "key" value
|
||||
shm_first_index_name: str
|
||||
shm_last_index_name: str
|
||||
dtype_descr: tuple
|
||||
size: int # in struct-array index / row terms
|
||||
|
||||
@property
|
||||
def dtype(self) -> np.dtype:
|
||||
return np.dtype(list(map(tuple, self.dtype_descr))).descr
|
||||
|
||||
def as_msg(self):
|
||||
return self.to_dict()
|
||||
return self.dict()
|
||||
|
||||
@classmethod
|
||||
def from_msg(cls, msg: dict) -> _Token:
|
||||
if isinstance(msg, _Token):
|
||||
return msg
|
||||
|
||||
# TODO: native struct decoding
|
||||
# return _token_dec.decode(msg)
|
||||
|
||||
msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
|
||||
return _Token(**msg)
|
||||
|
||||
|
||||
# _token_dec = msgspec.msgpack.Decoder(_Token)
|
||||
|
||||
# TODO: this api?
|
||||
# _known_tokens = tractor.ActorVar('_shm_tokens', {})
|
||||
# _known_tokens = tractor.ContextStack('_known_tokens', )
|
||||
|
|
@ -163,7 +155,6 @@ def get_shm_token(key: str) -> _Token:
|
|||
|
||||
def _make_token(
|
||||
key: str,
|
||||
size: int,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
) -> _Token:
|
||||
'''
|
||||
|
|
@ -176,8 +167,7 @@ def _make_token(
|
|||
shm_name=key,
|
||||
shm_first_index_name=key + "_first",
|
||||
shm_last_index_name=key + "_last",
|
||||
dtype_descr=tuple(np.dtype(dtype).descr),
|
||||
size=size,
|
||||
dtype_descr=np.dtype(dtype).descr
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -229,7 +219,6 @@ class ShmArray:
|
|||
shm_first_index_name=self._first._shm.name,
|
||||
shm_last_index_name=self._last._shm.name,
|
||||
dtype_descr=tuple(self._array.dtype.descr),
|
||||
size=self._len,
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
@ -444,7 +433,7 @@ class ShmArray:
|
|||
def open_shm_array(
|
||||
|
||||
key: Optional[str] = None,
|
||||
size: int = _default_size, # see above
|
||||
size: int = _default_size,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
readonly: bool = False,
|
||||
|
||||
|
|
@ -475,8 +464,7 @@ def open_shm_array(
|
|||
|
||||
token = _make_token(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
dtype=dtype
|
||||
)
|
||||
|
||||
# create single entry arrays for storing an first and last indices
|
||||
|
|
@ -528,15 +516,15 @@ def open_shm_array(
|
|||
# "unlink" created shm on process teardown by
|
||||
# pushing teardown calls onto actor context stack
|
||||
|
||||
stack = tractor.current_actor().lifetime_stack
|
||||
stack.callback(shmarr.close)
|
||||
stack.callback(shmarr.destroy)
|
||||
tractor._actor._lifetime_stack.callback(shmarr.close)
|
||||
tractor._actor._lifetime_stack.callback(shmarr.destroy)
|
||||
|
||||
return shmarr
|
||||
|
||||
|
||||
def attach_shm_array(
|
||||
token: tuple[str, str, tuple[str, str]],
|
||||
size: int = _default_size,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> ShmArray:
|
||||
|
|
@ -575,7 +563,7 @@ def attach_shm_array(
|
|||
raise _err
|
||||
|
||||
shmarr = np.ndarray(
|
||||
(token.size,),
|
||||
(size,),
|
||||
dtype=token.dtype,
|
||||
buffer=shm.buf
|
||||
)
|
||||
|
|
@ -614,8 +602,8 @@ def attach_shm_array(
|
|||
if key not in _known_tokens:
|
||||
_known_tokens[key] = token
|
||||
|
||||
# "close" attached shm on actor teardown
|
||||
tractor.current_actor().lifetime_stack.callback(sha.close)
|
||||
# "close" attached shm on process teardown
|
||||
tractor._actor._lifetime_stack.callback(sha.close)
|
||||
|
||||
return sha
|
||||
|
||||
|
|
@ -643,23 +631,18 @@ def maybe_open_shm_array(
|
|||
use ``attach_shm_array``.
|
||||
|
||||
'''
|
||||
size = kwargs.pop('size', _default_size)
|
||||
try:
|
||||
# see if we already know this key
|
||||
token = _known_tokens[key]
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except KeyError:
|
||||
log.debug(f"Could not find {key} in shms cache")
|
||||
log.warning(f"Could not find {key} in shms cache")
|
||||
if dtype:
|
||||
token = _make_token(
|
||||
key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
)
|
||||
token = _make_token(key, dtype)
|
||||
try:
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except FileNotFoundError:
|
||||
log.debug(f"Could not attach to shm with token {token}")
|
||||
log.warning(f"Could not attach to shm with token {token}")
|
||||
|
||||
# This actor does not know about memory
|
||||
# associated with the provided "key".
|
||||
|
|
|
|||
|
|
@ -18,22 +18,14 @@
|
|||
numpy data source coversion helpers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from decimal import (
|
||||
Decimal,
|
||||
ROUND_HALF_EVEN,
|
||||
)
|
||||
from typing import Any
|
||||
import decimal
|
||||
|
||||
from bidict import bidict
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
# from numba import from_dtype
|
||||
|
||||
from .types import Struct
|
||||
from ..accounting._mktinfo import (
|
||||
# mkfqsn,
|
||||
unpack_fqsn,
|
||||
# digits_to_dec,
|
||||
float_digits,
|
||||
)
|
||||
|
||||
ohlc_fields = [
|
||||
('time', float),
|
||||
|
|
@ -54,7 +46,6 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
|
|||
|
||||
# TODO: for now need to construct this manually for readonly arrays, see
|
||||
# https://github.com/numba/numba/issues/4511
|
||||
# from numba import from_dtype
|
||||
# numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
|
||||
|
||||
# map time frame "keys" to seconds values
|
||||
|
|
@ -69,6 +60,28 @@ tf_in_1s = bidict({
|
|||
})
|
||||
|
||||
|
||||
def mk_fqsn(
|
||||
provider: str,
|
||||
symbol: str,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Generate a "fully qualified symbol name" which is
|
||||
a reverse-hierarchical cross broker/provider symbol
|
||||
|
||||
'''
|
||||
return '.'.join([symbol, provider]).lower()
|
||||
|
||||
|
||||
def float_digits(
|
||||
value: float,
|
||||
) -> int:
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(-decimal.Decimal(str(value)).as_tuple().exponent)
|
||||
|
||||
|
||||
def ohlc_zeros(length: int) -> np.ndarray:
|
||||
"""Construct an OHLC field formatted structarray.
|
||||
|
||||
|
|
@ -79,6 +92,168 @@ def ohlc_zeros(length: int) -> np.ndarray:
|
|||
return np.zeros(length, dtype=base_ohlc_dtype)
|
||||
|
||||
|
||||
def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqsn.split('.')
|
||||
if len(tokens) < 3:
|
||||
# probably crypto
|
||||
symbol, broker = tokens
|
||||
return (
|
||||
broker,
|
||||
symbol,
|
||||
'',
|
||||
)
|
||||
|
||||
elif len(tokens) > 3:
|
||||
symbol, venue, suffix, broker = tokens
|
||||
else:
|
||||
symbol, venue, broker = tokens
|
||||
suffix = ''
|
||||
|
||||
# head, _, broker = fqsn.rpartition('.')
|
||||
# symbol, _, suffix = head.rpartition('.')
|
||||
return (
|
||||
broker,
|
||||
'.'.join([symbol, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
class Symbol(BaseModel):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
tick_size: float = 0.01
|
||||
lot_tick_size: float = 0.0 # "volume" precision as min step value
|
||||
tick_size_digits: int = 2
|
||||
lot_size_digits: int = 0
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# specifies a "class" of financial instrument
|
||||
# ex. stock, futer, option, bond etc.
|
||||
|
||||
# @validate_arguments
|
||||
@classmethod
|
||||
def from_broker_info(
|
||||
cls,
|
||||
broker: str,
|
||||
symbol: str,
|
||||
info: dict[str, Any],
|
||||
suffix: str = '',
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_tick_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
key=symbol,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_tick_size,
|
||||
tick_size_digits=float_digits(tick_size),
|
||||
lot_size_digits=float_digits(lot_tick_size),
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_fqsn(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
broker, key, suffix = unpack_fqsn(fqsn)
|
||||
return cls.from_broker_info(
|
||||
broker,
|
||||
key,
|
||||
info=info,
|
||||
suffix=suffix,
|
||||
)
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def brokers(self) -> list[str]:
|
||||
return list(self.broker_info.keys())
|
||||
|
||||
def nearest_tick(self, value: float) -> float:
|
||||
'''
|
||||
Return the nearest tick value based on mininum increment.
|
||||
|
||||
'''
|
||||
mult = 1 / self.tick_size
|
||||
return round(value * mult) / mult
|
||||
|
||||
def front_feed(self) -> tuple[str, str]:
|
||||
'''
|
||||
Return the "current" feed key for this symbol.
|
||||
|
||||
(i.e. the broker + symbol key in a tuple).
|
||||
|
||||
'''
|
||||
return (
|
||||
list(self.broker_info.keys())[0],
|
||||
self.key,
|
||||
)
|
||||
|
||||
def tokens(self) -> tuple[str]:
|
||||
broker, key = self.front_feed()
|
||||
if self.suffix:
|
||||
return (key, self.suffix, broker)
|
||||
else:
|
||||
return (key, broker)
|
||||
|
||||
def front_fqsn(self) -> str:
|
||||
'''
|
||||
fqsn = "fully qualified symbol name"
|
||||
|
||||
Basically the idea here is for all client-ish code (aka programs/actors
|
||||
that ask the provider agnostic layers in the stack for data) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string key of the current form:
|
||||
|
||||
<instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.):
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
'''
|
||||
tokens = self.tokens()
|
||||
fqsn = '.'.join(tokens)
|
||||
return fqsn
|
||||
|
||||
def iterfqsns(self) -> list[str]:
|
||||
keys = []
|
||||
for broker in self.broker_info.keys():
|
||||
fqsn = mk_fqsn(self.key, broker)
|
||||
if self.suffix:
|
||||
fqsn += f'.{self.suffix}'
|
||||
keys.append(fqsn)
|
||||
|
||||
return keys
|
||||
|
||||
|
||||
def _nan_to_closest_num(array: np.ndarray):
|
||||
"""Return interpolated values instead of NaN.
|
||||
|
||||
|
|
|
|||
|
|
@ -18,24 +18,13 @@
|
|||
ToOlS fOr CoPInG wITh "tHE wEB" protocols.
|
||||
|
||||
"""
|
||||
from contextlib import (
|
||||
asynccontextmanager,
|
||||
AsyncExitStack,
|
||||
)
|
||||
from itertools import count
|
||||
from contextlib import asynccontextmanager, AsyncExitStack
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
AsyncGenerator,
|
||||
Iterable,
|
||||
)
|
||||
from typing import Any, Callable, AsyncGenerator
|
||||
import json
|
||||
|
||||
import trio
|
||||
import trio_websocket
|
||||
from wsproto.utilities import LocalProtocolError
|
||||
from trio_websocket._impl import (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
|
|
@ -46,53 +35,43 @@ from trio_websocket._impl import (
|
|||
|
||||
from ..log import get_logger
|
||||
|
||||
from .types import Struct
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class NoBsWs:
|
||||
'''
|
||||
Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
"""Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
|
||||
You can provide a ``fixture`` async-context-manager which will be
|
||||
enter/exitted around each reconnect operation.
|
||||
'''
|
||||
"""
|
||||
recon_errors = (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
ConnectionRejected,
|
||||
HandshakeError,
|
||||
ConnectionTimeout,
|
||||
LocalProtocolError,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
token: str,
|
||||
stack: AsyncExitStack,
|
||||
fixture: Optional[Callable] = None,
|
||||
serializer: ModuleType = json
|
||||
fixture: Callable,
|
||||
serializer: ModuleType = json,
|
||||
):
|
||||
self.url = url
|
||||
self.token = token
|
||||
self.fixture = fixture
|
||||
self._stack = stack
|
||||
self._ws: 'WebSocketConnection' = None # noqa
|
||||
|
||||
# TODO: is there some method we can call
|
||||
# on the underlying `._ws` to get this?
|
||||
self._connected: bool = False
|
||||
|
||||
async def _connect(
|
||||
self,
|
||||
tries: int = 1000,
|
||||
) -> None:
|
||||
|
||||
self._connected = False
|
||||
while True:
|
||||
try:
|
||||
await self._stack.aclose()
|
||||
except self.recon_errors:
|
||||
except (DisconnectionTimeout, RuntimeError):
|
||||
await trio.sleep(0.5)
|
||||
else:
|
||||
break
|
||||
|
|
@ -103,18 +82,19 @@ class NoBsWs:
|
|||
self._ws = await self._stack.enter_async_context(
|
||||
trio_websocket.open_websocket_url(self.url)
|
||||
)
|
||||
|
||||
if self.fixture is not None:
|
||||
# rerun user code fixture
|
||||
if self.token == '':
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self)
|
||||
)
|
||||
else:
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self, self.token)
|
||||
)
|
||||
|
||||
assert ret is None
|
||||
|
||||
log.info(f'Connection success: {self.url}')
|
||||
|
||||
self._connected = True
|
||||
return self._ws
|
||||
|
||||
except self.recon_errors as err:
|
||||
|
|
@ -124,15 +104,11 @@ class NoBsWs:
|
|||
f'{type(err)}...retry attempt {i}'
|
||||
)
|
||||
await trio.sleep(0.5)
|
||||
self._connected = False
|
||||
continue
|
||||
else:
|
||||
log.exception('ws connection fail...')
|
||||
raise last_err
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
async def send_msg(
|
||||
self,
|
||||
data: Any,
|
||||
|
|
@ -152,26 +128,21 @@ class NoBsWs:
|
|||
except self.recon_errors:
|
||||
await self._connect()
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
return await self.recv_msg()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def open_autorecon_ws(
|
||||
url: str,
|
||||
|
||||
# TODO: proper type cannot smh
|
||||
fixture: Optional[Callable] = None,
|
||||
|
||||
# TODO: proper type annot smh
|
||||
fixture: Callable,
|
||||
# used for authenticated websockets
|
||||
token: str = '',
|
||||
) -> AsyncGenerator[tuple[...], NoBsWs]:
|
||||
"""Apparently we can QoS for all sorts of reasons..so catch em.
|
||||
|
||||
"""
|
||||
async with AsyncExitStack() as stack:
|
||||
ws = NoBsWs(url, stack, fixture=fixture)
|
||||
ws = NoBsWs(url, token, stack, fixture=fixture)
|
||||
await ws._connect()
|
||||
|
||||
try:
|
||||
|
|
@ -179,114 +150,3 @@ async def open_autorecon_ws(
|
|||
|
||||
finally:
|
||||
await stack.aclose()
|
||||
|
||||
|
||||
'''
|
||||
JSONRPC response-request style machinery for transparent multiplexing of msgs
|
||||
over a NoBsWs.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
id: int
|
||||
jsonrpc: str = '2.0'
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def open_jsonrpc_session(
|
||||
url: str,
|
||||
start_id: int = 0,
|
||||
response_type: type = JSONRPCResult,
|
||||
request_type: Optional[type] = None,
|
||||
request_hook: Optional[Callable] = None,
|
||||
error_hook: Optional[Callable] = None,
|
||||
) -> Callable[[str, dict], dict]:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_autorecon_ws(url) as ws
|
||||
):
|
||||
rpc_id: Iterable = count(start_id)
|
||||
rpc_results: dict[int, dict] = {}
|
||||
|
||||
async def json_rpc(method: str, params: dict) -> dict:
|
||||
'''
|
||||
perform a json rpc call and wait for the result, raise exception in
|
||||
case of error field present on response
|
||||
'''
|
||||
msg = {
|
||||
'jsonrpc': '2.0',
|
||||
'id': next(rpc_id),
|
||||
'method': method,
|
||||
'params': params
|
||||
}
|
||||
_id = msg['id']
|
||||
|
||||
rpc_results[_id] = {
|
||||
'result': None,
|
||||
'event': trio.Event()
|
||||
}
|
||||
|
||||
await ws.send_msg(msg)
|
||||
|
||||
await rpc_results[_id]['event'].wait()
|
||||
|
||||
ret = rpc_results[_id]['result']
|
||||
|
||||
del rpc_results[_id]
|
||||
|
||||
if ret.error is not None:
|
||||
raise Exception(json.dumps(ret.error, indent=4))
|
||||
|
||||
return ret
|
||||
|
||||
async def recv_task():
|
||||
'''
|
||||
receives every ws message and stores it in its corresponding
|
||||
result field, then sets the event to wakeup original sender
|
||||
tasks. also recieves responses to requests originated from
|
||||
the server side.
|
||||
|
||||
'''
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if res_entry := rpc_results.get(mid):
|
||||
|
||||
res_entry['result'] = response_type(**msg)
|
||||
res_entry['event'].set()
|
||||
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if not rpc_results.get(mid):
|
||||
log.warning(
|
||||
f'Unexpected ws msg: {json.dumps(msg, indent=4)}'
|
||||
)
|
||||
|
||||
case {
|
||||
'method': _,
|
||||
'params': _,
|
||||
}:
|
||||
log.debug(f'Recieved\n{msg}')
|
||||
if request_hook:
|
||||
await request_hook(request_type(**msg))
|
||||
|
||||
case {
|
||||
'error': error
|
||||
}:
|
||||
log.warning(f'Recieved\n{error}')
|
||||
if error_hook:
|
||||
await error_hook(response_type(**msg))
|
||||
|
||||
case _:
|
||||
log.warning(f'Unhandled JSON-RPC msg!?\n{msg}')
|
||||
|
||||
n.start_soon(recv_task)
|
||||
yield json_rpc
|
||||
n.cancel_scope.cancel()
|
||||
|
|
|
|||
|
|
@ -18,22 +18,31 @@
|
|||
marketstore cli.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
from pprint import pformat
|
||||
|
||||
from anyio_marketstore import open_marketstore_client
|
||||
import trio
|
||||
import tractor
|
||||
import click
|
||||
import numpy as np
|
||||
|
||||
from ..service.marketstore import (
|
||||
# get_client,
|
||||
from .marketstore import (
|
||||
get_client,
|
||||
# stream_quotes,
|
||||
ingest_quote_stream,
|
||||
# _url,
|
||||
# _tick_tbk_ids,
|
||||
# mk_tbk,
|
||||
_tick_tbk_ids,
|
||||
mk_tbk,
|
||||
)
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ..log import (
|
||||
get_logger,
|
||||
from ..log import get_logger
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
)
|
||||
from ._source import (
|
||||
base_iohlc_dtype,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -104,11 +113,15 @@ def ms_stream(
|
|||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--tsdb_host',
|
||||
'--tl',
|
||||
is_flag=True,
|
||||
help='Enable tractor logging')
|
||||
@click.option(
|
||||
'--host',
|
||||
default='localhost'
|
||||
)
|
||||
@click.option(
|
||||
'--tsdb_port',
|
||||
'--port',
|
||||
default=5993
|
||||
)
|
||||
@click.argument('symbols', nargs=-1)
|
||||
|
|
@ -124,93 +137,18 @@ def storesh(
|
|||
Start an IPython shell ready to query the local marketstore db.
|
||||
|
||||
'''
|
||||
from piker.data.marketstore import open_tsdb_client
|
||||
from piker.service import open_piker_runtime
|
||||
from piker.data.marketstore import tsdb_history_update
|
||||
from piker._daemon import open_piker_runtime
|
||||
|
||||
async def main():
|
||||
nonlocal symbols
|
||||
|
||||
async with open_piker_runtime(
|
||||
'storesh',
|
||||
enable_modules=['piker.service._ahab'],
|
||||
enable_modules=['piker.data._ahab'],
|
||||
):
|
||||
symbol = symbols[0]
|
||||
|
||||
async with open_tsdb_client(symbol):
|
||||
# TODO: ask if user wants to write history for detected
|
||||
# available shm buffers?
|
||||
from tractor.trionics import ipython_embed
|
||||
await ipython_embed()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--host',
|
||||
default='localhost'
|
||||
)
|
||||
@click.option(
|
||||
'--port',
|
||||
default=5993
|
||||
)
|
||||
@click.option(
|
||||
'--delete',
|
||||
'-d',
|
||||
is_flag=True,
|
||||
help='Delete history (1 Min) for symbol(s)',
|
||||
)
|
||||
@click.argument('symbols', nargs=-1)
|
||||
@click.pass_obj
|
||||
def storage(
|
||||
config,
|
||||
host,
|
||||
port,
|
||||
symbols: list[str],
|
||||
delete: bool,
|
||||
|
||||
):
|
||||
'''
|
||||
Start an IPython shell ready to query the local marketstore db.
|
||||
|
||||
'''
|
||||
from piker.service.marketstore import open_tsdb_client
|
||||
from piker.service import open_piker_runtime
|
||||
|
||||
async def main():
|
||||
nonlocal symbols
|
||||
|
||||
async with open_piker_runtime(
|
||||
'tsdb_storage',
|
||||
enable_modules=['piker.service._ahab'],
|
||||
):
|
||||
symbol = symbols[0]
|
||||
async with open_tsdb_client(symbol) as storage:
|
||||
if delete:
|
||||
for fqsn in symbols:
|
||||
syms = await storage.client.list_symbols()
|
||||
|
||||
resp60s = await storage.delete_ts(fqsn, 60)
|
||||
|
||||
msgish = resp60s.ListFields()[0][1]
|
||||
if 'error' in str(msgish):
|
||||
|
||||
# TODO: MEGA LOL, apparently the symbols don't
|
||||
# flush out until you refresh something or other
|
||||
# (maybe the WALFILE)... #lelandorlulzone, classic
|
||||
# alpaca(Rtm) design here ..
|
||||
# well, if we ever can make this work we
|
||||
# probably want to dogsplain the real reason
|
||||
# for the delete errurz..llululu
|
||||
if fqsn not in syms:
|
||||
log.error(f'Pair {fqsn} dne in DB')
|
||||
|
||||
log.error(f'Deletion error: {fqsn}\n{msgish}')
|
||||
|
||||
resp1s = await storage.delete_ts(fqsn, 1)
|
||||
msgish = resp1s.ListFields()[0][1]
|
||||
if 'error' in str(msgish):
|
||||
log.error(f'Deletion error: {fqsn}\n{msgish}')
|
||||
await tsdb_history_update(symbol)
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
|
|
|||
1795
piker/data/feed.py
1795
piker/data/feed.py
File diff suppressed because it is too large
Load Diff
|
|
@ -1,238 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
abstractions for organizing, managing and generally operating-on
|
||||
real-time data processing data-structures.
|
||||
|
||||
"Streams, flumes, cascades and flows.."
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
# from decimal import Decimal
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import pendulum
|
||||
import numpy as np
|
||||
|
||||
from ..accounting._mktinfo import (
|
||||
MktPair,
|
||||
Symbol,
|
||||
)
|
||||
from .types import Struct
|
||||
from ._sharedmem import (
|
||||
attach_shm_array,
|
||||
ShmArray,
|
||||
_Token,
|
||||
)
|
||||
# from .._profile import (
|
||||
# Profiler,
|
||||
# pg_profile_enabled,
|
||||
# )
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# from pyqtgraph import PlotItem
|
||||
from .feed import Feed
|
||||
|
||||
|
||||
# TODO: ideas for further abstractions as per
|
||||
# https://github.com/pikers/piker/issues/216 and
|
||||
# https://github.com/pikers/piker/issues/270:
|
||||
# - a ``Cascade`` would be the minimal "connection" of 2 ``Flumes``
|
||||
# as per circuit parlance:
|
||||
# https://en.wikipedia.org/wiki/Two-port_network#Cascade_connection
|
||||
# - could cover the combination of our `FspAdmin` and the
|
||||
# backend `.fsp._engine` related machinery to "connect" one flume
|
||||
# to another?
|
||||
# - a (financial signal) ``Flow`` would be the a "collection" of such
|
||||
# minmial cascades. Some engineering based jargon concepts:
|
||||
# - https://en.wikipedia.org/wiki/Signal_chain
|
||||
# - https://en.wikipedia.org/wiki/Daisy_chain_(electrical_engineering)
|
||||
# - https://en.wikipedia.org/wiki/Audio_signal_flow
|
||||
# - https://en.wikipedia.org/wiki/Digital_signal_processing#Implementation
|
||||
# - https://en.wikipedia.org/wiki/Dataflow_programming
|
||||
# - https://en.wikipedia.org/wiki/Signal_programming
|
||||
# - https://en.wikipedia.org/wiki/Incremental_computing
|
||||
|
||||
|
||||
class Flume(Struct):
|
||||
'''
|
||||
Composite reference type which points to all the addressing handles
|
||||
and other meta-data necessary for the read, measure and management
|
||||
of a set of real-time updated data flows.
|
||||
|
||||
Can be thought of as a "flow descriptor" or "flow frame" which
|
||||
describes the high level properties of a set of data flows that can
|
||||
be used seamlessly across process-memory boundaries.
|
||||
|
||||
Each instance's sub-components normally includes:
|
||||
- a msg oriented quote stream provided via an IPC transport
|
||||
- history and real-time shm buffers which are both real-time
|
||||
updated and backfilled.
|
||||
- associated startup indexing information related to both buffer
|
||||
real-time-append and historical prepend addresses.
|
||||
- low level APIs to read and measure the updated data and manage
|
||||
queuing properties.
|
||||
|
||||
'''
|
||||
symbol: Symbol | MktPair
|
||||
first_quote: dict
|
||||
_rt_shm_token: _Token
|
||||
|
||||
# optional since some data flows won't have a "downsampled" history
|
||||
# buffer/stream (eg. FSPs).
|
||||
_hist_shm_token: _Token | None = None
|
||||
|
||||
# private shm refs loaded dynamically from tokens
|
||||
_hist_shm: ShmArray | None = None
|
||||
_rt_shm: ShmArray | None = None
|
||||
|
||||
stream: tractor.MsgStream | None = None
|
||||
izero_hist: int = 0
|
||||
izero_rt: int = 0
|
||||
throttle_rate: int | None = None
|
||||
|
||||
# TODO: do we need this really if we can pull the `Portal` from
|
||||
# ``tractor``'s internals?
|
||||
feed: Feed | None = None
|
||||
|
||||
@property
|
||||
def rt_shm(self) -> ShmArray:
|
||||
|
||||
if self._rt_shm is None:
|
||||
self._rt_shm = attach_shm_array(
|
||||
token=self._rt_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._rt_shm
|
||||
|
||||
@property
|
||||
def hist_shm(self) -> ShmArray:
|
||||
|
||||
if self._hist_shm_token is None:
|
||||
raise RuntimeError(
|
||||
'No shm token has been set for the history buffer?'
|
||||
)
|
||||
|
||||
if (
|
||||
self._hist_shm is None
|
||||
):
|
||||
self._hist_shm = attach_shm_array(
|
||||
token=self._hist_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._hist_shm
|
||||
|
||||
async def receive(self) -> dict:
|
||||
return await self.stream.receive()
|
||||
|
||||
def get_ds_info(
|
||||
self,
|
||||
) -> tuple[float, float, float]:
|
||||
'''
|
||||
Compute the "downsampling" ratio info between the historical shm
|
||||
buffer and the real-time (HFT) one.
|
||||
|
||||
Return a tuple of the fast sample period, historical sample
|
||||
period and ratio between them.
|
||||
|
||||
'''
|
||||
times = self.hist_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
hist_step_size_s = (end - start).seconds
|
||||
|
||||
times = self.rt_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
rt_step_size_s = (end - start).seconds
|
||||
|
||||
ratio = hist_step_size_s / rt_step_size_s
|
||||
return (
|
||||
rt_step_size_s,
|
||||
hist_step_size_s,
|
||||
ratio,
|
||||
)
|
||||
|
||||
# TODO: get native msgspec decoding for these workinn
|
||||
def to_msg(self) -> dict:
|
||||
|
||||
msg = self.to_dict()
|
||||
|
||||
# TODO: do we even need to convert to dict
|
||||
# first now?
|
||||
# TODO: drop the former.
|
||||
msg['symbol'] = msg['symbol'].to_dict()
|
||||
mktpair = msg.get('mktpair')
|
||||
if mktpair:
|
||||
msg['mktpair'] = mktpair.to_dict()
|
||||
|
||||
# can't serialize the stream or feed objects, it's expected
|
||||
# you'll have a ref to it since this msg should be rxed on
|
||||
# a stream on whatever far end IPC..
|
||||
msg.pop('stream')
|
||||
msg.pop('feed')
|
||||
return msg
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Load from an IPC msg presumably in either `dict` or
|
||||
`msgspec.Struct` form.
|
||||
|
||||
'''
|
||||
sym_msg = msg.pop('symbol')
|
||||
|
||||
if 'dst' in sym_msg:
|
||||
mkt = MktPair.from_msg(sym_msg)
|
||||
|
||||
else:
|
||||
# XXX NOTE: ``msgspec`` can encode `Decimal`
|
||||
# but it doesn't decide to it by default since
|
||||
# we aren't spec-cing these msgs as structs, SO
|
||||
# we have to ensure we do a struct type case (which `.copy()`
|
||||
# does) to ensure we get the right type!
|
||||
mkt = Symbol(**sym_msg).copy()
|
||||
|
||||
return cls(symbol=mkt, **msg)
|
||||
|
||||
def get_index(
|
||||
self,
|
||||
time_s: float,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> int | float:
|
||||
'''
|
||||
Return array shm-buffer index for for epoch time.
|
||||
|
||||
'''
|
||||
times = array['time']
|
||||
first = np.searchsorted(
|
||||
times,
|
||||
time_s,
|
||||
side='left',
|
||||
)
|
||||
imx = times.shape[0] - 1
|
||||
return min(first, imx)
|
||||
|
|
@ -26,6 +26,7 @@
|
|||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
|
|
@ -34,11 +35,10 @@ from typing import (
|
|||
)
|
||||
import time
|
||||
from math import isnan
|
||||
from pathlib import Path
|
||||
|
||||
from bidict import bidict
|
||||
from msgspec.msgpack import encode, decode
|
||||
# import pyqtgraph as pg
|
||||
import msgpack
|
||||
import pyqtgraph as pg
|
||||
import numpy as np
|
||||
import tractor
|
||||
from trio_websocket import open_websocket_url
|
||||
|
|
@ -54,20 +54,18 @@ if TYPE_CHECKING:
|
|||
import docker
|
||||
from ._ahab import DockerContainer
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ..data.feed import maybe_open_feed
|
||||
from .._profile import Profiler
|
||||
from .feed import maybe_open_feed
|
||||
from ..log import get_logger, get_console_log
|
||||
|
||||
|
||||
# ahabd-supervisor and container level config
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# container level config
|
||||
_config = {
|
||||
'grpc_listen_port': 5995,
|
||||
'ws_listen_port': 5993,
|
||||
'log_level': 'debug',
|
||||
'startup_timeout': 2,
|
||||
}
|
||||
|
||||
_yaml_config = '''
|
||||
|
|
@ -133,10 +131,7 @@ def start_marketstore(
|
|||
|
||||
mktsdir = os.path.join(config._config_dir, 'marketstore')
|
||||
|
||||
# create dirs when dne
|
||||
if not os.path.isdir(config._config_dir):
|
||||
Path(config._config_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# create when dne
|
||||
if not os.path.isdir(mktsdir):
|
||||
os.mkdir(mktsdir)
|
||||
|
||||
|
|
@ -185,29 +180,18 @@ def start_marketstore(
|
|||
config_dir_mnt,
|
||||
data_dir_mnt,
|
||||
],
|
||||
|
||||
# XXX: this must be set to allow backgrounding/non-blocking
|
||||
# usage interaction with the container's process.
|
||||
detach=True,
|
||||
|
||||
# stop_signal='SIGINT',
|
||||
init=True,
|
||||
# remove=True,
|
||||
)
|
||||
|
||||
async def start_matcher(msg: str):
|
||||
return "launching tcp listener for all services..." in msg
|
||||
|
||||
async def stop_matcher(msg: str):
|
||||
return "exiting..." in msg
|
||||
|
||||
return (
|
||||
dcntr,
|
||||
_config,
|
||||
|
||||
# expected startup and stop msgs
|
||||
start_matcher,
|
||||
stop_matcher,
|
||||
"launching tcp listener for all services...",
|
||||
"exiting...",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -328,7 +312,7 @@ def quote_to_marketstore_structarray(
|
|||
@acm
|
||||
async def get_client(
|
||||
host: str = 'localhost',
|
||||
port: int = _config['grpc_listen_port'],
|
||||
port: int = 5995
|
||||
|
||||
) -> MarketstoreClient:
|
||||
'''
|
||||
|
|
@ -403,54 +387,50 @@ class Storage:
|
|||
async def load(
|
||||
self,
|
||||
fqsn: str,
|
||||
timeframe: int,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # timeframe sampled array-series
|
||||
dict[int, np.ndarray], # timeframe (in secs) to series
|
||||
Optional[datetime], # first dt
|
||||
Optional[datetime], # last dt
|
||||
]:
|
||||
|
||||
first_tsdb_dt, last_tsdb_dt = None, None
|
||||
hist = await self.read_ohlcv(
|
||||
tsdb_arrays = await self.read_ohlcv(
|
||||
fqsn,
|
||||
# on first load we don't need to pull the max
|
||||
# history per request size worth.
|
||||
limit=3000,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
log.info(f'Loaded tsdb history {hist}')
|
||||
log.info(f'Loaded tsdb history {tsdb_arrays}')
|
||||
|
||||
if len(hist):
|
||||
times = hist['Epoch']
|
||||
if tsdb_arrays:
|
||||
fastest = list(tsdb_arrays.values())[0]
|
||||
times = fastest['Epoch']
|
||||
first, last = times[0], times[-1]
|
||||
first_tsdb_dt, last_tsdb_dt = map(
|
||||
pendulum.from_timestamp, [first, last]
|
||||
)
|
||||
|
||||
return (
|
||||
hist, # array-data
|
||||
first_tsdb_dt, # start of query-frame
|
||||
last_tsdb_dt, # most recent
|
||||
)
|
||||
return tsdb_arrays, first_tsdb_dt, last_tsdb_dt
|
||||
|
||||
async def read_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
timeframe: int | str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
end: Optional[int] = None,
|
||||
limit: int = int(800e3),
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
) -> tuple[
|
||||
MarketstoreClient,
|
||||
Union[dict, np.ndarray]
|
||||
]:
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
|
||||
if fqsn not in syms:
|
||||
return {}
|
||||
|
||||
# use the provided timeframe or 1s by default
|
||||
tfstr = tf_in_1s.get(timeframe, tf_in_1s[1])
|
||||
tfstr = tf_in_1s[1]
|
||||
|
||||
params = Params(
|
||||
symbols=fqsn,
|
||||
|
|
@ -464,71 +444,58 @@ class Storage:
|
|||
limit=limit,
|
||||
)
|
||||
|
||||
if timeframe is None:
|
||||
log.info(f'starting {fqsn} tsdb granularity scan..')
|
||||
# loop through and try to find highest granularity
|
||||
for tfstr in tf_in_1s.values():
|
||||
try:
|
||||
log.info(f'querying for {tfstr}@{fqsn}')
|
||||
params.set('timeframe', tfstr)
|
||||
result = await client.query(params)
|
||||
except purerpc.grpclib.exceptions.UnknownError as err:
|
||||
# indicate there is no history for this timeframe
|
||||
log.exception(
|
||||
f'Unknown mkts QUERY error: {params}\n'
|
||||
f'{err.args}'
|
||||
)
|
||||
break
|
||||
|
||||
except purerpc.grpclib.exceptions.UnknownError:
|
||||
# XXX: this is already logged by the container and
|
||||
# thus shows up through `marketstored` logs relay.
|
||||
# log.warning(f'{tfstr}@{fqsn} not found')
|
||||
continue
|
||||
else:
|
||||
return {}
|
||||
|
||||
else:
|
||||
result = await client.query(params)
|
||||
|
||||
# TODO: it turns out column access on recarrays is actually slower:
|
||||
# https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
|
||||
# it might make sense to make these structured arrays?
|
||||
data_set = result.by_symbols()[fqsn]
|
||||
array = data_set.array
|
||||
# Fill out a `numpy` array-results map
|
||||
arrays = {}
|
||||
for fqsn, data_set in result.by_symbols().items():
|
||||
arrays.setdefault(fqsn, {})[
|
||||
tf_in_1s.inverse[data_set.timeframe]
|
||||
] = data_set.array
|
||||
|
||||
# XXX: ensure sample rate is as expected
|
||||
time = data_set.array['Epoch']
|
||||
if len(time) > 1:
|
||||
time_step = time[-1] - time[-2]
|
||||
ts = tf_in_1s.inverse[data_set.timeframe]
|
||||
|
||||
if time_step != ts:
|
||||
log.warning(
|
||||
f'MKTS BUG: wrong timeframe loaded: {time_step}'
|
||||
'YOUR DATABASE LIKELY CONTAINS BAD DATA FROM AN OLD BUG'
|
||||
f'WIPING HISTORY FOR {ts}s'
|
||||
)
|
||||
await self.delete_ts(fqsn, timeframe)
|
||||
|
||||
# try reading again..
|
||||
return await self.read_ohlcv(
|
||||
fqsn,
|
||||
timeframe,
|
||||
end,
|
||||
limit,
|
||||
)
|
||||
|
||||
return array
|
||||
return arrays[fqsn][timeframe] if timeframe else arrays[fqsn]
|
||||
|
||||
async def delete_ts(
|
||||
self,
|
||||
key: str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
fmt: str = 'OHLCV',
|
||||
|
||||
) -> bool:
|
||||
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
if key not in syms:
|
||||
raise KeyError(f'`{key}` table key not found in\n{syms}?')
|
||||
print(syms)
|
||||
# if key not in syms:
|
||||
# raise KeyError(f'`{fqsn}` table key not found?')
|
||||
|
||||
tbk = mk_tbk((
|
||||
key,
|
||||
tf_in_1s.get(timeframe, tf_in_1s[60]),
|
||||
fmt,
|
||||
))
|
||||
return await client.destroy(tbk=tbk)
|
||||
return await client.destroy(tbk=key)
|
||||
|
||||
async def write_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
ohlcv: np.ndarray,
|
||||
timeframe: int,
|
||||
append_and_duplicate: bool = True,
|
||||
limit: int = int(800e3),
|
||||
|
||||
|
|
@ -552,18 +519,17 @@ class Storage:
|
|||
|
||||
m, r = divmod(len(mkts_array), limit)
|
||||
|
||||
tfkey = tf_in_1s[timeframe]
|
||||
for i in range(m, 1):
|
||||
to_push = mkts_array[i-1:i*limit]
|
||||
|
||||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/{tfkey}/OHLCV',
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
# TODO: pre-deduplicate?
|
||||
# TODO: pre deduplicate?
|
||||
isvariablelength=append_and_duplicate,
|
||||
)
|
||||
|
||||
|
|
@ -582,7 +548,7 @@ class Storage:
|
|||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/{tfkey}/OHLCV',
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
|
|
@ -611,7 +577,6 @@ class Storage:
|
|||
# def delete_range(self, start_dt, end_dt) -> None:
|
||||
# ...
|
||||
|
||||
|
||||
@acm
|
||||
async def open_storage_client(
|
||||
fqsn: str,
|
||||
|
|
@ -630,10 +595,10 @@ async def open_storage_client(
|
|||
yield Storage(client)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_tsdb_client(
|
||||
fqsn: str,
|
||||
) -> Storage:
|
||||
async def tsdb_history_update(
|
||||
fqsn: Optional[str] = None,
|
||||
|
||||
) -> list[str]:
|
||||
|
||||
# TODO: real-time dedicated task for ensuring
|
||||
# history consistency between the tsdb, shm and real-time feed..
|
||||
|
|
@ -661,8 +626,8 @@ async def open_tsdb_client(
|
|||
# * the original data feed arch blurb:
|
||||
# - https://github.com/pikers/piker/issues/98
|
||||
#
|
||||
profiler = Profiler(
|
||||
disabled=True, # not pg_profile_enabled(),
|
||||
profiler = pg.debug.Profiler(
|
||||
disabled=False, # not pg_profile_enabled(),
|
||||
delayed=False,
|
||||
)
|
||||
|
||||
|
|
@ -673,40 +638,43 @@ async def open_tsdb_client(
|
|||
[fqsn],
|
||||
start_stream=False,
|
||||
|
||||
) as feed,
|
||||
) as (feed, stream),
|
||||
):
|
||||
profiler(f'opened feed for {fqsn}')
|
||||
|
||||
# to_append = feed.hist_shm.array
|
||||
# to_prepend = None
|
||||
to_append = feed.shm.array
|
||||
to_prepend = None
|
||||
|
||||
if fqsn:
|
||||
flume = feed.flumes[fqsn]
|
||||
symbol = flume.symbol
|
||||
symbol = feed.symbols.get(fqsn)
|
||||
if symbol:
|
||||
fqsn = symbol.fqsn
|
||||
fqsn = symbol.front_fqsn()
|
||||
|
||||
# diff db history with shm and only write the missing portions
|
||||
# ohlcv = flume.hist_shm.array
|
||||
ohlcv = feed.shm.array
|
||||
|
||||
# TODO: use pg profiler
|
||||
# for secs in (1, 60):
|
||||
# tsdb_array = await storage.read_ohlcv(
|
||||
# fqsn,
|
||||
# timeframe=timeframe,
|
||||
# )
|
||||
# # hist diffing:
|
||||
# # these aren't currently used but can be referenced from
|
||||
# # within the embedded ipython shell below.
|
||||
# to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
|
||||
# to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
|
||||
tsdb_arrays = await storage.read_ohlcv(fqsn)
|
||||
# hist diffing
|
||||
if tsdb_arrays:
|
||||
for secs in (1, 60):
|
||||
ts = tsdb_arrays.get(secs)
|
||||
if ts is not None and len(ts):
|
||||
# these aren't currently used but can be referenced from
|
||||
# within the embedded ipython shell below.
|
||||
to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
|
||||
to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
|
||||
|
||||
# profiler('Finished db arrays diffs')
|
||||
profiler('Finished db arrays diffs')
|
||||
|
||||
_ = await storage.client.list_symbols()
|
||||
# log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
|
||||
# profiler(f'listed symbols {syms}')
|
||||
yield storage
|
||||
syms = await storage.client.list_symbols()
|
||||
log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
|
||||
profiler(f'listed symbols {syms}')
|
||||
|
||||
# TODO: ask if user wants to write history for detected
|
||||
# available shm buffers?
|
||||
from tractor.trionics import ipython_embed
|
||||
await ipython_embed()
|
||||
|
||||
# for array in [to_append, to_prepend]:
|
||||
# if array is None:
|
||||
|
|
@ -806,13 +774,12 @@ async def stream_quotes(
|
|||
async with open_websocket_url(f'ws://{host}:{port}/ws') as ws:
|
||||
# send subs topics to server
|
||||
resp = await ws.send_message(
|
||||
|
||||
encode({'streams': list(tbks.values())})
|
||||
msgpack.dumps({'streams': list(tbks.values())})
|
||||
)
|
||||
log.info(resp)
|
||||
|
||||
async def recv() -> dict[str, Any]:
|
||||
return decode((await ws.get_message()), encoding='utf-8')
|
||||
return msgpack.loads((await ws.get_message()), encoding='utf-8')
|
||||
|
||||
streams = (await recv())['streams']
|
||||
log.info(f"Subscribed to {streams}")
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Built-in (extension) types.
|
||||
|
||||
"""
|
||||
import sys
|
||||
from pprint import pformat
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class Struct(
|
||||
msgspec.Struct,
|
||||
|
||||
# https://jcristharif.com/msgspec/structs.html#tagged-unions
|
||||
# tag='pikerstruct',
|
||||
# tag=True,
|
||||
):
|
||||
'''
|
||||
A "human friendlier" (aka repl buddy) struct subtype.
|
||||
|
||||
'''
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
f: getattr(self, f)
|
||||
for f in self.__struct_fields__
|
||||
}
|
||||
|
||||
# Lul, doesn't seem to work that well..
|
||||
# def __repr__(self):
|
||||
# # only turn on pprint when we detect a python REPL
|
||||
# # at runtime B)
|
||||
# if (
|
||||
# hasattr(sys, 'ps1')
|
||||
# # TODO: check if we're in pdb
|
||||
# ):
|
||||
# return self.pformat()
|
||||
|
||||
# return super().__repr__()
|
||||
|
||||
def pformat(self) -> str:
|
||||
return f'Struct({pformat(self.to_dict())})'
|
||||
|
||||
def copy(
|
||||
self,
|
||||
update: dict | None = None,
|
||||
|
||||
) -> msgspec.Struct:
|
||||
'''
|
||||
Validate-typecast all self defined fields, return a copy of us
|
||||
with all such fields.
|
||||
|
||||
This is kinda like the default behaviour in `pydantic.BaseModel`.
|
||||
|
||||
'''
|
||||
if update:
|
||||
for k, v in update.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
# roundtrip serialize to validate
|
||||
return msgspec.msgpack.Decoder(
|
||||
type=type(self)
|
||||
).decode(
|
||||
msgspec.msgpack.Encoder().encode(self)
|
||||
)
|
||||
|
||||
# NOTE XXX: this won't work on frozen types!
|
||||
# use ``.copy()`` above in such cases.
|
||||
def typecast(
|
||||
self,
|
||||
# fields: list[str] | None = None,
|
||||
) -> None:
|
||||
for fname, ftype in self.__annotations__.items():
|
||||
setattr(self, fname, ftype(getattr(self, fname)))
|
||||
|
|
@ -78,8 +78,7 @@ class Fsp:
|
|||
# + the consuming fsp *to* the consumers output
|
||||
# shm flow.
|
||||
_flow_registry: dict[
|
||||
tuple[_Token, str],
|
||||
tuple[_Token, Optional[ShmArray]],
|
||||
tuple[_Token, str], _Token,
|
||||
] = {}
|
||||
|
||||
def __init__(
|
||||
|
|
@ -121,6 +120,7 @@ class Fsp:
|
|||
):
|
||||
return self.func(*args, **kwargs)
|
||||
|
||||
# TODO: lru_cache this? prettty sure it'll work?
|
||||
def get_shm(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
|
|
@ -131,27 +131,12 @@ class Fsp:
|
|||
for this "instance" of a signal processor for
|
||||
the given ``key``.
|
||||
|
||||
The destination shm "token" and array are cached if possible to
|
||||
minimize multiple stdlib/system calls.
|
||||
|
||||
'''
|
||||
dst_token, maybe_array = self._flow_registry[
|
||||
dst_token = self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
]
|
||||
if maybe_array is None:
|
||||
self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
] = (
|
||||
dst_token,
|
||||
# "cache" the ``ShmArray`` such that
|
||||
# we call the underlying "attach" code as few
|
||||
# times as possible as per:
|
||||
# - https://github.com/pikers/piker/issues/359
|
||||
# - https://github.com/pikers/piker/issues/332
|
||||
maybe_array := attach_shm_array(dst_token)
|
||||
)
|
||||
|
||||
return maybe_array
|
||||
shm = attach_shm_array(dst_token)
|
||||
return shm
|
||||
|
||||
|
||||
def fsp(
|
||||
|
|
@ -199,10 +184,7 @@ def maybe_mk_fsp_shm(
|
|||
# TODO: load output types from `Fsp`
|
||||
# - should `index` be a required internal field?
|
||||
fsp_dtype = np.dtype(
|
||||
[('index', int)]
|
||||
+
|
||||
[('time', float)]
|
||||
+
|
||||
[('index', int)] +
|
||||
[(field_name, float) for field_name in target.outputs]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ core task logic for processing chains
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import (
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Optional,
|
||||
AsyncIterator, Callable, Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
|
@ -36,22 +35,14 @@ from tractor.msg import NamespacePath
|
|||
from ..log import get_logger, get_console_log
|
||||
from .. import data
|
||||
from ..data import attach_shm_array
|
||||
from ..data.feed import (
|
||||
Flume,
|
||||
Feed,
|
||||
)
|
||||
from ..data.feed import Feed
|
||||
from ..data._sharedmem import ShmArray
|
||||
from ..data._sampling import (
|
||||
_default_delay_s,
|
||||
open_sample_stream,
|
||||
)
|
||||
from ..accounting._mktinfo import Symbol
|
||||
from ..data._source import Symbol
|
||||
from ._api import (
|
||||
Fsp,
|
||||
_load_builtins,
|
||||
_Token,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -86,7 +77,7 @@ async def filter_quotes_by_sym(
|
|||
async def fsp_compute(
|
||||
|
||||
symbol: Symbol,
|
||||
flume: Flume,
|
||||
feed: Feed,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
|
||||
src: ShmArray,
|
||||
|
|
@ -99,12 +90,12 @@ async def fsp_compute(
|
|||
|
||||
) -> None:
|
||||
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=True
|
||||
)
|
||||
|
||||
fqsn = symbol.fqme
|
||||
fqsn = symbol.front_fqsn()
|
||||
out_stream = func(
|
||||
|
||||
# TODO: do we even need this if we do the feed api right?
|
||||
|
|
@ -114,17 +105,16 @@ async def fsp_compute(
|
|||
filter_quotes_by_sym(fqsn, quote_stream),
|
||||
|
||||
# XXX: currently the ``ohlcv`` arg
|
||||
flume.rt_shm,
|
||||
feed.shm,
|
||||
)
|
||||
|
||||
# HISTORY COMPUTE PHASE
|
||||
# conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output.
|
||||
# Conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output
|
||||
history_output: Union[
|
||||
dict[str, np.ndarray], # multi-output case
|
||||
np.ndarray, # single output case
|
||||
]
|
||||
history_output = await anext(out_stream)
|
||||
history_output = await out_stream.__anext__()
|
||||
|
||||
func_name = func.__name__
|
||||
profiler(f'{func_name} generated history')
|
||||
|
|
@ -136,13 +126,9 @@ async def fsp_compute(
|
|||
# each respective field.
|
||||
fields = getattr(dst.array.dtype, 'fields', None).copy()
|
||||
fields.pop('index')
|
||||
history_by_field: Optional[np.ndarray] = None
|
||||
src_time = src.array['time']
|
||||
history: Optional[np.ndarray] = None # TODO: nptyping here!
|
||||
|
||||
if (
|
||||
fields and
|
||||
len(fields) > 1
|
||||
):
|
||||
if fields and len(fields) > 1 and fields:
|
||||
if not isinstance(history_output, dict):
|
||||
raise ValueError(
|
||||
f'`{func_name}` is a multi-output FSP and should yield a '
|
||||
|
|
@ -153,7 +139,7 @@ async def fsp_compute(
|
|||
if key in history_output:
|
||||
output = history_output[key]
|
||||
|
||||
if history_by_field is None:
|
||||
if history is None:
|
||||
|
||||
if output is None:
|
||||
length = len(src.array)
|
||||
|
|
@ -163,7 +149,7 @@ async def fsp_compute(
|
|||
# using the first output, determine
|
||||
# the length of the struct-array that
|
||||
# will be pushed to shm.
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
length,
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
|
|
@ -171,7 +157,7 @@ async def fsp_compute(
|
|||
if output is None:
|
||||
continue
|
||||
|
||||
history_by_field[key] = output
|
||||
history[key] = output
|
||||
|
||||
# single-key output stream
|
||||
else:
|
||||
|
|
@ -180,15 +166,11 @@ async def fsp_compute(
|
|||
f'`{func_name}` is a single output FSP and should yield an '
|
||||
'`np.ndarray` for history'
|
||||
)
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
len(history_output),
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
history_by_field[func_name] = history_output
|
||||
|
||||
history_by_field['time'] = src_time[-len(history_by_field):]
|
||||
|
||||
history_output['time'] = src.array['time']
|
||||
history[func_name] = history_output
|
||||
|
||||
# TODO: XXX:
|
||||
# THERE'S A BIG BUG HERE WITH THE `index` field since we're
|
||||
|
|
@ -205,10 +187,7 @@ async def fsp_compute(
|
|||
|
||||
# TODO: can we use this `start` flag instead of the manual
|
||||
# setting above?
|
||||
index = dst.push(
|
||||
history_by_field,
|
||||
start=first,
|
||||
)
|
||||
index = dst.push(history, start=first)
|
||||
|
||||
profiler(f'{func_name} pushed history')
|
||||
profiler.finish()
|
||||
|
|
@ -234,14 +213,8 @@ async def fsp_compute(
|
|||
|
||||
log.debug(f"{func_name}: {processed}")
|
||||
key, output = processed
|
||||
# dst.array[-1][key] = output
|
||||
dst.array[[key, 'time']][-1] = (
|
||||
output,
|
||||
# TODO: what about pushing ``time.time_ns()``
|
||||
# in which case we'll need to round at the graphics
|
||||
# processing / sampling layer?
|
||||
src.array[-1]['time']
|
||||
)
|
||||
index = src.index
|
||||
dst.array[-1][key] = output
|
||||
|
||||
# NOTE: for now we aren't streaming this to the consumer
|
||||
# stream latest array index entry which basically just acts
|
||||
|
|
@ -252,7 +225,6 @@ async def fsp_compute(
|
|||
# N-consumers who subscribe for the real-time output,
|
||||
# which we'll likely want to implement using local-mem
|
||||
# chans for the fan out?
|
||||
# index = src.index
|
||||
# if attach_stream:
|
||||
# await client_stream.send(index)
|
||||
|
||||
|
|
@ -289,7 +261,7 @@ async def cascade(
|
|||
destination shm array buffer.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=False
|
||||
)
|
||||
|
|
@ -312,10 +284,9 @@ async def cascade(
|
|||
# TODO: ugh i hate this wind/unwind to list over the wire
|
||||
# but not sure how else to do it.
|
||||
for (token, fsp_name, dst_token) in shm_registry:
|
||||
Fsp._flow_registry[(
|
||||
_Token.from_msg(token),
|
||||
fsp_name,
|
||||
)] = _Token.from_msg(dst_token), None
|
||||
Fsp._flow_registry[
|
||||
(_Token.from_msg(token), fsp_name)
|
||||
] = _Token.from_msg(dst_token)
|
||||
|
||||
fsp: Fsp = reg.get(
|
||||
NamespacePath(ns_path)
|
||||
|
|
@ -327,7 +298,6 @@ async def cascade(
|
|||
raise ValueError(f'Unknown fsp target: {ns_path}')
|
||||
|
||||
# open a data feed stream with requested broker
|
||||
feed: Feed
|
||||
async with data.feed.maybe_open_feed(
|
||||
[fqsn],
|
||||
|
||||
|
|
@ -337,13 +307,14 @@ async def cascade(
|
|||
# needs to get throttled the ticks we generate.
|
||||
# tick_throttle=60,
|
||||
|
||||
) as feed:
|
||||
) as (feed, quote_stream):
|
||||
symbol = feed.symbols[fqsn]
|
||||
|
||||
flume = feed.flumes[fqsn]
|
||||
symbol = flume.symbol
|
||||
assert src.token == flume.rt_shm.token
|
||||
profiler(f'{func}: feed up')
|
||||
|
||||
assert src.token == feed.shm.token
|
||||
# last_len = new_len = len(src.array)
|
||||
|
||||
func_name = func.__name__
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
|
|
@ -353,8 +324,8 @@ async def cascade(
|
|||
|
||||
fsp_compute,
|
||||
symbol=symbol,
|
||||
flume=flume,
|
||||
quote_stream=flume.stream,
|
||||
feed=feed,
|
||||
quote_stream=quote_stream,
|
||||
|
||||
# shm
|
||||
src=src,
|
||||
|
|
@ -390,7 +361,7 @@ async def cascade(
|
|||
) -> tuple[TaskTracker, int]:
|
||||
# TODO: adopt an incremental update engine/approach
|
||||
# where possible here eventually!
|
||||
log.info(f're-syncing fsp {func_name} to source')
|
||||
log.debug(f're-syncing fsp {func_name} to source')
|
||||
tracker.cs.cancel()
|
||||
await tracker.complete.wait()
|
||||
tracker, index = await n.start(fsp_target)
|
||||
|
|
@ -403,16 +374,14 @@ async def cascade(
|
|||
'key': dst_shm_token,
|
||||
'first': dst._first.value,
|
||||
'last': dst._last.value,
|
||||
}
|
||||
})
|
||||
}})
|
||||
return tracker, index
|
||||
|
||||
def is_synced(
|
||||
src: ShmArray,
|
||||
dst: ShmArray
|
||||
) -> tuple[bool, int, int]:
|
||||
'''
|
||||
Predicate to dertmine if a destination FSP
|
||||
'''Predicate to dertmine if a destination FSP
|
||||
output array is aligned to its source array.
|
||||
|
||||
'''
|
||||
|
|
@ -421,15 +390,16 @@ async def cascade(
|
|||
return not (
|
||||
# the source is likely backfilling and we must
|
||||
# sync history calculations
|
||||
len_diff > 2
|
||||
len_diff > 2 or
|
||||
|
||||
# we aren't step synced to the source and may be
|
||||
# leading/lagging by a step
|
||||
or step_diff > 1
|
||||
or step_diff < 0
|
||||
step_diff > 1 or
|
||||
step_diff < 0
|
||||
), step_diff, len_diff
|
||||
|
||||
async def poll_and_sync_to_step(
|
||||
|
||||
tracker: TaskTracker,
|
||||
src: ShmArray,
|
||||
dst: ShmArray,
|
||||
|
|
@ -448,23 +418,18 @@ async def cascade(
|
|||
# detect sample period step for subscription to increment
|
||||
# signal
|
||||
times = src.array['time']
|
||||
if len(times) > 1:
|
||||
last_ts = times[-1]
|
||||
delay_s = float(last_ts - times[times != last_ts][-1])
|
||||
else:
|
||||
# our default "HFT" sample rate.
|
||||
delay_s = _default_delay_s
|
||||
delay_s = times[-1] - times[times != times[-1]][-1]
|
||||
|
||||
# sub and increment the underlying shared memory buffer
|
||||
# on every step msg received from the global `samplerd`
|
||||
# service.
|
||||
async with open_sample_stream(float(delay_s)) as istream:
|
||||
# Increment the underlying shared memory buffer on every
|
||||
# "increment" msg received from the underlying data feed.
|
||||
async with feed.index_stream(
|
||||
int(delay_s)
|
||||
) as istream:
|
||||
|
||||
profiler(f'{func_name}: sample stream up')
|
||||
profiler.finish()
|
||||
|
||||
async for i in istream:
|
||||
# print(f'FSP incrementing {i}')
|
||||
async for _ in istream:
|
||||
|
||||
# respawn the compute task if the source
|
||||
# array has been updated such that we compute
|
||||
|
|
@ -493,23 +458,3 @@ async def cascade(
|
|||
last = array[-1:].copy()
|
||||
|
||||
dst.push(last)
|
||||
|
||||
# sync with source buffer's time step
|
||||
src_l2 = src.array[-2:]
|
||||
src_li, src_lt = src_l2[-1][['index', 'time']]
|
||||
src_2li, src_2lt = src_l2[-2][['index', 'time']]
|
||||
dst._array['time'][src_li] = src_lt
|
||||
dst._array['time'][src_2li] = src_2lt
|
||||
|
||||
# last2 = dst.array[-2:]
|
||||
# if (
|
||||
# last2[-1]['index'] != src_li
|
||||
# or last2[-2]['index'] != src_2li
|
||||
# ):
|
||||
# dstl2 = list(last2)
|
||||
# srcl2 = list(src_l2)
|
||||
# print(
|
||||
# # f'{dst.token}\n'
|
||||
# f'src: {srcl2}\n'
|
||||
# f'dst: {dstl2}\n'
|
||||
# )
|
||||
|
|
|
|||
|
|
@ -234,7 +234,7 @@ async def flow_rates(
|
|||
# FSPs, user input, and possibly any general event stream in
|
||||
# real-time. Hint: ideally implemented with caching until mutated
|
||||
# ;)
|
||||
period: 'Param[int]' = 1, # noqa
|
||||
period: 'Param[int]' = 6, # noqa
|
||||
|
||||
# TODO: support other means by providing a map
|
||||
# to weights `partial()`-ed with `wma()`?
|
||||
|
|
@ -268,7 +268,8 @@ async def flow_rates(
|
|||
'dark_dvlm_rate': None,
|
||||
}
|
||||
|
||||
quote = await anext(source)
|
||||
# TODO: 3.10 do ``anext()``
|
||||
quote = await source.__anext__()
|
||||
|
||||
# ltr = 0
|
||||
# lvr = 0
|
||||
|
|
|
|||
46
piker/log.py
46
piker/log.py
|
|
@ -21,11 +21,7 @@ import logging
|
|||
import json
|
||||
|
||||
import tractor
|
||||
from pygments import (
|
||||
highlight,
|
||||
lexers,
|
||||
formatters,
|
||||
)
|
||||
from pygments import highlight, lexers, formatters
|
||||
|
||||
# Makes it so we only see the full module name when using ``__name__``
|
||||
# without the extra "piker." prefix.
|
||||
|
|
@ -36,48 +32,26 @@ def get_logger(
|
|||
name: str = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''
|
||||
Return the package log or a sub-log for `name` if provided.
|
||||
|
||||
'''Return the package log or a sub-log for `name` if provided.
|
||||
'''
|
||||
return tractor.log.get_logger(name=name, _root_name=_proj_name)
|
||||
|
||||
|
||||
def get_console_log(
|
||||
level: str | None = None,
|
||||
name: str | None = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''
|
||||
Get the package logger and enable a handler which writes to stderr.
|
||||
def get_console_log(level: str = None, name: str = None) -> logging.Logger:
|
||||
'''Get the package logger and enable a handler which writes to stderr.
|
||||
|
||||
Yeah yeah, i know we can use ``DictConfig``. You do it...
|
||||
|
||||
'''
|
||||
return tractor.log.get_console_log(
|
||||
level,
|
||||
name=name,
|
||||
_root_name=_proj_name,
|
||||
) # our root logger
|
||||
level, name=name, _root_name=_proj_name) # our root logger
|
||||
|
||||
|
||||
def colorize_json(
|
||||
data: dict,
|
||||
style='algol_nu',
|
||||
):
|
||||
'''
|
||||
Colorize json output using ``pygments``.
|
||||
|
||||
'''
|
||||
formatted_json = json.dumps(
|
||||
data,
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
)
|
||||
def colorize_json(data, style='algol_nu'):
|
||||
"""Colorize json output using ``pygments``.
|
||||
"""
|
||||
formatted_json = json.dumps(data, sort_keys=True, indent=4)
|
||||
return highlight(
|
||||
formatted_json,
|
||||
lexers.JsonLexer(),
|
||||
|
||||
formatted_json, lexers.JsonLexer(),
|
||||
# likeable styles: algol_nu, tango, monokai
|
||||
formatters.TerminalTrueColorFormatter(style=style)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,61 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Actor-runtime service orchestration machinery.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from ._util import log
|
||||
from ._mngr import Services
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
open_registry,
|
||||
find_service,
|
||||
check_for_service,
|
||||
)
|
||||
from ._daemon import ( # noqa
|
||||
maybe_spawn_daemon,
|
||||
spawn_brokerd,
|
||||
maybe_spawn_brokerd,
|
||||
spawn_emsd,
|
||||
maybe_open_emsd,
|
||||
)
|
||||
from ._actor_runtime import (
|
||||
open_piker_runtime,
|
||||
maybe_open_pikerd,
|
||||
open_pikerd,
|
||||
get_tractor_runtime_kwargs,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'check_for_service',
|
||||
'Services',
|
||||
'maybe_spawn_daemon',
|
||||
'spawn_brokerd',
|
||||
'maybe_spawn_brokerd',
|
||||
'spawn_emsd',
|
||||
'maybe_open_emsd',
|
||||
'open_piker_runtime',
|
||||
'maybe_open_pikerd',
|
||||
'open_pikerd',
|
||||
'get_tractor_runtime_kwargs',
|
||||
]
|
||||
|
|
@ -1,348 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
``tractor`` wrapping + default config to bootstrap the `pikerd`.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from pprint import pformat
|
||||
from functools import partial
|
||||
import os
|
||||
from typing import (
|
||||
Optional,
|
||||
Any,
|
||||
ClassVar,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
open_registry,
|
||||
)
|
||||
|
||||
|
||||
def get_tractor_runtime_kwargs() -> dict[str, Any]:
|
||||
'''
|
||||
Deliver ``tractor`` related runtime variables in a `dict`.
|
||||
|
||||
'''
|
||||
return _tractor_kwargs
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX NOTE XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
# TODO: once we have `rsyscall` support we will read a config
|
||||
# and spawn the service tree distributed per that.
|
||||
start_method: str = 'trio',
|
||||
|
||||
tractor_runtime_overrides: dict | None = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> tuple[
|
||||
tractor.Actor,
|
||||
tuple[str, int],
|
||||
]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically sync with
|
||||
existing piker actors on the local link based on configuration.
|
||||
|
||||
Can be called from a subactor or any program that needs to start
|
||||
a root actor.
|
||||
|
||||
'''
|
||||
try:
|
||||
# check for existing runtime
|
||||
actor = tractor.current_actor().uid
|
||||
|
||||
except tractor._exceptions.NoRuntime:
|
||||
tractor._state._runtime_vars[
|
||||
'piker_vars'] = tractor_runtime_overrides
|
||||
|
||||
registry_addr = registry_addr or _default_reg_addr
|
||||
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=enable_modules,
|
||||
|
||||
**tractor_kwargs,
|
||||
) as _,
|
||||
|
||||
open_registry(registry_addr, ensure_exists=False) as addr,
|
||||
):
|
||||
yield (
|
||||
tractor.current_actor(),
|
||||
addr,
|
||||
)
|
||||
else:
|
||||
async with open_registry(registry_addr) as addr:
|
||||
yield (
|
||||
actor,
|
||||
addr,
|
||||
)
|
||||
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.service._daemon',
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
'piker.data._sampling',
|
||||
]
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
# db init flags
|
||||
tsdb: bool = False,
|
||||
es: bool = False,
|
||||
drop_root_perms_for_ahab: bool = True,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> Services:
|
||||
'''
|
||||
Start a root piker daemon with an indefinite lifetime.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
|
||||
name=_root_dname,
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) as (root_actor, reg_addr),
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
trio.open_nursery() as service_nursery,
|
||||
):
|
||||
if root_actor.accept_addr != reg_addr:
|
||||
raise RuntimeError(
|
||||
f'`pikerd` failed to bind on {reg_addr}!\n'
|
||||
'Maybe you have another daemon already running?'
|
||||
)
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
Services.actor_n = actor_nursery
|
||||
Services.service_n = service_nursery
|
||||
Services.debug_mode = debug_mode
|
||||
|
||||
if tsdb:
|
||||
from ._ahab import start_ahab
|
||||
from .marketstore import start_marketstore
|
||||
|
||||
log.info('Spawning `marketstore` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await service_nursery.start(
|
||||
partial(
|
||||
start_ahab,
|
||||
'marketstored',
|
||||
start_marketstore,
|
||||
loglevel=loglevel,
|
||||
drop_root_perms=drop_root_perms_for_ahab,
|
||||
)
|
||||
|
||||
)
|
||||
log.info(
|
||||
f'`marketstored` up!\n'
|
||||
f'pid: {pid}\n'
|
||||
f'container id: {cid[:12]}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
if es:
|
||||
from ._ahab import start_ahab
|
||||
from .elastic import start_elasticsearch
|
||||
|
||||
log.info('Spawning `elasticsearch` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await service_nursery.start(
|
||||
partial(
|
||||
start_ahab,
|
||||
'elasticsearch',
|
||||
start_elasticsearch,
|
||||
loglevel=loglevel,
|
||||
drop_root_perms=drop_root_perms_for_ahab,
|
||||
)
|
||||
)
|
||||
|
||||
log.info(
|
||||
f'`elasticsearch` up!\n'
|
||||
f'pid: {pid}\n'
|
||||
f'container id: {cid[:12]}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
try:
|
||||
yield Services
|
||||
|
||||
finally:
|
||||
# TODO: is this more clever/efficient?
|
||||
# if 'samplerd' in Services.service_tasks:
|
||||
# await Services.cancel_service('samplerd')
|
||||
service_nursery.cancel_scope.cancel()
|
||||
|
||||
|
||||
# TODO: do we even need this?
|
||||
# @acm
|
||||
# async def maybe_open_runtime(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> None:
|
||||
# '''
|
||||
# Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
# '''
|
||||
# name = kwargs.pop('name')
|
||||
|
||||
# if not tractor.current_actor(err_on_no_runtime=False):
|
||||
# async with open_piker_runtime(
|
||||
# name,
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
# ) as (_, addr):
|
||||
# yield addr,
|
||||
# else:
|
||||
# async with open_registry() as addr:
|
||||
# yield addr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
registry_addr: None | tuple = None,
|
||||
tsdb: bool = False,
|
||||
es: bool = False,
|
||||
drop_root_perms_for_ahab: bool = True,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal | ClassVar[Services]:
|
||||
'''
|
||||
If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
query_name = kwargs.pop(
|
||||
'name',
|
||||
f'piker_query_{os.getpid()}',
|
||||
)
|
||||
|
||||
# TODO: if we need to make the query part faster we could not init
|
||||
# an actor runtime and instead just hit the socket?
|
||||
# from tractor._ipc import _connect_chan, Channel
|
||||
# async with _connect_chan(host, port) as chan:
|
||||
# async with open_portal(chan) as arb_portal:
|
||||
# yield arb_portal
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name=query_name,
|
||||
registry_addr=registry_addr,
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
) as _,
|
||||
|
||||
tractor.find_actor(
|
||||
_root_dname,
|
||||
arbiter_sockaddr=registry_addr,
|
||||
) as portal
|
||||
):
|
||||
# connect to any existing daemon presuming
|
||||
# its registry socket was selected.
|
||||
if (
|
||||
portal is not None
|
||||
):
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
loglevel=loglevel,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
# ahabd (docker super) specific controls
|
||||
tsdb=tsdb,
|
||||
es=es,
|
||||
drop_root_perms_for_ahab=drop_root_perms_for_ahab,
|
||||
|
||||
# passthrough to ``tractor`` init
|
||||
**kwargs,
|
||||
|
||||
) as service_manager:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
assert service_manager
|
||||
yield service_manager
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Daemon-actor spawning "endpoint-hooks".
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..brokers import get_brokermod
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._actor_runtime import maybe_open_pikerd
|
||||
from ._registry import find_service
|
||||
|
||||
# `brokerd` enabled modules
|
||||
# TODO: move this def to the `.data` subpkg..
|
||||
# NOTE: keeping this list as small as possible is part of our caps-sec
|
||||
# model and should be treated with utmost care!
|
||||
_data_mods = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
|
||||
spawn_args: dict[str, Any],
|
||||
|
||||
loglevel: str | None = None,
|
||||
singleton: bool = False,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Services.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(
|
||||
f"Couldn't find any existing {service_name}\n"
|
||||
'Attempting to spawn new daemon-service..'
|
||||
)
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
started: bool
|
||||
if pikerd_portal is None:
|
||||
started = await service_task_target(
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
else:
|
||||
# request a remote `pikerd` (service manager) to start the
|
||||
# target daemon-task, the target can't return
|
||||
# a non-serializable value since it is expected that service
|
||||
# starting is non-blocking and the target task will persist
|
||||
# running "under" or "within" the `pikerd` actor tree after
|
||||
# the questing client disconnects. in other words this
|
||||
# spawns a persistent daemon actor that continues to live
|
||||
# for the lifespan of whatever the service manager inside
|
||||
# `pikerd` says it should.
|
||||
started = await pikerd_portal.run(
|
||||
service_task_target,
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
if started:
|
||||
log.info(f'Service {service_name} started!')
|
||||
|
||||
# block until we can discover (by IPC connection) to the newly
|
||||
# spawned daemon-actor and then deliver the portal to the
|
||||
# caller.
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
dname = f'brokerd.{brokername}'
|
||||
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
modpath = brokermod.__name__
|
||||
broker_enable = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
broker_enable.append(subpath)
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + broker_enable,
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from ..data import _setup_persistent_brokerd
|
||||
|
||||
await Services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
_setup_persistent_brokerd,
|
||||
brokername=brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client
|
||||
who wishes to use the sub-actor-daemon.
|
||||
|
||||
'''
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={
|
||||
'brokername': brokername,
|
||||
},
|
||||
loglevel=loglevel,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from ..clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await Services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
_setup_persistent_emsd,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={},
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
daemon-service management API.
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
|
||||
# TODO: factor this into a ``tractor.highlevel`` extension
|
||||
# pack for the library.
|
||||
class Services:
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[
|
||||
str,
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
tractor.Portal,
|
||||
trio.Event,
|
||||
]
|
||||
] = {}
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
@classmethod
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
**ctx_kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
trio.Event,
|
||||
Any,
|
||||
]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
async with portal.open_context(
|
||||
target,
|
||||
**ctx_kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
complete = trio.Event()
|
||||
task_status.started((cs, complete, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
# and any final portal result from the
|
||||
# sub-actor.
|
||||
ctx_res = await ctx.result()
|
||||
|
||||
# NOTE: blocks indefinitely until cancelled
|
||||
# either by error from the target context
|
||||
# function or by being cancelled here by the
|
||||
# surrounding cancel scope.
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
finally:
|
||||
await portal.cancel_actor()
|
||||
complete.set()
|
||||
self.service_tasks.pop(name)
|
||||
|
||||
cs, complete, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal, complete)
|
||||
|
||||
return cs, first
|
||||
|
||||
@classmethod
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
) -> Any:
|
||||
'''
|
||||
Cancel the service task and actor for the given ``name``.
|
||||
|
||||
'''
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal, complete = self.service_tasks[name]
|
||||
cs.cancel()
|
||||
await complete.wait()
|
||||
assert name not in self.service_tasks, \
|
||||
f'Serice task for {name} not terminated?'
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Inter-actor "discovery" (protocol) layer.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
_default_registry_host: str = '127.0.0.1'
|
||||
_default_registry_port: int = 6116
|
||||
_default_reg_addr: tuple[str, int] = (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
|
||||
|
||||
# NOTE: this value is set as an actor-global once the first endpoint
|
||||
# who is capable, spawns a `pikerd` service tree.
|
||||
_registry: Registry | None = None
|
||||
|
||||
|
||||
class Registry:
|
||||
addr: None | tuple[str, int] = None
|
||||
|
||||
# TODO: table of uids to sockaddrs
|
||||
peers: dict[
|
||||
tuple[str, str],
|
||||
tuple[str, int],
|
||||
] = {}
|
||||
|
||||
|
||||
_tractor_kwargs: dict[str, Any] = {}
|
||||
|
||||
|
||||
@acm
|
||||
async def open_registry(
|
||||
addr: None | tuple[str, int] = None,
|
||||
ensure_exists: bool = True,
|
||||
|
||||
) -> tuple[str, int]:
|
||||
|
||||
global _tractor_kwargs
|
||||
actor = tractor.current_actor()
|
||||
uid = actor.uid
|
||||
if (
|
||||
Registry.addr is not None
|
||||
and addr
|
||||
):
|
||||
raise RuntimeError(
|
||||
f'`{uid}` registry addr already bound @ {_registry.sockaddr}'
|
||||
)
|
||||
|
||||
was_set: bool = False
|
||||
|
||||
if (
|
||||
not tractor.is_root_process()
|
||||
and Registry.addr is None
|
||||
):
|
||||
Registry.addr = actor._arb_addr
|
||||
|
||||
if (
|
||||
ensure_exists
|
||||
and Registry.addr is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"`{uid}` registry should already exist bug doesn't?"
|
||||
)
|
||||
|
||||
if (
|
||||
Registry.addr is None
|
||||
):
|
||||
was_set = True
|
||||
Registry.addr = addr or _default_reg_addr
|
||||
|
||||
_tractor_kwargs['arbiter_addr'] = Registry.addr
|
||||
|
||||
try:
|
||||
yield Registry.addr
|
||||
finally:
|
||||
# XXX: always clear the global addr if we set it so that the
|
||||
# next (set of) calls will apply whatever new one is passed
|
||||
# in.
|
||||
if was_set:
|
||||
Registry.addr = None
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> tractor.Portal | None:
|
||||
|
||||
async with open_registry() as reg_addr:
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> None | tuple[str, int]:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with open_registry(ensure_exists=False) as reg_addr:
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Sub-sys module commons.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.service'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Any,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import asks
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import docker
|
||||
from ._ahab import DockerContainer
|
||||
|
||||
from . import log # sub-sys logger
|
||||
from ._util import (
|
||||
get_console_log,
|
||||
)
|
||||
|
||||
|
||||
# container level config
|
||||
_config = {
|
||||
'port': 19200,
|
||||
'log_level': 'debug',
|
||||
|
||||
# hardcoded to our image version
|
||||
'version': '7.17.4',
|
||||
}
|
||||
|
||||
|
||||
def start_elasticsearch(
|
||||
client: docker.DockerClient,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tuple[DockerContainer, dict[str, Any]]:
|
||||
'''
|
||||
Start and supervise an elasticsearch instance with its config bind-mounted
|
||||
in from the piker config directory on the system.
|
||||
|
||||
The equivalent cli cmd to this code is:
|
||||
|
||||
sudo docker run \
|
||||
-itd \
|
||||
--rm \
|
||||
--network=host \
|
||||
--mount type=bind,source="$(pwd)"/elastic,\
|
||||
target=/usr/share/elasticsearch/data \
|
||||
--env "elastic_username=elastic" \
|
||||
--env "elastic_password=password" \
|
||||
--env "xpack.security.enabled=false" \
|
||||
elastic
|
||||
|
||||
'''
|
||||
get_console_log('info', name=__name__)
|
||||
|
||||
dcntr: DockerContainer = client.containers.run(
|
||||
'piker:elastic',
|
||||
name='piker-elastic',
|
||||
network='host',
|
||||
detach=True,
|
||||
remove=True
|
||||
)
|
||||
|
||||
async def health_query(msg: str | None = None):
|
||||
if (
|
||||
msg
|
||||
and _config['version'] in msg
|
||||
):
|
||||
return True
|
||||
|
||||
try:
|
||||
health = (await asks.get(
|
||||
'http://localhost:19200/_cat/health',
|
||||
params={'format': 'json'}
|
||||
)).json()
|
||||
log.info(
|
||||
'ElasticSearch cntr health:\n'
|
||||
f'{health}'
|
||||
)
|
||||
|
||||
except OSError:
|
||||
log.exception('couldnt reach elastic container')
|
||||
return False
|
||||
|
||||
log.info(health)
|
||||
return health[0]['status'] == 'green'
|
||||
|
||||
async def chk_for_closed_msg(msg: str):
|
||||
return msg == 'closed'
|
||||
|
||||
return (
|
||||
dcntr,
|
||||
{
|
||||
# apparently we're REALLY tolerant of startup latency
|
||||
# for CI XD
|
||||
'startup_timeout': 240.0,
|
||||
|
||||
# XXX: decrease http poll period bc docker
|
||||
# is shite at handling fast poll rates..
|
||||
'startup_query_period': 0.1,
|
||||
|
||||
'log_msg_key': 'message',
|
||||
|
||||
# 'started_afunc': health_query,
|
||||
},
|
||||
# expected startup and stop msgs
|
||||
health_query,
|
||||
chk_for_closed_msg,
|
||||
)
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
Annotations for ur faces.
|
||||
|
||||
"""
|
||||
from typing import Callable
|
||||
from typing import Callable, Optional
|
||||
|
||||
from PyQt5 import QtCore, QtGui, QtWidgets
|
||||
from PyQt5.QtCore import QPointF, QRectF
|
||||
|
|
@ -32,22 +32,16 @@ def mk_marker_path(
|
|||
style: str,
|
||||
|
||||
) -> QGraphicsPathItem:
|
||||
'''
|
||||
Add a marker to be displayed on the line wrapped in
|
||||
a ``QGraphicsPathItem`` ready to be placed using scene coordinates
|
||||
(not view).
|
||||
"""Add a marker to be displayed on the line wrapped in a ``QGraphicsPathItem``
|
||||
ready to be placed using scene coordinates (not view).
|
||||
|
||||
**Arguments**
|
||||
style String indicating the style of marker to add:
|
||||
``'<|'``, ``'|>'``, ``'>|'``, ``'|<'``, ``'<|>'``,
|
||||
``'>|<'``, ``'^'``, ``'v'``, ``'o'``
|
||||
size Size of the marker in pixels.
|
||||
|
||||
This code is taken nearly verbatim from the
|
||||
`InfiniteLine.addMarker()` method but does not attempt do be aware
|
||||
of low(er) level graphics controls and expects for the output
|
||||
polygon to be applied to a ``QGraphicsPathItem``.
|
||||
|
||||
'''
|
||||
"""
|
||||
path = QtGui.QPainterPath()
|
||||
|
||||
if style == 'o':
|
||||
|
|
@ -93,8 +87,7 @@ def mk_marker_path(
|
|||
|
||||
|
||||
class LevelMarker(QGraphicsPathItem):
|
||||
'''
|
||||
An arrow marker path graphich which redraws itself
|
||||
'''An arrow marker path graphich which redraws itself
|
||||
to the specified view coordinate level on each paint cycle.
|
||||
|
||||
'''
|
||||
|
|
@ -105,14 +98,13 @@ class LevelMarker(QGraphicsPathItem):
|
|||
get_level: Callable[..., float],
|
||||
size: float = 20,
|
||||
keep_in_view: bool = True,
|
||||
on_paint: Callable | None = None,
|
||||
on_paint: Optional[Callable] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
# get polygon and scale
|
||||
super().__init__()
|
||||
# self.setScale(size, size)
|
||||
self.setScale(size)
|
||||
self.scale(size, size)
|
||||
|
||||
# interally generates path
|
||||
self._style = None
|
||||
|
|
@ -122,7 +114,6 @@ class LevelMarker(QGraphicsPathItem):
|
|||
|
||||
self.get_level = get_level
|
||||
self._on_paint = on_paint
|
||||
|
||||
self.scene_x = lambda: chart.marker_right_points()[1]
|
||||
self.level: float = 0
|
||||
self.keep_in_view = keep_in_view
|
||||
|
|
@ -158,9 +149,12 @@ class LevelMarker(QGraphicsPathItem):
|
|||
def w(self) -> float:
|
||||
return self.path_br().width()
|
||||
|
||||
def position_in_view(self) -> None:
|
||||
'''
|
||||
Show a pp off-screen indicator for a level label.
|
||||
def position_in_view(
|
||||
self,
|
||||
# level: float,
|
||||
|
||||
) -> None:
|
||||
'''Show a pp off-screen indicator for a level label.
|
||||
|
||||
This is like in fps games where you have a gps "nav" indicator
|
||||
but your teammate is outside the range of view, except in 2D, on
|
||||
|
|
@ -168,6 +162,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
|
||||
'''
|
||||
level = self.get_level()
|
||||
|
||||
view = self.chart.getViewBox()
|
||||
vr = view.state['viewRange']
|
||||
ymn, ymx = vr[1]
|
||||
|
|
@ -191,6 +186,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
)
|
||||
|
||||
elif level < ymn: # pin to bottom of view
|
||||
|
||||
self.setPos(
|
||||
QPointF(
|
||||
x,
|
||||
|
|
@ -215,8 +211,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
w: QtWidgets.QWidget
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Core paint which we override to always update
|
||||
'''Core paint which we override to always update
|
||||
our marker position in scene coordinates from a
|
||||
view cooridnate "level".
|
||||
|
||||
|
|
@ -240,12 +235,11 @@ def qgo_draw_markers(
|
|||
right_offset: float,
|
||||
|
||||
) -> float:
|
||||
'''
|
||||
Paint markers in ``pg.GraphicsItem`` style by first
|
||||
"""Paint markers in ``pg.GraphicsItem`` style by first
|
||||
removing the view transform for the painter, drawing the markers
|
||||
in scene coords, then restoring the view coords.
|
||||
|
||||
'''
|
||||
"""
|
||||
# paint markers in native coordinate system
|
||||
orig_tr = p.transform()
|
||||
|
||||
|
|
|
|||
|
|
@ -19,16 +19,15 @@ Main app startup and run.
|
|||
|
||||
'''
|
||||
from functools import partial
|
||||
from types import ModuleType
|
||||
|
||||
from PyQt5.QtCore import QEvent
|
||||
import trio
|
||||
|
||||
from ..service import maybe_spawn_brokerd
|
||||
from .._daemon import maybe_spawn_brokerd
|
||||
from ..brokers import get_brokermod
|
||||
from . import _event
|
||||
from ._exec import run_qtractor
|
||||
from ..data.feed import install_brokerd_search
|
||||
from ..accounting._mktinfo import unpack_fqsn
|
||||
from . import _search
|
||||
from ._chart import GodWidget
|
||||
from ..log import get_logger
|
||||
|
|
@ -37,26 +36,27 @@ log = get_logger(__name__)
|
|||
|
||||
|
||||
async def load_provider_search(
|
||||
brokermod: str,
|
||||
|
||||
broker: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> None:
|
||||
|
||||
name = brokermod.name
|
||||
log.info(f'loading brokerd for {name}..')
|
||||
log.info(f'loading brokerd for {broker}..')
|
||||
|
||||
async with (
|
||||
|
||||
maybe_spawn_brokerd(
|
||||
name,
|
||||
broker,
|
||||
loglevel=loglevel
|
||||
) as portal,
|
||||
|
||||
install_brokerd_search(
|
||||
portal,
|
||||
brokermod,
|
||||
get_brokermod(broker),
|
||||
),
|
||||
):
|
||||
|
||||
# keep search engine stream up until cancelled
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
|
@ -66,8 +66,8 @@ async def _async_main(
|
|||
# implicit required argument provided by ``qtractor_run()``
|
||||
main_widget: GodWidget,
|
||||
|
||||
syms: list[str],
|
||||
brokers: dict[str, ModuleType],
|
||||
sym: str,
|
||||
brokernames: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -78,8 +78,6 @@ async def _async_main(
|
|||
|
||||
"""
|
||||
from . import _display
|
||||
from ._pg_overrides import _do_overrides
|
||||
_do_overrides()
|
||||
|
||||
godwidget = main_widget
|
||||
|
||||
|
|
@ -99,11 +97,6 @@ async def _async_main(
|
|||
sbar = godwidget.window.status_bar
|
||||
starting_done = sbar.open_status('starting ze sexy chartz')
|
||||
|
||||
needed_brokermods: dict[str, ModuleType] = {}
|
||||
for fqsn in syms:
|
||||
brokername, *_ = unpack_fqsn(fqsn)
|
||||
needed_brokermods[brokername] = brokers[brokername]
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as root_n,
|
||||
):
|
||||
|
|
@ -114,14 +107,18 @@ async def _async_main(
|
|||
# setup search widget and focus main chart view at startup
|
||||
# search widget is a singleton alongside the godwidget
|
||||
search = _search.SearchWidget(godwidget=godwidget)
|
||||
# search.bar.unfocus()
|
||||
# godwidget.hbox.addWidget(search)
|
||||
search.bar.unfocus()
|
||||
|
||||
godwidget.hbox.addWidget(search)
|
||||
godwidget.search = search
|
||||
|
||||
symbol, _, provider = sym.rpartition('.')
|
||||
|
||||
# this internally starts a ``display_symbol_data()`` task above
|
||||
order_mode_ready = await godwidget.load_symbols(
|
||||
fqsns=syms,
|
||||
loglevel=loglevel,
|
||||
order_mode_ready = await godwidget.load_symbol(
|
||||
provider,
|
||||
symbol,
|
||||
loglevel
|
||||
)
|
||||
|
||||
# spin up a search engine for the local cached symbol set
|
||||
|
|
@ -138,12 +135,8 @@ async def _async_main(
|
|||
):
|
||||
# load other providers into search **after**
|
||||
# the chart's select cache
|
||||
for brokername, mod in needed_brokermods.items():
|
||||
root_n.start_soon(
|
||||
load_provider_search,
|
||||
mod,
|
||||
loglevel,
|
||||
)
|
||||
for broker in brokernames:
|
||||
root_n.start_soon(load_provider_search, broker, loglevel)
|
||||
|
||||
await order_mode_ready.wait()
|
||||
|
||||
|
|
@ -172,22 +165,19 @@ async def _async_main(
|
|||
|
||||
|
||||
def _main(
|
||||
syms: list[str],
|
||||
brokermods: list[ModuleType],
|
||||
sym: str,
|
||||
brokernames: [str],
|
||||
piker_loglevel: str,
|
||||
tractor_kwargs,
|
||||
) -> None:
|
||||
'''
|
||||
Sync entry point to start a chart: a ``tractor`` + Qt runtime.
|
||||
Sync entry point to start a chart: a ``tractor`` + Qt runtime
|
||||
entry point
|
||||
|
||||
'''
|
||||
run_qtractor(
|
||||
func=_async_main,
|
||||
args=(
|
||||
syms,
|
||||
{mod.name: mod for mod in brokermods},
|
||||
piker_loglevel,
|
||||
),
|
||||
main_widget_type=GodWidget,
|
||||
args=(sym, brokernames, piker_loglevel),
|
||||
main_widget=GodWidget,
|
||||
tractor_kwargs=tractor_kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,9 +18,8 @@
|
|||
Chart axes graphics and behavior.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from functools import lru_cache
|
||||
from typing import Callable
|
||||
from typing import Optional, Callable
|
||||
from math import floor
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -28,8 +27,7 @@ import pyqtgraph as pg
|
|||
from PyQt5 import QtCore, QtGui, QtWidgets
|
||||
from PyQt5.QtCore import QPointF
|
||||
|
||||
from . import _pg_overrides as pgo
|
||||
from ..accounting._mktinfo import float_digits
|
||||
from ..data._source import float_digits
|
||||
from ._label import Label
|
||||
from ._style import DpiAwareFont, hcolor, _font
|
||||
from ._interaction import ChartView
|
||||
|
|
@ -41,17 +39,12 @@ class Axis(pg.AxisItem):
|
|||
'''
|
||||
A better axis that sizes tick contents considering font size.
|
||||
|
||||
Also includes tick values lru caching originally proposed in but never
|
||||
accepted upstream:
|
||||
https://github.com/pyqtgraph/pyqtgraph/pull/2160
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
plotitem: pgo.PlotItem,
|
||||
typical_max_str: str = '100 000.000 ',
|
||||
linkedsplits,
|
||||
typical_max_str: str = '100 000.000',
|
||||
text_color: str = 'bracket',
|
||||
lru_cache_tick_strings: bool = True,
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
|
@ -60,82 +53,44 @@ class Axis(pg.AxisItem):
|
|||
**kwargs
|
||||
)
|
||||
|
||||
# XXX: pretty sure this makes things slower!
|
||||
# no idea why given we only move labels for the most part?
|
||||
# XXX: pretty sure this makes things slower
|
||||
# self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
self.pi = plotitem
|
||||
self.linkedsplits = linkedsplits
|
||||
self._dpi_font = _font
|
||||
|
||||
self.setTickFont(_font.font)
|
||||
font_size = self._dpi_font.font.pixelSize()
|
||||
|
||||
style_conf = {
|
||||
'textFillLimits': [(0, 0.5)],
|
||||
'tickFont': self._dpi_font.font,
|
||||
|
||||
}
|
||||
text_offset = None
|
||||
if self.orientation in ('bottom',):
|
||||
text_offset = floor(0.25 * font_size)
|
||||
|
||||
elif self.orientation in ('left', 'right'):
|
||||
text_offset = floor(font_size / 2)
|
||||
|
||||
if text_offset:
|
||||
style_conf.update({
|
||||
self.setStyle(**{
|
||||
'textFillLimits': [(0, 0.5)],
|
||||
'tickFont': self._dpi_font.font,
|
||||
|
||||
# offset of text *away from* axis line in px
|
||||
# use approx. half the font pixel size (height)
|
||||
'tickTextOffset': text_offset,
|
||||
})
|
||||
|
||||
self.setStyle(**style_conf)
|
||||
self.setTickFont(_font.font)
|
||||
|
||||
# NOTE: this is for surrounding "border"
|
||||
self.setPen(_axis_pen)
|
||||
|
||||
# this is the text color
|
||||
# self.setTextPen(pg.mkPen(hcolor(text_color)))
|
||||
self.text_color = text_color
|
||||
|
||||
# generate a bounding rect based on sizing to a "typical"
|
||||
# maximum length-ed string defined as init default.
|
||||
self.typical_br = _font._qfm.boundingRect(typical_max_str)
|
||||
|
||||
# size the pertinent axis dimension to a "typical value"
|
||||
self.size_to_values()
|
||||
|
||||
# NOTE: requires override ``.tickValues()`` method seen below.
|
||||
if lru_cache_tick_strings:
|
||||
self.tickStrings = lru_cache(
|
||||
maxsize=2**20
|
||||
)(self.tickStrings)
|
||||
|
||||
# axis "sticky" labels
|
||||
self._stickies: dict[str, YAxisLabel] = {}
|
||||
|
||||
# NOTE: only overriden to cast tick values entries into tuples
|
||||
# for use with the lru caching.
|
||||
def tickValues(
|
||||
self,
|
||||
minVal: float,
|
||||
maxVal: float,
|
||||
size: int,
|
||||
|
||||
) -> list[tuple[float, tuple[str]]]:
|
||||
'''
|
||||
Repack tick values into tuples for lru caching.
|
||||
|
||||
'''
|
||||
ticks = []
|
||||
for scalar, values in super().tickValues(minVal, maxVal, size):
|
||||
ticks.append((
|
||||
scalar,
|
||||
tuple(values), # this
|
||||
))
|
||||
|
||||
return ticks
|
||||
|
||||
@property
|
||||
def text_color(self) -> str:
|
||||
return self._text_color
|
||||
|
|
@ -151,38 +106,6 @@ class Axis(pg.AxisItem):
|
|||
def txt_offsets(self) -> tuple[int, int]:
|
||||
return tuple(self.style['tickTextOffset'])
|
||||
|
||||
def add_sticky(
|
||||
self,
|
||||
pi: pgo.PlotItem,
|
||||
name: None | str = None,
|
||||
digits: None | int = 2,
|
||||
bg_color='default',
|
||||
fg_color='black',
|
||||
|
||||
) -> YAxisLabel:
|
||||
|
||||
# if the sticky is for our symbol
|
||||
# use the tick size precision for display
|
||||
name = name or pi.name
|
||||
digits = digits or 2
|
||||
|
||||
# TODO: ``._ysticks`` should really be an attr on each
|
||||
# ``PlotItem`` now instead of the containing widget (because of
|
||||
# overlays) ?
|
||||
|
||||
# add y-axis "last" value label
|
||||
sticky = self._stickies[name] = YAxisLabel(
|
||||
pi=pi,
|
||||
parent=self,
|
||||
digits=digits, # TODO: pass this from symbol data
|
||||
opacity=0.9, # slight see-through
|
||||
bg_color=bg_color,
|
||||
fg_color=fg_color,
|
||||
)
|
||||
|
||||
pi.sigRangeChanged.connect(sticky.update_on_resize)
|
||||
return sticky
|
||||
|
||||
|
||||
class PriceAxis(Axis):
|
||||
|
||||
|
|
@ -191,7 +114,7 @@ class PriceAxis(Axis):
|
|||
*args,
|
||||
min_tick: int = 2,
|
||||
title: str = '',
|
||||
formatter: Callable[[float], str] | None = None,
|
||||
formatter: Optional[Callable[[float], str]] = None,
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
|
@ -203,8 +126,8 @@ class PriceAxis(Axis):
|
|||
def set_title(
|
||||
self,
|
||||
title: str,
|
||||
view: ChartView | None = None,
|
||||
color: str | None = None,
|
||||
view: Optional[ChartView] = None,
|
||||
color: Optional[str] = None,
|
||||
|
||||
) -> Label:
|
||||
'''
|
||||
|
|
@ -244,6 +167,7 @@ class PriceAxis(Axis):
|
|||
self._min_tick = size
|
||||
|
||||
def size_to_values(self) -> None:
|
||||
# self.typical_br = _font._qfm.boundingRect(typical_max_str)
|
||||
self.setWidth(self.typical_br.width())
|
||||
|
||||
# XXX: drop for now since it just eats up h space
|
||||
|
|
@ -298,53 +222,28 @@ class DynamicDateAxis(Axis):
|
|||
|
||||
) -> list[str]:
|
||||
|
||||
# XX: ARGGGGG AG:LKSKDJF:LKJSDFD
|
||||
chart = self.pi.chart_widget
|
||||
|
||||
viz = chart._vizs[chart.name]
|
||||
shm = viz.shm
|
||||
array = shm.array
|
||||
ifield = viz.index_field
|
||||
index = array[ifield]
|
||||
i_0, i_l = index[0], index[-1]
|
||||
|
||||
# edge cases
|
||||
if (
|
||||
not indexes
|
||||
or
|
||||
(indexes[0] < i_0
|
||||
and indexes[-1] < i_l)
|
||||
or
|
||||
(indexes[0] > i_0
|
||||
and indexes[-1] > i_l)
|
||||
):
|
||||
# print(f"x-label indexes empty edge case: {indexes}")
|
||||
return []
|
||||
|
||||
if ifield == 'index':
|
||||
arr_len = index.shape[0]
|
||||
chart = self.linkedsplits.chart
|
||||
flow = chart._flows[chart.name]
|
||||
shm = flow.shm
|
||||
bars = shm.array
|
||||
first = shm._first.value
|
||||
times = array['time']
|
||||
epochs = times[
|
||||
list(
|
||||
|
||||
bars_len = len(bars)
|
||||
times = bars['time']
|
||||
|
||||
epochs = times[list(
|
||||
map(
|
||||
int,
|
||||
filter(
|
||||
lambda i: i > 0 and i < arr_len,
|
||||
(i - first for i in indexes)
|
||||
lambda i: i > 0 and i < bars_len,
|
||||
(i-first for i in indexes)
|
||||
)
|
||||
)
|
||||
)
|
||||
]
|
||||
else:
|
||||
epochs = list(map(int, indexes))
|
||||
)]
|
||||
|
||||
# TODO: **don't** have this hard coded shift to EST
|
||||
# delay = times[-1] - times[-2]
|
||||
dts = np.array(
|
||||
epochs,
|
||||
dtype='datetime64[s]',
|
||||
)
|
||||
dts = np.array(epochs, dtype='datetime64[s]')
|
||||
|
||||
# see units listing:
|
||||
# https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
|
||||
|
|
@ -362,39 +261,24 @@ class DynamicDateAxis(Axis):
|
|||
spacing: float,
|
||||
|
||||
) -> list[str]:
|
||||
|
||||
return self._indexes_to_timestrs(values)
|
||||
|
||||
# NOTE: handy for debugging the lru cache
|
||||
# info = self.tickStrings.cache_info()
|
||||
# print(info)
|
||||
return self._indexes_to_timestrs(values)
|
||||
|
||||
|
||||
class AxisLabel(pg.GraphicsObject):
|
||||
|
||||
# relative offsets *OF* the bounding rect relative
|
||||
# to parent graphics object.
|
||||
# eg. <parent>| => <_x_br_offset> => | <text> |
|
||||
_x_br_offset: float = 0
|
||||
_y_br_offset: float = 0
|
||||
|
||||
# relative offsets of text *within* bounding rect
|
||||
# eg. | <_x_margin> => <text> |
|
||||
_x_margin: float = 0
|
||||
_y_margin: float = 0
|
||||
|
||||
# multiplier of the text content's height in order
|
||||
# to force a larger (y-dimension) bounding rect.
|
||||
_y_txt_h_scaling: float = 1
|
||||
_x_margin = 0
|
||||
_y_margin = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parent: pg.GraphicsItem,
|
||||
digits: int = 2,
|
||||
|
||||
bg_color: str = 'default',
|
||||
bg_color: str = 'bracket',
|
||||
fg_color: str = 'black',
|
||||
opacity: int = .8, # XXX: seriously don't set this to 0
|
||||
opacity: int = 1, # XXX: seriously don't set this to 0
|
||||
font_size: str = 'default',
|
||||
|
||||
use_arrow: bool = True,
|
||||
|
|
@ -405,7 +289,6 @@ class AxisLabel(pg.GraphicsObject):
|
|||
self.setParentItem(parent)
|
||||
|
||||
self.setFlag(self.ItemIgnoresTransformations)
|
||||
self.setZValue(100)
|
||||
|
||||
# XXX: pretty sure this is faster
|
||||
self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
|
@ -437,14 +320,14 @@ class AxisLabel(pg.GraphicsObject):
|
|||
p: QtGui.QPainter,
|
||||
opt: QtWidgets.QStyleOptionGraphicsItem,
|
||||
w: QtWidgets.QWidget
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Draw a filled rectangle based on the size of ``.label_str`` text.
|
||||
"""Draw a filled rectangle based on the size of ``.label_str`` text.
|
||||
|
||||
Subtypes can customize further by overloading ``.draw()``.
|
||||
|
||||
'''
|
||||
"""
|
||||
# p.setCompositionMode(QtWidgets.QPainter.CompositionMode_SourceOver)
|
||||
|
||||
if self.label_str:
|
||||
|
||||
# if not self.rect:
|
||||
|
|
@ -455,11 +338,7 @@ class AxisLabel(pg.GraphicsObject):
|
|||
|
||||
p.setFont(self._dpifont.font)
|
||||
p.setPen(self.fg_color)
|
||||
p.drawText(
|
||||
self.rect,
|
||||
self.text_flags,
|
||||
self.label_str,
|
||||
)
|
||||
p.drawText(self.rect, self.text_flags, self.label_str)
|
||||
|
||||
def draw(
|
||||
self,
|
||||
|
|
@ -467,8 +346,6 @@ class AxisLabel(pg.GraphicsObject):
|
|||
rect: QtCore.QRectF
|
||||
) -> None:
|
||||
|
||||
p.setOpacity(self.opacity)
|
||||
|
||||
if self._use_arrow:
|
||||
if not self.path:
|
||||
self._draw_arrow_path()
|
||||
|
|
@ -476,13 +353,15 @@ class AxisLabel(pg.GraphicsObject):
|
|||
p.drawPath(self.path)
|
||||
p.fillPath(self.path, pg.mkBrush(self.bg_color))
|
||||
|
||||
# this adds a nice black outline around the label for some odd
|
||||
# reason; ok by us
|
||||
p.setOpacity(self.opacity)
|
||||
|
||||
# this cause the L1 labels to glitch out if used in the subtype
|
||||
# and it will leave a small black strip with the arrow path if
|
||||
# done before the above
|
||||
p.fillRect(
|
||||
self.rect,
|
||||
self.bg_color,
|
||||
)
|
||||
p.fillRect(self.rect, self.bg_color)
|
||||
|
||||
|
||||
def boundingRect(self): # noqa
|
||||
'''
|
||||
|
|
@ -526,18 +405,15 @@ class AxisLabel(pg.GraphicsObject):
|
|||
txt_h, txt_w = txt_br.height(), txt_br.width()
|
||||
# print(f'wsw: {self._dpifont.boundingRect(" ")}')
|
||||
|
||||
# allow subtypes to override width and height
|
||||
# allow subtypes to specify a static width and height
|
||||
h, w = self.size_hint()
|
||||
# print(f'axis size: {self._parent.size()}')
|
||||
# print(f'axis geo: {self._parent.geometry()}')
|
||||
|
||||
self.rect = QtCore.QRectF(
|
||||
|
||||
# relative bounds offsets
|
||||
self._x_br_offset,
|
||||
self._y_br_offset,
|
||||
|
||||
0, 0,
|
||||
(w or txt_w) + self._x_margin / 2,
|
||||
|
||||
(h or txt_h) * self._y_txt_h_scaling + (self._y_margin / 2),
|
||||
(h or txt_h) + self._y_margin / 2,
|
||||
)
|
||||
# print(self.rect)
|
||||
# hb = self.path.controlPointRect()
|
||||
|
|
@ -613,7 +489,7 @@ class XAxisLabel(AxisLabel):
|
|||
|
||||
|
||||
class YAxisLabel(AxisLabel):
|
||||
_y_margin: int = 4
|
||||
_y_margin = 4
|
||||
|
||||
text_flags = (
|
||||
QtCore.Qt.AlignLeft
|
||||
|
|
@ -624,19 +500,19 @@ class YAxisLabel(AxisLabel):
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
pi: pgo.PlotItem,
|
||||
chart,
|
||||
*args,
|
||||
**kwargs
|
||||
) -> None:
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self._pi = pi
|
||||
pi.sigRangeChanged.connect(self.update_on_resize)
|
||||
self._chart = chart
|
||||
|
||||
chart.sigRangeChanged.connect(self.update_on_resize)
|
||||
|
||||
self._last_datum = (None, None)
|
||||
|
||||
self.x_offset = 0
|
||||
# pull text offset from axis from parent axis
|
||||
if getattr(self._parent, 'txt_offsets', False):
|
||||
self.x_offset, y_offset = self._parent.txt_offsets()
|
||||
|
|
@ -655,8 +531,7 @@ class YAxisLabel(AxisLabel):
|
|||
value: float, # data for text
|
||||
|
||||
# on odd dimension and/or adds nice black line
|
||||
x_offset: int = 0,
|
||||
|
||||
x_offset: Optional[int] = None
|
||||
) -> None:
|
||||
|
||||
# this is read inside ``.paint()``
|
||||
|
|
@ -702,7 +577,7 @@ class YAxisLabel(AxisLabel):
|
|||
self._last_datum = (index, value)
|
||||
|
||||
self.update_label(
|
||||
self._pi.mapFromView(QPointF(index, value)),
|
||||
self._chart.mapFromView(QPointF(index, value)),
|
||||
value
|
||||
)
|
||||
|
||||
|
|
|
|||
1016
piker/ui/_chart.py
1016
piker/ui/_chart.py
File diff suppressed because it is too large
Load Diff
|
|
@ -15,30 +15,17 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Graphics downsampling using the infamous M4 algorithm.
|
||||
|
||||
This is one of ``piker``'s secret weapons allowing us to boss all other
|
||||
charting platforms B)
|
||||
|
||||
(AND DON'T YOU DARE TAKE THIS CODE WITHOUT CREDIT OR WE'LL SUE UR F#&@* ASS).
|
||||
|
||||
NOTES: this method is a so called "visualization driven data
|
||||
aggregation" approach. It gives error-free line chart
|
||||
downsampling, see
|
||||
further scientific paper resources:
|
||||
- http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
- http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
|
||||
|
||||
Details on implementation of this algo are based in,
|
||||
https://github.com/pikers/piker/issues/109
|
||||
Graphics related downsampling routines for compressing to pixel
|
||||
limits on the display device.
|
||||
|
||||
'''
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
from numba import (
|
||||
njit,
|
||||
jit,
|
||||
# float64, optional, int64,
|
||||
)
|
||||
|
||||
|
|
@ -48,6 +35,109 @@ from ..log import get_logger
|
|||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
|
||||
'''
|
||||
Convert a OHLC struct-array containing 'high'/'low' columns
|
||||
to a "joined" max/min 1-d array.
|
||||
|
||||
'''
|
||||
index = ohlc['index']
|
||||
hls = ohlc[[
|
||||
'low',
|
||||
'high',
|
||||
]]
|
||||
|
||||
mxmn = np.empty(2*hls.size, dtype=np.float64)
|
||||
x = np.empty(2*hls.size, dtype=np.float64)
|
||||
trace_hl(hls, mxmn, x, index[0])
|
||||
x = x + index[0]
|
||||
|
||||
return mxmn, x
|
||||
|
||||
|
||||
@jit(
|
||||
# TODO: the type annots..
|
||||
# float64[:](float64[:],),
|
||||
nopython=True,
|
||||
)
|
||||
def trace_hl(
|
||||
hl: 'np.ndarray',
|
||||
out: np.ndarray,
|
||||
x: np.ndarray,
|
||||
start: int,
|
||||
|
||||
# the "offset" values in the x-domain which
|
||||
# place the 2 output points around each ``int``
|
||||
# master index.
|
||||
margin: float = 0.43,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
"Trace" the outline of the high-low values of an ohlc sequence
|
||||
as a line such that the maximum deviation (aka disperaion) between
|
||||
bars if preserved.
|
||||
|
||||
This routine is expected to modify input arrays in-place.
|
||||
|
||||
'''
|
||||
last_l = hl['low'][0]
|
||||
last_h = hl['high'][0]
|
||||
|
||||
for i in range(hl.size):
|
||||
row = hl[i]
|
||||
l, h = row['low'], row['high']
|
||||
|
||||
up_diff = h - last_l
|
||||
down_diff = last_h - l
|
||||
|
||||
if up_diff > down_diff:
|
||||
out[2*i + 1] = h
|
||||
out[2*i] = last_l
|
||||
else:
|
||||
out[2*i + 1] = l
|
||||
out[2*i] = last_h
|
||||
|
||||
last_l = l
|
||||
last_h = h
|
||||
|
||||
x[2*i] = int(i) - margin
|
||||
x[2*i + 1] = int(i) + margin
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def ohlc_flatten(
|
||||
ohlc: np.ndarray,
|
||||
use_mxmn: bool = True,
|
||||
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
'''
|
||||
Convert an OHLCV struct-array into a flat ready-for-line-plotting
|
||||
1-d array that is 4 times the size with x-domain values distributed
|
||||
evenly (by 0.5 steps) over each index.
|
||||
|
||||
'''
|
||||
index = ohlc['index']
|
||||
|
||||
if use_mxmn:
|
||||
# traces a line optimally over highs to lows
|
||||
# using numba. NOTE: pretty sure this is faster
|
||||
# and looks about the same as the below output.
|
||||
flat, x = hl2mxmn(ohlc)
|
||||
|
||||
else:
|
||||
flat = rfn.structured_to_unstructured(
|
||||
ohlc[['open', 'high', 'low', 'close']]
|
||||
).flatten()
|
||||
|
||||
x = np.linspace(
|
||||
start=index[0] - 0.5,
|
||||
stop=index[-1] + 0.5,
|
||||
num=len(flat),
|
||||
)
|
||||
return x, flat
|
||||
|
||||
|
||||
def ds_m4(
|
||||
x: np.ndarray,
|
||||
y: np.ndarray,
|
||||
|
|
@ -70,6 +160,16 @@ def ds_m4(
|
|||
This is more or less an OHLC style sampling of a line-style series.
|
||||
|
||||
'''
|
||||
# NOTE: this method is a so called "visualization driven data
|
||||
# aggregation" approach. It gives error-free line chart
|
||||
# downsampling, see
|
||||
# further scientific paper resources:
|
||||
# - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
# - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
|
||||
|
||||
# Details on implementation of this algo are based in,
|
||||
# https://github.com/pikers/piker/issues/109
|
||||
|
||||
# XXX: from infinite on downsampling viewable graphics:
|
||||
# "one thing i remembered about the binning - if you are
|
||||
# picking a range within your timeseries the start and end bin
|
||||
|
|
@ -91,14 +191,6 @@ def ds_m4(
|
|||
x_end = x[-1] # x end value/highest in domain
|
||||
xrange = (x_end - x_start)
|
||||
|
||||
if xrange < 0:
|
||||
log.error(f'-VE M4 X-RANGE: {x_start} -> {x_end}')
|
||||
# XXX: broken x-range calc-case, likely the x-end points
|
||||
# are wrong and have some default value set (such as
|
||||
# x_end -> <some epoch float> while x_start -> 0.5).
|
||||
# breakpoint()
|
||||
return None
|
||||
|
||||
# XXX: always round up on the input pixels
|
||||
# lnx = len(x)
|
||||
# uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
|
||||
|
|
@ -131,20 +223,14 @@ def ds_m4(
|
|||
assert frames >= (xrange / uppx)
|
||||
|
||||
# call into ``numba``
|
||||
(
|
||||
nb,
|
||||
x_out,
|
||||
y_out,
|
||||
ymn,
|
||||
ymx,
|
||||
) = _m4(
|
||||
nb, i_win, y_out = _m4(
|
||||
x,
|
||||
y,
|
||||
|
||||
frames,
|
||||
|
||||
# TODO: see func below..
|
||||
# x_out,
|
||||
# i_win,
|
||||
# y_out,
|
||||
|
||||
# first index in x data to start at
|
||||
|
|
@ -157,14 +243,14 @@ def ds_m4(
|
|||
# filter out any overshoot in the input allocation arrays by
|
||||
# removing zero-ed tail entries which should start at a certain
|
||||
# index.
|
||||
x_out = x_out[x_out != 0]
|
||||
y_out = y_out[:x_out.size]
|
||||
i_win = i_win[i_win != 0]
|
||||
y_out = y_out[:i_win.size]
|
||||
|
||||
# print(f'M4 output ymn, ymx: {ymn},{ymx}')
|
||||
return nb, x_out, y_out, ymn, ymx
|
||||
return nb, i_win, y_out
|
||||
|
||||
|
||||
@njit(
|
||||
@jit(
|
||||
nopython=True,
|
||||
nogil=True,
|
||||
)
|
||||
def _m4(
|
||||
|
|
@ -174,8 +260,8 @@ def _m4(
|
|||
|
||||
frames: int,
|
||||
|
||||
# TODO: using this approach, having the ``.zeros()`` alloc lines
|
||||
# below in pure python, there were segs faults and alloc crashes..
|
||||
# TODO: using this approach by having the ``.zeros()`` alloc lines
|
||||
# below, in put python was causing segs faults and alloc crashes..
|
||||
# we might need to see how it behaves with shm arrays and consider
|
||||
# allocating them once at startup?
|
||||
|
||||
|
|
@ -188,22 +274,14 @@ def _m4(
|
|||
x_start: int,
|
||||
step: float,
|
||||
|
||||
) -> tuple[
|
||||
int,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
float,
|
||||
float,
|
||||
]:
|
||||
'''
|
||||
Implementation of the m4 algorithm in ``numba``:
|
||||
http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
) -> int:
|
||||
# nbins = len(i_win)
|
||||
# count = len(xs)
|
||||
|
||||
'''
|
||||
# these are pre-allocated and mutated by ``numba``
|
||||
# code in-place.
|
||||
y_out = np.zeros((frames, 4), ys.dtype)
|
||||
x_out = np.zeros(frames, xs.dtype)
|
||||
i_win = np.zeros(frames, xs.dtype)
|
||||
|
||||
bincount = 0
|
||||
x_left = x_start
|
||||
|
|
@ -217,34 +295,24 @@ def _m4(
|
|||
|
||||
# set all bins in the left-most entry to the starting left-most x value
|
||||
# (aka a row broadcast).
|
||||
x_out[bincount] = x_left
|
||||
i_win[bincount] = x_left
|
||||
# set all y-values to the first value passed in.
|
||||
y_out[bincount] = ys[0]
|
||||
|
||||
# full input y-data mx and mn
|
||||
mx: float = -np.inf
|
||||
mn: float = np.inf
|
||||
|
||||
# compute OHLC style max / min values per window sized x-frame.
|
||||
for i in range(len(xs)):
|
||||
|
||||
x = xs[i]
|
||||
y = ys[i]
|
||||
|
||||
if x < x_left + step: # the current window "step" is [bin, bin+1)
|
||||
ymn = y_out[bincount, 1] = min(y, y_out[bincount, 1])
|
||||
ymx = y_out[bincount, 2] = max(y, y_out[bincount, 2])
|
||||
y_out[bincount, 1] = min(y, y_out[bincount, 1])
|
||||
y_out[bincount, 2] = max(y, y_out[bincount, 2])
|
||||
y_out[bincount, 3] = y
|
||||
mx = max(mx, ymx)
|
||||
mn = min(mn, ymn)
|
||||
|
||||
else:
|
||||
# Find the next bin
|
||||
while x >= x_left + step:
|
||||
x_left += step
|
||||
|
||||
bincount += 1
|
||||
x_out[bincount] = x_left
|
||||
i_win[bincount] = x_left
|
||||
y_out[bincount] = y
|
||||
|
||||
return bincount, x_out, y_out, mn, mx
|
||||
return bincount, i_win, y_out
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,12 +18,8 @@
|
|||
Mouse interaction graphics
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from functools import partial
|
||||
from typing import (
|
||||
Callable,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import Optional, Callable
|
||||
|
||||
import inspect
|
||||
import numpy as np
|
||||
|
|
@ -37,18 +33,9 @@ from ._style import (
|
|||
_font_small,
|
||||
_font,
|
||||
)
|
||||
from ._axes import (
|
||||
YAxisLabel,
|
||||
XAxisLabel,
|
||||
)
|
||||
from ._axes import YAxisLabel, XAxisLabel
|
||||
from ..log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._chart import (
|
||||
ChartPlotWidget,
|
||||
LinkedSplits,
|
||||
)
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -71,9 +58,9 @@ class LineDot(pg.CurvePoint):
|
|||
curve: pg.PlotCurveItem,
|
||||
index: int,
|
||||
|
||||
plot: ChartPlotWidget, # type: ingore # noqa
|
||||
plot: 'ChartPlotWidget', # type: ingore # noqa
|
||||
pos=None,
|
||||
color: str = 'bracket',
|
||||
color: str = 'default_light',
|
||||
|
||||
) -> None:
|
||||
# scale from dpi aware font size
|
||||
|
|
@ -164,12 +151,12 @@ class ContentsLabel(pg.LabelItem):
|
|||
def __init__(
|
||||
self,
|
||||
|
||||
# chart: ChartPlotWidget, # noqa
|
||||
# chart: 'ChartPlotWidget', # noqa
|
||||
view: pg.ViewBox,
|
||||
|
||||
anchor_at: str = ('top', 'right'),
|
||||
justify_text: str = 'left',
|
||||
font_size: int | None = None,
|
||||
font_size: Optional[int] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -200,11 +187,12 @@ class ContentsLabel(pg.LabelItem):
|
|||
self,
|
||||
|
||||
name: str,
|
||||
ix: int,
|
||||
index: int,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> None:
|
||||
# this being "html" is the dumbest shit :eyeroll:
|
||||
first = array[0]['index']
|
||||
|
||||
self.setText(
|
||||
"<b>i</b>:{index}<br/>"
|
||||
|
|
@ -217,7 +205,7 @@ class ContentsLabel(pg.LabelItem):
|
|||
"<b>C</b>:{}<br/>"
|
||||
"<b>V</b>:{}<br/>"
|
||||
"<b>wap</b>:{}".format(
|
||||
*array[ix][
|
||||
*array[index - first][
|
||||
[
|
||||
'time',
|
||||
'open',
|
||||
|
|
@ -229,7 +217,7 @@ class ContentsLabel(pg.LabelItem):
|
|||
]
|
||||
],
|
||||
name=name,
|
||||
index=ix,
|
||||
index=index,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -237,11 +225,14 @@ class ContentsLabel(pg.LabelItem):
|
|||
self,
|
||||
|
||||
name: str,
|
||||
ix: int,
|
||||
index: int,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> None:
|
||||
data = array[ix][name]
|
||||
|
||||
first = array[0]['index']
|
||||
if index < array[-1]['index'] and index > first:
|
||||
data = array[index - first][name]
|
||||
self.setText(f"{name}: {data:.2f}")
|
||||
|
||||
|
||||
|
|
@ -253,7 +244,7 @@ class ContentsLabels:
|
|||
'''
|
||||
def __init__(
|
||||
self,
|
||||
linkedsplits: LinkedSplits, # type: ignore # noqa
|
||||
linkedsplits: 'LinkedSplits', # type: ignore # noqa
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -267,20 +258,17 @@ class ContentsLabels:
|
|||
|
||||
def update_labels(
|
||||
self,
|
||||
x_in: int,
|
||||
index: int,
|
||||
|
||||
) -> None:
|
||||
for chart, name, label, update in self._labels:
|
||||
|
||||
viz = chart.get_viz(name)
|
||||
array = viz.shm.array
|
||||
index = array[viz.index_field]
|
||||
start = index[0]
|
||||
stop = index[-1]
|
||||
flow = chart._flows[name]
|
||||
array = flow.shm.array
|
||||
|
||||
if not (
|
||||
x_in >= start
|
||||
and x_in <= stop
|
||||
index >= 0
|
||||
and index < array[-1]['index']
|
||||
):
|
||||
# out of range
|
||||
print('WTF out of range?')
|
||||
|
|
@ -289,10 +277,7 @@ class ContentsLabels:
|
|||
# call provided update func with data point
|
||||
try:
|
||||
label.show()
|
||||
ix = np.searchsorted(index, x_in)
|
||||
if ix > len(array):
|
||||
breakpoint()
|
||||
update(ix, array)
|
||||
update(index, array)
|
||||
|
||||
except IndexError:
|
||||
log.exception(f"Failed to update label: {name}")
|
||||
|
|
@ -304,7 +289,7 @@ class ContentsLabels:
|
|||
def add_label(
|
||||
|
||||
self,
|
||||
chart: ChartPlotWidget, # type: ignore # noqa
|
||||
chart: 'ChartPlotWidget', # type: ignore # noqa
|
||||
name: str,
|
||||
anchor_at: tuple[str, str] = ('top', 'left'),
|
||||
update_func: Callable = ContentsLabel.update_from_value,
|
||||
|
|
@ -331,7 +316,7 @@ class Cursor(pg.GraphicsObject):
|
|||
def __init__(
|
||||
|
||||
self,
|
||||
linkedsplits: LinkedSplits, # noqa
|
||||
linkedsplits: 'LinkedSplits', # noqa
|
||||
digits: int = 0
|
||||
|
||||
) -> None:
|
||||
|
|
@ -340,8 +325,6 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
self.linked = linkedsplits
|
||||
self.graphics: dict[str, pg.GraphicsObject] = {}
|
||||
self.xaxis_label: XAxisLabel | None = None
|
||||
self.always_show_xlabel: bool = True
|
||||
self.plots: list['PlotChartWidget'] = [] # type: ignore # noqa
|
||||
self.active_plot = None
|
||||
self.digits: int = digits
|
||||
|
|
@ -353,7 +336,7 @@ class Cursor(pg.GraphicsObject):
|
|||
# XXX: not sure why these are instance variables?
|
||||
# It's not like we can change them on the fly..?
|
||||
self.pen = pg.mkPen(
|
||||
color=hcolor('bracket'),
|
||||
color=hcolor('default'),
|
||||
style=QtCore.Qt.DashLine,
|
||||
)
|
||||
self.lines_pen = pg.mkPen(
|
||||
|
|
@ -363,14 +346,13 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
# value used for rounding y-axis discreet tick steps
|
||||
# computing once, up front, here cuz why not
|
||||
mkt = self.linked._symbol
|
||||
self._y_tick_mult = 1/float(mkt.price_tick)
|
||||
self._y_incr_mult = 1 / self.linked._symbol.tick_size
|
||||
|
||||
# line width in view coordinates
|
||||
self._lw = self.pixelWidth() * self.lines_pen.width()
|
||||
|
||||
# xhair label's color name
|
||||
self.label_color: str = 'bracket'
|
||||
self.label_color: str = 'default'
|
||||
|
||||
self._y_label_update: bool = True
|
||||
|
||||
|
|
@ -403,7 +385,7 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
def add_plot(
|
||||
self,
|
||||
plot: ChartPlotWidget, # noqa
|
||||
plot: 'ChartPlotWidget', # noqa
|
||||
digits: int = 0,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -423,7 +405,7 @@ class Cursor(pg.GraphicsObject):
|
|||
hl.hide()
|
||||
|
||||
yl = YAxisLabel(
|
||||
pi=plot.plotItem,
|
||||
chart=plot,
|
||||
# parent=plot.getAxis('right'),
|
||||
parent=plot.pi_overlay.get_axis(plot.plotItem, 'right'),
|
||||
digits=digits or self.digits,
|
||||
|
|
@ -487,58 +469,39 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
def add_curve_cursor(
|
||||
self,
|
||||
chart: ChartPlotWidget, # noqa
|
||||
plot: 'ChartPlotWidget', # noqa
|
||||
curve: 'PlotCurveItem', # noqa
|
||||
|
||||
) -> LineDot:
|
||||
# if this chart contains curves add line dot "cursors" to denote
|
||||
# if this plot contains curves add line dot "cursors" to denote
|
||||
# the current sample under the mouse
|
||||
main_viz = chart.get_viz(chart.name)
|
||||
|
||||
main_flow = plot._flows[plot.name]
|
||||
# read out last index
|
||||
i = main_viz.shm.array[-1]['index']
|
||||
i = main_flow.shm.array[-1]['index']
|
||||
cursor = LineDot(
|
||||
curve,
|
||||
index=i,
|
||||
plot=chart
|
||||
plot=plot
|
||||
)
|
||||
chart.addItem(cursor)
|
||||
self.graphics[chart].setdefault('cursors', []).append(cursor)
|
||||
plot.addItem(cursor)
|
||||
self.graphics[plot].setdefault('cursors', []).append(cursor)
|
||||
return cursor
|
||||
|
||||
def mouseAction(
|
||||
self,
|
||||
action: str,
|
||||
plot: ChartPlotWidget,
|
||||
|
||||
) -> None: # noqa
|
||||
|
||||
def mouseAction(self, action, plot): # noqa
|
||||
log.debug(f"{(action, plot.name)}")
|
||||
if action == 'Enter':
|
||||
self.active_plot = plot
|
||||
plot.linked.godwidget._active_cursor = self
|
||||
|
||||
# show horiz line and y-label
|
||||
self.graphics[plot]['hl'].show()
|
||||
self.graphics[plot]['yl'].show()
|
||||
|
||||
if (
|
||||
not self.always_show_xlabel
|
||||
and not self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.show()
|
||||
else: # Leave
|
||||
|
||||
# Leave: hide horiz line and y-label
|
||||
else:
|
||||
# hide horiz line and y-label
|
||||
self.graphics[plot]['hl'].hide()
|
||||
self.graphics[plot]['yl'].hide()
|
||||
|
||||
if (
|
||||
not self.always_show_xlabel
|
||||
and self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.hide()
|
||||
|
||||
def mouseMoved(
|
||||
self,
|
||||
coords: tuple[QPointF], # noqa
|
||||
|
|
@ -572,15 +535,9 @@ class Cursor(pg.GraphicsObject):
|
|||
line_offset = self._lw / 2
|
||||
|
||||
# round y value to nearest tick step
|
||||
m = self._y_tick_mult
|
||||
m = self._y_incr_mult
|
||||
iy = round(y * m) / m
|
||||
vl_y = iy - line_offset
|
||||
# print(
|
||||
# f'tick: {self._y_tick}\n'
|
||||
# f'y: {y}\n'
|
||||
# f'iy: {iy}\n'
|
||||
# f'vl_y: {vl_y}\n'
|
||||
# )
|
||||
|
||||
# update y-range items
|
||||
if iy != last_iy:
|
||||
|
|
@ -633,10 +590,6 @@ class Cursor(pg.GraphicsObject):
|
|||
left_axis_width += left.width()
|
||||
|
||||
# map back to abs (label-local) coordinates
|
||||
if (
|
||||
self.always_show_xlabel
|
||||
or self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.update_label(
|
||||
abs_pos=(
|
||||
plot.mapFromView(QPointF(vl_x, iy)) -
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ Fast, smooth, sexy curves.
|
|||
|
||||
"""
|
||||
from contextlib import contextmanager as cm
|
||||
from typing import Callable
|
||||
from typing import Optional, Callable
|
||||
|
||||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
|
|
@ -28,7 +28,10 @@ from PyQt5.QtWidgets import QGraphicsItem
|
|||
from PyQt5.QtCore import (
|
||||
Qt,
|
||||
QLineF,
|
||||
QSizeF,
|
||||
QRectF,
|
||||
# QRect,
|
||||
QPointF,
|
||||
)
|
||||
from PyQt5.QtGui import (
|
||||
QPainter,
|
||||
|
|
@ -36,8 +39,11 @@ from PyQt5.QtGui import (
|
|||
)
|
||||
from .._profile import pg_profile_enabled, ms_slower_then
|
||||
from ._style import hcolor
|
||||
# from ._compression import (
|
||||
# # ohlc_to_m4_line,
|
||||
# ds_m4,
|
||||
# )
|
||||
from ..log import get_logger
|
||||
from .._profile import Profiler
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
@ -51,144 +57,11 @@ _line_styles: dict[str, int] = {
|
|||
}
|
||||
|
||||
|
||||
class FlowGraphic(pg.GraphicsObject):
|
||||
'''
|
||||
Base class with minimal interface for `QPainterPath` implemented,
|
||||
real-time updated "data flow" graphics.
|
||||
|
||||
See subtypes below.
|
||||
|
||||
'''
|
||||
# sub-type customization methods
|
||||
declare_paintables: Callable | None = None
|
||||
sub_paint: Callable | None = None
|
||||
|
||||
# XXX-NOTE-XXX: graphics caching B)
|
||||
# see explanation for different caching modes:
|
||||
# https://stackoverflow.com/a/39410081
|
||||
cache_mode: int = QGraphicsItem.DeviceCoordinateCache
|
||||
# XXX: WARNING item caching seems to only be useful
|
||||
# if we don't re-generate the entire QPainterPath every time
|
||||
# don't ever use this - it's a colossal nightmare of artefacts
|
||||
# and is disastrous for performance.
|
||||
# QGraphicsItem.ItemCoordinateCache
|
||||
# TODO: still questions todo with coord-cacheing that we should
|
||||
# probably talk to a core dev about:
|
||||
# - if this makes trasform interactions slower (such as zooming)
|
||||
# and if so maybe if/when we implement a "history" mode for the
|
||||
# view we disable this in that mode?
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
name: str | None = None,
|
||||
|
||||
# line styling
|
||||
color: str = 'bracket',
|
||||
last_step_color: str | None = None,
|
||||
fill_color: str | None = None,
|
||||
style: str = 'solid',
|
||||
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
||||
self._name = name
|
||||
|
||||
# primary graphics item used for history
|
||||
self.path: QPainterPath = QPainterPath()
|
||||
|
||||
# additional path that can be optionally used for appends which
|
||||
# tries to avoid triggering an update/redraw of the presumably
|
||||
# larger historical ``.path`` above. the flag to enable
|
||||
# this behaviour is found in `Renderer.render()`.
|
||||
self.fast_path: QPainterPath | None = None
|
||||
|
||||
# TODO: evaluating the path capacity stuff and see
|
||||
# if it really makes much diff pre-allocating it.
|
||||
# self._last_cap: int = 0
|
||||
# cap = path.capacity()
|
||||
# if cap != self._last_cap:
|
||||
# print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
|
||||
# self._last_cap = cap
|
||||
|
||||
# all history of curve is drawn in single px thickness
|
||||
self._color: str = color
|
||||
pen = pg.mkPen(hcolor(color), width=1)
|
||||
pen.setStyle(_line_styles[style])
|
||||
|
||||
if 'dash' in style:
|
||||
pen.setDashPattern([8, 3])
|
||||
|
||||
self._pen = pen
|
||||
self._brush = pg.functions.mkBrush(
|
||||
hcolor(fill_color or color)
|
||||
)
|
||||
|
||||
# last segment is drawn in 2px thickness for emphasis
|
||||
if last_step_color:
|
||||
self.last_step_pen = pg.mkPen(
|
||||
hcolor(last_step_color),
|
||||
width=2,
|
||||
)
|
||||
else:
|
||||
self.last_step_pen = pg.mkPen(
|
||||
self._pen,
|
||||
width=2,
|
||||
)
|
||||
|
||||
self._last_line: QLineF = QLineF()
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# apply cache mode
|
||||
self.setCacheMode(self.cache_mode)
|
||||
|
||||
def x_uppx(self) -> int:
|
||||
|
||||
px_vecs = self.pixelVectors()[0]
|
||||
if px_vecs:
|
||||
return px_vecs.x()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def x_last(self) -> float | None:
|
||||
'''
|
||||
Return the last most x value of the last line segment or if not
|
||||
drawn yet, ``None``.
|
||||
|
||||
'''
|
||||
if self._last_line:
|
||||
return self._last_line.x1()
|
||||
|
||||
return None
|
||||
|
||||
# XXX: due to a variety of weird jitter bugs and "smearing"
|
||||
# artifacts when click-drag panning and viewing history time series,
|
||||
# we offer this ctx-mngr interface to allow temporarily disabling
|
||||
# Qt's graphics caching mode; this is now currently used from
|
||||
# ``ChartView.start/signal_ic()`` methods which also disable the
|
||||
# rt-display loop when the user is moving around a view.
|
||||
@cm
|
||||
def reset_cache(self) -> None:
|
||||
try:
|
||||
none = QGraphicsItem.NoCache
|
||||
log.debug(
|
||||
f'{self._name} -> CACHE DISABLE: {none}'
|
||||
)
|
||||
self.setCacheMode(none)
|
||||
yield
|
||||
finally:
|
||||
mode = self.cache_mode
|
||||
log.debug(f'{self._name} -> CACHE ENABLE {mode}')
|
||||
self.setCacheMode(mode)
|
||||
|
||||
|
||||
class Curve(FlowGraphic):
|
||||
class Curve(pg.GraphicsObject):
|
||||
'''
|
||||
A faster, simpler, append friendly version of
|
||||
``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
|
||||
updates; a graphics object to render a simple "line" plot.
|
||||
updates.
|
||||
|
||||
This type is a much stripped down version of a ``pyqtgraph`` style
|
||||
"graphics object" in the sense that the internal lower level
|
||||
|
|
@ -201,7 +74,7 @@ class Curve(FlowGraphic):
|
|||
lower level graphics data can be rendered in different threads and
|
||||
then read and drawn in this main thread without having to worry
|
||||
about dealing with Qt's concurrency primitives. See
|
||||
``piker.ui._render.Renderer`` for details and logic related to lower
|
||||
``piker.ui._flows.Renderer`` for details and logic related to lower
|
||||
level path generation and incremental update. The main differences in
|
||||
the path generation code include:
|
||||
|
||||
|
|
@ -213,38 +86,127 @@ class Curve(FlowGraphic):
|
|||
updates don't trigger a full path redraw.
|
||||
|
||||
'''
|
||||
# TODO: can we remove this?
|
||||
# sub_br: Callable | None = None
|
||||
|
||||
# sub-type customization methods
|
||||
sub_br: Optional[Callable] = None
|
||||
sub_paint: Optional[Callable] = None
|
||||
declare_paintables: Optional[Callable] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
|
||||
# color: str = 'default_lightest',
|
||||
# fill_color: str | None = None,
|
||||
# style: str = 'solid',
|
||||
step_mode: bool = False,
|
||||
color: str = 'default_lightest',
|
||||
fill_color: Optional[str] = None,
|
||||
style: str = 'solid',
|
||||
name: Optional[str] = None,
|
||||
use_fpath: bool = True,
|
||||
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
||||
self._name = name
|
||||
|
||||
# brutaaalll, see comments within..
|
||||
self.yData = None
|
||||
self.xData = None
|
||||
|
||||
# self._last_cap: int = 0
|
||||
self.path: Optional[QPainterPath] = None
|
||||
|
||||
# additional path used for appends which tries to avoid
|
||||
# triggering an update/redraw of the presumably larger
|
||||
# historical ``.path`` above.
|
||||
self.use_fpath = use_fpath
|
||||
self.fast_path: Optional[QPainterPath] = None
|
||||
|
||||
# TODO: we can probably just dispense with the parent since
|
||||
# we're basically only using the pen setting now...
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self._last_line: QLineF = QLineF()
|
||||
# all history of curve is drawn in single px thickness
|
||||
pen = pg.mkPen(hcolor(color))
|
||||
pen.setStyle(_line_styles[style])
|
||||
|
||||
if 'dash' in style:
|
||||
pen.setDashPattern([8, 3])
|
||||
|
||||
self._pen = pen
|
||||
|
||||
# last segment is drawn in 2px thickness for emphasis
|
||||
# self.last_step_pen = pg.mkPen(hcolor(color), width=2)
|
||||
self.last_step_pen = pg.mkPen(pen, width=2)
|
||||
|
||||
# self._last_line: Optional[QLineF] = None
|
||||
self._last_line = QLineF()
|
||||
self._last_w: float = 1
|
||||
|
||||
# flat-top style histogram-like discrete curve
|
||||
# self._step_mode: bool = step_mode
|
||||
|
||||
# self._fill = True
|
||||
self._brush = pg.functions.mkBrush(hcolor(fill_color or color))
|
||||
|
||||
# NOTE: this setting seems to mostly prevent redraws on mouse
|
||||
# interaction which is a huge boon for avg interaction latency.
|
||||
|
||||
# TODO: one question still remaining is if this makes trasform
|
||||
# interactions slower (such as zooming) and if so maybe if/when
|
||||
# we implement a "history" mode for the view we disable this in
|
||||
# that mode?
|
||||
# don't enable caching by default for the case where the
|
||||
# only thing drawn is the "last" line segment which can
|
||||
# have a weird artifact where it won't be fully drawn to its
|
||||
# endpoint (something we saw on trade rate curves)
|
||||
self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
# XXX: see explanation for different caching modes:
|
||||
# https://stackoverflow.com/a/39410081
|
||||
# seems to only be useful if we don't re-generate the entire
|
||||
# QPainterPath every time
|
||||
# curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
# don't ever use this - it's a colossal nightmare of artefacts
|
||||
# and is disastrous for performance.
|
||||
# curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
|
||||
|
||||
# allow sub-type customization
|
||||
declare = self.declare_paintables
|
||||
if declare:
|
||||
declare()
|
||||
|
||||
# TODO: probably stick this in a new parent
|
||||
# type which will contain our own version of
|
||||
# what ``PlotCurveItem`` had in terms of base
|
||||
# functionality? A `FlowGraphic` maybe?
|
||||
def x_uppx(self) -> int:
|
||||
|
||||
px_vecs = self.pixelVectors()[0]
|
||||
if px_vecs:
|
||||
xs_in_px = px_vecs.x()
|
||||
return round(xs_in_px)
|
||||
else:
|
||||
return 0
|
||||
|
||||
def px_width(self) -> float:
|
||||
|
||||
vb = self.getViewBox()
|
||||
if not vb:
|
||||
return 0
|
||||
|
||||
vr = self.viewRect()
|
||||
l, r = int(vr.left()), int(vr.right())
|
||||
|
||||
start, stop = self._xrange
|
||||
lbar = max(l, start)
|
||||
rbar = min(r, stop)
|
||||
|
||||
return vb.mapViewToDevice(
|
||||
QLineF(lbar, 0, rbar, 0)
|
||||
).length()
|
||||
|
||||
# XXX: lol brutal, the internals of `CurvePoint` (inherited by
|
||||
# our `LineDot`) required ``.getData()`` to work..
|
||||
def getData(self):
|
||||
|
|
@ -268,8 +230,14 @@ class Curve(FlowGraphic):
|
|||
self.path.clear()
|
||||
|
||||
if self.fast_path:
|
||||
self.fast_path.clear()
|
||||
# self.fast_path = None
|
||||
# self.fast_path.clear()
|
||||
self.fast_path = None
|
||||
|
||||
@cm
|
||||
def reset_cache(self) -> None:
|
||||
self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
|
||||
yield
|
||||
self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
def boundingRect(self):
|
||||
'''
|
||||
|
|
@ -283,65 +251,77 @@ class Curve(FlowGraphic):
|
|||
self.boundingRect = self._path_br
|
||||
return self._path_br()
|
||||
|
||||
# Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
|
||||
def _path_br(self):
|
||||
'''
|
||||
Post init ``.boundingRect()```.
|
||||
|
||||
'''
|
||||
# profiler = Profiler(
|
||||
# msg=f'Curve.boundingRect(): `{self._name}`',
|
||||
# disabled=not pg_profile_enabled(),
|
||||
# ms_threshold=ms_slower_then,
|
||||
# hb = self.path.boundingRect()
|
||||
hb = self.path.controlPointRect()
|
||||
hb_size = hb.size()
|
||||
|
||||
fp = self.fast_path
|
||||
if fp:
|
||||
fhb = fp.controlPointRect()
|
||||
hb_size = fhb.size() + hb_size
|
||||
|
||||
# print(f'hb_size: {hb_size}')
|
||||
|
||||
# if self._last_step_rect:
|
||||
# hb_size += self._last_step_rect.size()
|
||||
|
||||
# if self._line:
|
||||
# br = self._last_step_rect.bottomRight()
|
||||
|
||||
# tl = QPointF(
|
||||
# # self._vr[0],
|
||||
# # hb.topLeft().y(),
|
||||
# # 0,
|
||||
# # hb_size.height() + 1
|
||||
# )
|
||||
pr = self.path.controlPointRect()
|
||||
hb_tl, hb_br = (
|
||||
pr.topLeft(),
|
||||
pr.bottomRight(),
|
||||
)
|
||||
mn_y = hb_tl.y()
|
||||
mx_y = hb_br.y()
|
||||
most_left = hb_tl.x()
|
||||
most_right = hb_br.x()
|
||||
# profiler('calc path vertices')
|
||||
|
||||
# TODO: if/when we get fast path appends working in the
|
||||
# `Renderer`, then we might need to actually use this..
|
||||
# fp = self.fast_path
|
||||
# if fp:
|
||||
# fhb = fp.controlPointRect()
|
||||
# # hb_size = fhb.size() + hb_size
|
||||
# br = pr.united(fhb)
|
||||
# br = self._last_step_rect.bottomRight()
|
||||
|
||||
# XXX: *was* a way to allow sub-types to extend the
|
||||
# boundingrect calc, but in the one use case for a step curve
|
||||
# doesn't seem like we need it as long as the last line segment
|
||||
# is drawn as it is?
|
||||
|
||||
# sbr = self.sub_br
|
||||
# if sbr:
|
||||
# # w, h = self.sub_br(w, h)
|
||||
# sub_br = sbr()
|
||||
# br = br.united(sub_br)
|
||||
w = hb_size.width()
|
||||
h = hb_size.height()
|
||||
|
||||
sbr = self.sub_br
|
||||
if sbr:
|
||||
w, h = self.sub_br(w, h)
|
||||
else:
|
||||
# assume plain line graphic and use
|
||||
# default unit step in each direction.
|
||||
ll = self._last_line
|
||||
y1, y2 = ll.y1(), ll.y2()
|
||||
x1, x2 = ll.x1(), ll.x2()
|
||||
|
||||
ymn = min(y1, y2, mn_y)
|
||||
ymx = max(y1, y2, mx_y)
|
||||
most_left = min(x1, x2, most_left)
|
||||
most_right = max(x1, x2, most_right)
|
||||
# profiler('calc last line vertices')
|
||||
# only on a plane line do we include
|
||||
# and extra index step's worth of width
|
||||
# since in the step case the end of the curve
|
||||
# actually terminates earlier so we don't need
|
||||
# this for the last step.
|
||||
w += self._last_w
|
||||
# ll = self._last_line
|
||||
h += 1 # ll.y2() - ll.y1()
|
||||
|
||||
return QRectF(
|
||||
most_left,
|
||||
ymn,
|
||||
most_right - most_left + 1,
|
||||
ymx,
|
||||
# br = QPointF(
|
||||
# self._vr[-1],
|
||||
# # tl.x() + w,
|
||||
# tl.y() + h,
|
||||
# )
|
||||
|
||||
br = QRectF(
|
||||
|
||||
# top left
|
||||
# hb.topLeft()
|
||||
# tl,
|
||||
QPointF(hb.topLeft()),
|
||||
|
||||
# br,
|
||||
# total size
|
||||
# QSizeF(hb_size)
|
||||
# hb_size,
|
||||
QSizeF(w, h)
|
||||
)
|
||||
# print(f'bounding rect: {br}')
|
||||
return br
|
||||
|
||||
def paint(
|
||||
self,
|
||||
|
|
@ -351,7 +331,7 @@ class Curve(FlowGraphic):
|
|||
|
||||
) -> None:
|
||||
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
msg=f'Curve.paint(): `{self._name}`',
|
||||
disabled=not pg_profile_enabled(),
|
||||
ms_threshold=ms_slower_then,
|
||||
|
|
@ -359,14 +339,18 @@ class Curve(FlowGraphic):
|
|||
|
||||
sub_paint = self.sub_paint
|
||||
if sub_paint:
|
||||
sub_paint(p)
|
||||
sub_paint(p, profiler)
|
||||
|
||||
p.setPen(self.last_step_pen)
|
||||
p.drawLine(self._last_line)
|
||||
profiler('last datum `.drawLine()`')
|
||||
|
||||
profiler('.drawLine()')
|
||||
p.setPen(self._pen)
|
||||
|
||||
path = self.path
|
||||
# cap = path.capacity()
|
||||
# if cap != self._last_cap:
|
||||
# print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
|
||||
# self._last_cap = cap
|
||||
|
||||
if path:
|
||||
p.drawPath(path)
|
||||
|
|
@ -389,29 +373,22 @@ class Curve(FlowGraphic):
|
|||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
) -> None:
|
||||
# default line draw last call
|
||||
x = src_data[index_field]
|
||||
y = src_data[array_key]
|
||||
|
||||
x_last = x[-1]
|
||||
x_2last = x[-2]
|
||||
# with self.reset_cache():
|
||||
x = render_data['index']
|
||||
y = render_data[array_key]
|
||||
|
||||
# draw the "current" step graphic segment so it
|
||||
# lines up with the "middle" of the current
|
||||
# (OHLC) sample.
|
||||
self._last_line = QLineF(
|
||||
|
||||
# NOTE: currently we draw in x-domain
|
||||
# from last datum to current such that
|
||||
# the end of line touches the "beginning"
|
||||
# of the current datum step span.
|
||||
x_2last, y[-2],
|
||||
x_last, y[-1],
|
||||
x[-2], y[-2],
|
||||
x[-1], y[-1],
|
||||
)
|
||||
|
||||
return x, y
|
||||
|
|
@ -422,31 +399,18 @@ class Curve(FlowGraphic):
|
|||
# element such that the current datum in view can be shown
|
||||
# (via it's max / min) even when highly zoomed out.
|
||||
class FlattenedOHLC(Curve):
|
||||
'''
|
||||
More or less the exact same as a standard line ``Curve`` above
|
||||
but meant to handle a traced-and-downsampled OHLC time series.
|
||||
_
|
||||
_| | _
|
||||
|_ | |_ | |
|
||||
_| => |_| |
|
||||
| |
|
||||
|_ |_
|
||||
|
||||
The main implementation different is that ``.draw_last_datum()``
|
||||
expects an underlying OHLC array for the ``src_data`` input.
|
||||
|
||||
'''
|
||||
def draw_last_datum(
|
||||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
) -> None:
|
||||
lasts = src_data[-2:]
|
||||
x = lasts[index_field]
|
||||
x = lasts['index']
|
||||
y = lasts['close']
|
||||
|
||||
# draw the "current" step graphic segment so it
|
||||
|
|
@ -460,19 +424,7 @@ class FlattenedOHLC(Curve):
|
|||
|
||||
|
||||
class StepCurve(Curve):
|
||||
'''
|
||||
A familiar rectangle-with-y-height-per-datum type curve:
|
||||
|
||||
||
|
||||
|| ||
|
||||
|| || ||||
|
||||
_||_||_||_||||_ where each datum's y-value is drawn as
|
||||
a nearly full rectangle, each "level" spans some x-step size.
|
||||
|
||||
This is most often used for vlm and option OI style curves and/or
|
||||
the very popular "bar chart".
|
||||
|
||||
'''
|
||||
def declare_paintables(
|
||||
self,
|
||||
) -> None:
|
||||
|
|
@ -482,9 +434,9 @@ class StepCurve(Curve):
|
|||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
w: float = 0.5,
|
||||
|
||||
|
|
@ -493,31 +445,40 @@ class StepCurve(Curve):
|
|||
# TODO: remove this and instead place all step curve
|
||||
# updating into pre-path data render callbacks.
|
||||
# full input data
|
||||
x = src_data[index_field]
|
||||
x = src_data['index']
|
||||
y = src_data[array_key]
|
||||
|
||||
x_last = x[-1]
|
||||
x_2last = x[-2]
|
||||
y_last = y[-1]
|
||||
step_size = x_last - x_2last
|
||||
|
||||
# lol, commenting this makes step curves
|
||||
# all "black" for me :eyeroll:..
|
||||
self._last_line = QLineF(
|
||||
x_2last, 0,
|
||||
x_last, 0,
|
||||
x_last - w, 0,
|
||||
x_last + w, 0,
|
||||
)
|
||||
self._last_step_rect = QRectF(
|
||||
x_last, 0,
|
||||
step_size, y_last,
|
||||
x_last - w, 0,
|
||||
x_last + w, y_last,
|
||||
)
|
||||
return x, y
|
||||
|
||||
def sub_paint(
|
||||
self,
|
||||
p: QPainter,
|
||||
profiler: pg.debug.Profiler,
|
||||
|
||||
) -> None:
|
||||
# p.drawLines(*tuple(filter(bool, self._last_step_lines)))
|
||||
# p.drawRect(self._last_step_rect)
|
||||
p.fillRect(self._last_step_rect, self._brush)
|
||||
profiler('.fillRect()')
|
||||
|
||||
def sub_br(
|
||||
self,
|
||||
path_w: float,
|
||||
path_h: float,
|
||||
|
||||
) -> (float, float):
|
||||
# passthrough
|
||||
return path_w, path_h
|
||||
|
|
|
|||
1472
piker/ui/_dataviz.py
1472
piker/ui/_dataviz.py
File diff suppressed because it is too large
Load Diff
1582
piker/ui/_display.py
1582
piker/ui/_display.py
File diff suppressed because it is too large
Load Diff
|
|
@ -18,26 +18,11 @@
|
|||
Higher level annotation editors.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
TYPE_CHECKING
|
||||
)
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
import pyqtgraph as pg
|
||||
from pyqtgraph import (
|
||||
ViewBox,
|
||||
Point,
|
||||
QtCore,
|
||||
QtWidgets,
|
||||
)
|
||||
from PyQt5.QtGui import (
|
||||
QColor,
|
||||
)
|
||||
from PyQt5.QtWidgets import (
|
||||
QLabel,
|
||||
)
|
||||
|
||||
from pyqtgraph import ViewBox, Point, QtCore, QtGui
|
||||
from pyqtgraph import functions as fn
|
||||
from PyQt5.QtCore import QPointF
|
||||
import numpy as np
|
||||
|
|
@ -45,34 +30,28 @@ import numpy as np
|
|||
from ._style import hcolor, _font
|
||||
from ._lines import LevelLine
|
||||
from ..log import get_logger
|
||||
from ..data.types import Struct
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._chart import GodWidget
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class ArrowEditor(Struct):
|
||||
@dataclass
|
||||
class ArrowEditor:
|
||||
|
||||
godw: GodWidget = None # type: ignore # noqa
|
||||
_arrows: dict[str, list[pg.ArrowItem]] = {}
|
||||
chart: 'ChartPlotWidget' # noqa
|
||||
_arrows: field(default_factory=dict)
|
||||
|
||||
def add(
|
||||
self,
|
||||
plot: pg.PlotItem,
|
||||
uid: str,
|
||||
x: float,
|
||||
y: float,
|
||||
color='default',
|
||||
pointing: str | None = None,
|
||||
|
||||
pointing: Optional[str] = None,
|
||||
) -> pg.ArrowItem:
|
||||
'''
|
||||
Add an arrow graphic to view at given (x, y).
|
||||
"""Add an arrow graphic to view at given (x, y).
|
||||
|
||||
'''
|
||||
"""
|
||||
angle = {
|
||||
'up': 90,
|
||||
'down': -90,
|
||||
|
|
@ -95,25 +74,25 @@ class ArrowEditor(Struct):
|
|||
brush=pg.mkBrush(hcolor(color)),
|
||||
)
|
||||
arrow.setPos(x, y)
|
||||
self._arrows.setdefault(uid, []).append(arrow)
|
||||
|
||||
self._arrows[uid] = arrow
|
||||
|
||||
# render to view
|
||||
plot.addItem(arrow)
|
||||
self.chart.plotItem.addItem(arrow)
|
||||
|
||||
return arrow
|
||||
|
||||
def remove(self, arrow) -> bool:
|
||||
for linked in self.godw.iter_linked():
|
||||
linked.chart.plotItem.removeItem(arrow)
|
||||
self.chart.plotItem.removeItem(arrow)
|
||||
|
||||
|
||||
class LineEditor(Struct):
|
||||
'''
|
||||
The great editor of linez.
|
||||
@dataclass
|
||||
class LineEditor:
|
||||
'''The great editor of linez.
|
||||
|
||||
'''
|
||||
godw: GodWidget = None # type: ignore # noqa
|
||||
_order_lines: defaultdict[str, LevelLine] = defaultdict(list)
|
||||
chart: 'ChartPlotWidget' = None # type: ignore # noqa
|
||||
_order_lines: dict[str, LevelLine] = field(default_factory=dict)
|
||||
_active_staged_line: LevelLine = None
|
||||
|
||||
def stage_line(
|
||||
|
|
@ -121,11 +100,11 @@ class LineEditor(Struct):
|
|||
line: LevelLine,
|
||||
|
||||
) -> LevelLine:
|
||||
'''
|
||||
Stage a line at the current chart's cursor position
|
||||
"""Stage a line at the current chart's cursor position
|
||||
and return it.
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
# add a "staged" cursor-tracking line to view
|
||||
# and cash it in a a var
|
||||
if self._active_staged_line:
|
||||
|
|
@ -136,25 +115,17 @@ class LineEditor(Struct):
|
|||
return line
|
||||
|
||||
def unstage_line(self) -> LevelLine:
|
||||
'''
|
||||
Inverse of ``.stage_line()``.
|
||||
"""Inverse of ``.stage_line()``.
|
||||
|
||||
'''
|
||||
cursor = self.godw.get_cursor()
|
||||
if not cursor:
|
||||
return None
|
||||
"""
|
||||
# chart = self.chart._cursor.active_plot
|
||||
# # chart.setCursor(QtCore.Qt.ArrowCursor)
|
||||
cursor = self.chart.linked.cursor
|
||||
|
||||
# delete "staged" cursor tracking line from view
|
||||
line = self._active_staged_line
|
||||
if line:
|
||||
try:
|
||||
cursor._trackers.remove(line)
|
||||
except KeyError:
|
||||
# when the current cursor doesn't have said line
|
||||
# registered (probably means that user held order mode
|
||||
# key while panning to another view) then we just
|
||||
# ignore the remove error.
|
||||
pass
|
||||
line.delete()
|
||||
|
||||
self._active_staged_line = None
|
||||
|
|
@ -162,65 +133,62 @@ class LineEditor(Struct):
|
|||
# show the crosshair y line and label
|
||||
cursor.show_xhair()
|
||||
|
||||
def submit_lines(
|
||||
def submit_line(
|
||||
self,
|
||||
lines: list[LevelLine],
|
||||
line: LevelLine,
|
||||
uuid: str,
|
||||
|
||||
) -> LevelLine:
|
||||
|
||||
# staged_line = self._active_staged_line
|
||||
# if not staged_line:
|
||||
# raise RuntimeError("No line is currently staged!?")
|
||||
staged_line = self._active_staged_line
|
||||
if not staged_line:
|
||||
raise RuntimeError("No line is currently staged!?")
|
||||
|
||||
# for now, until submission reponse arrives
|
||||
for line in lines:
|
||||
line.hide_labels()
|
||||
|
||||
# register for later lookup/deletion
|
||||
self._order_lines[uuid] += lines
|
||||
self._order_lines[uuid] = line
|
||||
|
||||
return lines
|
||||
return line
|
||||
|
||||
def commit_line(self, uuid: str) -> list[LevelLine]:
|
||||
'''
|
||||
Commit a "staged line" to view.
|
||||
def commit_line(self, uuid: str) -> LevelLine:
|
||||
"""Commit a "staged line" to view.
|
||||
|
||||
Submits the line graphic under the cursor as a (new) permanent
|
||||
graphic in view.
|
||||
|
||||
'''
|
||||
lines = self._order_lines[uuid]
|
||||
if lines:
|
||||
for line in lines:
|
||||
"""
|
||||
try:
|
||||
line = self._order_lines[uuid]
|
||||
except KeyError:
|
||||
log.warning(f'No line for {uuid} could be found?')
|
||||
return
|
||||
else:
|
||||
line.show_labels()
|
||||
line.hide_markers()
|
||||
log.debug(f'Level active for level: {line.value()}')
|
||||
|
||||
# TODO: other flashy things to indicate the order is active
|
||||
|
||||
return lines
|
||||
log.debug(f'Level active for level: {line.value()}')
|
||||
|
||||
return line
|
||||
|
||||
def lines_under_cursor(self) -> list[LevelLine]:
|
||||
'''
|
||||
Get the line(s) under the cursor position.
|
||||
"""Get the line(s) under the cursor position.
|
||||
|
||||
'''
|
||||
"""
|
||||
# Delete any hoverable under the cursor
|
||||
return self.godw.get_cursor()._hovered
|
||||
return self.chart.linked.cursor._hovered
|
||||
|
||||
def all_lines(self) -> list[LevelLine]:
|
||||
all_lines = []
|
||||
for lines in list(self._order_lines.values()):
|
||||
all_lines.extend(lines)
|
||||
|
||||
return all_lines
|
||||
def all_lines(self) -> tuple[LevelLine]:
|
||||
return tuple(self._order_lines.values())
|
||||
|
||||
def remove_line(
|
||||
self,
|
||||
line: LevelLine = None,
|
||||
uuid: str = None,
|
||||
|
||||
) -> LevelLine | None:
|
||||
) -> Optional[LevelLine]:
|
||||
'''Remove a line by refernce or uuid.
|
||||
|
||||
If no lines or ids are provided remove all lines under the
|
||||
|
|
@ -228,30 +196,29 @@ class LineEditor(Struct):
|
|||
|
||||
'''
|
||||
# try to look up line from our registry
|
||||
lines = self._order_lines.pop(uuid, None)
|
||||
if lines:
|
||||
cursor = self.godw.get_cursor()
|
||||
if cursor:
|
||||
for line in lines:
|
||||
line = self._order_lines.pop(uuid, line)
|
||||
if line:
|
||||
|
||||
# if hovered remove from cursor set
|
||||
cursor = self.chart.linked.cursor
|
||||
hovered = cursor._hovered
|
||||
if line in hovered:
|
||||
hovered.remove(line)
|
||||
|
||||
log.debug(f'deleting {line} with oid: {uuid}')
|
||||
line.delete()
|
||||
|
||||
# make sure the xhair doesn't get left off
|
||||
# just because we never got a un-hover event
|
||||
cursor.show_xhair()
|
||||
|
||||
log.debug(f'deleting {line} with oid: {uuid}')
|
||||
line.delete()
|
||||
|
||||
else:
|
||||
log.warning(f'Could not find line for {line}')
|
||||
|
||||
return lines
|
||||
return line
|
||||
|
||||
|
||||
class SelectRect(QtWidgets.QGraphicsRectItem):
|
||||
class SelectRect(QtGui.QGraphicsRectItem):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -260,12 +227,12 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
|
|||
) -> None:
|
||||
super().__init__(0, 0, 1, 1)
|
||||
|
||||
# self.rbScaleBox = QGraphicsRectItem(0, 0, 1, 1)
|
||||
# self.rbScaleBox = QtGui.QGraphicsRectItem(0, 0, 1, 1)
|
||||
self.vb = viewbox
|
||||
self._chart: 'ChartPlotWidget' = None # noqa
|
||||
|
||||
# override selection box color
|
||||
color = QColor(hcolor(color))
|
||||
color = QtGui.QColor(hcolor(color))
|
||||
self.setPen(fn.mkPen(color, width=1))
|
||||
color.setAlpha(66)
|
||||
self.setBrush(fn.mkBrush(color))
|
||||
|
|
@ -273,7 +240,7 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
|
|||
self.hide()
|
||||
self._label = None
|
||||
|
||||
label = self._label = QLabel()
|
||||
label = self._label = QtGui.QLabel()
|
||||
label.setTextFormat(0) # markdown
|
||||
label.setFont(_font.font)
|
||||
label.setMargin(0)
|
||||
|
|
@ -310,8 +277,8 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
|
|||
# TODO: get bg color working
|
||||
palette.setColor(
|
||||
self._label.backgroundRole(),
|
||||
# QColor(chart.backgroundBrush()),
|
||||
QColor(hcolor('papas_special')),
|
||||
# QtGui.QColor(chart.backgroundBrush()),
|
||||
QtGui.QColor(hcolor('papas_special')),
|
||||
)
|
||||
|
||||
def update_on_resize(self, vr, r):
|
||||
|
|
@ -359,7 +326,7 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
|
|||
|
||||
self.setPos(r.topLeft())
|
||||
self.resetTransform()
|
||||
self.setRect(r)
|
||||
self.scale(r.width(), r.height())
|
||||
self.show()
|
||||
|
||||
y1, y2 = start_pos.y(), end_pos.y()
|
||||
|
|
@ -376,7 +343,7 @@ class SelectRect(QtWidgets.QGraphicsRectItem):
|
|||
nbars = ixmx - ixmn + 1
|
||||
|
||||
chart = self._chart
|
||||
data = chart.get_viz(chart.name).shm.array[ixmn:ixmx]
|
||||
data = chart._flows[chart.name].shm.array[ixmn:ixmx]
|
||||
|
||||
if len(data):
|
||||
std = data['close'].std()
|
||||
|
|
|
|||
|
|
@ -18,11 +18,11 @@
|
|||
Qt event proxying and processing using ``trio`` mem chans.
|
||||
|
||||
"""
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from contextlib import asynccontextmanager, AsyncExitStack
|
||||
from typing import Callable
|
||||
|
||||
from pydantic import BaseModel
|
||||
import trio
|
||||
from tractor.trionics import gather_contexts
|
||||
from PyQt5 import QtCore
|
||||
from PyQt5.QtCore import QEvent, pyqtBoundSignal
|
||||
from PyQt5.QtWidgets import QWidget
|
||||
|
|
@ -30,8 +30,6 @@ from PyQt5.QtWidgets import (
|
|||
QGraphicsSceneMouseEvent as gs_mouse,
|
||||
)
|
||||
|
||||
from ..data.types import Struct
|
||||
|
||||
|
||||
MOUSE_EVENTS = {
|
||||
gs_mouse.GraphicsSceneMousePress,
|
||||
|
|
@ -45,10 +43,13 @@ MOUSE_EVENTS = {
|
|||
# TODO: maybe consider some constrained ints down the road?
|
||||
# https://pydantic-docs.helpmanual.io/usage/types/#constrained-types
|
||||
|
||||
class KeyboardMsg(Struct):
|
||||
class KeyboardMsg(BaseModel):
|
||||
'''Unpacked Qt keyboard event data.
|
||||
|
||||
'''
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
event: QEvent
|
||||
etype: int
|
||||
key: int
|
||||
|
|
@ -56,13 +57,16 @@ class KeyboardMsg(Struct):
|
|||
txt: str
|
||||
|
||||
def to_tuple(self) -> tuple:
|
||||
return tuple(self.to_dict().values())
|
||||
return tuple(self.dict().values())
|
||||
|
||||
|
||||
class MouseMsg(Struct):
|
||||
class MouseMsg(BaseModel):
|
||||
'''Unpacked Qt keyboard event data.
|
||||
|
||||
'''
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
event: QEvent
|
||||
etype: int
|
||||
button: int
|
||||
|
|
@ -156,7 +160,7 @@ class EventRelay(QtCore.QObject):
|
|||
return False
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_event_stream(
|
||||
|
||||
source_widget: QWidget,
|
||||
|
|
@ -182,7 +186,7 @@ async def open_event_stream(
|
|||
source_widget.removeEventFilter(kc)
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_signal_handler(
|
||||
|
||||
signal: pyqtBoundSignal,
|
||||
|
|
@ -207,7 +211,7 @@ async def open_signal_handler(
|
|||
yield
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_handlers(
|
||||
|
||||
source_widgets: list[QWidget],
|
||||
|
|
@ -216,14 +220,16 @@ async def open_handlers(
|
|||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
gather_contexts([
|
||||
open_event_stream(widget, event_types, **kwargs)
|
||||
for widget in source_widgets
|
||||
]) as streams,
|
||||
AsyncExitStack() as stack,
|
||||
):
|
||||
for widget, event_recv_stream in zip(source_widgets, streams):
|
||||
for widget in source_widgets:
|
||||
|
||||
event_recv_stream = await stack.enter_async_context(
|
||||
open_event_stream(widget, event_types, **kwargs)
|
||||
)
|
||||
n.start_soon(async_handler, widget, event_recv_stream)
|
||||
|
||||
yield
|
||||
|
|
|
|||
|
|
@ -20,24 +20,16 @@ Trio - Qt integration
|
|||
Run ``trio`` in guest mode on top of the Qt event loop.
|
||||
All global Qt runtime settings are mostly defined here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
Type,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import Tuple, Callable, Dict, Any
|
||||
import platform
|
||||
import traceback
|
||||
|
||||
# Qt specific
|
||||
import PyQt5 # noqa
|
||||
from PyQt5.QtWidgets import (
|
||||
QWidget,
|
||||
QMainWindow,
|
||||
QApplication,
|
||||
)
|
||||
import pyqtgraph as pg
|
||||
from pyqtgraph import QtGui
|
||||
from PyQt5 import QtCore
|
||||
# from PyQt5.QtGui import QLabel, QStatusBar
|
||||
from PyQt5.QtCore import (
|
||||
pyqtRemoveInputHook,
|
||||
Qt,
|
||||
|
|
@ -45,19 +37,15 @@ from PyQt5.QtCore import (
|
|||
)
|
||||
import qdarkstyle
|
||||
from qdarkstyle import DarkPalette
|
||||
# import qdarkgraystyle # TODO: play with it
|
||||
# import qdarkgraystyle
|
||||
import trio
|
||||
from outcome import Error
|
||||
|
||||
from ..service import (
|
||||
maybe_open_pikerd,
|
||||
get_tractor_runtime_kwargs,
|
||||
)
|
||||
from .._daemon import maybe_open_pikerd, _tractor_kwargs
|
||||
from ..log import get_logger
|
||||
from ._pg_overrides import _do_overrides
|
||||
from . import _style
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# pyqtgraph global config
|
||||
|
|
@ -84,18 +72,17 @@ if platform.system() == "Windows":
|
|||
|
||||
def run_qtractor(
|
||||
func: Callable,
|
||||
args: tuple,
|
||||
main_widget_type: Type[QWidget],
|
||||
tractor_kwargs: dict[str, Any] = {},
|
||||
window_type: QMainWindow = None,
|
||||
|
||||
args: Tuple,
|
||||
main_widget: QtGui.QWidget,
|
||||
tractor_kwargs: Dict[str, Any] = {},
|
||||
window_type: QtGui.QMainWindow = None,
|
||||
) -> None:
|
||||
# avoids annoying message when entering debugger from qt loop
|
||||
pyqtRemoveInputHook()
|
||||
|
||||
app = QApplication.instance()
|
||||
app = QtGui.QApplication.instance()
|
||||
if app is None:
|
||||
app = QApplication([])
|
||||
app = PyQt5.QtWidgets.QApplication([])
|
||||
|
||||
# TODO: we might not need this if it's desired
|
||||
# to cancel the tractor machinery on Qt loop
|
||||
|
|
@ -169,11 +156,11 @@ def run_qtractor(
|
|||
# hook into app focus change events
|
||||
app.focusChanged.connect(window.on_focus_change)
|
||||
|
||||
instance = main_widget_type()
|
||||
instance = main_widget()
|
||||
instance.window = window
|
||||
|
||||
# override tractor's defaults
|
||||
tractor_kwargs.update(get_tractor_runtime_kwargs())
|
||||
tractor_kwargs.update(_tractor_kwargs)
|
||||
|
||||
# define tractor entrypoint
|
||||
async def main():
|
||||
|
|
@ -191,7 +178,7 @@ def run_qtractor(
|
|||
# restrict_keyboard_interrupt_to_checkpoints=True,
|
||||
)
|
||||
|
||||
window.godwidget: GodWidget = instance
|
||||
window.main_widget = main_widget
|
||||
window.setCentralWidget(instance)
|
||||
if is_windows:
|
||||
window.configure_to_desktop()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -23,9 +23,7 @@ from contextlib import asynccontextmanager
|
|||
from functools import partial
|
||||
from math import floor
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
Awaitable,
|
||||
Optional, Any, Callable, Awaitable
|
||||
)
|
||||
|
||||
import trio
|
||||
|
|
@ -265,7 +263,7 @@ class Selection(QComboBox):
|
|||
def set_icon(
|
||||
self,
|
||||
key: str,
|
||||
icon_name: str | None,
|
||||
icon_name: Optional[str],
|
||||
|
||||
) -> None:
|
||||
self.setItemIcon(
|
||||
|
|
@ -346,7 +344,7 @@ class FieldsForm(QWidget):
|
|||
|
||||
name: str,
|
||||
|
||||
font_size: int | None = None,
|
||||
font_size: Optional[int] = None,
|
||||
font_color: str = 'default_lightest',
|
||||
|
||||
) -> QtGui.QLabel:
|
||||
|
|
@ -471,7 +469,7 @@ def mk_form(
|
|||
|
||||
parent: QWidget,
|
||||
fields_schema: dict,
|
||||
font_size: int | None = None,
|
||||
font_size: Optional[int] = None,
|
||||
|
||||
) -> FieldsForm:
|
||||
|
||||
|
|
@ -621,7 +619,7 @@ class FillStatusBar(QProgressBar):
|
|||
# color: #19232D;
|
||||
# width: 10px;
|
||||
|
||||
self.setRange(0, int(slots))
|
||||
self.setRange(0, slots)
|
||||
self.setValue(value)
|
||||
|
||||
|
||||
|
|
@ -630,7 +628,7 @@ def mk_fill_status_bar(
|
|||
parent_pane: QWidget,
|
||||
form: FieldsForm,
|
||||
pane_vbox: QVBoxLayout,
|
||||
label_font_size: int | None = None,
|
||||
label_font_size: Optional[int] = None,
|
||||
|
||||
) -> (
|
||||
# TODO: turn this into a composite?
|
||||
|
|
@ -646,7 +644,7 @@ def mk_fill_status_bar(
|
|||
|
||||
# TODO: calc this height from the ``ChartnPane``
|
||||
chart_h = round(parent_pane.height() * 5/8)
|
||||
bar_h = chart_h * 0.375*0.9
|
||||
bar_h = chart_h * 0.375
|
||||
|
||||
# TODO: once things are sized to screen
|
||||
bar_label_font_size = label_font_size or _font.px_size - 2
|
||||
|
|
@ -740,7 +738,7 @@ def mk_fill_status_bar(
|
|||
def mk_order_pane_layout(
|
||||
|
||||
parent: QWidget,
|
||||
# accounts: dict[str, str | None],
|
||||
# accounts: dict[str, Optional[str]],
|
||||
|
||||
) -> FieldsForm:
|
||||
|
||||
|
|
|
|||
332
piker/ui/_fsp.py
332
piker/ui/_fsp.py
|
|
@ -24,19 +24,15 @@ from contextlib import asynccontextmanager as acm
|
|||
from functools import partial
|
||||
import inspect
|
||||
from itertools import cycle
|
||||
from typing import (
|
||||
AsyncGenerator,
|
||||
Any,
|
||||
)
|
||||
from typing import Optional, AsyncGenerator, Any
|
||||
|
||||
import numpy as np
|
||||
import msgspec
|
||||
from pydantic import create_model
|
||||
import tractor
|
||||
import pyqtgraph as pg
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
|
||||
from piker.data.types import Struct
|
||||
from ._axes import PriceAxis
|
||||
from .._cacheables import maybe_open_context
|
||||
from ..calc import humanize
|
||||
|
|
@ -45,8 +41,6 @@ from ..data._sharedmem import (
|
|||
_Token,
|
||||
try_read,
|
||||
)
|
||||
from ..data.feed import Flume
|
||||
from ..accounting._mktinfo import Symbol
|
||||
from ._chart import (
|
||||
ChartPlotWidget,
|
||||
LinkedSplits,
|
||||
|
|
@ -56,18 +50,14 @@ from ._forms import (
|
|||
mk_form,
|
||||
open_form_input_handling,
|
||||
)
|
||||
from ..fsp._api import (
|
||||
maybe_mk_fsp_shm,
|
||||
Fsp,
|
||||
)
|
||||
from ..fsp._api import maybe_mk_fsp_shm, Fsp
|
||||
from ..fsp import cascade
|
||||
from ..fsp._volume import (
|
||||
# tina_vwap,
|
||||
tina_vwap,
|
||||
dolla_vlm,
|
||||
flow_rates,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from .._profile import Profiler
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -81,14 +71,15 @@ def has_vlm(ohlcv: ShmArray) -> bool:
|
|||
|
||||
|
||||
def update_fsp_chart(
|
||||
viz,
|
||||
chart: ChartPlotWidget,
|
||||
flow,
|
||||
graphics_name: str,
|
||||
array_key: str | None,
|
||||
array_key: Optional[str],
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
|
||||
shm = viz.shm
|
||||
shm = flow.shm
|
||||
if not shm:
|
||||
return
|
||||
|
||||
|
|
@ -103,15 +94,18 @@ def update_fsp_chart(
|
|||
# update graphics
|
||||
# NOTE: this does a length check internally which allows it
|
||||
# staying above the last row check below..
|
||||
viz.update_graphics()
|
||||
chart.update_graphics_from_flow(
|
||||
graphics_name,
|
||||
array_key=array_key or graphics_name,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# XXX: re: ``array_key``: fsp func names must be unique meaning we
|
||||
# can't have duplicates of the underlying data even if multiple
|
||||
# sub-charts reference it under different 'named charts'.
|
||||
|
||||
# read from last calculated value and update any label
|
||||
last_val_sticky = viz.plot.getAxis(
|
||||
'right')._stickies.get(graphics_name)
|
||||
last_val_sticky = chart._ysticks.get(graphics_name)
|
||||
if last_val_sticky:
|
||||
last = last_row[array_key]
|
||||
last_val_sticky.update_from_data(-1, last)
|
||||
|
|
@ -159,13 +153,12 @@ async def open_fsp_sidepane(
|
|||
)
|
||||
|
||||
# https://pydantic-docs.helpmanual.io/usage/models/#dynamic-model-creation
|
||||
FspConfig = msgspec.defstruct(
|
||||
"Point",
|
||||
[('name', name)] + list(params.items()),
|
||||
bases=(Struct,),
|
||||
FspConfig = create_model(
|
||||
'FspConfig',
|
||||
name=name,
|
||||
**params,
|
||||
)
|
||||
model = FspConfig(name=name, **params)
|
||||
sidepane.model = model
|
||||
sidepane.model = FspConfig()
|
||||
|
||||
# just a logger for now until we get fsp configs up and running.
|
||||
async def settings_change(
|
||||
|
|
@ -195,7 +188,7 @@ async def open_fsp_actor_cluster(
|
|||
|
||||
from tractor._clustering import open_actor_cluster
|
||||
|
||||
# profiler = Profiler(
|
||||
# profiler = pg.debug.Profiler(
|
||||
# delayed=False,
|
||||
# disabled=False
|
||||
# )
|
||||
|
|
@ -212,12 +205,12 @@ async def open_fsp_actor_cluster(
|
|||
async def run_fsp_ui(
|
||||
|
||||
linkedsplits: LinkedSplits,
|
||||
flume: Flume,
|
||||
shm: ShmArray,
|
||||
started: trio.Event,
|
||||
target: Fsp,
|
||||
conf: dict[str, dict],
|
||||
loglevel: str,
|
||||
# profiler: Profiler,
|
||||
# profiler: pg.debug.Profiler,
|
||||
# _quote_throttle_rate: int = 58,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -249,11 +242,9 @@ async def run_fsp_ui(
|
|||
else:
|
||||
chart = linkedsplits.subplots[overlay_with]
|
||||
|
||||
shm = flume.rt_shm
|
||||
chart.draw_curve(
|
||||
name,
|
||||
shm,
|
||||
flume,
|
||||
name=name,
|
||||
shm=shm,
|
||||
overlay=True,
|
||||
color='default_light',
|
||||
array_key=name,
|
||||
|
|
@ -263,9 +254,8 @@ async def run_fsp_ui(
|
|||
else:
|
||||
# create a new sub-chart widget for this fsp
|
||||
chart = linkedsplits.add_plot(
|
||||
name,
|
||||
shm,
|
||||
flume,
|
||||
name=name,
|
||||
shm=shm,
|
||||
|
||||
array_key=name,
|
||||
sidepane=sidepane,
|
||||
|
|
@ -285,10 +275,9 @@ async def run_fsp_ui(
|
|||
# profiler(f'fsp:{name} chart created')
|
||||
|
||||
# first UI update, usually from shm pushed history
|
||||
viz = chart.get_viz(array_key)
|
||||
update_fsp_chart(
|
||||
chart,
|
||||
viz,
|
||||
chart._flows[array_key],
|
||||
name,
|
||||
array_key=array_key,
|
||||
)
|
||||
|
|
@ -315,7 +304,7 @@ async def run_fsp_ui(
|
|||
# level_line(chart, 70, orient_v='bottom')
|
||||
# level_line(chart, 80, orient_v='top')
|
||||
|
||||
chart.view._set_yrange(viz=viz)
|
||||
chart.view._set_yrange()
|
||||
# done() # status updates
|
||||
|
||||
# profiler(f'fsp:{func_name} starting update loop')
|
||||
|
|
@ -356,9 +345,6 @@ async def run_fsp_ui(
|
|||
# last = time.time()
|
||||
|
||||
|
||||
# TODO: maybe this should be our ``Viz`` type since it maps
|
||||
# one flume to the next? The machinery for task/actor mgmt should
|
||||
# be part of the instantiation API?
|
||||
class FspAdmin:
|
||||
'''
|
||||
Client API for orchestrating FSP actors and displaying
|
||||
|
|
@ -370,7 +356,7 @@ class FspAdmin:
|
|||
tn: trio.Nursery,
|
||||
cluster: dict[str, tractor.Portal],
|
||||
linked: LinkedSplits,
|
||||
flume: Flume,
|
||||
src_shm: ShmArray,
|
||||
|
||||
) -> None:
|
||||
self.tn = tn
|
||||
|
|
@ -382,11 +368,7 @@ class FspAdmin:
|
|||
tuple[tractor.MsgStream, ShmArray]
|
||||
] = {}
|
||||
self._flow_registry: dict[_Token, str] = {}
|
||||
|
||||
# TODO: make this a `.src_flume` and add
|
||||
# a `dst_flume`?
|
||||
# (=> but then wouldn't this be the most basic `Viz`?)
|
||||
self.flume = flume
|
||||
self.src_shm = src_shm
|
||||
|
||||
def rr_next_portal(self) -> tractor.Portal:
|
||||
name, portal = next(self._rr_next_actor)
|
||||
|
|
@ -399,7 +381,7 @@ class FspAdmin:
|
|||
complete: trio.Event,
|
||||
started: trio.Event,
|
||||
fqsn: str,
|
||||
dst_fsp_flume: Flume,
|
||||
dst_shm: ShmArray,
|
||||
conf: dict,
|
||||
target: Fsp,
|
||||
loglevel: str,
|
||||
|
|
@ -420,10 +402,9 @@ class FspAdmin:
|
|||
# data feed key
|
||||
fqsn=fqsn,
|
||||
|
||||
# TODO: pass `Flume.to_msg()`s here?
|
||||
# mems
|
||||
src_shm_token=self.flume.rt_shm.token,
|
||||
dst_shm_token=dst_fsp_flume.rt_shm.token,
|
||||
src_shm_token=self.src_shm.token,
|
||||
dst_shm_token=dst_shm.token,
|
||||
|
||||
# target
|
||||
ns_path=ns_path,
|
||||
|
|
@ -440,14 +421,12 @@ class FspAdmin:
|
|||
ctx.open_stream() as stream,
|
||||
):
|
||||
|
||||
dst_fsp_flume.stream: tractor.MsgStream = stream
|
||||
|
||||
# register output data
|
||||
self._registry[
|
||||
(fqsn, ns_path)
|
||||
] = (
|
||||
stream,
|
||||
dst_fsp_flume.rt_shm,
|
||||
dst_shm,
|
||||
complete
|
||||
)
|
||||
|
||||
|
|
@ -461,9 +440,7 @@ class FspAdmin:
|
|||
# if the chart isn't hidden try to update
|
||||
# the data on screen.
|
||||
if not self.linked.isHidden():
|
||||
log.debug(
|
||||
f'Re-syncing graphics for fsp: {ns_path}'
|
||||
)
|
||||
log.debug(f'Re-syncing graphics for fsp: {ns_path}')
|
||||
self.linked.graphics_cycle(
|
||||
trigger_all=True,
|
||||
prepend_update_index=info['first'],
|
||||
|
|
@ -479,12 +456,12 @@ class FspAdmin:
|
|||
target: Fsp,
|
||||
conf: dict[str, dict[str, Any]],
|
||||
|
||||
worker_name: str | None = None,
|
||||
worker_name: Optional[str] = None,
|
||||
loglevel: str = 'info',
|
||||
|
||||
) -> (Flume, trio.Event):
|
||||
) -> (ShmArray, trio.Event):
|
||||
|
||||
fqsn = self.flume.symbol.fqsn
|
||||
fqsn = self.linked.symbol.front_fqsn()
|
||||
|
||||
# allocate an output shm array
|
||||
key, dst_shm, opened = maybe_mk_fsp_shm(
|
||||
|
|
@ -492,36 +469,16 @@ class FspAdmin:
|
|||
target=target,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
portal = self.cluster.get(worker_name) or self.rr_next_portal()
|
||||
provider_tag = portal.channel.uid
|
||||
|
||||
symbol = Symbol(
|
||||
key=key,
|
||||
broker_info={
|
||||
provider_tag: {'asset_type': 'fsp'},
|
||||
},
|
||||
)
|
||||
dst_fsp_flume = Flume(
|
||||
symbol=symbol,
|
||||
_rt_shm_token=dst_shm.token,
|
||||
first_quote={},
|
||||
|
||||
# set to 0 presuming for now that we can't load
|
||||
# FSP history (though we should eventually).
|
||||
izero_hist=0,
|
||||
izero_rt=0,
|
||||
)
|
||||
self._flow_registry[(
|
||||
self.flume.rt_shm._token,
|
||||
target.name
|
||||
)] = dst_shm._token
|
||||
self._flow_registry[
|
||||
(self.src_shm._token, target.name)
|
||||
] = dst_shm._token
|
||||
|
||||
# if not opened:
|
||||
# raise RuntimeError(
|
||||
# f'Already started FSP `{fqsn}:{func_name}`'
|
||||
# )
|
||||
|
||||
portal = self.cluster.get(worker_name) or self.rr_next_portal()
|
||||
complete = trio.Event()
|
||||
started = trio.Event()
|
||||
self.tn.start_soon(
|
||||
|
|
@ -530,13 +487,13 @@ class FspAdmin:
|
|||
complete,
|
||||
started,
|
||||
fqsn,
|
||||
dst_fsp_flume,
|
||||
dst_shm,
|
||||
conf,
|
||||
target,
|
||||
loglevel,
|
||||
)
|
||||
|
||||
return dst_fsp_flume, started
|
||||
return dst_shm, started
|
||||
|
||||
async def open_fsp_chart(
|
||||
self,
|
||||
|
|
@ -548,7 +505,7 @@ class FspAdmin:
|
|||
|
||||
) -> (trio.Event, ChartPlotWidget):
|
||||
|
||||
flume, started = await self.start_engine_task(
|
||||
shm, started = await self.start_engine_task(
|
||||
target,
|
||||
conf,
|
||||
loglevel,
|
||||
|
|
@ -560,7 +517,7 @@ class FspAdmin:
|
|||
run_fsp_ui,
|
||||
|
||||
self.linked,
|
||||
flume,
|
||||
shm,
|
||||
started,
|
||||
target,
|
||||
|
||||
|
|
@ -574,7 +531,7 @@ class FspAdmin:
|
|||
@acm
|
||||
async def open_fsp_admin(
|
||||
linked: LinkedSplits,
|
||||
flume: Flume,
|
||||
src_shm: ShmArray,
|
||||
**kwargs,
|
||||
|
||||
) -> AsyncGenerator[dict, dict[str, tractor.Portal]]:
|
||||
|
|
@ -595,7 +552,7 @@ async def open_fsp_admin(
|
|||
tn,
|
||||
cluster_map,
|
||||
linked,
|
||||
flume,
|
||||
src_shm,
|
||||
)
|
||||
try:
|
||||
yield admin
|
||||
|
|
@ -609,13 +566,12 @@ async def open_fsp_admin(
|
|||
async def open_vlm_displays(
|
||||
|
||||
linked: LinkedSplits,
|
||||
flume: Flume,
|
||||
ohlcv: ShmArray,
|
||||
dvlm: bool = True,
|
||||
loglevel: str = 'info',
|
||||
|
||||
task_status: TaskStatus[ChartPlotWidget] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
) -> ChartPlotWidget:
|
||||
'''
|
||||
Volume subchart displays.
|
||||
|
||||
|
|
@ -632,8 +588,6 @@ async def open_vlm_displays(
|
|||
sig = inspect.signature(flow_rates.func)
|
||||
params = sig.parameters
|
||||
|
||||
ohlcv: ShmArray = flume.rt_shm
|
||||
|
||||
async with (
|
||||
open_fsp_sidepane(
|
||||
linked, {
|
||||
|
|
@ -653,7 +607,7 @@ async def open_vlm_displays(
|
|||
}
|
||||
},
|
||||
) as sidepane,
|
||||
open_fsp_admin(linked, flume) as admin,
|
||||
open_fsp_admin(linked, ohlcv) as admin,
|
||||
):
|
||||
# TODO: support updates
|
||||
# period_field = sidepane.fields['period']
|
||||
|
|
@ -661,20 +615,12 @@ async def open_vlm_displays(
|
|||
# str(period_param.default)
|
||||
# )
|
||||
|
||||
# use slightly less light (then bracket) gray
|
||||
# for volume from "main exchange" and a more "bluey"
|
||||
# gray for "dark" vlm.
|
||||
vlm_color = 'i3'
|
||||
dark_vlm_color = 'charcoal'
|
||||
|
||||
# built-in vlm which we plot ASAP since it's
|
||||
# usually data provided directly with OHLC history.
|
||||
shm = ohlcv
|
||||
|
||||
vlm_chart = linked.add_plot(
|
||||
chart = linked.add_plot(
|
||||
name='volume',
|
||||
shm=shm,
|
||||
flume=flume,
|
||||
|
||||
array_key='volume',
|
||||
sidepane=sidepane,
|
||||
|
|
@ -687,45 +633,63 @@ async def open_vlm_displays(
|
|||
# the curve item internals are pretty convoluted.
|
||||
style='step',
|
||||
)
|
||||
vlm_viz = vlm_chart._vizs['volume']
|
||||
|
||||
# force 0 to always be in view
|
||||
def multi_maxmin(
|
||||
names: list[str],
|
||||
|
||||
) -> tuple[float, float]:
|
||||
|
||||
mx = 0
|
||||
for name in names:
|
||||
|
||||
mxmn = chart.maxmin(name=name)
|
||||
if mxmn:
|
||||
ymax = mxmn[1]
|
||||
if ymax > mx:
|
||||
mx = ymax
|
||||
|
||||
return 0, mx
|
||||
|
||||
chart.view.maxmin = partial(multi_maxmin, names=['volume'])
|
||||
|
||||
# TODO: fix the x-axis label issue where if you put
|
||||
# the axis on the left it's totally not lined up...
|
||||
# show volume units value on LHS (for dinkus)
|
||||
# vlm_chart.hideAxis('right')
|
||||
vlm_chart.hideAxis('left')
|
||||
|
||||
# TODO: is it worth being able to remove axes (from i guess
|
||||
# a perf perspective) enough that we can actually do this and
|
||||
# other axis related calls (for eg. label upddates in the
|
||||
# display loop) don't raise when a the axis can't be loaded and
|
||||
# thus would normally cause many label related calls to crash?
|
||||
# axis = vlm_chart.removeAxis('left')
|
||||
# chart.hideAxis('right')
|
||||
# chart.showAxis('left')
|
||||
|
||||
# send back new chart to caller
|
||||
task_status.started(vlm_chart)
|
||||
task_status.started(chart)
|
||||
|
||||
# should **not** be the same sub-chart widget
|
||||
assert vlm_chart.name != linked.chart.name
|
||||
assert chart.name != linked.chart.name
|
||||
|
||||
# sticky only on sub-charts atm
|
||||
last_val_sticky = vlm_chart.plotItem.getAxis(
|
||||
'right')._stickies.get(vlm_chart.name)
|
||||
last_val_sticky = chart._ysticks[chart.name]
|
||||
|
||||
# read from last calculated value
|
||||
value = shm.array['volume'][-1]
|
||||
|
||||
last_val_sticky.update_from_data(-1, value)
|
||||
|
||||
_, _, vlm_curve = vlm_viz.update_graphics()
|
||||
vlm_curve = chart.update_graphics_from_flow(
|
||||
'volume',
|
||||
# shm.array,
|
||||
)
|
||||
|
||||
# size view to data once at outset
|
||||
chart.view._set_yrange()
|
||||
|
||||
# add axis title
|
||||
axis = vlm_chart.getAxis('right')
|
||||
axis = chart.getAxis('right')
|
||||
axis.set_title(' vlm')
|
||||
|
||||
if dvlm:
|
||||
|
||||
tasks_ready = []
|
||||
# spawn and overlay $ vlm on the same subchart
|
||||
dvlm_flume, started = await admin.start_engine_task(
|
||||
dvlm_shm, started = await admin.start_engine_task(
|
||||
dolla_vlm,
|
||||
|
||||
{ # fsp engine conf
|
||||
|
|
@ -737,32 +701,43 @@ async def open_vlm_displays(
|
|||
},
|
||||
},
|
||||
},
|
||||
loglevel,
|
||||
# loglevel,
|
||||
)
|
||||
tasks_ready.append(started)
|
||||
|
||||
# FIXME: we should error on starting the same fsp right
|
||||
# since it might collide with existing shm.. or wait we
|
||||
# had this before??
|
||||
# dolla_vlm,
|
||||
|
||||
tasks_ready.append(started)
|
||||
# profiler(f'created shm for fsp actor: {display_name}')
|
||||
|
||||
# wait for all engine tasks to startup
|
||||
async with trio.open_nursery() as n:
|
||||
for event in tasks_ready:
|
||||
n.start_soon(event.wait)
|
||||
|
||||
# dolla vlm overlay
|
||||
# XXX: the main chart already contains a vlm "units" axis
|
||||
# so here we add an overlay wth a y-range in
|
||||
# $ liquidity-value units (normally a fiat like USD).
|
||||
dvlm_pi = vlm_chart.overlay_plotitem(
|
||||
dvlm_pi = chart.overlay_plotitem(
|
||||
'dolla_vlm',
|
||||
index=0, # place axis on inside (nearest to chart)
|
||||
|
||||
axis_title=' $vlm',
|
||||
axis_side='left',
|
||||
|
||||
axis_side='right',
|
||||
axis_kwargs={
|
||||
'typical_max_str': ' 100.0 M ',
|
||||
'formatter': partial(
|
||||
humanize,
|
||||
digits=2,
|
||||
),
|
||||
'text_color': vlm_color,
|
||||
},
|
||||
)
|
||||
|
||||
# all to be overlayed curve names
|
||||
dvlm_fields = [
|
||||
fields = [
|
||||
'dolla_vlm',
|
||||
'dark_vlm',
|
||||
]
|
||||
|
|
@ -775,18 +750,32 @@ async def open_vlm_displays(
|
|||
'dark_trade_rate',
|
||||
]
|
||||
|
||||
group_mxmn = partial(
|
||||
multi_maxmin,
|
||||
# keep both regular and dark vlm in view
|
||||
names=fields,
|
||||
# names=fields + dvlm_rate_fields,
|
||||
)
|
||||
|
||||
# add custom auto range handler
|
||||
dvlm_pi.vb._maxmin = group_mxmn
|
||||
|
||||
# use slightly less light (then bracket) gray
|
||||
# for volume from "main exchange" and a more "bluey"
|
||||
# gray for "dark" vlm.
|
||||
vlm_color = 'i3'
|
||||
dark_vlm_color = 'charcoal'
|
||||
|
||||
# add dvlm (step) curves to common view
|
||||
def chart_curves(
|
||||
names: list[str],
|
||||
pi: pg.PlotItem,
|
||||
shm: ShmArray,
|
||||
flume: Flume,
|
||||
step_mode: bool = False,
|
||||
style: str = 'solid',
|
||||
|
||||
) -> None:
|
||||
for name in names:
|
||||
|
||||
if 'dark' in name:
|
||||
color = dark_vlm_color
|
||||
elif 'rate' in name:
|
||||
|
|
@ -794,13 +783,9 @@ async def open_vlm_displays(
|
|||
else:
|
||||
color = 'bracket'
|
||||
|
||||
assert isinstance(shm, ShmArray)
|
||||
assert isinstance(flume, Flume)
|
||||
|
||||
viz = vlm_chart.draw_curve(
|
||||
name,
|
||||
shm,
|
||||
flume,
|
||||
curve, _ = chart.draw_curve(
|
||||
name=name,
|
||||
shm=shm,
|
||||
array_key=name,
|
||||
overlay=pi,
|
||||
color=color,
|
||||
|
|
@ -808,32 +793,38 @@ async def open_vlm_displays(
|
|||
style=style,
|
||||
pi=pi,
|
||||
)
|
||||
assert viz.plot is pi
|
||||
|
||||
await started.wait()
|
||||
# TODO: we need a better API to do this..
|
||||
# specially store ref to shm for lookup in display loop
|
||||
# since only a placeholder of `None` is entered in
|
||||
# ``.draw_curve()``.
|
||||
flow = chart._flows[name]
|
||||
assert flow.plot is pi
|
||||
|
||||
chart_curves(
|
||||
dvlm_fields,
|
||||
fields,
|
||||
dvlm_pi,
|
||||
dvlm_flume.rt_shm,
|
||||
dvlm_flume,
|
||||
dvlm_shm,
|
||||
step_mode=True,
|
||||
)
|
||||
|
||||
# NOTE: spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
|
||||
# up since calculating vlm "rates" obvs first requires the
|
||||
# underlying vlm event feed ;)
|
||||
fr_flume, started = await admin.start_engine_task(
|
||||
# spawn flow rates fsp **ONLY AFTER** the 'dolla_vlm' fsp is
|
||||
# up since this one depends on it.
|
||||
|
||||
fr_shm, started = await admin.start_engine_task(
|
||||
flow_rates,
|
||||
{ # fsp engine conf
|
||||
'func_name': 'flow_rates',
|
||||
'zero_on_step': True,
|
||||
'zero_on_step': False,
|
||||
},
|
||||
loglevel,
|
||||
# loglevel,
|
||||
)
|
||||
await started.wait()
|
||||
|
||||
# chart_curves(
|
||||
# dvlm_rate_fields,
|
||||
# dvlm_pi,
|
||||
# fr_flume.rt_shm,
|
||||
# fr_shm,
|
||||
# )
|
||||
|
||||
# TODO: is there a way to "sync" the dual axes such that only
|
||||
|
|
@ -841,27 +832,25 @@ async def open_vlm_displays(
|
|||
# hide the original vlm curve since the $vlm one is now
|
||||
# displayed and the curves are effectively the same minus
|
||||
# liquidity events (well at least on low OHLC periods - 1s).
|
||||
# vlm_curve.hide()
|
||||
vlm_chart.removeItem(vlm_curve)
|
||||
vlm_viz = vlm_chart._vizs['volume']
|
||||
vlm_chart.view.disable_auto_yrange()
|
||||
# NOTE: DON'T DO THIS.
|
||||
# WHY: we want range sorting on volume for the RHS label!
|
||||
# -> if you don't want that then use this but likely you
|
||||
# only will if we decide to drop unit vlm..
|
||||
# vlm_viz.render = False
|
||||
vlm_curve.hide()
|
||||
chart.removeItem(vlm_curve)
|
||||
vflow = chart._flows['volume']
|
||||
vflow.render = False
|
||||
|
||||
# avoid range sorting on volume once disabled
|
||||
chart.view.disable_auto_yrange()
|
||||
|
||||
# Trade rate overlay
|
||||
# XXX: requires an additional overlay for
|
||||
# a trades-per-period (time) y-range.
|
||||
tr_pi = vlm_chart.overlay_plotitem(
|
||||
tr_pi = chart.overlay_plotitem(
|
||||
'trade_rates',
|
||||
|
||||
# TODO: dynamically update period (and thus this axis?)
|
||||
# title from user input.
|
||||
axis_title='clears',
|
||||
axis_side='left',
|
||||
|
||||
axis_side='left',
|
||||
axis_kwargs={
|
||||
'typical_max_str': ' 10.0 M ',
|
||||
'formatter': partial(
|
||||
|
|
@ -872,13 +861,17 @@ async def open_vlm_displays(
|
|||
},
|
||||
|
||||
)
|
||||
# add custom auto range handler
|
||||
tr_pi.vb.maxmin = partial(
|
||||
multi_maxmin,
|
||||
# keep both regular and dark vlm in view
|
||||
names=trade_rate_fields,
|
||||
)
|
||||
|
||||
await started.wait()
|
||||
chart_curves(
|
||||
trade_rate_fields,
|
||||
tr_pi,
|
||||
fr_flume.rt_shm,
|
||||
fr_flume,
|
||||
fr_shm,
|
||||
# step_mode=True,
|
||||
|
||||
# dashed line to represent "individual trades" being
|
||||
|
|
@ -912,7 +905,7 @@ async def open_vlm_displays(
|
|||
async def start_fsp_displays(
|
||||
|
||||
linked: LinkedSplits,
|
||||
flume: Flume,
|
||||
ohlcv: ShmArray,
|
||||
group_status_key: str,
|
||||
loglevel: str,
|
||||
|
||||
|
|
@ -947,7 +940,7 @@ async def start_fsp_displays(
|
|||
# },
|
||||
# },
|
||||
}
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=False
|
||||
)
|
||||
|
|
@ -955,10 +948,7 @@ async def start_fsp_displays(
|
|||
async with (
|
||||
|
||||
# NOTE: this admin internally opens an actor cluster
|
||||
open_fsp_admin(
|
||||
linked,
|
||||
flume,
|
||||
) as admin,
|
||||
open_fsp_admin(linked, ohlcv) as admin,
|
||||
):
|
||||
statuses = []
|
||||
for target, conf in fsp_conf.items():
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue