Compare commits
No commits in common. "rekt_pps" and "310_plus" have entirely different histories.
|
|
@ -3,9 +3,10 @@ name: CI
|
|||
|
||||
on:
|
||||
# Triggers the workflow on push or pull request events but only for the master branch
|
||||
pull_request:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
# Allows you to run this workflow manually from the Actions tab
|
||||
workflow_dispatch:
|
||||
|
|
@ -13,46 +14,21 @@ on:
|
|||
|
||||
jobs:
|
||||
|
||||
# test that we can generate a software distribution and install it
|
||||
# thus avoid missing file issues after packaging.
|
||||
sdist-linux:
|
||||
name: 'sdist'
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Build sdist
|
||||
run: python setup.py sdist --formats=zip
|
||||
|
||||
- name: Install sdist from .zips
|
||||
run: python -m pip install dist/*.zip
|
||||
|
||||
testing:
|
||||
name: 'install + test-suite'
|
||||
timeout-minutes: 10
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Build DB container
|
||||
run: docker build -t piker:elastic dockering/elastic
|
||||
|
||||
- name: Setup python
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -U .[es] -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
run: pip install -U . -r requirements-test.txt -r requirements.txt --upgrade-strategy eager
|
||||
|
||||
- name: Test suite
|
||||
run: pytest tests -rs
|
||||
|
|
|
|||
13
README.rst
13
README.rst
|
|
@ -71,19 +71,6 @@ for a development install::
|
|||
source ./env/bin/activate
|
||||
pip install -r requirements.txt -e .
|
||||
|
||||
install for nixos
|
||||
*****************
|
||||
for users of `NixOS` we offer a development shell envoirment that can be
|
||||
loaded with::
|
||||
|
||||
nix-shell develop.nix
|
||||
|
||||
this will setup the required python environment to run piker, make sure to
|
||||
run::
|
||||
|
||||
pip install -r requirements.txt -e .
|
||||
|
||||
once after loading the shell
|
||||
|
||||
install for tinas
|
||||
*****************
|
||||
|
|
|
|||
|
|
@ -50,8 +50,3 @@ prefer_data_account = [
|
|||
paper = "XX0000000"
|
||||
margin = "X0000000"
|
||||
ira = "X0000000"
|
||||
|
||||
|
||||
[deribit]
|
||||
key_id = 'XXXXXXXX'
|
||||
key_secret = 'Xx_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
|
|
|
|||
32
develop.nix
32
develop.nix
|
|
@ -1,32 +0,0 @@
|
|||
with (import <nixpkgs> {});
|
||||
with python310Packages;
|
||||
stdenv.mkDerivation {
|
||||
name = "pip-env";
|
||||
buildInputs = [
|
||||
# System requirements.
|
||||
readline
|
||||
|
||||
# Python requirements (enough to get a virtualenv going).
|
||||
python310Full
|
||||
virtualenv
|
||||
setuptools
|
||||
pyqt5
|
||||
pip
|
||||
];
|
||||
src = null;
|
||||
shellHook = ''
|
||||
# Allow the use of wheels.
|
||||
SOURCE_DATE_EPOCH=$(date +%s)
|
||||
|
||||
# Augment the dynamic linker path
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${R}/lib/R/lib:${readline}/lib
|
||||
|
||||
export QT_QPA_PLATFORM_PLUGIN_PATH="${qt5.qtbase.bin}/lib/qt-${qt5.qtbase.version}/plugins";
|
||||
|
||||
if [ ! -d "venv" ]; then
|
||||
virtualenv venv
|
||||
fi
|
||||
|
||||
source venv/bin/activate
|
||||
'';
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
FROM elasticsearch:7.17.4
|
||||
|
||||
ENV ES_JAVA_OPTS "-Xms2g -Xmx2g"
|
||||
ENV ELASTIC_USERNAME "elastic"
|
||||
ENV ELASTIC_PASSWORD "password"
|
||||
|
||||
COPY elasticsearch.yml /usr/share/elasticsearch/config/
|
||||
|
||||
RUN printf "password" | ./bin/elasticsearch-keystore add -f -x "bootstrap.password"
|
||||
|
||||
EXPOSE 19200
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
network.host: 0.0.0.0
|
||||
|
||||
http.port: 19200
|
||||
|
||||
discovery.type: single-node
|
||||
|
|
@ -2,26 +2,12 @@
|
|||
# https://github.com/waytrade/ib-gateway-docker/blob/master/docker-compose.yml
|
||||
version: "3.5"
|
||||
|
||||
|
||||
services:
|
||||
|
||||
ib_gw_paper:
|
||||
|
||||
# apparently java is a mega cukc:
|
||||
# https://stackoverflow.com/a/56895801
|
||||
# https://bugs.openjdk.org/browse/JDK-8150460
|
||||
ulimits:
|
||||
# nproc: 65535
|
||||
nproc: 6000
|
||||
nofile:
|
||||
soft: 2000
|
||||
hard: 3000
|
||||
|
||||
ib-gateway:
|
||||
# other image tags available:
|
||||
# https://github.com/waytrade/ib-gateway-docker#supported-tags
|
||||
# image: waytrade/ib-gateway:981.3j
|
||||
image: waytrade/ib-gateway:1012.2i
|
||||
restart: 'no' # restart on boot whenev there's a crash or user clicsk
|
||||
image: waytrade/ib-gateway:981.3j
|
||||
restart: always
|
||||
network_mode: 'host'
|
||||
|
||||
volumes:
|
||||
|
|
@ -53,12 +39,14 @@ services:
|
|||
# this compose file which looks something like:
|
||||
# TWS_USERID='myuser'
|
||||
# TWS_PASSWORD='guest'
|
||||
# TRADING_MODE=paper (or live)
|
||||
# VNC_SERVER_PASSWORD='diggity'
|
||||
|
||||
environment:
|
||||
TWS_USERID: ${TWS_USERID}
|
||||
TWS_PASSWORD: ${TWS_PASSWORD}
|
||||
TRADING_MODE: 'paper'
|
||||
VNC_SERVER_PASSWORD: 'doggy'
|
||||
VNC_SERVER_PORT: '3003'
|
||||
TRADING_MODE: ${TRADING_MODE:-paper}
|
||||
VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
|
||||
|
||||
# ports:
|
||||
# - target: 4002
|
||||
|
|
@ -74,40 +62,3 @@ services:
|
|||
# - "127.0.0.1:4001:4001"
|
||||
# - "127.0.0.1:4002:4002"
|
||||
# - "127.0.0.1:5900:5900"
|
||||
|
||||
# ib_gw_live:
|
||||
# image: waytrade/ib-gateway:1012.2i
|
||||
# restart: no
|
||||
# network_mode: 'host'
|
||||
|
||||
# volumes:
|
||||
# - type: bind
|
||||
# source: ./jts_live.ini
|
||||
# target: /root/jts/jts.ini
|
||||
# # don't let ibc clobber this file for
|
||||
# # the main reason of not having a stupid
|
||||
# # timezone set..
|
||||
# read_only: true
|
||||
|
||||
# # force our own ibc config
|
||||
# - type: bind
|
||||
# source: ./ibc.ini
|
||||
# target: /root/ibc/config.ini
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./fork_ports_delayed.sh
|
||||
# target: /root/scripts/fork_ports_delayed.sh
|
||||
|
||||
# # force our noop script - socat isn't needed in host mode.
|
||||
# - type: bind
|
||||
# source: ./run_x11_vnc.sh
|
||||
# target: /root/scripts/run_x11_vnc.sh
|
||||
# read_only: true
|
||||
|
||||
# # NOTE: to fill these out, define an `.env` file in the same dir as
|
||||
# # this compose file which looks something like:
|
||||
# environment:
|
||||
# TRADING_MODE: 'live'
|
||||
# VNC_SERVER_PASSWORD: 'doggy'
|
||||
# VNC_SERVER_PORT: '3004'
|
||||
|
|
|
|||
|
|
@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes
|
|||
#
|
||||
# The default value is 60.
|
||||
|
||||
LoginDialogDisplayTimeout=20
|
||||
LoginDialogDisplayTimeout = 60
|
||||
|
||||
|
||||
|
||||
|
|
@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary
|
|||
# be set dynamically at run-time: most users will never need it,
|
||||
# so don't use it unless you know you need it.
|
||||
|
||||
; OverrideTwsApiPort=4002
|
||||
OverrideTwsApiPort=4002
|
||||
|
||||
|
||||
# Read-only Login
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
[IBGateway]
|
||||
ApiOnly=true
|
||||
LocalServerPort=4001
|
||||
# NOTE: must be set if using IBC's "reject" mode
|
||||
TrustedIPs=127.0.0.1
|
||||
; RemoteHostOrderRouting=ndc1.ibllc.com
|
||||
; WriteDebug=true
|
||||
; RemotePortOrderRouting=4001
|
||||
; useRemoteSettings=false
|
||||
; tradingMode=p
|
||||
; Steps=8
|
||||
; colorPalletName=dark
|
||||
|
||||
# window geo, this may be useful for sending `xdotool` commands?
|
||||
; MainWindow.Width=1986
|
||||
; screenHeight=3960
|
||||
|
||||
|
||||
[Logon]
|
||||
Locale=en
|
||||
# most markets are oriented around this zone
|
||||
# so might as well hard code it.
|
||||
TimeZone=America/New_York
|
||||
UseSSL=true
|
||||
displayedproxymsg=1
|
||||
os_titlebar=true
|
||||
s3store=true
|
||||
useRemoteSettings=false
|
||||
|
||||
[Communication]
|
||||
ctciAutoEncrypt=true
|
||||
Region=usr
|
||||
; Peer=cdc1.ibllc.com:4001
|
||||
|
|
@ -1,35 +1,16 @@
|
|||
#!/bin/sh
|
||||
# start vnc server and listen for connections
|
||||
# on port specced in `$VNC_SERVER_PORT`
|
||||
|
||||
# start VNC server
|
||||
x11vnc \
|
||||
-listen 127.0.0.1 \
|
||||
-allow 127.0.0.1 \
|
||||
-rfbport "${VNC_SERVER_PORT}" \
|
||||
-ncache_cr \
|
||||
-listen localhost \
|
||||
-display :1 \
|
||||
-forever \
|
||||
-shared \
|
||||
-logappend /var/log/x11vnc.log \
|
||||
-bg \
|
||||
-nowf \
|
||||
-noxdamage \
|
||||
-noxfixes \
|
||||
-no6 \
|
||||
-noipv6 \
|
||||
|
||||
|
||||
# -nowcr \
|
||||
# TODO: can't use this because of ``asyncvnc`` issue:
|
||||
-autoport 3003 \
|
||||
# can't use this because of ``asyncvnc`` issue:
|
||||
# https://github.com/barneygale/asyncvnc/issues/1
|
||||
# -passwd 'ibcansmbz'
|
||||
|
||||
# XXX: optional graphics caching flags that seem to rekt the overlay
|
||||
# of the 2 gw windows? When running a single gateway
|
||||
# this seems to maybe optimize some memory usage?
|
||||
# -ncache_cr \
|
||||
# -ncache \
|
||||
|
||||
# NOTE: this will prevent logs from going to the console.
|
||||
# -logappend /var/log/x11vnc.log \
|
||||
|
||||
# where to start allocating ports
|
||||
# -autoport "${VNC_SERVER_PORT}" \
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers.
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright 2020-eternity Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,14 +14,7 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"""
|
||||
piker: trading gear for hackers.
|
||||
|
||||
'''
|
||||
from .service import open_piker_runtime
|
||||
from .data.feed import open_feed
|
||||
|
||||
__all__ = [
|
||||
'open_piker_runtime',
|
||||
'open_feed',
|
||||
]
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,20 +14,14 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"""
|
||||
Cacheing apis and toolz.
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from typing import (
|
||||
Awaitable,
|
||||
Callable,
|
||||
ParamSpec,
|
||||
TypeVar,
|
||||
asynccontextmanager,
|
||||
)
|
||||
|
||||
from tractor.trionics import maybe_open_context
|
||||
|
|
@ -38,54 +32,19 @@ from .log import get_logger
|
|||
|
||||
log = get_logger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
P = ParamSpec("P")
|
||||
|
||||
|
||||
# TODO: move this to `tractor.trionics`..
|
||||
# - egs. to replicate for tests: https://github.com/aio-libs/async-lru#usage
|
||||
# - their suite as well:
|
||||
# https://github.com/aio-libs/async-lru/tree/master/tests
|
||||
# - asked trio_util about it too:
|
||||
# https://github.com/groove-x/trio-util/issues/21
|
||||
def async_lifo_cache(
|
||||
maxsize=128,
|
||||
|
||||
# NOTE: typing style was learned from:
|
||||
# https://stackoverflow.com/a/71132186
|
||||
) -> Callable[
|
||||
Callable[P, Awaitable[T]],
|
||||
Callable[
|
||||
Callable[P, Awaitable[T]],
|
||||
Callable[P, Awaitable[T]],
|
||||
],
|
||||
]:
|
||||
'''
|
||||
Async ``cache`` with a LIFO policy.
|
||||
def async_lifo_cache(maxsize=128):
|
||||
"""Async ``cache`` with a LIFO policy.
|
||||
|
||||
Implemented my own since no one else seems to have
|
||||
a standard. I'll wait for the smarter people to come
|
||||
up with one, but until then...
|
||||
|
||||
NOTE: when decorating, due to this simple/naive implementation, you
|
||||
MUST call the decorator like,
|
||||
|
||||
.. code:: python
|
||||
|
||||
@async_lifo_cache()
|
||||
async def cache_target():
|
||||
|
||||
'''
|
||||
"""
|
||||
cache = OrderedDict()
|
||||
|
||||
def decorator(
|
||||
fn: Callable[P, Awaitable[T]],
|
||||
) -> Callable[P, Awaitable[T]]:
|
||||
def decorator(fn):
|
||||
|
||||
async def decorated(
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> T:
|
||||
async def wrapper(*args):
|
||||
key = args
|
||||
try:
|
||||
return cache[key]
|
||||
|
|
@ -94,20 +53,16 @@ def async_lifo_cache(
|
|||
# discard last added new entry
|
||||
cache.popitem()
|
||||
|
||||
# call underlying
|
||||
cache[key] = await fn(
|
||||
*args,
|
||||
**kwargs,
|
||||
)
|
||||
# do it
|
||||
cache[key] = await fn(*args)
|
||||
return cache[key]
|
||||
|
||||
return decorated
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
# TODO: move this to `.brokers.utils`..
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_cached_client(
|
||||
brokername: str,
|
||||
) -> 'Client': # noqa
|
||||
|
|
|
|||
|
|
@ -0,0 +1,561 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Structured, daemon tree service management.
|
||||
|
||||
"""
|
||||
from typing import Optional, Union, Callable, Any
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from collections import defaultdict
|
||||
|
||||
from pydantic import BaseModel
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from .log import get_logger, get_console_log
|
||||
from .brokers import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
|
||||
_registry_addr = ('127.0.0.1', 6116)
|
||||
_tractor_kwargs: dict[str, Any] = {
|
||||
# use a different registry addr then tractor's default
|
||||
'arbiter_addr': _registry_addr
|
||||
}
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
]
|
||||
|
||||
|
||||
class Services(BaseModel):
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[str, tuple[trio.CancelScope, tractor.Portal]] = {}
|
||||
|
||||
class Config:
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
**kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
async with portal.open_context(
|
||||
target,
|
||||
**kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
task_status.started((cs, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
ctx_res = await ctx.result()
|
||||
except tractor.ContextCancelled:
|
||||
return await self.cancel_service(name)
|
||||
else:
|
||||
# wait on any error from the sub-actor
|
||||
# NOTE: this will block indefinitely until
|
||||
# cancelled either by error from the target
|
||||
# context function or by being cancelled here by
|
||||
# the surrounding cancel scope
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
cs, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal)
|
||||
|
||||
return cs, first
|
||||
|
||||
# TODO: per service cancellation by scope, we aren't using this
|
||||
# anywhere right?
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
) -> Any:
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal = self.service_tasks[name]
|
||||
# XXX: not entirely sure why this is required,
|
||||
# and should probably be better fine tuned in
|
||||
# ``tractor``?
|
||||
cs.cancel()
|
||||
return await portal.cancel_actor()
|
||||
|
||||
|
||||
_services: Optional[Services] = None
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a root piker daemon who's lifetime extends indefinitely
|
||||
until cancelled.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=_root_dname,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
):
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
|
||||
# # setup service mngr singleton instance
|
||||
# async with AsyncExitStack() as stack:
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
_services = Services(
|
||||
actor_n=actor_nursery,
|
||||
service_n=service_nursery,
|
||||
debug_mode=debug_mode,
|
||||
)
|
||||
|
||||
yield _services
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
start_method: str = 'trio',
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
) -> Optional[tractor._portal.Portal]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically
|
||||
sync with existing piker actors in local network
|
||||
based on configuration.
|
||||
|
||||
'''
|
||||
global _services
|
||||
assert _services is None
|
||||
|
||||
# XXX: this may open a root actor as well
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=_registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
) as _,
|
||||
):
|
||||
yield tractor.current_actor()
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_runtime(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
"""
|
||||
Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
"""
|
||||
settings = _tractor_kwargs
|
||||
settings.update(kwargs)
|
||||
|
||||
if not tractor.current_actor(err_on_no_runtime=False):
|
||||
async with tractor.open_root_actor(
|
||||
loglevel=loglevel,
|
||||
**settings,
|
||||
):
|
||||
yield
|
||||
else:
|
||||
yield
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> Union[tractor._portal.Portal, Services]:
|
||||
"""If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
"""
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
async with maybe_open_runtime(loglevel, **kwargs):
|
||||
|
||||
async with tractor.find_actor(_root_dname) as portal:
|
||||
# assert portal is not None
|
||||
if portal is not None:
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
debug_mode=kwargs.get('debug_mode', False),
|
||||
|
||||
) as _:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
yield None
|
||||
|
||||
|
||||
# brokerd enabled modules
|
||||
_data_mods = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
class Brokerd:
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> Optional[tractor.Portal]:
|
||||
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=_registry_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
spawn_args: dict[str, Any],
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Brokerd.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(f"Couldn't find any existing {service_name}")
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
if pikerd_portal is None:
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
await service_task_target(**spawn_args)
|
||||
|
||||
else:
|
||||
# tell the remote `pikerd` to start the target,
|
||||
# the target can't return a non-serializable value
|
||||
# since it is expected that service startingn is
|
||||
# non-blocking and the target task will persist running
|
||||
# on `pikerd` after the client requesting it's start
|
||||
# disconnects.
|
||||
await pikerd_portal.run(
|
||||
service_task_target,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
dname = f'brokerd.{brokername}'
|
||||
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
modpath = brokermod.__name__
|
||||
broker_enable = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
broker_enable.append(subpath)
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + broker_enable,
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from .data import _setup_persistent_brokerd
|
||||
|
||||
await _services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
_setup_persistent_brokerd,
|
||||
brokername=brokername,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client
|
||||
who wishes to use the sub-actor-daemon.
|
||||
|
||||
'''
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={'brokername': brokername, 'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: Optional[str] = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
global _services
|
||||
assert _services
|
||||
|
||||
portal = await _services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=_services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from .clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await _services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
_setup_persistent_emsd,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: Optional[str] = None,
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={'loglevel': loglevel},
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
# TODO: ideally we can start the tsdb "on demand" but it's
|
||||
# probably going to require "rootless" docker, at least if we don't
|
||||
# want to expect the user to start ``pikerd`` with root perms all the
|
||||
# time.
|
||||
# async def maybe_open_marketstored(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> tractor._portal.Portal: # noqa
|
||||
|
||||
# async with maybe_spawn_daemon(
|
||||
|
||||
# 'marketstored',
|
||||
# service_task_target=spawn_emsd,
|
||||
# spawn_args={'loglevel': loglevel},
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
|
||||
# ) as portal:
|
||||
# yield portal
|
||||
|
|
@ -18,10 +18,7 @@
|
|||
Profiling wrappers for internal libs.
|
||||
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from time import perf_counter
|
||||
from functools import wraps
|
||||
|
||||
# NOTE: you can pass a flag to enable this:
|
||||
|
|
@ -47,193 +44,3 @@ def timeit(fn):
|
|||
return res
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# Modified version of ``pyqtgraph.debug.Profiler`` that
|
||||
# core seems hesitant to land in:
|
||||
# https://github.com/pyqtgraph/pyqtgraph/pull/2281
|
||||
class Profiler(object):
|
||||
'''
|
||||
Simple profiler allowing measurement of multiple time intervals.
|
||||
|
||||
By default, profilers are disabled. To enable profiling, set the
|
||||
environment variable `PYQTGRAPHPROFILE` to a comma-separated list of
|
||||
fully-qualified names of profiled functions.
|
||||
|
||||
Calling a profiler registers a message (defaulting to an increasing
|
||||
counter) that contains the time elapsed since the last call. When the
|
||||
profiler is about to be garbage-collected, the messages are passed to the
|
||||
outer profiler if one is running, or printed to stdout otherwise.
|
||||
|
||||
If `delayed` is set to False, messages are immediately printed instead.
|
||||
|
||||
Example:
|
||||
def function(...):
|
||||
profiler = Profiler()
|
||||
... do stuff ...
|
||||
profiler('did stuff')
|
||||
... do other stuff ...
|
||||
profiler('did other stuff')
|
||||
# profiler is garbage-collected and flushed at function end
|
||||
|
||||
If this function is a method of class C, setting `PYQTGRAPHPROFILE` to
|
||||
"C.function" (without the module name) will enable this profiler.
|
||||
|
||||
For regular functions, use the qualified name of the function, stripping
|
||||
only the initial "pyqtgraph." prefix from the module.
|
||||
'''
|
||||
|
||||
_profilers = os.environ.get("PYQTGRAPHPROFILE", None)
|
||||
_profilers = _profilers.split(",") if _profilers is not None else []
|
||||
|
||||
_depth = 0
|
||||
|
||||
# NOTE: without this defined at the class level
|
||||
# you won't see apprpriately "nested" sub-profiler
|
||||
# instance calls.
|
||||
_msgs = []
|
||||
|
||||
# set this flag to disable all or individual profilers at runtime
|
||||
disable = False
|
||||
|
||||
class DisabledProfiler(object):
|
||||
def __init__(self, *args, **kwds):
|
||||
pass
|
||||
|
||||
def __call__(self, *args):
|
||||
pass
|
||||
|
||||
def finish(self):
|
||||
pass
|
||||
|
||||
def mark(self, msg=None):
|
||||
pass
|
||||
|
||||
_disabledProfiler = DisabledProfiler()
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
msg=None,
|
||||
disabled='env',
|
||||
delayed=True,
|
||||
ms_threshold: float = 0.0,
|
||||
):
|
||||
"""Optionally create a new profiler based on caller's qualname.
|
||||
|
||||
``ms_threshold`` can be set to value in ms for which, if the
|
||||
total measured time of the lifetime of this profiler is **less
|
||||
than** this value, then no profiling messages will be printed.
|
||||
Setting ``delayed=False`` disables this feature since messages
|
||||
are emitted immediately.
|
||||
|
||||
"""
|
||||
if (
|
||||
disabled is True
|
||||
or (
|
||||
disabled == 'env'
|
||||
and len(cls._profilers) == 0
|
||||
)
|
||||
):
|
||||
return cls._disabledProfiler
|
||||
|
||||
# determine the qualified name of the caller function
|
||||
caller_frame = sys._getframe(1)
|
||||
try:
|
||||
caller_object_type = type(caller_frame.f_locals["self"])
|
||||
|
||||
except KeyError: # we are in a regular function
|
||||
qualifier = caller_frame.f_globals["__name__"].split(".", 1)[-1]
|
||||
|
||||
else: # we are in a method
|
||||
qualifier = caller_object_type.__name__
|
||||
func_qualname = qualifier + "." + caller_frame.f_code.co_name
|
||||
|
||||
if disabled == 'env' and func_qualname not in cls._profilers:
|
||||
# don't do anything
|
||||
return cls._disabledProfiler
|
||||
|
||||
cls._depth += 1
|
||||
obj = super(Profiler, cls).__new__(cls)
|
||||
obj._msgs = []
|
||||
|
||||
# create an actual profiling object
|
||||
if cls._depth < 1:
|
||||
cls._msgs = []
|
||||
|
||||
obj._name = msg or func_qualname
|
||||
obj._delayed = delayed
|
||||
obj._markCount = 0
|
||||
obj._finished = False
|
||||
obj._firstTime = obj._lastTime = perf_counter()
|
||||
obj._mt = ms_threshold
|
||||
obj._newMsg("> Entering " + obj._name)
|
||||
return obj
|
||||
|
||||
def __call__(self, msg=None):
|
||||
"""Register or print a new message with timing information.
|
||||
"""
|
||||
if self.disable:
|
||||
return
|
||||
if msg is None:
|
||||
msg = str(self._markCount)
|
||||
|
||||
self._markCount += 1
|
||||
newTime = perf_counter()
|
||||
tot_ms = (newTime - self._firstTime) * 1000
|
||||
ms = (newTime - self._lastTime) * 1000
|
||||
self._newMsg(
|
||||
f" {msg}: {ms:0.4f}, tot:{tot_ms:0.4f}"
|
||||
)
|
||||
|
||||
self._lastTime = newTime
|
||||
|
||||
def mark(self, msg=None):
|
||||
self(msg)
|
||||
|
||||
def _newMsg(self, msg, *args):
|
||||
msg = " " * (self._depth - 1) + msg
|
||||
if self._delayed:
|
||||
self._msgs.append((msg, args))
|
||||
else:
|
||||
print(msg % args)
|
||||
|
||||
def __del__(self):
|
||||
self.finish()
|
||||
|
||||
def finish(self, msg=None):
|
||||
"""Add a final message; flush the message list if no parent profiler.
|
||||
"""
|
||||
if self._finished or self.disable:
|
||||
return
|
||||
|
||||
self._finished = True
|
||||
if msg is not None:
|
||||
self(msg)
|
||||
|
||||
tot_ms = (perf_counter() - self._firstTime) * 1000
|
||||
self._newMsg(
|
||||
"< Exiting %s, total time: %0.4f ms",
|
||||
self._name,
|
||||
tot_ms,
|
||||
)
|
||||
|
||||
if tot_ms < self._mt:
|
||||
# print(f'{tot_ms} < {self._mt}, clearing')
|
||||
# NOTE: this list **must** be an instance var to avoid
|
||||
# deleting common messages during GC I think?
|
||||
self._msgs.clear()
|
||||
# else:
|
||||
# print(f'{tot_ms} > {self._mt}, not clearing')
|
||||
|
||||
# XXX: why is this needed?
|
||||
# don't we **want to show** nested profiler messages?
|
||||
if self._msgs: # and self._depth < 1:
|
||||
|
||||
# if self._msgs:
|
||||
print("\n".join([m[0] % m[1] for m in self._msgs]))
|
||||
|
||||
# clear all entries
|
||||
self._msgs.clear()
|
||||
# type(self)._msgs = []
|
||||
|
||||
type(self)._depth -= 1
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
"Accounting for degens": count dem numberz that tracks how much you got
|
||||
for tendiez.
|
||||
|
||||
'''
|
||||
from ..log import get_logger
|
||||
|
||||
from ._ledger import (
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
)
|
||||
from ._pos import (
|
||||
load_pps_from_ledger,
|
||||
open_pps,
|
||||
Position,
|
||||
PpTable,
|
||||
)
|
||||
from ._mktinfo import (
|
||||
Asset,
|
||||
dec_digits,
|
||||
digits_to_dec,
|
||||
MktPair,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'Asset',
|
||||
'dec_digits',
|
||||
'digits_to_dec',
|
||||
'MktPair',
|
||||
'Position',
|
||||
'PpTable',
|
||||
'Transaction',
|
||||
'TransactionLedger',
|
||||
'load_pps_from_ledger',
|
||||
'open_pps',
|
||||
'open_trade_ledger',
|
||||
]
|
||||
|
||||
|
||||
def get_likely_pair(
|
||||
src: str,
|
||||
dst: str,
|
||||
bs_mktid: str,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Attempt to get the likely trading pair matching a given destination
|
||||
asset `dst: str`.
|
||||
|
||||
'''
|
||||
try:
|
||||
src_name_start = bs_mktid.rindex(src)
|
||||
except (
|
||||
ValueError, # substr not found
|
||||
):
|
||||
# TODO: handle nested positions..(i.e.
|
||||
# positions where the src fiat was used to
|
||||
# buy some other dst which was furhter used
|
||||
# to buy another dst..)
|
||||
log.warning(
|
||||
f'No src fiat {src} found in {bs_mktid}?'
|
||||
)
|
||||
return
|
||||
|
||||
likely_dst = bs_mktid[:src_name_start]
|
||||
if likely_dst == dst:
|
||||
return bs_mktid
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
from pprint import pformat
|
||||
|
||||
args = sys.argv
|
||||
assert len(args) > 1, 'Specifiy account(s) from `brokers.toml`'
|
||||
args = args[1:]
|
||||
for acctid in args:
|
||||
broker, name = acctid.split('.')
|
||||
trans, updated_pps = load_pps_from_ledger(broker, name)
|
||||
print(
|
||||
f'Processing transactions into pps for {broker}:{acctid}\n'
|
||||
f'{pformat(trans)}\n\n'
|
||||
f'{pformat(updated_pps)}'
|
||||
)
|
||||
|
|
@ -1,278 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Trade and transaction ledger processing.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from collections import UserDict
|
||||
from contextlib import contextmanager as cm
|
||||
from pathlib import Path
|
||||
import time
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
Union,
|
||||
Generator
|
||||
)
|
||||
|
||||
from pendulum import (
|
||||
datetime,
|
||||
parse,
|
||||
)
|
||||
import tomli
|
||||
import toml
|
||||
|
||||
from .. import config
|
||||
from ..data.types import Struct
|
||||
from ..log import get_logger
|
||||
from ._mktinfo import (
|
||||
Symbol, # legacy
|
||||
MktPair,
|
||||
Asset,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Transaction(Struct, frozen=True):
|
||||
|
||||
# TODO: unify this with the `MktPair`,
|
||||
# once we have that as a required field,
|
||||
# we don't really need the fqsn any more..
|
||||
fqsn: str
|
||||
|
||||
tid: Union[str, int] # unique transaction id
|
||||
size: float
|
||||
price: float
|
||||
cost: float # commisions or other additional costs
|
||||
dt: datetime
|
||||
|
||||
# TODO: we can drop this right since we
|
||||
# can instead expect the backend to provide this
|
||||
# via the `MktPair`?
|
||||
expiry: datetime | None = None
|
||||
|
||||
# remap for back-compat
|
||||
@property
|
||||
def fqme(self) -> str:
|
||||
return self.fqsn
|
||||
|
||||
# TODO: drop the Symbol type, construct using
|
||||
# t.sys (the transaction system)
|
||||
|
||||
# the underlying "transaction system", normally one of a ``MktPair``
|
||||
# (a description of a tradable double auction) or a ledger-recorded
|
||||
# ("ledger" in any sense as long as you can record transfers) of any
|
||||
# sort) ``Asset``.
|
||||
sym: MktPair | Asset | Symbol | None = None
|
||||
|
||||
@property
|
||||
def sys(self) -> Symbol:
|
||||
return self.sym
|
||||
|
||||
# (optional) key-id defined by the broker-service backend which
|
||||
# ensures the instrument-symbol market key for this record is unique
|
||||
# in the "their backend/system" sense; i.e. this uid for the market
|
||||
# as defined (internally) in some namespace defined by the broker
|
||||
# service.
|
||||
bs_mktid: str | int | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
dct = super().to_dict()
|
||||
|
||||
# TODO: switch to sys!
|
||||
dct.pop('sym')
|
||||
|
||||
# ensure we use a pendulum formatted
|
||||
# ISO style str here!@
|
||||
dct['dt'] = str(self.dt)
|
||||
return dct
|
||||
|
||||
|
||||
class TransactionLedger(UserDict):
|
||||
'''
|
||||
Very simple ``dict`` wrapper + ``pathlib.Path`` handle to
|
||||
a TOML formatted transaction file for enabling file writes
|
||||
dynamically whilst still looking exactly like a ``dict`` from the
|
||||
outside.
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
ledger_dict: dict,
|
||||
file_path: Path,
|
||||
|
||||
) -> None:
|
||||
self.file_path = file_path
|
||||
super().__init__(ledger_dict)
|
||||
|
||||
def write_config(self) -> None:
|
||||
'''
|
||||
Render the self.data ledger dict to it's TML file form.
|
||||
|
||||
'''
|
||||
with self.file_path.open(mode='w') as fp:
|
||||
|
||||
# rewrite the key name to fqme if needed
|
||||
fqsn: str = self.data.get('fqsn')
|
||||
if fqsn:
|
||||
self.data['fqme'] = fqsn
|
||||
|
||||
toml.dump(self.data, fp)
|
||||
|
||||
def update_from_t(
|
||||
self,
|
||||
t: Transaction,
|
||||
) -> None:
|
||||
self.data[t.tid] = t.to_dict()
|
||||
|
||||
def iter_trans(
|
||||
self,
|
||||
mkt_by_fqme: dict[str, MktPair],
|
||||
broker: str = 'paper',
|
||||
|
||||
) -> Generator[
|
||||
tuple[str, Transaction],
|
||||
None,
|
||||
None,
|
||||
]:
|
||||
'''
|
||||
Deliver trades records in ``(key: str, t: Transaction)``
|
||||
form via generator.
|
||||
|
||||
'''
|
||||
if broker != 'paper':
|
||||
raise NotImplementedError('Per broker support not dun yet!')
|
||||
|
||||
# TODO: lookup some standard normalizer
|
||||
# func in the backend?
|
||||
# from ..brokers import get_brokermod
|
||||
# mod = get_brokermod(broker)
|
||||
# trans_dict = mod.norm_trade_records(self.data)
|
||||
|
||||
# NOTE: instead i propose the normalizer is
|
||||
# a one shot routine (that can be lru cached)
|
||||
# and instead call it for each entry incrementally:
|
||||
# normer = mod.norm_trade_record(txdict)
|
||||
|
||||
for tid, txdict in self.data.items():
|
||||
# special field handling for datetimes
|
||||
# to ensure pendulum is used!
|
||||
fqme = txdict.get('fqme', txdict['fqsn'])
|
||||
dt = parse(txdict['dt'])
|
||||
expiry = txdict.get('expiry')
|
||||
|
||||
mkt = mkt_by_fqme.get(fqme)
|
||||
if not mkt:
|
||||
# we can't build a trans if we don't have
|
||||
# the ``.sys: MktPair`` info, so skip.
|
||||
continue
|
||||
|
||||
yield (
|
||||
tid,
|
||||
Transaction(
|
||||
fqsn=fqme,
|
||||
tid=txdict['tid'],
|
||||
dt=dt,
|
||||
price=txdict['price'],
|
||||
size=txdict['size'],
|
||||
cost=txdict.get('cost', 0),
|
||||
bs_mktid=txdict['bs_mktid'],
|
||||
|
||||
# TODO: change to .sys!
|
||||
sym=mkt,
|
||||
expiry=parse(expiry) if expiry else None,
|
||||
)
|
||||
)
|
||||
|
||||
def to_trans(
|
||||
self,
|
||||
**kwargs,
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Return entire output from ``.iter_trans()`` in a ``dict``.
|
||||
|
||||
'''
|
||||
return dict(self.iter_trans(**kwargs))
|
||||
|
||||
|
||||
@cm
|
||||
def open_trade_ledger(
|
||||
broker: str,
|
||||
account: str,
|
||||
|
||||
) -> Generator[dict, None, None]:
|
||||
'''
|
||||
Indempotently create and read in a trade log file from the
|
||||
``<configuration_dir>/ledgers/`` directory.
|
||||
|
||||
Files are named per broker account of the form
|
||||
``<brokername>_<accountname>.toml``. The ``accountname`` here is the
|
||||
name as defined in the user's ``brokers.toml`` config.
|
||||
|
||||
'''
|
||||
ldir: Path = config._config_dir / 'ledgers'
|
||||
if not ldir.is_dir():
|
||||
ldir.mkdir()
|
||||
|
||||
fname = f'trades_{broker}_{account}.toml'
|
||||
tradesfile: Path = ldir / fname
|
||||
|
||||
if not tradesfile.is_file():
|
||||
log.info(
|
||||
f'Creating new local trades ledger: {tradesfile}'
|
||||
)
|
||||
tradesfile.touch()
|
||||
|
||||
with tradesfile.open(mode='rb') as cf:
|
||||
start = time.time()
|
||||
ledger_dict = tomli.load(cf)
|
||||
log.info(f'Ledger load took {time.time() - start}s')
|
||||
cpy = ledger_dict.copy()
|
||||
|
||||
ledger = TransactionLedger(
|
||||
ledger_dict=cpy,
|
||||
file_path=tradesfile,
|
||||
)
|
||||
|
||||
try:
|
||||
yield ledger
|
||||
finally:
|
||||
if ledger.data != ledger_dict:
|
||||
|
||||
# TODO: show diff output?
|
||||
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
|
||||
log.info(f'Updating ledger for {tradesfile}:\n')
|
||||
ledger.write_config()
|
||||
|
||||
|
||||
def iter_by_dt(
|
||||
clears: dict[str, Any],
|
||||
|
||||
) -> Iterator[tuple[str, dict]]:
|
||||
'''
|
||||
Iterate entries of a ``clears: dict`` table sorted by entry recorded
|
||||
datetime presumably set at the ``'dt'`` field in each entry.
|
||||
|
||||
'''
|
||||
for tid, data in sorted(
|
||||
list(clears.items()),
|
||||
key=lambda item: item[1]['dt'],
|
||||
):
|
||||
yield tid, data
|
||||
|
|
@ -1,630 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Market (pair) meta-info layer: sane addressing semantics and meta-data
|
||||
for cross-provider marketplaces.
|
||||
|
||||
We intoduce the concept of,
|
||||
|
||||
- a FQMA: fully qualified market address,
|
||||
- a sane schema for FQMAs including derivatives,
|
||||
- a msg-serializeable description of markets for
|
||||
easy sharing with other pikers B)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from decimal import (
|
||||
Decimal,
|
||||
ROUND_HALF_EVEN,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
Literal,
|
||||
)
|
||||
|
||||
from ..data.types import Struct
|
||||
|
||||
|
||||
_underlyings: list[str] = [
|
||||
'stock',
|
||||
'bond',
|
||||
'crypto',
|
||||
'fiat',
|
||||
'commodity',
|
||||
]
|
||||
|
||||
|
||||
_derivs: list[str] = [
|
||||
'swap',
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
|
||||
# if we can't figure it out, presume the worst XD
|
||||
'unknown',
|
||||
]
|
||||
|
||||
# NOTE: a tag for other subsystems to try
|
||||
# and do default settings for certain things:
|
||||
# - allocator does unit vs. dolla size limiting.
|
||||
AssetTypeName: Literal[
|
||||
_underlyings
|
||||
+
|
||||
_derivs
|
||||
]
|
||||
|
||||
# egs. stock, futer, option, bond etc.
|
||||
|
||||
|
||||
def dec_digits(
|
||||
value: float | str | Decimal,
|
||||
|
||||
) -> int:
|
||||
'''
|
||||
Return the number of precision digits read from a decimal or float
|
||||
value.
|
||||
|
||||
'''
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(
|
||||
-Decimal(str(value)).as_tuple().exponent
|
||||
)
|
||||
|
||||
|
||||
float_digits = dec_digits
|
||||
|
||||
|
||||
def digits_to_dec(
|
||||
ndigits: int,
|
||||
) -> Decimal:
|
||||
'''
|
||||
Return the minimum float value for an input integer value.
|
||||
|
||||
eg. 3 -> 0.001
|
||||
|
||||
'''
|
||||
if ndigits == 0:
|
||||
return Decimal('0')
|
||||
|
||||
return Decimal('0.' + '0'*(ndigits-1) + '1')
|
||||
|
||||
|
||||
class Asset(Struct, frozen=True):
|
||||
'''
|
||||
Container type describing any transactable asset and its
|
||||
contract-like and/or underlying technology meta-info.
|
||||
|
||||
'''
|
||||
name: str
|
||||
atype: str # AssetTypeName
|
||||
|
||||
# minimum transaction size / precision.
|
||||
# eg. for buttcoin this is a "satoshi".
|
||||
tx_tick: Decimal
|
||||
|
||||
# NOTE: additional info optionally packed in by the backend, but
|
||||
# should not be explicitly required in our generic API.
|
||||
info: dict = {} # make it frozen?
|
||||
|
||||
# TODO?
|
||||
# _to_dict_skip = {'info'}
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
quantized form of the digit precision defined
|
||||
by ``self.lot_tick_size``.
|
||||
|
||||
'''
|
||||
digits = float_digits(self.tx_tick)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def guess_from_mkt_ep_key(
|
||||
cls,
|
||||
mkt_ep_key: str,
|
||||
atype: str | None = None,
|
||||
|
||||
) -> Asset:
|
||||
'''
|
||||
A hacky guess method for presuming a (target) asset's properties
|
||||
based on either the actualy market endpoint key, or config settings
|
||||
from the user.
|
||||
|
||||
'''
|
||||
atype = atype or 'unknown'
|
||||
|
||||
# attempt to strip off any source asset
|
||||
# via presumed syntax of:
|
||||
# - <dst>/<src>
|
||||
# - <dst>.<src>
|
||||
# - etc.
|
||||
for char in ['/', '.']:
|
||||
dst, _, src = mkt_ep_key.partition(char)
|
||||
if src:
|
||||
if not atype:
|
||||
atype = 'fiat'
|
||||
break
|
||||
|
||||
return Asset(
|
||||
name=dst,
|
||||
atype=atype,
|
||||
tx_tick=Decimal('0.01'),
|
||||
)
|
||||
|
||||
|
||||
def maybe_cons_tokens(
|
||||
tokens: list[Any],
|
||||
delim_char: str = '.',
|
||||
) -> str:
|
||||
'''
|
||||
Construct `str` output from a maybe-concatenation of input
|
||||
sequence of elements in ``tokens``.
|
||||
|
||||
'''
|
||||
return delim_char.join(filter(bool, tokens)).lower()
|
||||
|
||||
|
||||
class MktPair(Struct, frozen=True):
|
||||
'''
|
||||
Market description for a pair of assets which are tradeable:
|
||||
a market which enables transactions of the form,
|
||||
buy: source asset -> destination asset
|
||||
sell: destination asset -> source asset
|
||||
|
||||
The main intention of this type is for a **simple** cross-asset
|
||||
venue/broker normalized descrption type from which all
|
||||
market-auctions can be mapped from FQME identifiers.
|
||||
|
||||
TODO: our eventual target fqme format/schema is:
|
||||
<dst>/<src>.<expiry>.<con_info_1>.<con_info_2>. -> .<venue>.<broker>
|
||||
^ -- optional tokens ------------------------------- ^
|
||||
|
||||
'''
|
||||
dst: str | Asset
|
||||
# "destination asset" (name) used to buy *to*
|
||||
# (or used to sell *from*)
|
||||
|
||||
price_tick: Decimal # minimum price increment
|
||||
size_tick: Decimal # minimum size (aka vlm) increment
|
||||
# the tick size is the number describing the smallest step in value
|
||||
# available in this market between the source and destination
|
||||
# assets.
|
||||
# https://en.wikipedia.org/wiki/Tick_size
|
||||
# https://en.wikipedia.org/wiki/Commodity_tick
|
||||
# https://en.wikipedia.org/wiki/Percentage_in_point
|
||||
|
||||
# unique "broker id" since every market endpoint provider
|
||||
# has their own nomenclature and schema for market maps.
|
||||
bs_mktid: str
|
||||
broker: str # the middle man giving access
|
||||
|
||||
# NOTE: to start this field is optional but should eventually be
|
||||
# required; the reason is for backward compat since more positioning
|
||||
# calculations were not originally stored with a src asset..
|
||||
|
||||
src: str | Asset = ''
|
||||
# "source asset" (name) used to buy *from*
|
||||
# (or used to sell *to*).
|
||||
|
||||
venue: str = '' # market venue provider name
|
||||
expiry: str = '' # for derivs, expiry datetime parseable str
|
||||
|
||||
# destination asset's financial type/classification name
|
||||
# NOTE: this is required for the order size allocator system,
|
||||
# since we use different default settings based on the type
|
||||
# of the destination asset, eg. futes use a units limits vs.
|
||||
# equities a $limit.
|
||||
# dst_type: AssetTypeName | None = None
|
||||
|
||||
# source asset's financial type/classification name
|
||||
# TODO: is a src type required for trading?
|
||||
# there's no reason to need any more then the one-way alloc-limiter
|
||||
# config right?
|
||||
# src_type: AssetTypeName
|
||||
|
||||
# for derivs, info describing contract, egs.
|
||||
# strike price, call or put, swap type, exercise model, etc.
|
||||
contract_info: list[str] | None = None
|
||||
|
||||
_atype: str = ''
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict[str, Any],
|
||||
|
||||
) -> MktPair:
|
||||
'''
|
||||
Constructor for a received msg-dict normally received over IPC.
|
||||
|
||||
'''
|
||||
dst_asset_msg = msg.pop('dst')
|
||||
src_asset_msg = msg.pop('src')
|
||||
|
||||
if isinstance(dst_asset_msg, str):
|
||||
src: str = str(src_asset_msg)
|
||||
assert isinstance(src, str)
|
||||
return cls.from_fqme(
|
||||
dst_asset_msg,
|
||||
src=src,
|
||||
**msg,
|
||||
)
|
||||
|
||||
else:
|
||||
# NOTE: we call `.copy()` here to ensure
|
||||
# type casting!
|
||||
dst = Asset(**dst_asset_msg).copy()
|
||||
if not isinstance(src_asset_msg, str):
|
||||
src = Asset(**src_asset_msg).copy()
|
||||
else:
|
||||
src = str(src_asset_msg)
|
||||
|
||||
return cls(
|
||||
dst=dst,
|
||||
src=src,
|
||||
**msg,
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def resolved(self) -> bool:
|
||||
return isinstance(self.dst, Asset)
|
||||
|
||||
@classmethod
|
||||
def from_fqme(
|
||||
cls,
|
||||
fqme: str,
|
||||
|
||||
price_tick: float | str,
|
||||
size_tick: float | str,
|
||||
bs_mktid: str,
|
||||
|
||||
broker: str | None = None,
|
||||
**kwargs,
|
||||
|
||||
) -> MktPair:
|
||||
|
||||
_fqme: str = fqme
|
||||
if (
|
||||
broker
|
||||
and broker not in fqme
|
||||
):
|
||||
_fqme = f'{fqme}.{broker}'
|
||||
|
||||
broker, mkt_ep_key, venue, suffix = unpack_fqme(_fqme)
|
||||
dst: Asset = Asset.guess_from_mkt_ep_key(
|
||||
mkt_ep_key,
|
||||
atype=kwargs.get('_atype'),
|
||||
)
|
||||
|
||||
# XXX: loading from a fqme string will
|
||||
# leave this pair as "un resolved" meaning
|
||||
# we don't yet have `.dst` set as an `Asset`
|
||||
# which we expect to be filled in by some
|
||||
# backend client with access to that data-info.
|
||||
return cls(
|
||||
# XXX: not resolved to ``Asset`` :(
|
||||
dst=dst,
|
||||
|
||||
broker=broker,
|
||||
venue=venue,
|
||||
# XXX NOTE: we presume this token
|
||||
# if the expiry for now!
|
||||
expiry=suffix,
|
||||
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
**kwargs,
|
||||
|
||||
).copy()
|
||||
|
||||
@property
|
||||
def key(self) -> str:
|
||||
'''
|
||||
The "endpoint key" for this market.
|
||||
|
||||
Eg. mnq/usd or btc/usdt or xmr/btc
|
||||
|
||||
In most other tina platforms this is referred to as the
|
||||
"symbol".
|
||||
|
||||
'''
|
||||
return maybe_cons_tokens(
|
||||
[str(self.dst),
|
||||
str(self.src)],
|
||||
delim_char='',
|
||||
)
|
||||
|
||||
@property
|
||||
def suffix(self) -> str:
|
||||
'''
|
||||
The "contract suffix" for this market.
|
||||
|
||||
Eg. mnq/usd.20230616.cme.ib
|
||||
^ ----- ^
|
||||
or tsla/usd.20230324.200c.cboe.ib
|
||||
^ ---------- ^
|
||||
|
||||
In most other tina platforms they only show you these details in
|
||||
some kinda "meta data" format, we have FQMEs so we do this up
|
||||
front and explicit.
|
||||
|
||||
'''
|
||||
field_strs = [self.expiry]
|
||||
con_info = self.contract_info
|
||||
if con_info is not None:
|
||||
field_strs.extend(con_info)
|
||||
|
||||
return maybe_cons_tokens(field_strs)
|
||||
|
||||
# NOTE: the main idea behind an fqme is to map a "market address"
|
||||
# to some endpoint from a transaction provider (eg. a broker) such
|
||||
# that we build a table of `fqme: str -> bs_mktid: Any` where any "piker
|
||||
# market address" maps 1-to-1 to some broker trading endpoint.
|
||||
# @cached_property
|
||||
@property
|
||||
def fqme(self) -> str:
|
||||
'''
|
||||
Return the fully qualified market endpoint-address for the
|
||||
pair of transacting assets.
|
||||
|
||||
fqme = "fully qualified market endpoint"
|
||||
|
||||
And yes, you pronounce it colloquially as read..
|
||||
|
||||
Basically the idea here is for all client code (consumers of piker's
|
||||
APIs which query the data/broker-provider agnostic layer(s)) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string-key of the current form:
|
||||
|
||||
<market-instrument-name>
|
||||
.<venue>
|
||||
.<expiry>
|
||||
.<derivative-suffix-info>
|
||||
.<brokerbackendname>
|
||||
|
||||
eg. for an explicit daq mini futes contract: mnq.cme.20230317.ib
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.)
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
TODO:
|
||||
See community discussion on naming and nomenclature, order
|
||||
of addressing hierarchy, general schema, internal representation:
|
||||
|
||||
https://github.com/pikers/piker/issues/467
|
||||
|
||||
'''
|
||||
return maybe_cons_tokens([
|
||||
self.key, # final "pair name" (eg. qqq[/usd], btcusdt)
|
||||
self.venue,
|
||||
self.suffix, # includes expiry and other con info
|
||||
self.broker,
|
||||
])
|
||||
|
||||
@property
|
||||
def bs_fqme(self) -> str:
|
||||
'''
|
||||
FQME sin broker part XD
|
||||
|
||||
'''
|
||||
return self.fqme.rstrip(f'.{self.broker}')
|
||||
|
||||
@property
|
||||
def fqsn(self) -> str:
|
||||
return self.fqme
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
|
||||
quantity_type: Literal['price', 'size'] = 'size',
|
||||
|
||||
) -> Decimal:
|
||||
'''
|
||||
Truncate input ``size: float`` using ``Decimal``
|
||||
and ``.size_tick``'s # of digits.
|
||||
|
||||
'''
|
||||
match quantity_type:
|
||||
case 'price':
|
||||
digits = float_digits(self.price_tick)
|
||||
case 'size':
|
||||
digits = float_digits(self.size_tick)
|
||||
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# TODO: BACKWARD COMPAT, TO REMOVE?
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
if isinstance(self.dst, Asset):
|
||||
return str(self.dst.atype)
|
||||
|
||||
return self._atype
|
||||
|
||||
@property
|
||||
def tick_size_digits(self) -> int:
|
||||
return float_digits(self.price_tick)
|
||||
|
||||
@property
|
||||
def lot_size_digits(self) -> int:
|
||||
return float_digits(self.size_tick)
|
||||
|
||||
|
||||
def unpack_fqme(
|
||||
fqme: str,
|
||||
|
||||
broker: str | None = None
|
||||
|
||||
) -> tuple[str, ...]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqme.split('.')
|
||||
|
||||
match tokens:
|
||||
case [mkt_ep, broker]:
|
||||
# probably crypto
|
||||
return (
|
||||
broker,
|
||||
mkt_ep,
|
||||
'',
|
||||
'',
|
||||
)
|
||||
|
||||
# TODO: swap venue and suffix/deriv-info here?
|
||||
case [mkt_ep, venue, suffix, broker]:
|
||||
pass
|
||||
|
||||
# handle `bs_mktid` + `broker` input case
|
||||
case [
|
||||
mkt_ep, venue, suffix
|
||||
] if (
|
||||
broker
|
||||
and suffix != broker
|
||||
):
|
||||
pass
|
||||
|
||||
case [mkt_ep, venue, broker]:
|
||||
suffix = ''
|
||||
|
||||
case _:
|
||||
raise ValueError(f'Invalid fqme: {fqme}')
|
||||
|
||||
return (
|
||||
broker,
|
||||
mkt_ep,
|
||||
venue,
|
||||
# '.'.join([mkt_ep, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
class Symbol(Struct):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
|
||||
# precision descriptors for price and vlm
|
||||
tick_size: Decimal = Decimal('0.01')
|
||||
lot_tick_size: Decimal = Decimal('0.0')
|
||||
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
@classmethod
|
||||
def from_fqme(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
) -> Symbol:
|
||||
broker, mktep, venue, suffix = unpack_fqme(fqsn)
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
key=mktep,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_size,
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def tick_size_digits(self) -> int:
|
||||
return float_digits(self.tick_size)
|
||||
|
||||
@property
|
||||
def lot_size_digits(self) -> int:
|
||||
return float_digits(self.lot_tick_size)
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return Decimal(str(self.tick_size))
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return Decimal(str(self.lot_tick_size))
|
||||
|
||||
@property
|
||||
def broker(self) -> str:
|
||||
return list(self.broker_info.keys())[0]
|
||||
|
||||
@property
|
||||
def fqsn(self) -> str:
|
||||
broker = self.broker
|
||||
key = self.key
|
||||
if self.suffix:
|
||||
tokens = (key, self.suffix, broker)
|
||||
else:
|
||||
tokens = (key, broker)
|
||||
|
||||
return '.'.join(tokens).lower()
|
||||
|
||||
fqme = fqsn
|
||||
|
||||
def quantize(
|
||||
self,
|
||||
size: float,
|
||||
) -> Decimal:
|
||||
digits = float_digits(self.lot_tick_size)
|
||||
return Decimal(size).quantize(
|
||||
Decimal(f'1.{"0".ljust(digits, "0")}'),
|
||||
rounding=ROUND_HALF_EVEN
|
||||
)
|
||||
|
||||
# NOTE: when cast to `str` return fqme
|
||||
def __str__(self) -> str:
|
||||
return self.fqme
|
||||
|
|
@ -1,946 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Personal/Private position parsing, calculating, summarizing in a way
|
||||
that doesn't try to cuk most humans who prefer to not lose their moneys..
|
||||
|
||||
(looking at you `ib` and dirt-bird friends)
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from contextlib import contextmanager as cm
|
||||
from decimal import Decimal
|
||||
from math import copysign
|
||||
from pprint import pformat
|
||||
from pathlib import Path
|
||||
from typing import (
|
||||
Any,
|
||||
Iterator,
|
||||
Union,
|
||||
Generator
|
||||
)
|
||||
|
||||
import pendulum
|
||||
from pendulum import datetime, now
|
||||
|
||||
from ._toml import (
|
||||
toml,
|
||||
PpsEncoder,
|
||||
)
|
||||
from ._ledger import (
|
||||
Transaction,
|
||||
iter_by_dt,
|
||||
open_trade_ledger,
|
||||
)
|
||||
from ._mktinfo import (
|
||||
Symbol,
|
||||
MktPair,
|
||||
Asset,
|
||||
unpack_fqme,
|
||||
)
|
||||
from .. import config
|
||||
from ..brokers import get_brokermod
|
||||
from ..clearing._messages import BrokerdPosition, Status
|
||||
from ..data.types import Struct
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Position(Struct):
|
||||
'''
|
||||
An asset "position" model with attached clearing transaction history.
|
||||
|
||||
A financial "position" in `piker` terms is a summary of accounting
|
||||
metrics computed from a transaction ledger; generally it describes
|
||||
some acumulative "size" and "average price" from the summarized
|
||||
underlying transaction set.
|
||||
|
||||
In piker we focus on the `.ppu` (price per unit) and the `.bep`
|
||||
(break even price) including all transaction entries and exits since
|
||||
the last "net-zero" size of the destination asset's holding.
|
||||
|
||||
This interface serves as an object API for computing and tracking
|
||||
positions as well as supports serialization for storage in the local
|
||||
file system (in TOML) and to interchange as a msg over IPC.
|
||||
|
||||
'''
|
||||
symbol: Symbol | MktPair
|
||||
|
||||
@property
|
||||
def mkt(self) -> MktPair:
|
||||
return self.symbol
|
||||
|
||||
# can be +ve or -ve for long/short
|
||||
size: float
|
||||
|
||||
# "price-per-unit price" above or below which pnl moves above and
|
||||
# below zero for the entirety of the current "trade state". The ppu
|
||||
# is only modified on "increases of" the absolute size of a position
|
||||
# in one of a long/short "direction" (i.e. abs(.size_i) > 0 after
|
||||
# the next transaction given .size was > 0 before that tx, and vice
|
||||
# versa for -ve sized positions).
|
||||
ppu: float
|
||||
|
||||
# TODO: break-even-price support!
|
||||
# bep: float
|
||||
|
||||
# unique "backend system market id"
|
||||
bs_mktid: str
|
||||
|
||||
split_ratio: int | None = None
|
||||
|
||||
# ordered record of known constituent trade messages
|
||||
clears: dict[
|
||||
Union[str, int, Status], # trade id
|
||||
dict[str, Any], # transaction history summaries
|
||||
] = {}
|
||||
first_clear_dt: datetime | None = None
|
||||
|
||||
expiry: datetime | None = None
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return pformat(self.to_dict())
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
f: getattr(self, f)
|
||||
for f in self.__struct_fields__
|
||||
}
|
||||
|
||||
def to_pretoml(self) -> tuple[str, dict]:
|
||||
'''
|
||||
Prep this position's data contents for export to toml including
|
||||
re-structuring of the ``.clears`` table to an array of
|
||||
inline-subtables for better ``pps.toml`` compactness.
|
||||
|
||||
'''
|
||||
d = self.to_dict()
|
||||
clears = d.pop('clears')
|
||||
expiry = d.pop('expiry')
|
||||
|
||||
if self.split_ratio is None:
|
||||
d.pop('split_ratio')
|
||||
|
||||
# should be obvious from clears/event table
|
||||
d.pop('first_clear_dt')
|
||||
|
||||
# TODO: we need to figure out how to have one top level
|
||||
# listing venue here even when the backend isn't providing
|
||||
# it via the trades ledger..
|
||||
# drop symbol obj in serialized form
|
||||
s = d.pop('symbol')
|
||||
fqme = s.fqme
|
||||
broker, mktep, venue, suffix = unpack_fqme(fqme)
|
||||
|
||||
if isinstance(s, Symbol):
|
||||
sym_info = s.broker_info[broker]
|
||||
d['asset_type'] = sym_info['asset_type']
|
||||
d['price_tick'] = (
|
||||
sym_info.get('price_tick_size')
|
||||
or
|
||||
s.tick_size
|
||||
)
|
||||
d['size_tick'] = (
|
||||
sym_info.get('lot_tick_size')
|
||||
or
|
||||
s.lot_tick_size
|
||||
)
|
||||
|
||||
# the newwww wayyy B)
|
||||
else:
|
||||
mkt = s
|
||||
assert isinstance(mkt, MktPair)
|
||||
|
||||
# an asset resolved mkt where we have ``Asset`` info about
|
||||
# each tradeable asset in the market.
|
||||
if mkt.resolved:
|
||||
dst: Asset = mkt.dst
|
||||
d['asset_type'] = dst.atype
|
||||
|
||||
d['price_tick'] = mkt.price_tick
|
||||
d['size_tick'] = mkt.size_tick
|
||||
|
||||
if self.expiry is None:
|
||||
d.pop('expiry', None)
|
||||
elif expiry:
|
||||
d['expiry'] = str(expiry)
|
||||
|
||||
toml_clears_list: list[dict[str, Any]] = []
|
||||
|
||||
# reverse sort so latest clears are at top of section?
|
||||
for tid, data in iter_by_dt(clears):
|
||||
inline_table = toml.TomlDecoder().get_empty_inline_table()
|
||||
|
||||
# serialize datetime to parsable `str`
|
||||
dtstr = inline_table['dt'] = str(data['dt'])
|
||||
assert 'Datetime' not in dtstr
|
||||
|
||||
# insert optional clear fields in column order
|
||||
for k in ['ppu', 'accum_size']:
|
||||
val = data.get(k)
|
||||
if val:
|
||||
inline_table[k] = val
|
||||
|
||||
# insert required fields
|
||||
for k in ['price', 'size', 'cost']:
|
||||
inline_table[k] = data[k]
|
||||
|
||||
inline_table['tid'] = tid
|
||||
toml_clears_list.append(inline_table)
|
||||
|
||||
d['clears'] = toml_clears_list
|
||||
|
||||
return fqme, d
|
||||
|
||||
def ensure_state(self) -> None:
|
||||
'''
|
||||
Audit either the `.size` and `.ppu` local instance vars against
|
||||
the clears table calculations and return the calc-ed values if
|
||||
they differ and log warnings to console.
|
||||
|
||||
'''
|
||||
clears = list(self.clears.values())
|
||||
self.first_clear_dt = min(
|
||||
list(entry['dt'] for entry in clears)
|
||||
)
|
||||
last_clear = clears[-1]
|
||||
|
||||
csize = self.calc_size()
|
||||
accum = last_clear['accum_size']
|
||||
if not self.expired():
|
||||
if (
|
||||
csize != accum
|
||||
and csize != round(accum * self.split_ratio or 1)
|
||||
):
|
||||
raise ValueError(f'Size mismatch: {csize}')
|
||||
else:
|
||||
assert csize == 0, 'Contract is expired but non-zero size?'
|
||||
|
||||
if self.size != csize:
|
||||
log.warning(
|
||||
'Position state mismatch:\n'
|
||||
f'{self.size} => {csize}'
|
||||
)
|
||||
self.size = csize
|
||||
|
||||
cppu = self.calc_ppu()
|
||||
ppu = last_clear['ppu']
|
||||
if (
|
||||
cppu != ppu
|
||||
and self.split_ratio is not None
|
||||
# handle any split info entered (for now) manually by user
|
||||
and cppu != (ppu / self.split_ratio)
|
||||
):
|
||||
raise ValueError(f'PPU mismatch: {cppu}')
|
||||
|
||||
if self.ppu != cppu:
|
||||
log.warning(
|
||||
'Position state mismatch:\n'
|
||||
f'{self.ppu} => {cppu}'
|
||||
)
|
||||
self.ppu = cppu
|
||||
|
||||
def update_from_msg(
|
||||
self,
|
||||
msg: BrokerdPosition,
|
||||
|
||||
) -> None:
|
||||
|
||||
# XXX: better place to do this?
|
||||
symbol = self.symbol
|
||||
|
||||
# TODO: switch to new fields..?
|
||||
# .size_tick_digits, .price_tick_digits
|
||||
size_tick_digits = symbol.lot_size_digits
|
||||
price_tick_digits = symbol.tick_size_digits
|
||||
|
||||
self.ppu = round(
|
||||
# TODO: change this to ppu?
|
||||
msg['avg_price'],
|
||||
ndigits=price_tick_digits,
|
||||
)
|
||||
self.size = round(
|
||||
msg['size'],
|
||||
ndigits=size_tick_digits,
|
||||
)
|
||||
|
||||
@property
|
||||
def dsize(self) -> float:
|
||||
'''
|
||||
The "dollar" size of the pp, normally in trading (fiat) unit
|
||||
terms.
|
||||
|
||||
'''
|
||||
return self.ppu * self.size
|
||||
|
||||
# TODO: idea: "real LIFO" dynamic positioning.
|
||||
# - when a trade takes place where the pnl for
|
||||
# the (set of) trade(s) is below the breakeven price
|
||||
# it may be that the trader took a +ve pnl on a short(er)
|
||||
# term trade in the same account.
|
||||
# - in this case we could recalc the be price to
|
||||
# be reverted back to it's prior value before the nearest term
|
||||
# trade was opened.?
|
||||
# def lifo_price() -> float:
|
||||
# ...
|
||||
|
||||
def iter_clears(self) -> Iterator[tuple[str, dict]]:
|
||||
'''
|
||||
Iterate the internally managed ``.clears: dict`` table in
|
||||
datetime-stamped order.
|
||||
|
||||
'''
|
||||
return iter_by_dt(self.clears)
|
||||
|
||||
def calc_ppu(
|
||||
self,
|
||||
# include transaction cost in breakeven price
|
||||
# and presume the worst case of the same cost
|
||||
# to exit this transaction (even though in reality
|
||||
# it will be dynamic based on exit stratetgy).
|
||||
cost_scalar: float = 2,
|
||||
|
||||
) -> float:
|
||||
'''
|
||||
Compute the "price-per-unit" price for the given non-zero sized
|
||||
rolling position.
|
||||
|
||||
The recurrence relation which computes this (exponential) mean
|
||||
per new clear which **increases** the accumulative postiion size
|
||||
is:
|
||||
|
||||
ppu[-1] = (
|
||||
ppu[-2] * accum_size[-2]
|
||||
+
|
||||
ppu[-1] * size
|
||||
) / accum_size[-1]
|
||||
|
||||
where `cost_basis` for the current step is simply the price
|
||||
* size of the most recent clearing transaction.
|
||||
|
||||
'''
|
||||
asize_h: list[float] = [] # historical accumulative size
|
||||
ppu_h: list[float] = [] # historical price-per-unit
|
||||
|
||||
tid: str
|
||||
entry: dict[str, Any]
|
||||
for (tid, entry) in self.iter_clears():
|
||||
clear_size = entry['size']
|
||||
clear_price = entry['price']
|
||||
|
||||
last_accum_size = asize_h[-1] if asize_h else 0
|
||||
accum_size = last_accum_size + clear_size
|
||||
accum_sign = copysign(1, accum_size)
|
||||
|
||||
sign_change: bool = False
|
||||
|
||||
if accum_size == 0:
|
||||
ppu_h.append(0)
|
||||
asize_h.append(0)
|
||||
continue
|
||||
|
||||
if accum_size == 0:
|
||||
ppu_h.append(0)
|
||||
asize_h.append(0)
|
||||
continue
|
||||
|
||||
# test if the pp somehow went "passed" a net zero size state
|
||||
# resulting in a change of the "sign" of the size (+ve for
|
||||
# long, -ve for short).
|
||||
sign_change = (
|
||||
copysign(1, last_accum_size) + accum_sign == 0
|
||||
and last_accum_size != 0
|
||||
)
|
||||
|
||||
# since we passed the net-zero-size state the new size
|
||||
# after sum should be the remaining size the new
|
||||
# "direction" (aka, long vs. short) for this clear.
|
||||
if sign_change:
|
||||
clear_size = accum_size
|
||||
abs_diff = abs(accum_size)
|
||||
asize_h.append(0)
|
||||
ppu_h.append(0)
|
||||
|
||||
else:
|
||||
# old size minus the new size gives us size diff with
|
||||
# +ve -> increase in pp size
|
||||
# -ve -> decrease in pp size
|
||||
abs_diff = abs(accum_size) - abs(last_accum_size)
|
||||
|
||||
# XXX: LIFO breakeven price update. only an increaze in size
|
||||
# of the position contributes the breakeven price,
|
||||
# a decrease does not (i.e. the position is being made
|
||||
# smaller).
|
||||
# abs_clear_size = abs(clear_size)
|
||||
abs_new_size = abs(accum_size)
|
||||
|
||||
if abs_diff > 0:
|
||||
|
||||
cost_basis = (
|
||||
# cost basis for this clear
|
||||
clear_price * abs(clear_size)
|
||||
+
|
||||
# transaction cost
|
||||
accum_sign * cost_scalar * entry['cost']
|
||||
)
|
||||
|
||||
if asize_h:
|
||||
size_last = abs(asize_h[-1])
|
||||
cb_last = ppu_h[-1] * size_last
|
||||
ppu = (cost_basis + cb_last) / abs_new_size
|
||||
|
||||
else:
|
||||
ppu = cost_basis / abs_new_size
|
||||
|
||||
ppu_h.append(ppu)
|
||||
asize_h.append(accum_size)
|
||||
|
||||
else:
|
||||
# on "exit" clears from a given direction,
|
||||
# only the size changes not the price-per-unit
|
||||
# need to be updated since the ppu remains constant
|
||||
# and gets weighted by the new size.
|
||||
asize_h.append(accum_size)
|
||||
ppu_h.append(ppu_h[-1])
|
||||
|
||||
final_ppu = ppu_h[-1] if ppu_h else 0
|
||||
|
||||
# handle any split info entered (for now) manually by user
|
||||
if self.split_ratio is not None:
|
||||
final_ppu /= self.split_ratio
|
||||
|
||||
return final_ppu
|
||||
|
||||
def expired(self) -> bool:
|
||||
'''
|
||||
Predicate which checks if the contract/instrument is past its expiry.
|
||||
|
||||
'''
|
||||
return bool(self.expiry) and self.expiry < now()
|
||||
|
||||
def calc_size(self) -> float:
|
||||
'''
|
||||
Calculate the unit size of this position in the destination
|
||||
asset using the clears/trade event table; zero if expired.
|
||||
|
||||
'''
|
||||
size: float = 0.
|
||||
|
||||
# time-expired pps (normally derivatives) are "closed"
|
||||
# and have a zero size.
|
||||
if self.expired():
|
||||
return 0.
|
||||
|
||||
for tid, entry in self.clears.items():
|
||||
size += entry['size']
|
||||
# XXX: do we need it every step?
|
||||
# no right since rounding is an LT?
|
||||
# size = self.mkt.quantize(
|
||||
# size + entry['size'],
|
||||
# quantity_type='size',
|
||||
# )
|
||||
|
||||
if self.split_ratio is not None:
|
||||
size = round(size * self.split_ratio)
|
||||
|
||||
return float(
|
||||
self.symbol.quantize(size),
|
||||
)
|
||||
|
||||
def minimize_clears(
|
||||
self,
|
||||
|
||||
) -> dict[str, dict]:
|
||||
'''
|
||||
Minimize the position's clears entries by removing
|
||||
all transactions before the last net zero size to avoid
|
||||
unecessary history irrelevant to the current pp state.
|
||||
|
||||
'''
|
||||
size: float = 0
|
||||
clears_since_zero: list[tuple(str, dict)] = []
|
||||
|
||||
# TODO: we might just want to always do this when iterating
|
||||
# a ledger? keep a state of the last net-zero and only do the
|
||||
# full iterate when no state was stashed?
|
||||
|
||||
# scan for the last "net zero" position by iterating
|
||||
# transactions until the next net-zero size, rinse, repeat.
|
||||
for tid, clear in self.clears.items():
|
||||
size = float(
|
||||
self.mkt.quantize(size + clear['size'])
|
||||
)
|
||||
clears_since_zero.append((tid, clear))
|
||||
|
||||
if size == 0:
|
||||
clears_since_zero.clear()
|
||||
|
||||
self.clears = dict(clears_since_zero)
|
||||
return self.clears
|
||||
|
||||
def add_clear(
|
||||
self,
|
||||
t: Transaction,
|
||||
) -> dict:
|
||||
'''
|
||||
Update clearing table and populate rolling ppu and accumulative
|
||||
size in both the clears entry and local attrs state.
|
||||
|
||||
'''
|
||||
clear = self.clears[t.tid] = {
|
||||
'cost': t.cost,
|
||||
'price': t.price,
|
||||
'size': t.size,
|
||||
'dt': t.dt
|
||||
}
|
||||
|
||||
# TODO: compute these incrementally instead
|
||||
# of re-looping through each time resulting in O(n**2)
|
||||
# behaviour..?
|
||||
|
||||
# NOTE: we compute these **after** adding the entry in order to
|
||||
# make the recurrence relation math work inside
|
||||
# ``.calc_size()``.
|
||||
self.size = clear['accum_size'] = self.calc_size()
|
||||
self.ppu = clear['ppu'] = self.calc_ppu()
|
||||
|
||||
return clear
|
||||
|
||||
# def sugest_split(self) -> float:
|
||||
# ...
|
||||
|
||||
|
||||
class PpTable(Struct):
|
||||
|
||||
brokername: str
|
||||
acctid: str
|
||||
pps: dict[str, Position]
|
||||
conf_path: Path
|
||||
conf: dict | None = {}
|
||||
|
||||
def update_from_trans(
|
||||
self,
|
||||
trans: dict[str, Transaction],
|
||||
cost_scalar: float = 2,
|
||||
|
||||
) -> dict[str, Position]:
|
||||
|
||||
pps = self.pps
|
||||
updated: dict[str, Position] = {}
|
||||
|
||||
# lifo update all pps from records, ensuring
|
||||
# we compute the PPU and size sorted in time!
|
||||
for t in sorted(
|
||||
trans.values(),
|
||||
key=lambda t: t.dt,
|
||||
reverse=True,
|
||||
):
|
||||
fqme = t.fqme
|
||||
bs_mktid = t.bs_mktid
|
||||
|
||||
# template the mkt-info presuming a legacy market ticks
|
||||
# if no info exists in the transactions..
|
||||
mkt: MktPair = t.sys
|
||||
pp = pps.get(bs_mktid)
|
||||
if not pp:
|
||||
# if no existing pp, allocate fresh one.
|
||||
pp = pps[bs_mktid] = Position(
|
||||
mkt,
|
||||
size=0.0,
|
||||
ppu=0.0,
|
||||
bs_mktid=bs_mktid,
|
||||
expiry=t.expiry,
|
||||
)
|
||||
else:
|
||||
# NOTE: if for some reason a "less resolved" mkt pair
|
||||
# info has been set (based on the `.fqme` being
|
||||
# a shorter string), instead use the one from the
|
||||
# transaction since it likely has (more) full
|
||||
# information from the provider.
|
||||
if len(pp.symbol.fqme) < len(fqme):
|
||||
pp.symbol = mkt
|
||||
|
||||
clears = pp.clears
|
||||
if clears:
|
||||
first_clear_dt = pp.first_clear_dt
|
||||
|
||||
# don't do updates for ledger records we already have
|
||||
# included in the current pps state.
|
||||
if (
|
||||
t.tid in clears
|
||||
or (
|
||||
first_clear_dt
|
||||
and t.dt < first_clear_dt
|
||||
)
|
||||
):
|
||||
# NOTE: likely you'll see repeats of the same
|
||||
# ``Transaction`` passed in here if/when you are restarting
|
||||
# a ``brokerd.ib`` where the API will re-report trades from
|
||||
# the current session, so we need to make sure we don't
|
||||
# "double count" these in pp calculations.
|
||||
continue
|
||||
|
||||
# update clearing table
|
||||
pp.add_clear(t)
|
||||
updated[t.bs_mktid] = pp
|
||||
|
||||
# minimize clears tables and update sizing.
|
||||
for bs_mktid, pp in updated.items():
|
||||
pp.ensure_state()
|
||||
|
||||
# deliver only the position entries that were actually updated
|
||||
# (modified the state) from the input transaction set.
|
||||
return updated
|
||||
|
||||
def dump_active(
|
||||
self,
|
||||
) -> tuple[
|
||||
dict[str, Position],
|
||||
dict[str, Position]
|
||||
]:
|
||||
'''
|
||||
Iterate all tabulated positions, render active positions to
|
||||
a ``dict`` format amenable to serialization (via TOML) and drop
|
||||
from state (``.pps``) as well as return in a ``dict`` all
|
||||
``Position``s which have recently closed.
|
||||
|
||||
'''
|
||||
# NOTE: newly closed position are also important to report/return
|
||||
# since a consumer, like an order mode UI ;), might want to react
|
||||
# based on the closure (for example removing the breakeven line
|
||||
# and clearing the entry from any lists/monitors).
|
||||
closed_pp_objs: dict[str, Position] = {}
|
||||
open_pp_objs: dict[str, Position] = {}
|
||||
|
||||
pp_objs = self.pps
|
||||
for bs_mktid in list(pp_objs):
|
||||
pp = pp_objs[bs_mktid]
|
||||
pp.ensure_state()
|
||||
|
||||
if (
|
||||
# "net-zero" is a "closed" position
|
||||
pp.size == 0
|
||||
|
||||
# time-expired pps (normally derivatives) are "closed"
|
||||
or (pp.expiry and pp.expiry < now())
|
||||
):
|
||||
# for expired cases
|
||||
pp.size = 0
|
||||
|
||||
# NOTE: we DO NOT pop the pp here since it can still be
|
||||
# used to check for duplicate clears that may come in as
|
||||
# new transaction from some backend API and need to be
|
||||
# ignored; the closed positions won't be written to the
|
||||
# ``pps.toml`` since ``pp_active_entries`` above is what's
|
||||
# written.
|
||||
closed_pp_objs[bs_mktid] = pp
|
||||
|
||||
else:
|
||||
open_pp_objs[bs_mktid] = pp
|
||||
|
||||
return open_pp_objs, closed_pp_objs
|
||||
|
||||
def to_toml(
|
||||
self,
|
||||
active: dict[str, Position] | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
|
||||
if active is None:
|
||||
active, _ = self.dump_active()
|
||||
|
||||
# ONLY dict-serialize all active positions; those that are
|
||||
# closed we don't store in the ``pps.toml``.
|
||||
to_toml_dict = {}
|
||||
|
||||
pos: Position
|
||||
for bs_mktid, pos in active.items():
|
||||
|
||||
# keep the minimal amount of clears that make up this
|
||||
# position since the last net-zero state.
|
||||
pos.minimize_clears()
|
||||
pos.ensure_state()
|
||||
|
||||
# serialize to pre-toml form
|
||||
fqme, asdict = pos.to_pretoml()
|
||||
|
||||
# assert 'Datetime' not in asdict['dt']
|
||||
log.info(f'Updating active pp: {fqme}')
|
||||
|
||||
# XXX: ugh, it's cuz we push the section under
|
||||
# the broker name.. maybe we need to rethink this?
|
||||
brokerless_key = fqme.removeprefix(f'{self.brokername}.')
|
||||
to_toml_dict[brokerless_key] = asdict
|
||||
|
||||
return to_toml_dict
|
||||
|
||||
def write_config(self) -> None:
|
||||
'''
|
||||
Write the current position table to the user's ``pps.toml``.
|
||||
|
||||
'''
|
||||
# TODO: show diff output?
|
||||
# https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
|
||||
# active, closed_pp_objs = table.dump_active()
|
||||
|
||||
active, closed = self.dump_active()
|
||||
pp_entries = self.to_toml(active=active)
|
||||
if pp_entries:
|
||||
log.info(
|
||||
f'Updating positions in ``{self.conf_path}``:\n'
|
||||
f'n{pformat(pp_entries)}'
|
||||
)
|
||||
|
||||
if self.brokername in self.conf:
|
||||
log.warning(
|
||||
f'Rewriting {self.conf_path} keys to drop <broker.acct>!'
|
||||
)
|
||||
# legacy key schema including <brokername.account>, so
|
||||
# rewrite all entries to drop those tables since we now
|
||||
# put that in the filename!
|
||||
accounts = self.conf.pop(self.brokername)
|
||||
assert len(accounts) == 1
|
||||
entries = accounts.pop(self.acctid)
|
||||
self.conf.update(entries)
|
||||
|
||||
self.conf.update(pp_entries)
|
||||
|
||||
# drop any entries that are computed as net-zero
|
||||
# we don't care about storing in the pps file.
|
||||
if closed:
|
||||
for fqme in closed:
|
||||
self.conf.pop(fqme, None)
|
||||
|
||||
# if there are no active position entries according
|
||||
# to the toml dump output above, then clear the config
|
||||
# file of all entries.
|
||||
elif self.conf:
|
||||
for entry in list(self.conf):
|
||||
del self.conf[entry]
|
||||
|
||||
# TODO: why tf haven't they already done this for inline
|
||||
# tables smh..
|
||||
enc = PpsEncoder(preserve=True)
|
||||
# table_bs_type = type(toml.TomlDecoder().get_empty_inline_table())
|
||||
enc.dump_funcs[
|
||||
toml.decoder.InlineTableDict
|
||||
] = enc.dump_inline_table
|
||||
|
||||
config.write(
|
||||
config=self.conf,
|
||||
path=self.conf_path,
|
||||
encoder=enc,
|
||||
fail_empty=False
|
||||
)
|
||||
|
||||
|
||||
def load_pps_from_ledger(
|
||||
|
||||
brokername: str,
|
||||
acctname: str,
|
||||
|
||||
# post normalization filter on ledger entries to be processed
|
||||
filter_by: list[dict] | None = None,
|
||||
|
||||
) -> tuple[
|
||||
dict[str, Transaction],
|
||||
dict[str, Position],
|
||||
]:
|
||||
'''
|
||||
Open a ledger file by broker name and account and read in and
|
||||
process any trade records into our normalized ``Transaction`` form
|
||||
and then update the equivalent ``Pptable`` and deliver the two
|
||||
bs_mktid-mapped dict-sets of the transactions and pps.
|
||||
|
||||
'''
|
||||
with (
|
||||
open_trade_ledger(brokername, acctname) as ledger,
|
||||
open_pps(brokername, acctname) as table,
|
||||
):
|
||||
if not ledger:
|
||||
# null case, no ledger file with content
|
||||
return {}
|
||||
|
||||
mod = get_brokermod(brokername)
|
||||
src_records: dict[str, Transaction] = mod.norm_trade_records(ledger)
|
||||
|
||||
if filter_by:
|
||||
records = {}
|
||||
bs_mktids = set(filter_by)
|
||||
for tid, r in src_records.items():
|
||||
if r.bs_mktid in bs_mktids:
|
||||
records[tid] = r
|
||||
else:
|
||||
records = src_records
|
||||
|
||||
updated = table.update_from_trans(records)
|
||||
|
||||
return records, updated
|
||||
|
||||
|
||||
@cm
|
||||
def open_pps(
|
||||
brokername: str,
|
||||
acctid: str,
|
||||
write_on_exit: bool = False,
|
||||
|
||||
) -> Generator[PpTable, None, None]:
|
||||
'''
|
||||
Read out broker-specific position entries from
|
||||
incremental update file: ``pps.toml``.
|
||||
|
||||
'''
|
||||
conf: dict
|
||||
conf_path: Path
|
||||
conf, conf_path = config.load(
|
||||
f'pps.{brokername}.{acctid}',
|
||||
)
|
||||
|
||||
if brokername in conf:
|
||||
log.warning(
|
||||
f'Rewriting {conf_path} keys to drop <broker.acct>!'
|
||||
)
|
||||
# legacy key schema including <brokername.account>, so
|
||||
# rewrite all entries to drop those tables since we now
|
||||
# put that in the filename!
|
||||
accounts = conf.pop(brokername)
|
||||
for acctid in accounts.copy():
|
||||
entries = accounts.pop(acctid)
|
||||
conf.update(entries)
|
||||
|
||||
# TODO: ideally we can pass in an existing
|
||||
# pps state to this right? such that we
|
||||
# don't have to do a ledger reload all the
|
||||
# time.. a couple ideas I can think of,
|
||||
# - mirror this in some client side actor which
|
||||
# does the actual ledger updates (say the paper
|
||||
# engine proc if we decide to always spawn it?),
|
||||
# - do diffs against updates from the ledger writer
|
||||
# actor and the in-mem state here?
|
||||
|
||||
pp_objs = {}
|
||||
table = PpTable(
|
||||
brokername,
|
||||
acctid,
|
||||
pp_objs,
|
||||
conf_path,
|
||||
conf=conf,
|
||||
)
|
||||
|
||||
# unmarshal/load ``pps.toml`` config entries into object form
|
||||
# and update `PpTable` obj entries.
|
||||
for fqme, entry in conf.items():
|
||||
|
||||
# atype = entry.get('asset_type', '<unknown>')
|
||||
|
||||
# unique broker market id
|
||||
bs_mktid = str(
|
||||
entry.get('bsuid')
|
||||
or entry.get('bs_mktid')
|
||||
)
|
||||
price_tick = Decimal(str(
|
||||
entry.get('price_tick_size')
|
||||
or entry.get('price_tick')
|
||||
or '0.01'
|
||||
))
|
||||
size_tick = Decimal(str(
|
||||
entry.get('lot_tick_size')
|
||||
or entry.get('size_tick')
|
||||
or '0.0'
|
||||
))
|
||||
|
||||
# load the pair using the fqme which
|
||||
# will make the pair "unresolved" until
|
||||
# the backend broker actually loads
|
||||
# the market and position info.
|
||||
mkt = MktPair.from_fqme(
|
||||
fqme,
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
bs_mktid=bs_mktid
|
||||
)
|
||||
|
||||
# TODO: RE: general "events" instead of just "clears":
|
||||
# - make this an `events` field and support more event types
|
||||
# such as 'split', 'name_change', 'mkt_info', etc..
|
||||
# - should be make a ``Struct`` for clear/event entries? convert
|
||||
# "clear events table" from the toml config (list of a dicts)
|
||||
# and load it into object form for use in position processing of
|
||||
# new clear events.
|
||||
|
||||
# convert clears sub-tables (only in this form
|
||||
# for toml re-presentation) back into a master table.
|
||||
toml_clears_list: list[dict[str, Any]] = entry['clears']
|
||||
trans: list[Transaction] = []
|
||||
for clears_table in toml_clears_list:
|
||||
|
||||
tid = clears_table.pop('tid')
|
||||
dtstr = clears_table['dt']
|
||||
dt = pendulum.parse(dtstr)
|
||||
clears_table['dt'] = dt
|
||||
|
||||
trans.append(Transaction(
|
||||
fqsn=bs_mktid,
|
||||
sym=mkt,
|
||||
bs_mktid=bs_mktid,
|
||||
tid=tid,
|
||||
size=clears_table['size'],
|
||||
price=clears_table['price'],
|
||||
cost=clears_table['cost'],
|
||||
dt=dt,
|
||||
))
|
||||
|
||||
size = entry['size']
|
||||
|
||||
# TODO: remove but, handle old field name for now
|
||||
ppu = entry.get(
|
||||
'ppu',
|
||||
entry.get('be_price', 0),
|
||||
)
|
||||
|
||||
split_ratio = entry.get('split_ratio')
|
||||
|
||||
expiry = entry.get('expiry')
|
||||
if expiry:
|
||||
expiry = pendulum.parse(expiry)
|
||||
|
||||
pp = pp_objs[bs_mktid] = Position(
|
||||
mkt,
|
||||
size=size,
|
||||
ppu=ppu,
|
||||
split_ratio=split_ratio,
|
||||
expiry=expiry,
|
||||
bs_mktid=bs_mktid,
|
||||
)
|
||||
|
||||
# XXX: super critical, we need to be sure to include
|
||||
# all pps.toml clears to avoid reusing clears that were
|
||||
# already included in the current incremental update
|
||||
# state, since today's records may have already been
|
||||
# processed!
|
||||
for t in trans:
|
||||
pp.add_clear(t)
|
||||
|
||||
# audit entries loaded from toml
|
||||
pp.ensure_state()
|
||||
|
||||
try:
|
||||
yield table
|
||||
finally:
|
||||
if write_on_exit:
|
||||
table.write_config()
|
||||
|
|
@ -1,156 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
TOML codec hacks to make position tables look decent.
|
||||
|
||||
(looking at you "`toml`-lib"..)
|
||||
|
||||
'''
|
||||
import re
|
||||
|
||||
import toml
|
||||
|
||||
|
||||
# TODO: instead see if we can hack tomli and tomli-w to do the same:
|
||||
# - https://github.com/hukkin/tomli
|
||||
# - https://github.com/hukkin/tomli-w
|
||||
class PpsEncoder(toml.TomlEncoder):
|
||||
'''
|
||||
Special "styled" encoder that makes a ``pps.toml`` redable and
|
||||
compact by putting `.clears` tables inline and everything else
|
||||
flat-ish.
|
||||
|
||||
'''
|
||||
separator = ','
|
||||
|
||||
def dump_list(self, v):
|
||||
'''
|
||||
Dump an inline list with a newline after every element and
|
||||
with consideration for denoted inline table types.
|
||||
|
||||
'''
|
||||
retval = "[\n"
|
||||
for u in v:
|
||||
if isinstance(u, toml.decoder.InlineTableDict):
|
||||
out = self.dump_inline_table(u)
|
||||
else:
|
||||
out = str(self.dump_value(u))
|
||||
|
||||
retval += " " + out + "," + "\n"
|
||||
retval += "]"
|
||||
return retval
|
||||
|
||||
def dump_inline_table(self, section):
|
||||
"""Preserve inline table in its compact syntax instead of expanding
|
||||
into subsection.
|
||||
https://github.com/toml-lang/toml#user-content-inline-table
|
||||
"""
|
||||
val_list = []
|
||||
for k, v in section.items():
|
||||
# if isinstance(v, toml.decoder.InlineTableDict):
|
||||
if isinstance(v, dict):
|
||||
val = self.dump_inline_table(v)
|
||||
else:
|
||||
val = str(self.dump_value(v))
|
||||
|
||||
val_list.append(k + " = " + val)
|
||||
|
||||
retval = "{ " + ", ".join(val_list) + " }"
|
||||
return retval
|
||||
|
||||
def dump_sections(self, o, sup):
|
||||
retstr = ""
|
||||
if sup != "" and sup[-1] != ".":
|
||||
sup += '.'
|
||||
retdict = self._dict()
|
||||
arraystr = ""
|
||||
for section in o:
|
||||
qsection = str(section)
|
||||
value = o[section]
|
||||
|
||||
if not re.match(r'^[A-Za-z0-9_-]+$', section):
|
||||
qsection = toml.encoder._dump_str(section)
|
||||
|
||||
# arrayoftables = False
|
||||
if (
|
||||
self.preserve
|
||||
and isinstance(value, toml.decoder.InlineTableDict)
|
||||
):
|
||||
retstr += (
|
||||
qsection
|
||||
+
|
||||
" = "
|
||||
+
|
||||
self.dump_inline_table(o[section])
|
||||
+
|
||||
'\n' # only on the final terminating left brace
|
||||
)
|
||||
|
||||
# XXX: this code i'm pretty sure is just blatantly bad
|
||||
# and/or wrong..
|
||||
# if isinstance(o[section], list):
|
||||
# for a in o[section]:
|
||||
# if isinstance(a, dict):
|
||||
# arrayoftables = True
|
||||
# if arrayoftables:
|
||||
# for a in o[section]:
|
||||
# arraytabstr = "\n"
|
||||
# arraystr += "[[" + sup + qsection + "]]\n"
|
||||
# s, d = self.dump_sections(a, sup + qsection)
|
||||
# if s:
|
||||
# if s[0] == "[":
|
||||
# arraytabstr += s
|
||||
# else:
|
||||
# arraystr += s
|
||||
# while d:
|
||||
# newd = self._dict()
|
||||
# for dsec in d:
|
||||
# s1, d1 = self.dump_sections(d[dsec], sup +
|
||||
# qsection + "." +
|
||||
# dsec)
|
||||
# if s1:
|
||||
# arraytabstr += ("[" + sup + qsection +
|
||||
# "." + dsec + "]\n")
|
||||
# arraytabstr += s1
|
||||
# for s1 in d1:
|
||||
# newd[dsec + "." + s1] = d1[s1]
|
||||
# d = newd
|
||||
# arraystr += arraytabstr
|
||||
|
||||
elif isinstance(value, dict):
|
||||
retdict[qsection] = o[section]
|
||||
|
||||
elif o[section] is not None:
|
||||
retstr += (
|
||||
qsection
|
||||
+
|
||||
" = "
|
||||
+
|
||||
str(self.dump_value(o[section]))
|
||||
)
|
||||
|
||||
# if not isinstance(value, dict):
|
||||
if not isinstance(value, toml.decoder.InlineTableDict):
|
||||
# inline tables should not contain newlines:
|
||||
# https://toml.io/en/v1.0.0#inline-table
|
||||
retstr += '\n'
|
||||
|
||||
else:
|
||||
raise ValueError(value)
|
||||
|
||||
retstr += arraystr
|
||||
return (retstr, retdict)
|
||||
|
|
@ -1,220 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
CLI front end for trades ledger and position tracking management.
|
||||
|
||||
'''
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
from rich.console import Console
|
||||
from rich.markdown import Markdown
|
||||
import tractor
|
||||
import trio
|
||||
import typer
|
||||
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
)
|
||||
from ..clearing._messages import BrokerdPosition
|
||||
from ..calc import humanize
|
||||
|
||||
|
||||
ledger = typer.Typer()
|
||||
|
||||
|
||||
def broker_init(
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**start_actor_kwargs,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Given an input broker name, load all named arguments
|
||||
which can be passed to a daemon + context spawn for
|
||||
the relevant `brokerd` service endpoint.
|
||||
|
||||
'''
|
||||
from ..brokers import get_brokermod
|
||||
brokermod = get_brokermod(brokername)
|
||||
modpath = brokermod.__name__
|
||||
|
||||
start_actor_kwargs['name'] = f'brokerd.{brokername}'
|
||||
start_actor_kwargs.update(
|
||||
getattr(
|
||||
brokermod,
|
||||
'_spawn_kwargs',
|
||||
{},
|
||||
)
|
||||
)
|
||||
|
||||
# lookup actor-enabled modules declared by the backend offering the
|
||||
# `brokerd` endpoint(s).
|
||||
enabled = start_actor_kwargs['enable_modules'] = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
enabled.append(subpath)
|
||||
|
||||
# TODO XXX: DO WE NEED THIS?
|
||||
# enabled.append('piker.data.feed')
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
from ..brokers import _setup_persistent_brokerd
|
||||
|
||||
return (
|
||||
start_actor_kwargs, # to `ActorNursery.start_actor()`
|
||||
_setup_persistent_brokerd, # service task ep
|
||||
getattr( # trades endpoint
|
||||
brokermod,
|
||||
'trades_dialogue',
|
||||
None,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@ledger.command()
|
||||
def sync(
|
||||
fully_qualified_account_name: str,
|
||||
pdb: bool = False,
|
||||
|
||||
loglevel: str = typer.Option(
|
||||
'error',
|
||||
"-l",
|
||||
),
|
||||
):
|
||||
console = Console()
|
||||
|
||||
try:
|
||||
brokername, account = fully_qualified_account_name.split('.')
|
||||
except ValueError:
|
||||
md = Markdown(
|
||||
f'=> `{fully_qualified_account_name}` <=\n\n'
|
||||
'is not a valid '
|
||||
'__fully qualified account name?__\n\n'
|
||||
'Your account name needs to be of the form '
|
||||
'`<brokername>.<account_name>`\n'
|
||||
)
|
||||
console.print(md)
|
||||
return
|
||||
|
||||
start_kwargs, _, trades_ep = broker_init(
|
||||
brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
|
||||
async def main():
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='ledger_cli',
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
|
||||
) as (actor, sockaddr),
|
||||
|
||||
tractor.open_nursery() as an,
|
||||
):
|
||||
portal = await an.start_actor(
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
**start_kwargs,
|
||||
)
|
||||
|
||||
if (
|
||||
brokername == 'paper'
|
||||
or trades_ep is None
|
||||
):
|
||||
from ..clearing import _paper_engine as paper
|
||||
open_trades_endpoint = paper.open_paperboi(
|
||||
fqme=None, # tell paper to not start clearing loop
|
||||
broker=brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
else:
|
||||
# open live brokerd trades endpoint
|
||||
open_trades_endpoint = portal.open_context(
|
||||
trades_ep,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
|
||||
positions: dict[str, Any]
|
||||
accounts: list[str]
|
||||
async with (
|
||||
open_trades_endpoint as (
|
||||
brokerd_ctx,
|
||||
(positions, accounts,),
|
||||
),
|
||||
):
|
||||
summary: str = (
|
||||
'[dim underline]Piker Position Summary[/] '
|
||||
f'[dim blue underline]{brokername}[/]'
|
||||
'[dim].[/]'
|
||||
f'[blue underline]{account}[/]'
|
||||
f'[dim underline] -> total pps: [/]'
|
||||
f'[green]{len(positions)}[/]\n'
|
||||
)
|
||||
for ppdict in positions:
|
||||
ppmsg = BrokerdPosition(**ppdict)
|
||||
size = ppmsg.size
|
||||
if size:
|
||||
ppu: float = round(
|
||||
ppmsg.avg_price,
|
||||
ndigits=2,
|
||||
)
|
||||
cost_basis: str = humanize(size * ppu)
|
||||
h_size: str = humanize(size)
|
||||
|
||||
if size < 0:
|
||||
pcolor = 'red'
|
||||
else:
|
||||
pcolor = 'green'
|
||||
|
||||
# sematic-highlight of fqme
|
||||
fqme = ppmsg.symbol
|
||||
tokens = fqme.split('.')
|
||||
styled_fqme = f'[blue underline]{tokens[0]}[/]'
|
||||
for tok in tokens[1:]:
|
||||
styled_fqme += '[dim].[/]'
|
||||
styled_fqme += f'[dim blue underline]{tok}[/]'
|
||||
|
||||
# TODO: instead display in a ``rich.Table``?
|
||||
summary += (
|
||||
styled_fqme +
|
||||
'[dim]: [/]'
|
||||
f'[{pcolor}]{h_size}[/]'
|
||||
'[dim blue]u @[/]'
|
||||
f'[{pcolor}]{ppu}[/]'
|
||||
'[dim blue] = [/]'
|
||||
f'[{pcolor}]$ {cost_basis}\n[/]'
|
||||
)
|
||||
|
||||
console.print(summary)
|
||||
await brokerd_ctx.cancel()
|
||||
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ledger()
|
||||
|
|
@ -20,30 +20,22 @@ Broker clients, daemons and general back end machinery.
|
|||
from importlib import import_module
|
||||
from types import ModuleType
|
||||
|
||||
# TODO: move to urllib3/requests once supported
|
||||
import asks
|
||||
asks.init('trio')
|
||||
|
||||
__brokers__ = [
|
||||
'binance',
|
||||
'questrade',
|
||||
'robinhood',
|
||||
'ib',
|
||||
'kraken',
|
||||
'kucoin'
|
||||
# broken but used to work
|
||||
# 'questrade',
|
||||
# 'robinhood',
|
||||
|
||||
# TODO: we should get on these stat!
|
||||
# alpaca
|
||||
# wstrade
|
||||
# iex
|
||||
|
||||
# deribit
|
||||
# bitso
|
||||
]
|
||||
|
||||
|
||||
def get_brokermod(brokername: str) -> ModuleType:
|
||||
'''
|
||||
Return the imported broker module by name.
|
||||
|
||||
'''
|
||||
"""Return the imported broker module by name.
|
||||
"""
|
||||
module = import_module('.' + brokername, 'piker.brokers')
|
||||
# we only allow monkeying because it's for internal keying
|
||||
module.name = module.__name__.split('.')[-1]
|
||||
|
|
@ -51,9 +43,7 @@ def get_brokermod(brokername: str) -> ModuleType:
|
|||
|
||||
|
||||
def iter_brokermods():
|
||||
'''
|
||||
Iterate all built-in broker modules.
|
||||
|
||||
'''
|
||||
"""Iterate all built-in broker modules.
|
||||
"""
|
||||
for name in __brokers__:
|
||||
yield get_brokermod(name)
|
||||
|
|
|
|||
|
|
@ -1,169 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Broker-daemon-actor "endpoint-hooks": the service task entry points for
|
||||
``brokerd``.
|
||||
|
||||
'''
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from . import _util
|
||||
from . import get_brokermod
|
||||
|
||||
# `brokerd` enabled modules
|
||||
# TODO: move this def to the `.data` subpkg..
|
||||
# NOTE: keeping this list as small as possible is part of our caps-sec
|
||||
# model and should be treated with utmost care!
|
||||
_data_mods = [
|
||||
'piker.brokers.core',
|
||||
'piker.brokers.data',
|
||||
'piker.brokers._daemon',
|
||||
'piker.data',
|
||||
'piker.data.feed',
|
||||
'piker.data._sampling'
|
||||
]
|
||||
|
||||
|
||||
# TODO: we should rename the daemon to datad prolly once we split up
|
||||
# broker vs. data tasks into separate actors?
|
||||
@tractor.context
|
||||
async def _setup_persistent_brokerd(
|
||||
ctx: tractor.Context,
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Allocate a actor-wide service nursery in ``brokerd``
|
||||
such that feeds can be run in the background persistently by
|
||||
the broker backend as needed.
|
||||
|
||||
'''
|
||||
log = _util.get_console_log(
|
||||
loglevel or tractor.current_actor().loglevel,
|
||||
name=f'{_util.subsys}.{brokername}',
|
||||
)
|
||||
# set global for this actor to this new process-wide instance B)
|
||||
_util.log = log
|
||||
|
||||
from piker.data.feed import (
|
||||
_bus,
|
||||
get_feed_bus,
|
||||
)
|
||||
global _bus
|
||||
assert not _bus
|
||||
|
||||
async with trio.open_nursery() as service_nursery:
|
||||
# assign a nursery to the feeds bus for spawning
|
||||
# background tasks from clients
|
||||
get_feed_bus(brokername, service_nursery)
|
||||
|
||||
# unblock caller
|
||||
await ctx.started()
|
||||
|
||||
# we pin this task to keep the feeds manager active until the
|
||||
# parent actor decides to tear it down
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
||||
async def spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> bool:
|
||||
|
||||
from piker.service import Services
|
||||
from piker.service._util import log # use service mngr log
|
||||
|
||||
log.info(f'Spawning {brokername} broker daemon')
|
||||
|
||||
brokermod = get_brokermod(brokername)
|
||||
dname = f'brokerd.{brokername}'
|
||||
|
||||
extra_tractor_kwargs = getattr(brokermod, '_spawn_kwargs', {})
|
||||
tractor_kwargs.update(extra_tractor_kwargs)
|
||||
|
||||
# ask `pikerd` to spawn a new sub-actor and manage it under its
|
||||
# actor nursery
|
||||
modpath = brokermod.__name__
|
||||
broker_enable = [modpath]
|
||||
for submodname in getattr(
|
||||
brokermod,
|
||||
'__enable_modules__',
|
||||
[],
|
||||
):
|
||||
subpath = f'{modpath}.{submodname}'
|
||||
broker_enable.append(subpath)
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=_data_mods + broker_enable,
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode,
|
||||
**tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of brokerd service nursery
|
||||
await Services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
_setup_persistent_brokerd,
|
||||
brokername=brokername,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_brokerd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
Helper to spawn a brokerd service *from* a client
|
||||
who wishes to use the sub-actor-daemon.
|
||||
|
||||
'''
|
||||
from piker.service import maybe_spawn_daemon
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
|
||||
f'brokerd.{brokername}',
|
||||
service_task_target=spawn_brokerd,
|
||||
spawn_args={
|
||||
'brokername': brokername,
|
||||
},
|
||||
loglevel=loglevel,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
|
@ -15,29 +15,13 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Handy cross-broker utils.
|
||||
|
||||
Handy utils.
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
import json
|
||||
import asks
|
||||
import logging
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
colorize_json,
|
||||
)
|
||||
subsys: str = 'piker.brokers'
|
||||
|
||||
# NOTE: level should be reset by any actor that is spawned
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
from ..log import colorize_json
|
||||
|
||||
|
||||
class BrokerError(Exception):
|
||||
|
|
@ -85,6 +69,7 @@ class DataThrottle(BrokerError):
|
|||
# TODO: add in throttle metrics/feedback
|
||||
|
||||
|
||||
|
||||
def resproc(
|
||||
resp: asks.response_objects.Response,
|
||||
log: logging.Logger,
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C)
|
||||
# Guillermo Rodriguez
|
||||
# Tyler Goodlet
|
||||
# (in stewardship for pikers)
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -21,13 +18,8 @@
|
|||
Binance backend
|
||||
|
||||
"""
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
aclosing,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
# from functools import lru_cache
|
||||
from decimal import Decimal
|
||||
from typing import (
|
||||
Any, Union, Optional,
|
||||
AsyncGenerator, Callable,
|
||||
|
|
@ -41,29 +33,17 @@ import asks
|
|||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
from pydantic.dataclasses import dataclass
|
||||
from pydantic import BaseModel
|
||||
import wsproto
|
||||
|
||||
from .._cacheables import async_lifo_cache
|
||||
from ..accounting._mktinfo import (
|
||||
Asset,
|
||||
MktPair,
|
||||
digits_to_dec,
|
||||
)
|
||||
from .._cacheables import open_cached_client
|
||||
from ._util import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
DataUnavailable,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..data.types import Struct
|
||||
from ..data.validate import FeedInit
|
||||
from ..data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
from ._util import resproc, SymbolNotFound
|
||||
from ..log import get_logger, get_console_log
|
||||
from ..data import ShmArray
|
||||
from ..data._web_bs import open_autorecon_ws, NoBsWs
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_url = 'https://api.binance.com'
|
||||
|
|
@ -99,17 +79,12 @@ _show_wap_in_history = False
|
|||
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
|
||||
# TODO: make this frozen again by pre-processing the
|
||||
# filters list to a dict at init time?
|
||||
class Pair(Struct, frozen=True):
|
||||
class Pair(BaseModel):
|
||||
symbol: str
|
||||
status: str
|
||||
|
||||
baseAsset: str
|
||||
baseAssetPrecision: int
|
||||
cancelReplaceAllowed: bool
|
||||
allowTrailingStop: bool
|
||||
quoteAsset: str
|
||||
quotePrecision: int
|
||||
quoteAssetPrecision: int
|
||||
|
|
@ -125,36 +100,18 @@ class Pair(Struct, frozen=True):
|
|||
isSpotTradingAllowed: bool
|
||||
isMarginTradingAllowed: bool
|
||||
|
||||
defaultSelfTradePreventionMode: str
|
||||
allowedSelfTradePreventionModes: list[str]
|
||||
|
||||
filters: dict[
|
||||
str,
|
||||
Union[str, int, float]
|
||||
]
|
||||
filters: list[dict[str, Union[str, int, float]]]
|
||||
permissions: list[str]
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
# XXX: lul, after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
step_size: str = self.filters['PRICE_FILTER']['tickSize'].rstrip('0')
|
||||
return Decimal(step_size)
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
step_size: str = self.filters['LOT_SIZE']['stepSize'].rstrip('0')
|
||||
return Decimal(step_size)
|
||||
|
||||
|
||||
class OHLC(Struct):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
@dataclass
|
||||
class OHLC:
|
||||
"""Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#kline-candlestick-streams
|
||||
|
||||
'''
|
||||
"""
|
||||
time: int
|
||||
|
||||
open: float
|
||||
|
|
@ -176,22 +133,8 @@ class OHLC(Struct):
|
|||
bar_wap: float = 0.0
|
||||
|
||||
|
||||
class L1(Struct):
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams
|
||||
|
||||
update_id: int
|
||||
sym: str
|
||||
|
||||
bid: float
|
||||
bsize: float
|
||||
ask: float
|
||||
asize: float
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def binance_timestamp(
|
||||
when: datetime
|
||||
) -> int:
|
||||
def binance_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
|
|
@ -200,7 +143,7 @@ class Client:
|
|||
def __init__(self) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._pairs: dict[str, Pair] = {}
|
||||
self._pairs: dict[str, Any] = {}
|
||||
|
||||
async def _api(
|
||||
self,
|
||||
|
|
@ -214,58 +157,48 @@ class Client:
|
|||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def exch_info(
|
||||
async def symbol_info(
|
||||
|
||||
self,
|
||||
sym: str | None = None,
|
||||
sym: Optional[str] = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
'''
|
||||
Fresh exchange-pairs info query for symbol ``sym: str``:
|
||||
https://binance-docs.github.io/apidocs/spot/en/#exchange-information
|
||||
) -> dict[str, Any]:
|
||||
'''Get symbol info for the exchange.
|
||||
|
||||
'''
|
||||
cached_pair = self._pairs.get(sym)
|
||||
if cached_pair:
|
||||
return cached_pair
|
||||
# TODO: we can load from our self._pairs cache
|
||||
# on repeat calls...
|
||||
|
||||
# retrieve all symbols by default
|
||||
# will retrieve all symbols by default
|
||||
params = {}
|
||||
|
||||
if sym is not None:
|
||||
sym = sym.lower()
|
||||
sym = sym.upper()
|
||||
params = {'symbol': sym}
|
||||
|
||||
resp = await self._api('exchangeInfo', params=params)
|
||||
entries = resp['symbols']
|
||||
if not entries:
|
||||
raise SymbolNotFound(f'{sym} not found:\n{resp}')
|
||||
|
||||
# pre-process .filters field into a table
|
||||
pairs = {}
|
||||
for item in entries:
|
||||
symbol = item['symbol']
|
||||
filters = {}
|
||||
filters_ls: list = item.pop('filters')
|
||||
for entry in filters_ls:
|
||||
ftype = entry['filterType']
|
||||
filters[ftype] = entry
|
||||
|
||||
pairs[symbol] = Pair(
|
||||
filters=filters,
|
||||
**item,
|
||||
resp = await self._api(
|
||||
'exchangeInfo',
|
||||
params=params,
|
||||
)
|
||||
|
||||
# pairs = {
|
||||
# item['symbol']: Pair(**item) for item in entries
|
||||
# }
|
||||
self._pairs.update(pairs)
|
||||
entries = resp['symbols']
|
||||
if not entries:
|
||||
raise SymbolNotFound(f'{sym} not found')
|
||||
|
||||
syms = {item['symbol']: item for item in entries}
|
||||
|
||||
if sym is not None:
|
||||
return pairs[sym]
|
||||
return syms[sym]
|
||||
else:
|
||||
return self._pairs
|
||||
return syms
|
||||
|
||||
symbol_info = exch_info
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
|
|
@ -275,7 +208,7 @@ class Client:
|
|||
if self._pairs is not None:
|
||||
data = self._pairs
|
||||
else:
|
||||
data = await self.exch_info()
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
|
|
@ -297,7 +230,7 @@ class Client:
|
|||
) -> dict:
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC').add(minutes=1)
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
|
|
@ -327,7 +260,6 @@ class Client:
|
|||
for i, bar in enumerate(bars):
|
||||
|
||||
bar = OHLC(*bar)
|
||||
bar.typecast()
|
||||
|
||||
row = []
|
||||
for j, (name, ftype) in enumerate(_ohlc_dtype[1:]):
|
||||
|
|
@ -350,13 +282,12 @@ class Client:
|
|||
@acm
|
||||
async def get_client() -> Client:
|
||||
client = Client()
|
||||
log.info('Caching exchange infos..')
|
||||
await client.exch_info()
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
|
||||
|
||||
# validation type
|
||||
class AggTrade(Struct):
|
||||
class AggTrade(BaseModel):
|
||||
e: str # Event type
|
||||
E: int # Event time
|
||||
s: str # Symbol
|
||||
|
|
@ -370,82 +301,51 @@ class AggTrade(Struct):
|
|||
M: bool # Ignore
|
||||
|
||||
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
) -> AsyncGenerator[NoBsWs, dict]:
|
||||
async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]:
|
||||
|
||||
timeouts = 0
|
||||
while True:
|
||||
|
||||
with trio.move_on_after(3) as cs:
|
||||
msg = await ws.recv_msg()
|
||||
|
||||
if cs.cancelled_caught:
|
||||
|
||||
timeouts += 1
|
||||
if timeouts > 2:
|
||||
log.error("binance feed seems down and slow af? rebooting...")
|
||||
await ws._connect()
|
||||
|
||||
continue
|
||||
|
||||
# TODO: match syntax here!
|
||||
msg: dict[str, Any]
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
# for l1 streams binance doesn't add an event type field so
|
||||
# identify those messages by matching keys
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams
|
||||
case {
|
||||
# NOTE: this is never an old value it seems, so
|
||||
# they are always sending real L1 spread updates.
|
||||
'u': upid, # update id
|
||||
's': sym,
|
||||
'b': bid,
|
||||
'B': bsize,
|
||||
'a': ask,
|
||||
'A': asize,
|
||||
}:
|
||||
# TODO: it would be super nice to have a `L1` piker type
|
||||
# which "renders" incremental tick updates from a packed
|
||||
# msg-struct:
|
||||
# - backend msgs after packed into the type such that we
|
||||
# can reduce IPC usage but without each backend having
|
||||
# to do that incremental update logic manually B)
|
||||
# - would it maybe be more efficient to use this instead?
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#diff-depth-stream
|
||||
l1 = L1(
|
||||
update_id=upid,
|
||||
sym=sym,
|
||||
bid=bid,
|
||||
bsize=bsize,
|
||||
ask=ask,
|
||||
asize=asize,
|
||||
)
|
||||
l1.typecast()
|
||||
|
||||
# repack into piker's tick-quote format
|
||||
if msg.get('u'):
|
||||
sym = msg['s']
|
||||
bid = float(msg['b'])
|
||||
bsize = float(msg['B'])
|
||||
ask = float(msg['a'])
|
||||
asize = float(msg['A'])
|
||||
|
||||
yield 'l1', {
|
||||
'symbol': l1.sym,
|
||||
'symbol': sym,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'bid',
|
||||
'price': l1.bid,
|
||||
'size': l1.bsize,
|
||||
},
|
||||
{
|
||||
'type': 'bsize',
|
||||
'price': l1.bid,
|
||||
'size': l1.bsize,
|
||||
},
|
||||
{
|
||||
'type': 'ask',
|
||||
'price': l1.ask,
|
||||
'size': l1.asize,
|
||||
},
|
||||
{
|
||||
'type': 'asize',
|
||||
'price': l1.ask,
|
||||
'size': l1.asize,
|
||||
}
|
||||
{'type': 'bid', 'price': bid, 'size': bsize},
|
||||
{'type': 'bsize', 'price': bid, 'size': bsize},
|
||||
{'type': 'ask', 'price': ask, 'size': asize},
|
||||
{'type': 'asize', 'price': ask, 'size': asize}
|
||||
]
|
||||
}
|
||||
|
||||
# https://binance-docs.github.io/apidocs/spot/en/#aggregate-trade-streams
|
||||
case {
|
||||
'e': 'aggTrade',
|
||||
}:
|
||||
# NOTE: this is purely for a definition,
|
||||
# ``msgspec.Struct`` does not runtime-validate until you
|
||||
# decode/encode, see:
|
||||
# https://jcristharif.com/msgspec/structs.html#type-validation
|
||||
elif msg.get('e') == 'aggTrade':
|
||||
|
||||
# validate
|
||||
msg = AggTrade(**msg)
|
||||
msg.typecast()
|
||||
|
||||
# TODO: type out and require this quote format
|
||||
# from all backends!
|
||||
yield 'trade', {
|
||||
'symbol': msg.s,
|
||||
'last': msg.p,
|
||||
|
|
@ -484,66 +384,39 @@ async def open_history_client(
|
|||
async with open_cached_client('binance') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: datetime | None = None,
|
||||
start_dt: datetime | None = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
if timeframe != 60:
|
||||
raise DataUnavailable('Only 1m bars are supported')
|
||||
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
times = array['time']
|
||||
if (
|
||||
end_dt is None
|
||||
):
|
||||
inow = round(time.time())
|
||||
if (inow - times[-1]) > 60:
|
||||
await tractor.breakpoint()
|
||||
|
||||
start_dt = pendulum.from_timestamp(times[0])
|
||||
end_dt = pendulum.from_timestamp(times[-1])
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
@async_lifo_cache()
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
|
||||
async def backfill_bars(
|
||||
sym: str,
|
||||
shm: ShmArray, # type: ignore # noqa
|
||||
task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
"""Fill historical bars into shared mem / storage afap.
|
||||
"""
|
||||
with trio.CancelScope() as cs:
|
||||
async with open_cached_client('binance') as client:
|
||||
|
||||
pair: Pair = await client.exch_info(fqme.upper())
|
||||
mkt = MktPair(
|
||||
dst=Asset(
|
||||
name=pair.baseAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.baseAssetPrecision),
|
||||
),
|
||||
src=Asset(
|
||||
name=pair.quoteAsset,
|
||||
atype='crypto',
|
||||
tx_tick=digits_to_dec(pair.quoteAssetPrecision),
|
||||
),
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=pair.symbol,
|
||||
broker='binance',
|
||||
)
|
||||
both = mkt, pair
|
||||
return both
|
||||
bars = await client.bars(symbol=sym)
|
||||
shm.push(bars)
|
||||
task_status.started(cs)
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
|
@ -560,25 +433,43 @@ async def stream_quotes(
|
|||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym_infos = {}
|
||||
uid = 0
|
||||
|
||||
async with (
|
||||
open_cached_client('binance') as client,
|
||||
send_chan as send_chan,
|
||||
):
|
||||
init_msgs: list[FeedInit] = []
|
||||
for sym in symbols:
|
||||
mkt, pair = await get_mkt_info(sym)
|
||||
|
||||
# build out init msgs according to latest spec
|
||||
init_msgs.append(
|
||||
FeedInit(
|
||||
mkt_info=mkt,
|
||||
shm_write_opts={'sum_tick_vml': False},
|
||||
)
|
||||
)
|
||||
# keep client cached for real-time section
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
for sym in symbols:
|
||||
d = cache[sym.upper()]
|
||||
syminfo = Pair(**d) # validation
|
||||
|
||||
si = sym_infos[sym] = syminfo.dict()
|
||||
|
||||
# XXX: after manually inspecting the response format we
|
||||
# just directly pick out the info we need
|
||||
si['price_tick_size'] = float(syminfo.filters[0]['tickSize'])
|
||||
si['lot_tick_size'] = float(syminfo.filters[2]['stepSize'])
|
||||
si['asset_type'] = 'crypto'
|
||||
|
||||
symbol = symbols[0]
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
symbol: {
|
||||
'symbol_info': sym_infos[sym],
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: NoBsWs):
|
||||
async def subscribe(ws: wsproto.WSConnection):
|
||||
# setup subs
|
||||
|
||||
# trade data (aka L1)
|
||||
|
|
@ -604,7 +495,6 @@ async def stream_quotes(
|
|||
subs.append("{sym}@bookTicker")
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
"method": "UNSUBSCRIBE",
|
||||
"params": subs,
|
||||
|
|
@ -614,23 +504,19 @@ async def stream_quotes(
|
|||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
# XXX: see api docs which show diff addr?
|
||||
# https://developers.binance.com/docs/binance-trading-api/websocket_api#general-api-information
|
||||
# 'wss://ws-api.binance.com:443/ws-api/v3',
|
||||
async with open_autorecon_ws(
|
||||
'wss://stream.binance.com/ws',
|
||||
fixture=subscribe,
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
aclosing(stream_messages(ws)) as msg_gen,
|
||||
):
|
||||
typ, quote = await anext(msg_gen)
|
||||
) as ws:
|
||||
|
||||
# pull a first quote and deliver
|
||||
msg_gen = stream_messages(ws)
|
||||
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
while typ != 'trade':
|
||||
typ, quote = await anext(msg_gen)
|
||||
# TODO: use ``anext()`` when it lands in 3.10!
|
||||
typ, quote = await msg_gen.__anext__()
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
|
|
@ -660,13 +546,13 @@ async def open_symbol_search(
|
|||
async with open_cached_client('binance') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.exch_info()
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# results = await client.exch_info(sym=pattern.upper())
|
||||
# results = await client.symbol_info(sym=pattern.upper())
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
|
|
@ -674,7 +560,7 @@ async def open_symbol_search(
|
|||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
item[0].symbol: item[0]
|
||||
for item in matches
|
||||
})
|
||||
await stream.send(
|
||||
{item[0]['symbol']: item[0]
|
||||
for item in matches}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,170 +28,17 @@ import tractor
|
|||
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..service import (
|
||||
maybe_spawn_brokerd,
|
||||
maybe_open_pikerd,
|
||||
)
|
||||
from ..brokers import (
|
||||
core,
|
||||
get_brokermod,
|
||||
data,
|
||||
)
|
||||
DEFAULT_BROKER = 'binance'
|
||||
from ..log import get_console_log, colorize_json, get_logger
|
||||
from .._daemon import maybe_spawn_brokerd, maybe_open_pikerd
|
||||
from ..brokers import core, get_brokermod, data
|
||||
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
_config_dir = click.get_app_dir('piker')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
|
||||
|
||||
OK = '\033[92m'
|
||||
WARNING = '\033[93m'
|
||||
FAIL = '\033[91m'
|
||||
ENDC = '\033[0m'
|
||||
|
||||
|
||||
def print_ok(s: str, **kwargs):
|
||||
print(OK + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def print_error(s: str, **kwargs):
|
||||
print(FAIL + s + ENDC, **kwargs)
|
||||
|
||||
|
||||
def get_method(client, meth_name: str):
|
||||
print(f'checking client for method \'{meth_name}\'...', end='', flush=True)
|
||||
method = getattr(client, meth_name, None)
|
||||
assert method
|
||||
print_ok('found!.')
|
||||
return method
|
||||
|
||||
|
||||
async def run_method(client, meth_name: str, **kwargs):
|
||||
method = get_method(client, meth_name)
|
||||
print('running...', end='', flush=True)
|
||||
result = await method(**kwargs)
|
||||
print_ok(f'done! result: {type(result)}')
|
||||
return result
|
||||
|
||||
|
||||
async def run_test(broker_name: str):
|
||||
brokermod = get_brokermod(broker_name)
|
||||
total = 0
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
print('getting client...', end='', flush=True)
|
||||
if not hasattr(brokermod, 'get_client'):
|
||||
print_error('fail! no \'get_client\' context manager found.')
|
||||
return
|
||||
|
||||
async with brokermod.get_client(is_brokercheck=True) as client:
|
||||
print_ok('done! inside client context.')
|
||||
|
||||
# check for methods present on brokermod
|
||||
method_list = [
|
||||
'backfill_bars',
|
||||
'get_client',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
|
||||
]
|
||||
|
||||
for method in method_list:
|
||||
print(
|
||||
f'checking brokermod for method \'{method}\'...',
|
||||
end='', flush=True)
|
||||
if not hasattr(brokermod, method):
|
||||
print_error(f'fail! method \'{method}\' not found.')
|
||||
failed += 1
|
||||
else:
|
||||
print_ok('done!')
|
||||
passed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
# for private methods only check is present
|
||||
method_list = [
|
||||
'get_balances',
|
||||
'get_assets',
|
||||
'get_trades',
|
||||
'get_xfers',
|
||||
'submit_limit',
|
||||
'submit_cancel',
|
||||
'search_symbols',
|
||||
]
|
||||
|
||||
for method_name in method_list:
|
||||
try:
|
||||
get_method(client, method_name)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
# check for methods present con brokermod.Client and their
|
||||
# results
|
||||
|
||||
syms = await run_method(client, 'symbol_info')
|
||||
total += 1
|
||||
|
||||
if len(syms) == 0:
|
||||
raise BaseException('Empty Symbol list?')
|
||||
|
||||
passed += 1
|
||||
|
||||
first_sym = tuple(syms.keys())[0]
|
||||
|
||||
method_list = [
|
||||
('cache_symbols', {}),
|
||||
('search_symbols', {'pattern': first_sym[:-1]}),
|
||||
('bars', {'symbol': first_sym})
|
||||
]
|
||||
|
||||
for method_name, method_kwargs in method_list:
|
||||
try:
|
||||
await run_method(client, method_name, **method_kwargs)
|
||||
passed += 1
|
||||
|
||||
except AssertionError:
|
||||
print_error(f'fail! method \'{method_name}\' not found.')
|
||||
failed += 1
|
||||
|
||||
total += 1
|
||||
|
||||
print(f'total: {total}, passed: {passed}, failed: {failed}')
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.argument('broker', nargs=1, required=True)
|
||||
@click.pass_obj
|
||||
def brokercheck(config, broker):
|
||||
'''
|
||||
Test broker apis for completeness.
|
||||
|
||||
'''
|
||||
async def bcheck_main():
|
||||
async with maybe_spawn_brokerd(broker) as portal:
|
||||
await portal.run(run_test, broker)
|
||||
await portal.cancel_actor()
|
||||
|
||||
trio.run(run_test, broker)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option('--keys', '-k', multiple=True,
|
||||
help='Return results only for these keys')
|
||||
|
|
|
|||
|
|
@ -26,12 +26,15 @@ from typing import List, Dict, Any, Optional
|
|||
|
||||
import trio
|
||||
|
||||
from ._util import log
|
||||
from ..log import get_logger
|
||||
from . import get_brokermod
|
||||
from ..service import maybe_spawn_brokerd
|
||||
from .._daemon import maybe_spawn_brokerd
|
||||
from .._cacheables import open_cached_client
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def api(brokername: str, methname: str, **kwargs) -> dict:
|
||||
"""Make (proxy through) a broker API call by name and return its result.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -41,13 +41,13 @@ import tractor
|
|||
from tractor.experimental import msgpub
|
||||
from async_generator import asynccontextmanager
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, get_console_log
|
||||
from . import get_brokermod
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
async def wait_for_network(
|
||||
net_func: Callable,
|
||||
sleep: int = 1
|
||||
|
|
@ -227,28 +227,26 @@ async def get_cached_feed(
|
|||
|
||||
@tractor.stream
|
||||
async def start_quote_stream(
|
||||
stream: tractor.Context, # marks this as a streaming func
|
||||
ctx: tractor.Context, # marks this as a streaming func
|
||||
broker: str,
|
||||
symbols: List[Any],
|
||||
feed_type: str = 'stock',
|
||||
rate: int = 3,
|
||||
) -> None:
|
||||
'''
|
||||
Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
"""Handle per-broker quote stream subscriptions using a "lazy" pub-sub
|
||||
pattern.
|
||||
|
||||
Spawns new quoter tasks for each broker backend on-demand.
|
||||
Since most brokers seems to support batch quote requests we
|
||||
limit to one task per process (for now).
|
||||
|
||||
'''
|
||||
"""
|
||||
# XXX: why do we need this again?
|
||||
get_console_log(tractor.current_actor().loglevel)
|
||||
|
||||
# pull global vars from local actor
|
||||
symbols = list(symbols)
|
||||
log.info(
|
||||
f"{stream.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
f"{ctx.chan.uid} subscribed to {broker} for symbols {symbols}")
|
||||
# another actor task may have already created it
|
||||
async with get_cached_feed(broker) as feed:
|
||||
|
||||
|
|
@ -292,13 +290,13 @@ async def start_quote_stream(
|
|||
assert fquote['displayable']
|
||||
payload[sym] = fquote
|
||||
|
||||
await stream.send_yield(payload)
|
||||
await ctx.send_yield(payload)
|
||||
|
||||
await stream_poll_requests(
|
||||
|
||||
# ``trionics.msgpub`` required kwargs
|
||||
task_name=feed_type,
|
||||
ctx=stream,
|
||||
ctx=ctx,
|
||||
topics=symbols,
|
||||
packetizer=feed.mod.packetizer,
|
||||
|
||||
|
|
@ -321,11 +319,9 @@ async def call_client(
|
|||
|
||||
|
||||
class DataFeed:
|
||||
'''
|
||||
Data feed client for streaming symbol data from and making API
|
||||
client calls to a (remote) ``brokerd`` daemon.
|
||||
|
||||
'''
|
||||
"""Data feed client for streaming symbol data from and making API client calls
|
||||
to a (remote) ``brokerd`` daemon.
|
||||
"""
|
||||
_allowed = ('stock', 'option')
|
||||
|
||||
def __init__(self, portal, brokermod):
|
||||
|
|
|
|||
|
|
@ -1,70 +0,0 @@
|
|||
``deribit`` backend
|
||||
------------------
|
||||
pretty good liquidity crypto derivatives, uses custom json rpc over ws for
|
||||
client methods, then `cryptofeed` for data streams.
|
||||
|
||||
status
|
||||
******
|
||||
- supports option charts
|
||||
- no order support yet
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[deribit]
|
||||
key_id = 'XXXXXXXX'
|
||||
key_secret = 'Xx_XxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXxXx'
|
||||
|
||||
To obtain an api id and secret you need to create an account, which can be a
|
||||
real market account over at:
|
||||
|
||||
- deribit.com (requires KYC for deposit address)
|
||||
|
||||
Or a testnet account over at:
|
||||
|
||||
- test.deribit.com
|
||||
|
||||
For testnet once the account is created here is how you deposit fake crypto to
|
||||
try it out:
|
||||
|
||||
1) Go to Wallet:
|
||||
|
||||
.. figure:: assets/0_wallet.png
|
||||
:align: center
|
||||
:target: assets/0_wallet.png
|
||||
:alt: wallet page
|
||||
|
||||
2) Then click on the elipsis menu and select deposit
|
||||
|
||||
.. figure:: assets/1_wallet_select_deposit.png
|
||||
:align: center
|
||||
:target: assets/1_wallet_select_deposit.png
|
||||
:alt: wallet deposit page
|
||||
|
||||
3) This will take you to the deposit address page
|
||||
|
||||
.. figure:: assets/2_gen_deposit_addr.png
|
||||
:align: center
|
||||
:target: assets/2_gen_deposit_addr.png
|
||||
:alt: generate deposit address page
|
||||
|
||||
4) After clicking generate you should see the address, copy it and go to the
|
||||
`coin faucet <https://test.deribit.com/dericoin/BTC/deposit>`_ and send fake
|
||||
coins to that address.
|
||||
|
||||
.. figure:: assets/3_deposit_address.png
|
||||
:align: center
|
||||
:target: assets/3_deposit_address.png
|
||||
:alt: generated address
|
||||
|
||||
5) Back in the deposit address page you should see the deposit in your history
|
||||
|
||||
.. figure:: assets/4_wallet_deposit_history.png
|
||||
:align: center
|
||||
:target: assets/4_wallet_deposit_history.png
|
||||
:alt: wallet deposit history
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
|
||||
from piker.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
backfill_bars
|
||||
)
|
||||
# from .broker import (
|
||||
# trades_dialogue,
|
||||
# norm_trade_records,
|
||||
# )
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
# 'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
# 'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
# 'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
# annotation to let backend agnostic code
|
||||
# know if ``brokerd`` should be spawned with
|
||||
# ``tractor``'s aio mode.
|
||||
_infect_asyncio: bool = True
|
||||
|
|
@ -1,672 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
|
||||
from contextlib import asynccontextmanager as acm, AsyncExitStack
|
||||
from functools import partial
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, Iterable, Callable
|
||||
|
||||
import pendulum
|
||||
import asks
|
||||
import trio
|
||||
from trio_typing import Nursery, TaskStatus
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
|
||||
from piker.data.types import Struct
|
||||
from piker.data._web_bs import (
|
||||
NoBsWs,
|
||||
open_autorecon_ws,
|
||||
open_jsonrpc_session
|
||||
)
|
||||
|
||||
from .._util import resproc
|
||||
|
||||
from piker import config
|
||||
from piker.log import get_logger
|
||||
|
||||
from tractor.trionics import (
|
||||
broadcast_receiver,
|
||||
BroadcastReceiver,
|
||||
maybe_open_context
|
||||
)
|
||||
from tractor import to_asyncio
|
||||
|
||||
from cryptofeed import FeedHandler
|
||||
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT,
|
||||
L1_BOOK, TRADES,
|
||||
OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
_url = 'https://www.deribit.com'
|
||||
_ws_url = 'wss://www.deribit.com/ws/api/v2'
|
||||
_testnet_ws_url = 'wss://test.deribit.com/ws/api/v2'
|
||||
|
||||
|
||||
# Broker specific ohlc schema (rest)
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('bar_wap', float), # will be zeroed by sampler if not filled
|
||||
]
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
id: int
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
usIn: int
|
||||
usOut: int
|
||||
usDiff: int
|
||||
testnet: bool
|
||||
|
||||
class JSONRPCChannel(Struct):
|
||||
jsonrpc: str = '2.0'
|
||||
method: str
|
||||
params: dict
|
||||
|
||||
|
||||
class KLinesResult(Struct):
|
||||
close: list[float]
|
||||
cost: list[float]
|
||||
high: list[float]
|
||||
low: list[float]
|
||||
open: list[float]
|
||||
status: str
|
||||
ticks: list[int]
|
||||
volume: list[float]
|
||||
|
||||
class Trade(Struct):
|
||||
trade_seq: int
|
||||
trade_id: str
|
||||
timestamp: int
|
||||
tick_direction: int
|
||||
price: float
|
||||
mark_price: float
|
||||
iv: float
|
||||
instrument_name: str
|
||||
index_price: float
|
||||
direction: str
|
||||
combo_trade_id: Optional[int] = 0,
|
||||
combo_id: Optional[str] = '',
|
||||
amount: float
|
||||
|
||||
class LastTradesResult(Struct):
|
||||
trades: list[Trade]
|
||||
has_more: bool
|
||||
|
||||
|
||||
# convert datetime obj timestamp to unixtime in milliseconds
|
||||
def deribit_timestamp(when):
|
||||
return int((when.timestamp() * 1000) + (when.microsecond / 1000))
|
||||
|
||||
|
||||
def str_to_cb_sym(name: str) -> Symbol:
|
||||
base, strike_price, expiry_date, option_type = name.split('-')
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'put':
|
||||
option_type = PUT
|
||||
elif option_type == 'call':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date,
|
||||
expiry_normalize=False)
|
||||
|
||||
|
||||
def piker_sym_to_cb_sym(name: str) -> Symbol:
|
||||
base, expiry_date, strike_price, option_type = tuple(
|
||||
name.upper().split('-'))
|
||||
|
||||
quote = base
|
||||
|
||||
if option_type == 'P':
|
||||
option_type = PUT
|
||||
elif option_type == 'C':
|
||||
option_type = CALL
|
||||
else:
|
||||
raise Exception("Couldn\'t parse option type")
|
||||
|
||||
return Symbol(
|
||||
base, quote,
|
||||
type=OPTION,
|
||||
strike_price=strike_price,
|
||||
option_type=option_type,
|
||||
expiry_date=expiry_date.upper())
|
||||
|
||||
|
||||
def cb_sym_to_deribit_inst(sym: Symbol):
|
||||
# cryptofeed normalized
|
||||
cb_norm = ['F', 'G', 'H', 'J', 'K', 'M', 'N', 'Q', 'U', 'V', 'X', 'Z']
|
||||
|
||||
# deribit specific
|
||||
months = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
|
||||
|
||||
exp = sym.expiry_date
|
||||
|
||||
# YYMDD
|
||||
# 01234
|
||||
year, month, day = (
|
||||
exp[:2], months[cb_norm.index(exp[2:3])], exp[3:])
|
||||
|
||||
otype = 'C' if sym.option_type == CALL else 'P'
|
||||
|
||||
return f'{sym.base}-{day}{month}{year}-{sym.strike_price}-{otype}'
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
|
||||
section = conf.get('deribit')
|
||||
|
||||
# TODO: document why we send this, basically because logging params for cryptofeed
|
||||
conf['log'] = {}
|
||||
conf['log']['disabled'] = True
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for deribit in {path}')
|
||||
|
||||
return conf
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
def __init__(self, json_rpc: Callable) -> None:
|
||||
self._pairs: dict[str, Any] = None
|
||||
|
||||
config = get_config().get('deribit', {})
|
||||
|
||||
if ('key_id' in config) and ('key_secret' in config):
|
||||
self._key_id = config['key_id']
|
||||
self._key_secret = config['key_secret']
|
||||
|
||||
else:
|
||||
self._key_id = None
|
||||
self._key_secret = None
|
||||
|
||||
self.json_rpc = json_rpc
|
||||
|
||||
@property
|
||||
def currencies(self):
|
||||
return ['btc', 'eth', 'sol', 'usd']
|
||||
|
||||
async def get_balances(self, kind: str = 'option') -> dict[str, float]:
|
||||
"""Return the set of positions for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_positions', params={
|
||||
'currency': currency.upper(),
|
||||
'kind': kind})
|
||||
|
||||
balances[currency] = resp.result
|
||||
|
||||
return balances
|
||||
|
||||
async def get_assets(self) -> dict[str, float]:
|
||||
"""Return the set of asset balances for this account
|
||||
by symbol.
|
||||
"""
|
||||
balances = {}
|
||||
|
||||
for currency in self.currencies:
|
||||
resp = await self.json_rpc(
|
||||
'private/get_account_summary', params={
|
||||
'currency': currency.upper()})
|
||||
|
||||
balances[currency] = resp.result['balance']
|
||||
|
||||
return balances
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float
|
||||
) -> dict:
|
||||
"""Place an order
|
||||
"""
|
||||
params = {
|
||||
'instrument_name': symbol.upper(),
|
||||
'amount': size,
|
||||
'type': 'limit',
|
||||
'price': price,
|
||||
}
|
||||
resp = await self.json_rpc(
|
||||
f'private/{action}', params)
|
||||
|
||||
return resp.result
|
||||
|
||||
async def submit_cancel(self, oid: str):
|
||||
"""Send cancel request for order id
|
||||
"""
|
||||
resp = await self.json_rpc(
|
||||
'private/cancel', {'order_id': oid})
|
||||
return resp.result
|
||||
|
||||
async def symbol_info(
|
||||
self,
|
||||
instrument: Optional[str] = None,
|
||||
currency: str = 'btc', # BTC, ETH, SOL, USDC
|
||||
kind: str = 'option',
|
||||
expired: bool = False
|
||||
) -> dict[str, Any]:
|
||||
"""Get symbol info for the exchange.
|
||||
|
||||
"""
|
||||
if self._pairs:
|
||||
return self._pairs
|
||||
|
||||
# will retrieve all symbols by default
|
||||
params = {
|
||||
'currency': currency.upper(),
|
||||
'kind': kind,
|
||||
'expired': str(expired).lower()
|
||||
}
|
||||
|
||||
resp = await self.json_rpc('public/get_instruments', params)
|
||||
results = resp.result
|
||||
|
||||
instruments = {
|
||||
item['instrument_name'].lower(): item
|
||||
for item in results
|
||||
}
|
||||
|
||||
if instrument is not None:
|
||||
return instruments[instrument]
|
||||
else:
|
||||
return instruments
|
||||
|
||||
async def cache_symbols(
|
||||
self,
|
||||
) -> dict:
|
||||
if not self._pairs:
|
||||
self._pairs = await self.symbol_info()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
data = await self.symbol_info()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
data,
|
||||
score_cutoff=35,
|
||||
limit=limit
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0]['instrument_name'].lower(): item[0]
|
||||
for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str,
|
||||
start_dt: Optional[datetime] = None,
|
||||
end_dt: Optional[datetime] = None,
|
||||
limit: int = 1000,
|
||||
as_np: bool = True,
|
||||
) -> dict:
|
||||
instrument = symbol
|
||||
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC')
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_time = deribit_timestamp(start_dt)
|
||||
end_time = deribit_timestamp(end_dt)
|
||||
|
||||
# https://docs.deribit.com/#public-get_tradingview_chart_data
|
||||
resp = await self.json_rpc(
|
||||
'public/get_tradingview_chart_data',
|
||||
params={
|
||||
'instrument_name': instrument.upper(),
|
||||
'start_timestamp': start_time,
|
||||
'end_timestamp': end_time,
|
||||
'resolution': '1'
|
||||
})
|
||||
|
||||
result = KLinesResult(**resp.result)
|
||||
new_bars = []
|
||||
for i in range(len(result.close)):
|
||||
|
||||
_open = result.open[i]
|
||||
high = result.high[i]
|
||||
low = result.low[i]
|
||||
close = result.close[i]
|
||||
volume = result.volume[i]
|
||||
|
||||
row = [
|
||||
(start_time + (i * (60 * 1000))) / 1000.0, # time
|
||||
result.open[i],
|
||||
result.high[i],
|
||||
result.low[i],
|
||||
result.close[i],
|
||||
result.volume[i],
|
||||
0
|
||||
]
|
||||
|
||||
new_bars.append((i,) + tuple(row))
|
||||
|
||||
array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else klines
|
||||
return array
|
||||
|
||||
async def last_trades(
|
||||
self,
|
||||
instrument: str,
|
||||
count: int = 10
|
||||
):
|
||||
resp = await self.json_rpc(
|
||||
'public/get_last_trades_by_instrument',
|
||||
params={
|
||||
'instrument_name': instrument,
|
||||
'count': count
|
||||
})
|
||||
|
||||
return LastTradesResult(**resp.result)
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client(
|
||||
is_brokercheck: bool = False
|
||||
) -> Client:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_jsonrpc_session(
|
||||
_testnet_ws_url, dtype=JSONRPCResult) as json_rpc
|
||||
):
|
||||
client = Client(json_rpc)
|
||||
|
||||
_refresh_token: Optional[str] = None
|
||||
_access_token: Optional[str] = None
|
||||
|
||||
async def _auth_loop(
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED
|
||||
):
|
||||
"""Background task that adquires a first access token and then will
|
||||
refresh the access token while the nursery isn't cancelled.
|
||||
|
||||
https://docs.deribit.com/?python#authentication-2
|
||||
"""
|
||||
renew_time = 10
|
||||
access_scope = 'trade:read_write'
|
||||
_expiry_time = time.time()
|
||||
got_access = False
|
||||
nonlocal _refresh_token
|
||||
nonlocal _access_token
|
||||
|
||||
while True:
|
||||
if time.time() - _expiry_time < renew_time:
|
||||
# if we are close to token expiry time
|
||||
|
||||
if _refresh_token != None:
|
||||
# if we have a refresh token already dont need to send
|
||||
# secret
|
||||
params = {
|
||||
'grant_type': 'refresh_token',
|
||||
'refresh_token': _refresh_token,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
else:
|
||||
# we don't have refresh token, send secret to initialize
|
||||
params = {
|
||||
'grant_type': 'client_credentials',
|
||||
'client_id': client._key_id,
|
||||
'client_secret': client._key_secret,
|
||||
'scope': access_scope
|
||||
}
|
||||
|
||||
resp = await json_rpc('public/auth', params)
|
||||
result = resp.result
|
||||
|
||||
_expiry_time = time.time() + result['expires_in']
|
||||
_refresh_token = result['refresh_token']
|
||||
|
||||
if 'access_token' in result:
|
||||
_access_token = result['access_token']
|
||||
|
||||
if not got_access:
|
||||
# first time this loop runs we must indicate task is
|
||||
# started, we have auth
|
||||
got_access = True
|
||||
task_status.started()
|
||||
|
||||
else:
|
||||
await trio.sleep(renew_time / 2)
|
||||
|
||||
# if we have client creds launch auth loop
|
||||
if client._key_id is not None:
|
||||
await n.start(_auth_loop)
|
||||
|
||||
await client.cache_symbols()
|
||||
yield client
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
@acm
|
||||
async def open_feed_handler():
|
||||
fh = FeedHandler(config=get_config())
|
||||
yield fh
|
||||
await to_asyncio.run_task(fh.stop_async)
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_feed_handler() -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_context(
|
||||
acm_func=open_feed_handler,
|
||||
key='feedhandler',
|
||||
) as (cache_hit, fh):
|
||||
yield fh
|
||||
|
||||
|
||||
async def aio_price_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _trade(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('trade', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'last': data,
|
||||
'broker_ts': time.time(),
|
||||
'data': data.to_dict(),
|
||||
'receipt': receipt_timestamp
|
||||
}))
|
||||
|
||||
async def _l1(data: dict, receipt_timestamp):
|
||||
to_trio.send_nowait(('l1', {
|
||||
'symbol': cb_sym_to_deribit_inst(
|
||||
str_to_cb_sym(data.symbol)).lower(),
|
||||
'ticks': [
|
||||
{'type': 'bid',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'bsize',
|
||||
'price': float(data.bid_price), 'size': float(data.bid_size)},
|
||||
{'type': 'ask',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)},
|
||||
{'type': 'asize',
|
||||
'price': float(data.ask_price), 'size': float(data.ask_size)}
|
||||
]
|
||||
}))
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[TRADES, L1_BOOK],
|
||||
symbols=[piker_sym_to_cb_sym(instrument)],
|
||||
callbacks={
|
||||
TRADES: _trade,
|
||||
L1_BOOK: _l1
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_price_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_price_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_price_feed,
|
||||
kwargs={
|
||||
'instrument': instrument
|
||||
},
|
||||
key=f'{instrument}-price',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
|
||||
|
||||
|
||||
async def aio_order_feed_relay(
|
||||
fh: FeedHandler,
|
||||
instrument: Symbol,
|
||||
from_trio: asyncio.Queue,
|
||||
to_trio: trio.abc.SendChannel,
|
||||
) -> None:
|
||||
async def _fill(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
async def _order_info(data: dict, receipt_timestamp):
|
||||
breakpoint()
|
||||
|
||||
fh.add_feed(
|
||||
DERIBIT,
|
||||
channels=[FILLS, ORDER_INFO],
|
||||
symbols=[instrument.upper()],
|
||||
callbacks={
|
||||
FILLS: _fill,
|
||||
ORDER_INFO: _order_info,
|
||||
})
|
||||
|
||||
if not fh.running:
|
||||
fh.run(
|
||||
start_loop=False,
|
||||
install_signal_handlers=False)
|
||||
|
||||
# sync with trio
|
||||
to_trio.send_nowait(None)
|
||||
|
||||
await asyncio.sleep(float('inf'))
|
||||
|
||||
|
||||
@acm
|
||||
async def open_order_feed(
|
||||
instrument: list[str]
|
||||
) -> trio.abc.ReceiveStream:
|
||||
async with maybe_open_feed_handler() as fh:
|
||||
async with to_asyncio.open_channel_from(
|
||||
partial(
|
||||
aio_order_feed_relay,
|
||||
fh,
|
||||
instrument
|
||||
)
|
||||
) as (first, chan):
|
||||
yield chan
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_order_feed(
|
||||
instrument: str
|
||||
) -> trio.abc.ReceiveStream:
|
||||
|
||||
# TODO: add a predicate to maybe_open_context
|
||||
async with maybe_open_context(
|
||||
acm_func=open_order_feed,
|
||||
kwargs={
|
||||
'instrument': instrument,
|
||||
'fh': fh
|
||||
},
|
||||
key=f'{instrument}-order',
|
||||
) as (cache_hit, feed):
|
||||
if cache_hit:
|
||||
yield broadcast_receiver(feed, 10)
|
||||
else:
|
||||
yield feed
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 169 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 106 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 59 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 132 KiB |
|
|
@ -1,185 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Deribit backend.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional, Callable
|
||||
import time
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import pendulum
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import tractor
|
||||
|
||||
from piker._cacheables import open_cached_client
|
||||
from piker.log import get_logger, get_console_log
|
||||
from piker.data import ShmArray
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataUnavailable,
|
||||
)
|
||||
|
||||
from cryptofeed import FeedHandler
|
||||
|
||||
from cryptofeed.defines import (
|
||||
DERIBIT, L1_BOOK, TRADES, OPTION, CALL, PUT
|
||||
)
|
||||
from cryptofeed.symbols import Symbol
|
||||
|
||||
from .api import (
|
||||
Client, Trade,
|
||||
get_config,
|
||||
str_to_cb_sym, piker_sym_to_cb_sym, cb_sym_to_deribit_inst,
|
||||
maybe_open_price_feed
|
||||
)
|
||||
|
||||
_spawn_kwargs = {
|
||||
'infect_asyncio': True,
|
||||
}
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
instrument: str,
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
async def get_ohlc(
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
array = await client.bars(
|
||||
instrument,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
if len(array) == 0:
|
||||
raise DataUnavailable
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 3, 'rate': 3}
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
# XXX: required to propagate ``tractor`` loglevel to piker logging
|
||||
get_console_log(loglevel or tractor.current_actor().loglevel)
|
||||
|
||||
sym = symbols[0]
|
||||
|
||||
async with (
|
||||
open_cached_client('deribit') as client,
|
||||
send_chan as send_chan
|
||||
):
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
sym: {
|
||||
'symbol_info': {
|
||||
'asset_type': 'option',
|
||||
'price_tick_size': 0.0005
|
||||
},
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
},
|
||||
}
|
||||
|
||||
nsym = piker_sym_to_cb_sym(sym)
|
||||
|
||||
async with maybe_open_price_feed(sym) as stream:
|
||||
|
||||
cache = await client.cache_symbols()
|
||||
|
||||
last_trades = (await client.last_trades(
|
||||
cb_sym_to_deribit_inst(nsym), count=1)).trades
|
||||
|
||||
if len(last_trades) == 0:
|
||||
last_trade = None
|
||||
async for typ, quote in stream:
|
||||
if typ == 'trade':
|
||||
last_trade = Trade(**(quote['data']))
|
||||
break
|
||||
|
||||
else:
|
||||
last_trade = Trade(**(last_trades[0]))
|
||||
|
||||
first_quote = {
|
||||
'symbol': sym,
|
||||
'last': last_trade.price,
|
||||
'brokerd_ts': last_trade.timestamp,
|
||||
'ticks': [{
|
||||
'type': 'trade',
|
||||
'price': last_trade.price,
|
||||
'size': last_trade.amount,
|
||||
'broker_ts': last_trade.timestamp
|
||||
}]
|
||||
}
|
||||
task_status.started((init_msgs, first_quote))
|
||||
|
||||
feed_is_live.set()
|
||||
|
||||
async for typ, quote in stream:
|
||||
topic = quote['symbol']
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> Client:
|
||||
async with open_cached_client('deribit') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
# repack in dict form
|
||||
await stream.send(
|
||||
await client.search_symbols(pattern))
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
``ib`` backend
|
||||
--------------
|
||||
more or less the "everything broker" for traditional and international
|
||||
markets. they are the "go to" provider for automatic retail trading
|
||||
and we interface to their APIs using the `ib_insync` project.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
currently there is not yet full support for:
|
||||
- options charting and trading
|
||||
- paxos based crypto rt feeds and trading
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib]
|
||||
hosts = [
|
||||
"127.0.0.1",
|
||||
]
|
||||
# TODO: when we eventually spawn gateways in our
|
||||
# container, we can just dynamically allocate these
|
||||
# using IBC.
|
||||
ports = [
|
||||
4002,
|
||||
4003,
|
||||
4006,
|
||||
4001,
|
||||
7497,
|
||||
]
|
||||
|
||||
# XXX: for a paper account the flex web query service
|
||||
# is not supported so you have to manually download
|
||||
# and XML report and put it in a location that can be
|
||||
# accessed by the ``brokerd.ib`` backend code for parsing.
|
||||
flex_token = '1111111111111111'
|
||||
flex_trades_query_id = '6969696' # live accounts only?
|
||||
|
||||
# 3rd party web-api token
|
||||
# (XXX: not sure if this works yet)
|
||||
trade_log_token = '111111111111111'
|
||||
|
||||
# when clients are being scanned this determines
|
||||
# which clients are preferred to be used for data feeds
|
||||
# based on account names which are detected as active
|
||||
# on each client.
|
||||
prefer_data_account = [
|
||||
# this has to be first in order to make data work with dual paper + live
|
||||
'main',
|
||||
'algopaper',
|
||||
]
|
||||
|
||||
[ib.accounts]
|
||||
main = 'U69696969'
|
||||
algopaper = 'DU9696969'
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_ib_algopaper.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
["0000e1a7.630f5e5a.01.01"]
|
||||
secType = "FUT"
|
||||
conId = 515416577
|
||||
symbol = "MNQ"
|
||||
lastTradeDateOrContractMonth = "20221216"
|
||||
strike = 0.0
|
||||
right = ""
|
||||
multiplier = "2"
|
||||
exchange = "GLOBEX"
|
||||
primaryExchange = ""
|
||||
currency = "USD"
|
||||
localSymbol = "MNQZ2"
|
||||
tradingClass = "MNQ"
|
||||
includeExpired = false
|
||||
secIdType = ""
|
||||
secId = ""
|
||||
comboLegsDescrip = ""
|
||||
comboLegs = []
|
||||
execId = "0000e1a7.630f5e5a.01.01"
|
||||
time = 1661972086.0
|
||||
acctNumber = "DU69696969"
|
||||
side = "BOT"
|
||||
shares = 1.0
|
||||
price = 12372.75
|
||||
permId = 441472655
|
||||
clientId = 6116
|
||||
orderId = 985
|
||||
liquidation = 0
|
||||
cumQty = 1.0
|
||||
avgPrice = 12372.75
|
||||
orderRef = ""
|
||||
evRule = ""
|
||||
evMultiplier = 0.0
|
||||
modelCode = ""
|
||||
lastLiquidity = 1
|
||||
broker_time = 1661972086.0
|
||||
name = "ib"
|
||||
commission = 0.57
|
||||
realizedPNL = 243.41
|
||||
yield_ = 0.0
|
||||
yieldRedemptionDate = 0
|
||||
listingExchange = "GLOBEX"
|
||||
date = "2022-08-31T18:54:46+00:00"
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[ib.algopaper."mnq.globex.20221216"]
|
||||
size = -1.0
|
||||
ppu = 12423.630576923071
|
||||
bs_mktid = 515416577
|
||||
expiry = "2022-12-16T00:00:00+00:00"
|
||||
clears = [
|
||||
{ dt = "2022-08-31T18:54:46+00:00", ppu = 12423.630576923071, accum_size = -19.0, price = 12372.75, size = 1.0, cost = 0.57, tid = "0000e1a7.630f5e5a.01.01" },
|
||||
]
|
||||
|
|
@ -20,10 +20,15 @@ Interactive Brokers API backend.
|
|||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
- ``data.py`` for real-time data feed endpoints
|
||||
|
||||
- ``client.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
- ``report.py`` for the hackery to build manual pp calcs
|
||||
to avoid ib's absolute bullshit FIFO style position
|
||||
tracking..
|
||||
|
||||
"""
|
||||
from .api import (
|
||||
get_client,
|
||||
|
|
@ -33,9 +38,7 @@ from .feed import (
|
|||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
trades_dialogue,
|
||||
)
|
||||
from .broker import trades_dialogue
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
|
|
@ -45,22 +48,13 @@ __all__ = [
|
|||
'stream_quotes',
|
||||
]
|
||||
|
||||
_brokerd_mods: list[str] = [
|
||||
'api',
|
||||
'broker',
|
||||
]
|
||||
|
||||
_datad_mods: list[str] = [
|
||||
'feed',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = (
|
||||
_brokerd_mods
|
||||
+
|
||||
_datad_mods
|
||||
)
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
||||
# passed to ``tractor.ActorNursery.start_actor()``
|
||||
_spawn_kwargs = {
|
||||
|
|
|
|||
|
|
@ -1,187 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
"FLEX" report processing utils.
|
||||
|
||||
"""
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
from pprint import pformat
|
||||
from typing import Any
|
||||
|
||||
from .api import (
|
||||
get_config,
|
||||
log,
|
||||
)
|
||||
from piker.accounting import (
|
||||
open_trade_ledger,
|
||||
)
|
||||
|
||||
|
||||
def parse_flex_dt(
|
||||
record: str,
|
||||
) -> pendulum.datetime:
|
||||
date, ts = record.split(';')
|
||||
dt = pendulum.parse(date)
|
||||
ts = f'{ts[:2]}:{ts[2:4]}:{ts[4:]}'
|
||||
tsdt = pendulum.parse(ts)
|
||||
return dt.set(hour=tsdt.hour, minute=tsdt.minute, second=tsdt.second)
|
||||
|
||||
|
||||
def flex_records_to_ledger_entries(
|
||||
accounts: bidict,
|
||||
trade_entries: list[object],
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Convert flex report entry objects into ``dict`` form, pretty much
|
||||
straight up without modification except add a `pydatetime` field
|
||||
from the parsed timestamp.
|
||||
|
||||
'''
|
||||
trades_by_account = {}
|
||||
for t in trade_entries:
|
||||
entry = t.__dict__
|
||||
|
||||
# XXX: LOL apparently ``toml`` has a bug
|
||||
# where a section key error will show up in the write
|
||||
# if you leave a table key as an `int`? So i guess
|
||||
# cast to strs for all keys..
|
||||
|
||||
# oddly for some so-called "BookTrade" entries
|
||||
# this field seems to be blank, no cuckin clue.
|
||||
# trade['ibExecID']
|
||||
tid = str(entry.get('ibExecID') or entry['tradeID'])
|
||||
# date = str(entry['tradeDate'])
|
||||
|
||||
# XXX: is it going to cause problems if a account name
|
||||
# get's lost? The user should be able to find it based
|
||||
# on the actual exec history right?
|
||||
acctid = accounts[str(entry['accountId'])]
|
||||
|
||||
# probably a flex record with a wonky non-std timestamp..
|
||||
dt = entry['pydatetime'] = parse_flex_dt(entry['dateTime'])
|
||||
entry['datetime'] = str(dt)
|
||||
|
||||
if not tid:
|
||||
# this is likely some kind of internal adjustment
|
||||
# transaction, likely one of the following:
|
||||
# - an expiry event that will show a "book trade" indicating
|
||||
# some adjustment to cash balances: zeroing or itm settle.
|
||||
# - a manual cash balance position adjustment likely done by
|
||||
# the user from the accounts window in TWS where they can
|
||||
# manually set the avg price and size:
|
||||
# https://api.ibkr.com/lib/cstools/faq/web1/index.html#/tag/DTWS_ADJ_AVG_COST
|
||||
log.warning(f'Skipping ID-less ledger entry:\n{pformat(entry)}')
|
||||
continue
|
||||
|
||||
trades_by_account.setdefault(
|
||||
acctid, {}
|
||||
)[tid] = entry
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_account[acctid] = dict(sorted(
|
||||
trades_by_account[acctid].items(),
|
||||
key=lambda entry: entry[1]['pydatetime'],
|
||||
))
|
||||
|
||||
return trades_by_account
|
||||
|
||||
|
||||
def load_flex_trades(
|
||||
path: str | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
|
||||
from ib_insync import flexreport, util
|
||||
|
||||
conf = get_config()
|
||||
|
||||
if not path:
|
||||
# load ``brokers.toml`` and try to get the flex
|
||||
# token and query id that must be previously defined
|
||||
# by the user.
|
||||
token = conf.get('flex_token')
|
||||
if not token:
|
||||
raise ValueError(
|
||||
'You must specify a ``flex_token`` field in your'
|
||||
'`brokers.toml` in order load your trade log, see our'
|
||||
'intructions for how to set this up here:\n'
|
||||
'PUT LINK HERE!'
|
||||
)
|
||||
|
||||
qid = conf['flex_trades_query_id']
|
||||
|
||||
# TODO: hack this into our logging
|
||||
# system like we do with the API client..
|
||||
util.logToConsole()
|
||||
|
||||
# TODO: rewrite the query part of this with async..httpx?
|
||||
report = flexreport.FlexReport(
|
||||
token=token,
|
||||
queryId=qid,
|
||||
)
|
||||
|
||||
else:
|
||||
# XXX: another project we could potentially look at,
|
||||
# https://pypi.org/project/ibflex/
|
||||
report = flexreport.FlexReport(path=path)
|
||||
|
||||
trade_entries = report.extract('Trade')
|
||||
ln = len(trade_entries)
|
||||
log.info(f'Loaded {ln} trades from flex query')
|
||||
|
||||
trades_by_account = flex_records_to_ledger_entries(
|
||||
conf['accounts'].inverse, # reverse map to user account names
|
||||
trade_entries,
|
||||
)
|
||||
|
||||
ledger_dict: dict | None = None
|
||||
|
||||
for acctid in trades_by_account:
|
||||
trades_by_id = trades_by_account[acctid]
|
||||
|
||||
with open_trade_ledger('ib', acctid) as ledger_dict:
|
||||
tid_delta = set(trades_by_id) - set(ledger_dict)
|
||||
log.info(
|
||||
'New trades detected\n'
|
||||
f'{pformat(tid_delta)}'
|
||||
)
|
||||
if tid_delta:
|
||||
sorted_delta = dict(sorted(
|
||||
{tid: trades_by_id[tid] for tid in tid_delta}.items(),
|
||||
key=lambda entry: entry[1].pop('pydatetime'),
|
||||
))
|
||||
ledger_dict.update(sorted_delta)
|
||||
|
||||
return ledger_dict
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
import os
|
||||
|
||||
args = sys.argv
|
||||
if len(args) > 1:
|
||||
args = args[1:]
|
||||
for arg in args:
|
||||
path = os.path.abspath(arg)
|
||||
load_flex_trades(path=path)
|
||||
else:
|
||||
# expect brokers.toml to have an entry and
|
||||
# pull from the web service.
|
||||
load_flex_trades()
|
||||
|
|
@ -1,189 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
``ib`` utilities and hacks suitable for use in the backend and/or as
|
||||
runnable script-programs.
|
||||
|
||||
'''
|
||||
from typing import Literal
|
||||
import subprocess
|
||||
|
||||
import tractor
|
||||
|
||||
from .._util import log
|
||||
|
||||
|
||||
_reset_tech: Literal[
|
||||
'vnc',
|
||||
'i3ipc_xdotool',
|
||||
|
||||
# TODO: in theory we can use a different linux DE API or
|
||||
# some other type of similar window scanning/mgmt client
|
||||
# (on other OSs) to do the same.
|
||||
|
||||
] = 'vnc'
|
||||
|
||||
|
||||
async def data_reset_hack(
|
||||
reset_type: str = 'data',
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Run key combos for resetting data feeds and yield back to caller
|
||||
when complete.
|
||||
|
||||
NOTE: this is a linux-only hack around!
|
||||
|
||||
There are multiple "techs" you can use depending on your infra setup:
|
||||
|
||||
- if running ib-gw in a container with a VNC server running the most
|
||||
performant method is the `'vnc'` option.
|
||||
|
||||
- if running ib-gw/tws locally, and you are using `i3` you can use
|
||||
the ``i3ipc`` lib and ``xdotool`` to send the appropriate click
|
||||
and key-combos automatically to your local desktop's java X-apps.
|
||||
|
||||
https://interactivebrokers.github.io/tws-api/historical_limitations.html#pacing_violations
|
||||
|
||||
TODOs:
|
||||
- a return type that hopefully determines if the hack was
|
||||
successful.
|
||||
- other OS support?
|
||||
- integration with ``ib-gw`` run in docker + Xorg?
|
||||
- is it possible to offer a local server that can be accessed by
|
||||
a client? Would be sure be handy for running native java blobs
|
||||
that need to be wrangle.
|
||||
|
||||
'''
|
||||
global _reset_tech
|
||||
|
||||
match _reset_tech:
|
||||
case 'vnc':
|
||||
try:
|
||||
await tractor.to_asyncio.run_task(vnc_click_hack)
|
||||
except OSError:
|
||||
_reset_tech = 'i3ipc_xdotool'
|
||||
try:
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
case 'i3ipc_xdotool':
|
||||
i3ipc_xdotool_manual_click_hack()
|
||||
|
||||
case _ as tech:
|
||||
raise RuntimeError(f'{tech} is not supported for reset tech!?')
|
||||
|
||||
# we don't really need the ``xdotool`` approach any more B)
|
||||
return True
|
||||
|
||||
|
||||
async def vnc_click_hack(
|
||||
reset_type: str = 'data'
|
||||
) -> None:
|
||||
'''
|
||||
Reset the data or netowork connection for the VNC attached
|
||||
ib gateway using magic combos.
|
||||
|
||||
'''
|
||||
key = {'data': 'f', 'connection': 'r'}[reset_type]
|
||||
|
||||
import asyncvnc
|
||||
|
||||
async with asyncvnc.connect(
|
||||
'localhost',
|
||||
port=3003,
|
||||
# password='ibcansmbz',
|
||||
) as client:
|
||||
|
||||
# move to middle of screen
|
||||
# 640x1800
|
||||
client.mouse.move(
|
||||
x=500,
|
||||
y=500,
|
||||
)
|
||||
client.mouse.click()
|
||||
client.keyboard.press('Ctrl', 'Alt', key) # keys are stacked
|
||||
|
||||
|
||||
def i3ipc_xdotool_manual_click_hack() -> None:
|
||||
import i3ipc
|
||||
|
||||
i3 = i3ipc.Connection()
|
||||
t = i3.get_tree()
|
||||
|
||||
orig_win_id = t.find_focused().window
|
||||
|
||||
# for tws
|
||||
win_names: list[str] = [
|
||||
'Interactive Brokers', # tws running in i3
|
||||
'IB Gateway', # gw running in i3
|
||||
# 'IB', # gw running in i3 (newer version?)
|
||||
]
|
||||
|
||||
try:
|
||||
for name in win_names:
|
||||
results = t.find_titled(name)
|
||||
print(f'results for {name}: {results}')
|
||||
if results:
|
||||
con = results[0]
|
||||
print(f'Resetting data feed for {name}')
|
||||
win_id = str(con.window)
|
||||
w, h = con.rect.width, con.rect.height
|
||||
|
||||
# TODO: seems to be a few libs for python but not sure
|
||||
# if they support all the sub commands we need, order of
|
||||
# most recent commit history:
|
||||
# https://github.com/rr-/pyxdotool
|
||||
# https://github.com/ShaneHutter/pyxdotool
|
||||
# https://github.com/cphyc/pyxdotool
|
||||
|
||||
# TODO: only run the reconnect (2nd) kc on a detected
|
||||
# disconnect?
|
||||
for key_combo, timeout in [
|
||||
# only required if we need a connection reset.
|
||||
# ('ctrl+alt+r', 12),
|
||||
# data feed reset.
|
||||
('ctrl+alt+f', 6)
|
||||
]:
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', win_id,
|
||||
|
||||
# move mouse to bottom left of window (where
|
||||
# there should be nothing to click).
|
||||
'mousemove_relative', '--sync', str(w-4), str(h-4),
|
||||
|
||||
# NOTE: we may need to stick a `--retry 3` in here..
|
||||
'click', '--window', win_id,
|
||||
'--repeat', '3', '1',
|
||||
|
||||
# hackzorzes
|
||||
'key', key_combo,
|
||||
],
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
# re-activate and focus original window
|
||||
subprocess.call([
|
||||
'xdotool',
|
||||
'windowactivate', '--sync', str(orig_win_id),
|
||||
'click', '--window', str(orig_win_id), '1',
|
||||
])
|
||||
except subprocess.TimeoutExpired:
|
||||
log.exception('xdotool timed out?')
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,64 +0,0 @@
|
|||
``kraken`` backend
|
||||
------------------
|
||||
though they don't have the most liquidity of all the cexes they sure are
|
||||
accommodating to those of us who appreciate a little ``xmr``.
|
||||
|
||||
status
|
||||
******
|
||||
current support is *production grade* and both real-time data and order
|
||||
management should be correct and fast. this backend is used by core devs
|
||||
for live trading.
|
||||
|
||||
|
||||
config
|
||||
******
|
||||
In order to get order mode support your ``brokers.toml``
|
||||
needs to have something like the following:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken]
|
||||
accounts.spot = 'spot'
|
||||
key_descr = "spot"
|
||||
api_key = "69696969696969696696969696969696969696969696969696969696"
|
||||
secret = "BOOBSBOOBSBOOBSBOOBSBOOBSSMBZ69696969696969669969696969696"
|
||||
|
||||
|
||||
If everything works correctly you should see any current positions
|
||||
loaded in the pps pane on chart load and you should also be able to
|
||||
check your trade records in the file::
|
||||
|
||||
<pikerk_conf_dir>/ledgers/trades_kraken_spot.toml
|
||||
|
||||
|
||||
An example ledger file will have entries written verbatim from the
|
||||
trade events schema:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[TFJBKK-SMBZS-VJ4UWS]
|
||||
ordertxid = "SMBZSA-7CNQU-3HWLNJ"
|
||||
postxid = "SMBZSE-M7IF5-CFI7LT"
|
||||
pair = "XXMRZEUR"
|
||||
time = 1655691993.4133966
|
||||
type = "buy"
|
||||
ordertype = "limit"
|
||||
price = "103.97000000"
|
||||
cost = "499.99999977"
|
||||
fee = "0.80000000"
|
||||
vol = "4.80907954"
|
||||
margin = "0.00000000"
|
||||
misc = ""
|
||||
|
||||
|
||||
your ``pps.toml`` file will have position entries like,
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[kraken.spot."xmreur.kraken"]
|
||||
size = 4.80907954
|
||||
ppu = 103.97000000
|
||||
bs_mktid = "XXMRZEUR"
|
||||
clears = [
|
||||
{ tid = "TFJBKK-SMBZS-VJ4UWS", cost = 0.8, price = 103.97, size = 4.80907954, dt = "2022-05-20T02:26:33.413397+00:00" },
|
||||
]
|
||||
|
|
@ -1,62 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken backend.
|
||||
|
||||
Sub-modules within break into the core functionalities:
|
||||
|
||||
- ``broker.py`` part for orders / trading endpoints
|
||||
- ``feed.py`` for real-time data feed endpoints
|
||||
- ``api.py`` for the core API machinery which is ``trio``-ized
|
||||
wrapping around ``ib_insync``.
|
||||
|
||||
'''
|
||||
|
||||
from piker.log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
from .api import (
|
||||
get_client,
|
||||
)
|
||||
from .feed import (
|
||||
get_mkt_info,
|
||||
open_history_client,
|
||||
open_symbol_search,
|
||||
stream_quotes,
|
||||
)
|
||||
from .broker import (
|
||||
trades_dialogue,
|
||||
norm_trade_records,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
'get_client',
|
||||
'trades_dialogue',
|
||||
'open_history_client',
|
||||
'open_symbol_search',
|
||||
'stream_quotes',
|
||||
'norm_trade_records',
|
||||
]
|
||||
|
||||
|
||||
# tractor RPC enable arg
|
||||
__enable_modules__: list[str] = [
|
||||
'api',
|
||||
'feed',
|
||||
'broker',
|
||||
]
|
||||
|
|
@ -1,677 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kraken web API wrapping.
|
||||
|
||||
'''
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
import itertools
|
||||
from typing import (
|
||||
Any,
|
||||
Union,
|
||||
)
|
||||
import time
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import asks
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import urllib.parse
|
||||
import hashlib
|
||||
import hmac
|
||||
import base64
|
||||
import trio
|
||||
|
||||
from piker import config
|
||||
from piker.data.types import Struct
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
digits_to_dec,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
resproc,
|
||||
SymbolNotFound,
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
)
|
||||
from piker.accounting import Transaction
|
||||
from . import log
|
||||
|
||||
# <uri>/<version>/
|
||||
_url = 'https://api.kraken.com/0'
|
||||
|
||||
|
||||
# Broker specific ohlc schema which includes a vwap field
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('count', int),
|
||||
('bar_wap', float),
|
||||
]
|
||||
|
||||
# UI components allow this to be declared such that additional
|
||||
# (historical) fields can be exposed.
|
||||
ohlc_dtype = np.dtype(_ohlc_dtype)
|
||||
|
||||
_show_wap_in_history = True
|
||||
_symbol_info_translation: dict[str, str] = {
|
||||
'tick_decimals': 'pair_decimals',
|
||||
}
|
||||
|
||||
|
||||
def get_config() -> dict[str, Any]:
|
||||
|
||||
conf, path = config.load()
|
||||
section = conf.get('kraken')
|
||||
|
||||
if section is None:
|
||||
log.warning(f'No config section found for kraken in {path}')
|
||||
return {}
|
||||
|
||||
return section
|
||||
|
||||
|
||||
def get_kraken_signature(
|
||||
urlpath: str,
|
||||
data: dict[str, Any],
|
||||
secret: str
|
||||
) -> str:
|
||||
postdata = urllib.parse.urlencode(data)
|
||||
encoded = (str(data['nonce']) + postdata).encode()
|
||||
message = urlpath.encode() + hashlib.sha256(encoded).digest()
|
||||
|
||||
mac = hmac.new(base64.b64decode(secret), message, hashlib.sha512)
|
||||
sigdigest = base64.b64encode(mac.digest())
|
||||
return sigdigest.decode()
|
||||
|
||||
|
||||
class InvalidKey(ValueError):
|
||||
'''
|
||||
EAPI:Invalid key
|
||||
This error is returned when the API key used for the call is
|
||||
either expired or disabled, please review the API key in your
|
||||
Settings -> API tab of account management or generate a new one
|
||||
and update your application.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
# https://www.kraken.com/features/api#get-tradable-pairs
|
||||
class Pair(Struct):
|
||||
altname: str # alternate pair name
|
||||
wsname: str # WebSocket pair name (if available)
|
||||
aclass_base: str # asset class of base component
|
||||
base: str # asset id of base component
|
||||
aclass_quote: str # asset class of quote component
|
||||
quote: str # asset id of quote component
|
||||
lot: str # volume lot size
|
||||
|
||||
cost_decimals: int
|
||||
costmin: float
|
||||
pair_decimals: int # scaling decimal places for pair
|
||||
lot_decimals: int # scaling decimal places for volume
|
||||
|
||||
# amount to multiply lot volume by to get currency volume
|
||||
lot_multiplier: float
|
||||
|
||||
# array of leverage amounts available when buying
|
||||
leverage_buy: list[int]
|
||||
# array of leverage amounts available when selling
|
||||
leverage_sell: list[int]
|
||||
|
||||
# fee schedule array in [volume, percent fee] tuples
|
||||
fees: list[tuple[int, float]]
|
||||
|
||||
# maker fee schedule array in [volume, percent fee] tuples (if on
|
||||
# maker/taker)
|
||||
fees_maker: list[tuple[int, float]]
|
||||
|
||||
fee_volume_currency: str # volume discount currency
|
||||
margin_call: str # margin call level
|
||||
margin_stop: str # stop-out/liquidation margin level
|
||||
ordermin: float # minimum order volume for pair
|
||||
tick_size: float # min price step size
|
||||
status: str
|
||||
|
||||
short_position_limit: float = 0
|
||||
long_position_limit: float = float('inf')
|
||||
|
||||
@property
|
||||
def price_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.pair_decimals)
|
||||
|
||||
@property
|
||||
def size_tick(self) -> Decimal:
|
||||
return digits_to_dec(self.lot_decimals)
|
||||
|
||||
|
||||
class Client:
|
||||
|
||||
# symbol mapping from all names to the altname
|
||||
_ntable: dict[str, str] = {}
|
||||
|
||||
# 2-way map of symbol names to their "alt names" ffs XD
|
||||
_altnames: bidict[str, str] = bidict()
|
||||
|
||||
_pairs: dict[str, Pair] = {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: dict[str, str],
|
||||
name: str = '',
|
||||
api_key: str = '',
|
||||
secret: str = ''
|
||||
) -> None:
|
||||
self._sesh = asks.Session(connections=4)
|
||||
self._sesh.base_location = _url
|
||||
self._sesh.headers.update({
|
||||
'User-Agent':
|
||||
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
|
||||
})
|
||||
self._name = name
|
||||
self._api_key = api_key
|
||||
self._secret = secret
|
||||
|
||||
self.conf: dict[str, str] = config
|
||||
self.assets: dict[str, Asset] = {}
|
||||
|
||||
@property
|
||||
def pairs(self) -> dict[str, Pair]:
|
||||
if self._pairs is None:
|
||||
raise RuntimeError(
|
||||
"Make sure to run `cache_symbols()` on startup!"
|
||||
)
|
||||
# retreive and cache all symbols
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def _public(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
) -> dict[str, Any]:
|
||||
resp = await self._sesh.post(
|
||||
path=f'/public/{method}',
|
||||
json=data,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def _private(
|
||||
self,
|
||||
method: str,
|
||||
data: dict,
|
||||
uri_path: str
|
||||
) -> dict[str, Any]:
|
||||
headers = {
|
||||
'Content-Type':
|
||||
'application/x-www-form-urlencoded',
|
||||
'API-Key':
|
||||
self._api_key,
|
||||
'API-Sign':
|
||||
get_kraken_signature(uri_path, data, self._secret)
|
||||
}
|
||||
resp = await self._sesh.post(
|
||||
path=f'/private/{method}',
|
||||
data=data,
|
||||
headers=headers,
|
||||
timeout=float('inf')
|
||||
)
|
||||
return resproc(resp, log)
|
||||
|
||||
async def endpoint(
|
||||
self,
|
||||
method: str,
|
||||
data: dict[str, Any]
|
||||
|
||||
) -> dict[str, Any]:
|
||||
uri_path = f'/0/private/{method}'
|
||||
data['nonce'] = str(int(1000*time.time()))
|
||||
return await self._private(method, data, uri_path)
|
||||
|
||||
async def get_balances(
|
||||
self,
|
||||
) -> dict[str, float]:
|
||||
'''
|
||||
Return the set of asset balances for this account
|
||||
by symbol.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'Balance',
|
||||
{},
|
||||
)
|
||||
by_bsmktid = resp['result']
|
||||
|
||||
# TODO: we need to pull out the "asset" decimals
|
||||
# data and return a `decimal.Decimal` instead here!
|
||||
# using the underlying Asset
|
||||
return {
|
||||
self._altnames[sym].lower(): float(bal)
|
||||
for sym, bal in by_bsmktid.items()
|
||||
}
|
||||
|
||||
async def get_assets(self) -> dict[str, Asset]:
|
||||
'''
|
||||
Load and cache all asset infos and pack into
|
||||
our native ``Asset`` struct.
|
||||
|
||||
https://docs.kraken.com/rest/#tag/Market-Data/operation/getAssetInfo
|
||||
|
||||
return msg:
|
||||
"asset1": {
|
||||
"aclass": "string",
|
||||
"altname": "string",
|
||||
"decimals": 0,
|
||||
"display_decimals": 0,
|
||||
"collateral_value": 0,
|
||||
"status": "string"
|
||||
}
|
||||
|
||||
'''
|
||||
resp = await self._public('Assets', {})
|
||||
assets = resp['result']
|
||||
|
||||
for bs_mktid, info in assets.items():
|
||||
altname = self._altnames[bs_mktid] = info['altname']
|
||||
aclass = info['aclass']
|
||||
|
||||
self.assets[bs_mktid] = Asset(
|
||||
name=altname.lower(),
|
||||
atype=f'crypto_{aclass}',
|
||||
tx_tick=digits_to_dec(info['decimals']),
|
||||
info=info,
|
||||
)
|
||||
|
||||
return self.assets
|
||||
|
||||
async def get_trades(
|
||||
self,
|
||||
fetch_limit: int | None = None,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Get the trades (aka cleared orders) history from the rest endpoint:
|
||||
https://docs.kraken.com/rest/#operation/getTradeHistory
|
||||
|
||||
'''
|
||||
ofs = 0
|
||||
trades_by_id: dict[str, Any] = {}
|
||||
|
||||
for i in itertools.count():
|
||||
if (
|
||||
fetch_limit
|
||||
and i >= fetch_limit
|
||||
):
|
||||
break
|
||||
|
||||
# increment 'ofs' pagination offset
|
||||
ofs = i*50
|
||||
|
||||
resp = await self.endpoint(
|
||||
'TradesHistory',
|
||||
{'ofs': ofs},
|
||||
)
|
||||
by_id = resp['result']['trades']
|
||||
trades_by_id.update(by_id)
|
||||
|
||||
# can get up to 50 results per query, see:
|
||||
# https://docs.kraken.com/rest/#tag/User-Data/operation/getTradeHistory
|
||||
if (
|
||||
len(by_id) < 50
|
||||
):
|
||||
err = resp.get('error')
|
||||
if err:
|
||||
raise BrokerError(err)
|
||||
|
||||
# we know we received the max amount of
|
||||
# trade results so there may be more history.
|
||||
# catch the end of the trades
|
||||
count = resp['result']['count']
|
||||
break
|
||||
|
||||
# santity check on update
|
||||
assert count == len(trades_by_id.values())
|
||||
return trades_by_id
|
||||
|
||||
async def get_xfers(
|
||||
self,
|
||||
asset: str,
|
||||
src_asset: str = '',
|
||||
|
||||
) -> dict[str, Transaction]:
|
||||
'''
|
||||
Get asset balance transfer transactions.
|
||||
|
||||
Currently only withdrawals are supported.
|
||||
|
||||
'''
|
||||
resp = await self.endpoint(
|
||||
'WithdrawStatus',
|
||||
{'asset': asset},
|
||||
)
|
||||
try:
|
||||
xfers: list[dict] = resp['result']
|
||||
except KeyError:
|
||||
log.exception(f'Kraken suxxx: {resp}')
|
||||
return []
|
||||
|
||||
# eg. resp schema:
|
||||
# 'result': [{'method': 'Bitcoin', 'aclass': 'currency', 'asset':
|
||||
# 'XXBT', 'refid': 'AGBJRMB-JHD2M4-NDI3NR', 'txid':
|
||||
# 'b95d66d3bb6fd76cbccb93f7639f99a505cb20752c62ea0acc093a0e46547c44',
|
||||
# 'info': 'bc1qc8enqjekwppmw3g80p56z5ns7ze3wraqk5rl9z',
|
||||
# 'amount': '0.00300726', 'fee': '0.00001000', 'time':
|
||||
# 1658347714, 'status': 'Success'}]}
|
||||
|
||||
trans: dict[str, Transaction] = {}
|
||||
for entry in xfers:
|
||||
|
||||
# look up the normalized name and asset info
|
||||
asset_key = entry['asset']
|
||||
asset = self.assets[asset_key]
|
||||
asset_key = self._altnames[asset_key].lower()
|
||||
|
||||
# XXX: this is in the asset units (likely) so it isn't
|
||||
# quite the same as a commisions cost necessarily..)
|
||||
cost = float(entry['fee'])
|
||||
|
||||
fqme = asset_key + '.kraken'
|
||||
|
||||
tx = Transaction(
|
||||
fqsn=fqme,
|
||||
sym=asset,
|
||||
tid=entry['txid'],
|
||||
dt=pendulum.from_timestamp(entry['time']),
|
||||
bs_mktid=f'{asset_key}{src_asset}',
|
||||
size=-1*(
|
||||
float(entry['amount'])
|
||||
+
|
||||
cost
|
||||
),
|
||||
# since this will be treated as a "sell" it
|
||||
# shouldn't be needed to compute the be price.
|
||||
price='NaN',
|
||||
|
||||
# XXX: see note above
|
||||
cost=cost,
|
||||
)
|
||||
trans[tx.tid] = tx
|
||||
|
||||
return trans
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
symbol: str,
|
||||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str = None,
|
||||
validate: bool = False # set True test call without a real submission
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
|
||||
'''
|
||||
# Build common data dict for common keys from both endpoints
|
||||
data = {
|
||||
"pair": symbol,
|
||||
"price": str(price),
|
||||
"validate": validate
|
||||
}
|
||||
if reqid is None:
|
||||
# Build order data for kraken api
|
||||
data |= {
|
||||
"ordertype": "limit",
|
||||
"type": action,
|
||||
"volume": str(size),
|
||||
}
|
||||
return await self.endpoint('AddOrder', data)
|
||||
|
||||
else:
|
||||
# Edit order data for kraken api
|
||||
data["txid"] = reqid
|
||||
return await self.endpoint('EditOrder', data)
|
||||
|
||||
async def submit_cancel(
|
||||
self,
|
||||
reqid: str,
|
||||
) -> dict:
|
||||
'''
|
||||
Send cancel request for order id ``reqid``.
|
||||
|
||||
'''
|
||||
# txid is a transaction id given by kraken
|
||||
return await self.endpoint('CancelOrder', {"txid": reqid})
|
||||
|
||||
async def pair_info(
|
||||
self,
|
||||
pair_patt: str | None = None,
|
||||
|
||||
) -> dict[str, Pair] | Pair:
|
||||
'''
|
||||
Query for a tradeable asset pair (info), or all if no input
|
||||
pattern is provided.
|
||||
|
||||
https://docs.kraken.com/rest/#tag/Market-Data/operation/getTradableAssetPairs
|
||||
|
||||
'''
|
||||
# get all pairs by default, or filter
|
||||
# to whatever pattern is provided as input.
|
||||
pairs: dict[str, str] | None = None
|
||||
if pair_patt is not None:
|
||||
pairs = {'pair': pair_patt}
|
||||
|
||||
resp = await self._public(
|
||||
'AssetPairs',
|
||||
pairs,
|
||||
)
|
||||
err = resp['error']
|
||||
if err:
|
||||
raise SymbolNotFound(pair_patt)
|
||||
|
||||
pairs: dict[str, Pair] = {
|
||||
|
||||
key: Pair(**data)
|
||||
for key, data in resp['result'].items()
|
||||
}
|
||||
# always cache so we can possibly do faster lookup
|
||||
self._pairs.update(pairs)
|
||||
|
||||
if pair_patt is not None:
|
||||
return next(iter(pairs.items()))[1]
|
||||
|
||||
return pairs
|
||||
|
||||
async def cache_symbols(self) -> dict:
|
||||
'''
|
||||
Load all market pair info build and cache it for downstream use.
|
||||
|
||||
A ``._ntable: dict[str, str]`` is available for mapping the
|
||||
websocket pair name-keys and their http endpoint API (smh)
|
||||
equivalents to the "alternative name" which is generally the one
|
||||
we actually want to use XD
|
||||
|
||||
'''
|
||||
if not self._pairs:
|
||||
pairs = await self.pair_info()
|
||||
assert self._pairs == pairs
|
||||
|
||||
# table of all ws and rest keys to their alt-name values.
|
||||
ntable: dict[str, str] = {}
|
||||
|
||||
for rest_key in list(pairs.keys()):
|
||||
|
||||
pair: Pair = pairs[rest_key]
|
||||
altname = pair.altname
|
||||
wsname = pair.wsname
|
||||
ntable[altname] = ntable[rest_key] = ntable[wsname] = altname
|
||||
|
||||
# register the pair under all monikers, a giant flat
|
||||
# surjection of all possible names to each info obj.
|
||||
self._pairs[altname] = self._pairs[wsname] = pair
|
||||
|
||||
self._ntable.update(ntable)
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
|
||||
) -> dict[str, Any]:
|
||||
'''
|
||||
Search for a symbol by "alt name"..
|
||||
|
||||
It is expected that the ``Client._pairs`` table
|
||||
gets populated before conducting the underlying fuzzy-search
|
||||
over the pair-key set.
|
||||
|
||||
'''
|
||||
if not len(self._pairs):
|
||||
await self.cache_symbols()
|
||||
assert self._pairs, '`Client.cache_symbols()` was never called!?'
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
self._pairs,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0].altname: item[0] for item in matches}
|
||||
|
||||
async def bars(
|
||||
self,
|
||||
symbol: str = 'XBTUSD',
|
||||
|
||||
# UTC 2017-07-02 12:53:20
|
||||
since: Union[int, datetime] | None = None,
|
||||
count: int = 720, # <- max allowed per query
|
||||
as_np: bool = True,
|
||||
|
||||
) -> dict:
|
||||
|
||||
if since is None:
|
||||
since = pendulum.now('UTC').start_of('minute').subtract(
|
||||
minutes=count).timestamp()
|
||||
|
||||
elif isinstance(since, int):
|
||||
since = pendulum.from_timestamp(since).timestamp()
|
||||
|
||||
else: # presumably a pendulum datetime
|
||||
since = since.timestamp()
|
||||
|
||||
# UTC 2017-07-02 12:53:20 is oldest seconds value
|
||||
since = str(max(1499000000, int(since)))
|
||||
json = await self._public(
|
||||
'OHLC',
|
||||
data={
|
||||
'pair': symbol,
|
||||
'since': since,
|
||||
},
|
||||
)
|
||||
try:
|
||||
res = json['result']
|
||||
res.pop('last')
|
||||
bars = next(iter(res.values()))
|
||||
|
||||
new_bars = []
|
||||
|
||||
first = bars[0]
|
||||
last_nz_vwap = first[-3]
|
||||
if last_nz_vwap == 0:
|
||||
# use close if vwap is zero
|
||||
last_nz_vwap = first[-4]
|
||||
|
||||
# convert all fields to native types
|
||||
for i, bar in enumerate(bars):
|
||||
# normalize weird zero-ed vwap values..cmon kraken..
|
||||
# indicates vwap didn't change since last bar
|
||||
vwap = float(bar.pop(-3))
|
||||
if vwap != 0:
|
||||
last_nz_vwap = vwap
|
||||
if vwap == 0:
|
||||
vwap = last_nz_vwap
|
||||
|
||||
# re-insert vwap as the last of the fields
|
||||
bar.append(vwap)
|
||||
|
||||
new_bars.append(
|
||||
(i,) + tuple(
|
||||
ftype(bar[j]) for j, (name, ftype) in enumerate(
|
||||
_ohlc_dtype[1:]
|
||||
)
|
||||
)
|
||||
)
|
||||
array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars
|
||||
return array
|
||||
except KeyError:
|
||||
errmsg = json['error'][0]
|
||||
|
||||
if 'not found' in errmsg:
|
||||
raise SymbolNotFound(errmsg + f': {symbol}')
|
||||
|
||||
elif 'Too many requests' in errmsg:
|
||||
raise DataThrottle(f'{symbol}')
|
||||
|
||||
else:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
@classmethod
|
||||
def normalize_symbol(
|
||||
cls,
|
||||
ticker: str
|
||||
) -> tuple[str, Pair]:
|
||||
'''
|
||||
Normalize symbol names to to a 3x3 pair from the global
|
||||
definition map which we build out from the data retreived from
|
||||
the 'AssetPairs' endpoint, see methods above.
|
||||
|
||||
'''
|
||||
return cls._ntable[ticker].lower()
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> Client:
|
||||
|
||||
conf = get_config()
|
||||
if conf:
|
||||
client = Client(
|
||||
conf,
|
||||
name=conf['key_descr'],
|
||||
api_key=conf['api_key'],
|
||||
secret=conf['secret']
|
||||
)
|
||||
else:
|
||||
client = Client({})
|
||||
|
||||
# at startup, load all symbols, and asset info in
|
||||
# batch requests.
|
||||
async with trio.open_nursery() as nurse:
|
||||
nurse.start_soon(client.get_assets)
|
||||
await client.cache_symbols()
|
||||
|
||||
yield client
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,488 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Real-time and historical data feed endpoints.
|
||||
|
||||
'''
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
aclosing,
|
||||
)
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
)
|
||||
import time
|
||||
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import numpy as np
|
||||
import pendulum
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from piker.accounting._mktinfo import (
|
||||
Asset,
|
||||
MktPair,
|
||||
)
|
||||
from piker._cacheables import (
|
||||
open_cached_client,
|
||||
async_lifo_cache,
|
||||
)
|
||||
from piker.brokers._util import (
|
||||
BrokerError,
|
||||
DataThrottle,
|
||||
DataUnavailable,
|
||||
)
|
||||
from piker.data.types import Struct
|
||||
from piker.data.validate import FeedInit
|
||||
from piker.data._web_bs import open_autorecon_ws, NoBsWs
|
||||
from . import log
|
||||
from .api import (
|
||||
Client,
|
||||
Pair,
|
||||
)
|
||||
|
||||
|
||||
class OHLC(Struct):
|
||||
'''
|
||||
Description of the flattened OHLC quote format.
|
||||
|
||||
For schema details see:
|
||||
https://docs.kraken.com/websockets/#message-ohlc
|
||||
|
||||
'''
|
||||
chan_id: int # internal kraken id
|
||||
chan_name: str # eg. ohlc-1 (name-interval)
|
||||
pair: str # fx pair
|
||||
time: float # Begin time of interval, in seconds since epoch
|
||||
etime: float # End time of interval, in seconds since epoch
|
||||
open: float # Open price of interval
|
||||
high: float # High price within interval
|
||||
low: float # Low price within interval
|
||||
close: float # Close price of interval
|
||||
vwap: float # Volume weighted average price within interval
|
||||
volume: float # Accumulated volume **within interval**
|
||||
count: int # Number of trades within interval
|
||||
# (sampled) generated tick data
|
||||
ticks: list[Any] = []
|
||||
|
||||
|
||||
async def stream_messages(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Message stream parser and heartbeat handler.
|
||||
|
||||
Deliver ws subscription messages as well as handle heartbeat logic
|
||||
though a single async generator.
|
||||
|
||||
'''
|
||||
last_hb: float = 0
|
||||
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
case {'event': 'heartbeat'}:
|
||||
now = time.time()
|
||||
delay = now - last_hb
|
||||
last_hb = now
|
||||
|
||||
# XXX: why tf is this not printing without --tl flag?
|
||||
log.debug(f"Heartbeat after {delay}")
|
||||
# print(f"Heartbeat after {delay}")
|
||||
|
||||
continue
|
||||
|
||||
case _:
|
||||
# passthrough sub msgs
|
||||
yield msg
|
||||
|
||||
|
||||
async def process_data_feed_msgs(
|
||||
ws: NoBsWs,
|
||||
):
|
||||
'''
|
||||
Parse and pack data feed messages.
|
||||
|
||||
'''
|
||||
async with aclosing(stream_messages(ws)) as ws_stream:
|
||||
async for msg in ws_stream:
|
||||
match msg:
|
||||
case {
|
||||
'errorMessage': errmsg
|
||||
}:
|
||||
raise BrokerError(errmsg)
|
||||
|
||||
case {
|
||||
'event': 'subscriptionStatus',
|
||||
} as sub:
|
||||
log.info(
|
||||
'WS subscription is active:\n'
|
||||
f'{sub}'
|
||||
)
|
||||
continue
|
||||
|
||||
case [
|
||||
chan_id,
|
||||
*payload_array,
|
||||
chan_name,
|
||||
pair
|
||||
]:
|
||||
if 'ohlc' in chan_name:
|
||||
ohlc = OHLC(
|
||||
chan_id,
|
||||
chan_name,
|
||||
pair,
|
||||
*payload_array[0]
|
||||
)
|
||||
ohlc.typecast()
|
||||
yield 'ohlc', ohlc
|
||||
|
||||
elif 'spread' in chan_name:
|
||||
|
||||
bid, ask, ts, bsize, asize = map(
|
||||
float, payload_array[0])
|
||||
|
||||
# TODO: really makes you think IB has a horrible API...
|
||||
quote = {
|
||||
'symbol': pair.replace('/', ''),
|
||||
'ticks': [
|
||||
{'type': 'bid', 'price': bid, 'size': bsize},
|
||||
{'type': 'bsize', 'price': bid, 'size': bsize},
|
||||
|
||||
{'type': 'ask', 'price': ask, 'size': asize},
|
||||
{'type': 'asize', 'price': ask, 'size': asize},
|
||||
],
|
||||
}
|
||||
yield 'l1', quote
|
||||
|
||||
# elif 'book' in msg[-2]:
|
||||
# chan_id, *payload_array, chan_name, pair = msg
|
||||
# print(msg)
|
||||
|
||||
case {
|
||||
'connectionID': conid,
|
||||
'event': 'systemStatus',
|
||||
'status': 'online',
|
||||
'version': ver,
|
||||
}:
|
||||
log.info(
|
||||
f'Established {ver} ws connection with id: {conid}'
|
||||
)
|
||||
continue
|
||||
|
||||
case _:
|
||||
print(f'UNHANDLED MSG: {msg}')
|
||||
# yield msg
|
||||
|
||||
|
||||
def normalize(
|
||||
ohlc: OHLC,
|
||||
|
||||
) -> dict:
|
||||
quote = ohlc.to_dict()
|
||||
quote['broker_ts'] = quote['time']
|
||||
quote['brokerd_ts'] = time.time()
|
||||
quote['symbol'] = quote['pair'] = quote['pair'].replace('/', '')
|
||||
quote['last'] = quote['close']
|
||||
quote['bar_wap'] = ohlc.vwap
|
||||
|
||||
# seriously eh? what's with this non-symmetry everywhere
|
||||
# in subscription systems...
|
||||
# XXX: piker style is always lowercases symbols.
|
||||
topic = quote['pair'].replace('/', '').lower()
|
||||
|
||||
# print(quote)
|
||||
return topic, quote
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
symbol: str,
|
||||
|
||||
) -> tuple[Callable, int]:
|
||||
|
||||
# TODO implement history getter for the new storage layer.
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# lol, kraken won't send any more then the "last"
|
||||
# 720 1m bars.. so we have to just ignore further
|
||||
# requests of this type..
|
||||
queries: int = 0
|
||||
|
||||
async def get_ohlc(
|
||||
timeframe: float,
|
||||
end_dt: Optional[datetime] = None,
|
||||
start_dt: Optional[datetime] = None,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
datetime, # start
|
||||
datetime, # end
|
||||
]:
|
||||
|
||||
nonlocal queries
|
||||
if (
|
||||
queries > 0
|
||||
or timeframe != 60
|
||||
):
|
||||
raise DataUnavailable(
|
||||
'Only a single query for 1m bars supported')
|
||||
|
||||
count = 0
|
||||
while count <= 3:
|
||||
try:
|
||||
array = await client.bars(
|
||||
symbol,
|
||||
since=end_dt,
|
||||
)
|
||||
count += 1
|
||||
queries += 1
|
||||
break
|
||||
except DataThrottle:
|
||||
log.warning(f'kraken OHLC throttle for {symbol}')
|
||||
await trio.sleep(1)
|
||||
|
||||
start_dt = pendulum.from_timestamp(array[0]['time'])
|
||||
end_dt = pendulum.from_timestamp(array[-1]['time'])
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc, {'erlangs': 1, 'rate': 1}
|
||||
|
||||
|
||||
@async_lifo_cache()
|
||||
async def get_mkt_info(
|
||||
fqme: str,
|
||||
|
||||
) -> tuple[MktPair, Pair]:
|
||||
'''
|
||||
Query for and return a `MktPair` and backend-native `Pair` (or
|
||||
wtv else) info.
|
||||
|
||||
If more then one fqme is provided return a ``dict`` of native
|
||||
key-strs to `MktPair`s.
|
||||
|
||||
'''
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# uppercase since kraken bs_mktid is always upper
|
||||
bs_fqme, _, broker = fqme.partition('.')
|
||||
pair_str: str = bs_fqme.upper()
|
||||
bs_mktid: str = Client.normalize_symbol(pair_str)
|
||||
pair: Pair = await client.pair_info(pair_str)
|
||||
|
||||
assets = client.assets
|
||||
dst_asset: Asset = assets[pair.base]
|
||||
src_asset: Asset = assets[pair.quote]
|
||||
|
||||
mkt = MktPair(
|
||||
dst=dst_asset,
|
||||
src=src_asset,
|
||||
|
||||
price_tick=pair.price_tick,
|
||||
size_tick=pair.size_tick,
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
broker='kraken',
|
||||
)
|
||||
return mkt, pair
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = None,
|
||||
|
||||
# backend specific
|
||||
sub_type: str = 'ohlc',
|
||||
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Subscribe for ohlc stream of quotes for ``pairs``.
|
||||
|
||||
``pairs`` must be formatted <crypto_symbol>/<fiat_symbol>.
|
||||
|
||||
'''
|
||||
|
||||
ws_pairs: list[str] = []
|
||||
init_msgs: list[FeedInit] = []
|
||||
|
||||
async with (
|
||||
send_chan as send_chan,
|
||||
):
|
||||
for sym_str in symbols:
|
||||
mkt, pair = await get_mkt_info(sym_str)
|
||||
init_msgs.append(
|
||||
FeedInit(
|
||||
mkt_info=mkt,
|
||||
shm_write_opts={
|
||||
'sum_tick_vml': False,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
ws_pairs.append(pair.wsname)
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: NoBsWs):
|
||||
|
||||
# XXX: setup subs
|
||||
# https://docs.kraken.com/websockets/#message-subscribe
|
||||
# specific logic for this in kraken's sync client:
|
||||
# https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188
|
||||
ohlc_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'ohlc',
|
||||
'interval': 1,
|
||||
},
|
||||
}
|
||||
|
||||
# TODO: we want to eventually allow unsubs which should
|
||||
# be completely fine to request from a separate task
|
||||
# since internally the ws methods appear to be FIFO
|
||||
# locked.
|
||||
await ws.send_msg(ohlc_sub)
|
||||
|
||||
# trade data (aka L1)
|
||||
l1_sub = {
|
||||
'event': 'subscribe',
|
||||
'pair': ws_pairs,
|
||||
'subscription': {
|
||||
'name': 'spread',
|
||||
# 'depth': 10}
|
||||
},
|
||||
}
|
||||
|
||||
# pull a first quote and deliver
|
||||
await ws.send_msg(l1_sub)
|
||||
|
||||
yield
|
||||
|
||||
# unsub from all pairs on teardown
|
||||
if ws.connected():
|
||||
await ws.send_msg({
|
||||
'pair': ws_pairs,
|
||||
'event': 'unsubscribe',
|
||||
'subscription': ['ohlc', 'spread'],
|
||||
})
|
||||
|
||||
# XXX: do we need to ack the unsub?
|
||||
# await ws.recv_msg()
|
||||
|
||||
# see the tips on reconnection logic:
|
||||
# https://support.kraken.com/hc/en-us/articles/360044504011-WebSocket-API-unexpected-disconnections-from-market-data-feeds
|
||||
ws: NoBsWs
|
||||
async with (
|
||||
open_autorecon_ws(
|
||||
'wss://ws.kraken.com/',
|
||||
fixture=subscribe,
|
||||
msg_recv_timeout=5,
|
||||
reset_after=20,
|
||||
) as ws,
|
||||
|
||||
# avoid stream-gen closure from breaking trio..
|
||||
# NOTE: not sure this actually works XD particularly
|
||||
# if we call `ws._connect()` manally in the streaming
|
||||
# async gen..
|
||||
aclosing(process_data_feed_msgs(ws)) as msg_gen,
|
||||
):
|
||||
# pull a first quote and deliver
|
||||
typ, ohlc_last = await anext(msg_gen)
|
||||
topic, quote = normalize(ohlc_last)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
|
||||
# lol, only "closes" when they're margin squeezing clients ;P
|
||||
feed_is_live.set()
|
||||
|
||||
# keep start of last interval for volume tracking
|
||||
last_interval_start = ohlc_last.etime
|
||||
|
||||
# start streaming
|
||||
async for typ, ohlc in msg_gen:
|
||||
|
||||
if typ == 'ohlc':
|
||||
|
||||
# TODO: can get rid of all this by using
|
||||
# ``trades`` subscription...
|
||||
|
||||
# generate tick values to match time & sales pane:
|
||||
# https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m
|
||||
volume = ohlc.volume
|
||||
|
||||
# new OHLC sample interval
|
||||
if ohlc.etime > last_interval_start:
|
||||
last_interval_start = ohlc.etime
|
||||
tick_volume = volume
|
||||
|
||||
else:
|
||||
# this is the tick volume *within the interval*
|
||||
tick_volume = volume - ohlc_last.volume
|
||||
|
||||
ohlc_last = ohlc
|
||||
last = ohlc.close
|
||||
|
||||
if tick_volume:
|
||||
ohlc.ticks.append({
|
||||
'type': 'trade',
|
||||
'price': last,
|
||||
'size': tick_volume,
|
||||
})
|
||||
|
||||
topic, quote = normalize(ohlc)
|
||||
|
||||
elif typ == 'l1':
|
||||
quote = ohlc
|
||||
topic = quote['symbol'].lower()
|
||||
|
||||
await send_chan.send({topic: quote})
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
|
||||
) -> Client:
|
||||
async with open_cached_client('kraken') as client:
|
||||
|
||||
# load all symbols locally for fast search
|
||||
cache = await client.cache_symbols()
|
||||
await ctx.started(cache)
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
|
||||
async for pattern in stream:
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern,
|
||||
cache,
|
||||
score_cutoff=50,
|
||||
)
|
||||
# repack in dict form
|
||||
await stream.send({
|
||||
pair[0].altname: pair[0]
|
||||
for pair in matches
|
||||
})
|
||||
|
|
@ -1,727 +0,0 @@
|
|||
# Copyright (C) Jared Goldman (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Kucoin broker backend
|
||||
|
||||
'''
|
||||
|
||||
from typing import Any, Callable, Literal, AsyncGenerator
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
import time
|
||||
import base64
|
||||
import hmac
|
||||
import hashlib
|
||||
import wsproto
|
||||
from uuid import uuid4
|
||||
|
||||
import asks
|
||||
import tractor
|
||||
import trio
|
||||
from trio_util import trio_async_generator
|
||||
from trio_typing import TaskStatus
|
||||
from fuzzywuzzy import process as fuzzy
|
||||
import pendulum
|
||||
import numpy as np
|
||||
|
||||
from piker._cacheables import open_cached_client
|
||||
from piker.log import get_logger
|
||||
from ._util import DataUnavailable
|
||||
from piker.pp import config
|
||||
from ..data.types import Struct
|
||||
from ..data._web_bs import (
|
||||
open_autorecon_ws,
|
||||
NoBsWs,
|
||||
)
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_ohlc_dtype = [
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('bar_wap', float), # will be zeroed by sampler if not filled
|
||||
]
|
||||
|
||||
|
||||
class KucoinMktPair(Struct, frozen=True):
|
||||
'''
|
||||
Kucoin's pair format:
|
||||
https://docs.kucoin.com/#get-symbols-list
|
||||
|
||||
'''
|
||||
|
||||
baseCurrency: str
|
||||
baseIncrement: float
|
||||
baseMaxSize: float
|
||||
baseMinSize: float
|
||||
enableTrading: bool
|
||||
feeCurrency: str
|
||||
isMarginEnabled: bool
|
||||
market: str
|
||||
minFunds: float
|
||||
name: str
|
||||
priceIncrement: float
|
||||
priceLimitRate: float
|
||||
quoteCurrency: str
|
||||
quoteIncrement: float
|
||||
quoteMaxSize: float
|
||||
quoteMinSize: float
|
||||
symbol: str
|
||||
|
||||
|
||||
class AccountTrade(Struct, frozen=True):
|
||||
'''
|
||||
Historical trade format:
|
||||
https://docs.kucoin.com/#get-account-ledgers
|
||||
|
||||
'''
|
||||
|
||||
id: str
|
||||
currency: str
|
||||
amount: float
|
||||
fee: float
|
||||
balance: float
|
||||
accountType: str
|
||||
bizType: str
|
||||
direction: Literal['in', 'out']
|
||||
createdAt: float
|
||||
context: list[str]
|
||||
|
||||
|
||||
class AccountResponse(Struct, frozen=True):
|
||||
'''
|
||||
https://docs.kucoin.com/#get-account-ledgers
|
||||
|
||||
'''
|
||||
|
||||
currentPage: int
|
||||
pageSize: int
|
||||
totalNum: int
|
||||
totalPage: int
|
||||
items: list[AccountTrade]
|
||||
|
||||
|
||||
class KucoinTrade(Struct, frozen=True):
|
||||
'''
|
||||
Real-time trade format:
|
||||
https://docs.kucoin.com/#symbol-ticker
|
||||
|
||||
'''
|
||||
|
||||
bestAsk: float
|
||||
bestAskSize: float
|
||||
bestBid: float
|
||||
bestBidSize: float
|
||||
price: float
|
||||
sequence: float
|
||||
size: float
|
||||
time: float
|
||||
|
||||
|
||||
class KucoinL2(Struct, frozen=True):
|
||||
'''
|
||||
Real-time L2 order book format:
|
||||
https://docs.kucoin.com/#level2-5-best-ask-bid-orders
|
||||
|
||||
'''
|
||||
|
||||
asks: list[list[float]]
|
||||
bids: list[list[float]]
|
||||
timestamp: float
|
||||
|
||||
|
||||
class KucoinMsg(Struct, frozen=True):
|
||||
'''
|
||||
Generic outer-wrapper for any Kucoin ws msg
|
||||
|
||||
'''
|
||||
|
||||
type: str
|
||||
topic: str
|
||||
subject: str
|
||||
data: list[KucoinTrade | KucoinL2]
|
||||
|
||||
|
||||
class BrokerConfig(Struct, frozen=True):
|
||||
key_id: str
|
||||
key_secret: str
|
||||
key_passphrase: str
|
||||
|
||||
|
||||
def get_config() -> BrokerConfig | None:
|
||||
conf, _ = config.load()
|
||||
|
||||
section = conf.get('kucoin')
|
||||
|
||||
if section is None:
|
||||
log.warning('No config section found for kucoin in config')
|
||||
return None
|
||||
|
||||
return BrokerConfig(**section).copy()
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self) -> None:
|
||||
self._pairs: dict[str, KucoinMktPair] = {}
|
||||
self._bars: list[list[float]] = []
|
||||
self._config: BrokerConfig | None = get_config()
|
||||
|
||||
def _gen_auth_req_headers(
|
||||
self,
|
||||
action: Literal['POST', 'GET'],
|
||||
endpoint: str,
|
||||
api_v: str = 'v2',
|
||||
) -> dict[str, str | bytes]:
|
||||
'''
|
||||
Generate authenticated request headers
|
||||
https://docs.kucoin.com/#authentication
|
||||
|
||||
'''
|
||||
|
||||
if not self._config:
|
||||
raise ValueError(
|
||||
'No config found when trying to send authenticated request')
|
||||
|
||||
str_to_sign = (
|
||||
str(int(time.time() * 1000))
|
||||
+ action + f'/api/{api_v}{endpoint}'
|
||||
)
|
||||
|
||||
signature = base64.b64encode(
|
||||
hmac.new(
|
||||
self._config.key_secret.encode('utf-8'),
|
||||
str_to_sign.encode('utf-8'),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
)
|
||||
|
||||
passphrase = base64.b64encode(
|
||||
hmac.new(
|
||||
self._config.key_secret.encode('utf-8'),
|
||||
self._config.key_passphrase.encode('utf-8'),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
)
|
||||
|
||||
return {
|
||||
'KC-API-SIGN': signature,
|
||||
'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000),
|
||||
'KC-API-KEY': self._config.key_id,
|
||||
'KC-API-PASSPHRASE': passphrase,
|
||||
# XXX: Even if using the v1 api - this stays the same
|
||||
'KC-API-KEY-VERSION': '2',
|
||||
}
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
action: Literal['POST', 'GET'],
|
||||
endpoint: str,
|
||||
api_v: str = 'v2',
|
||||
headers: dict = {},
|
||||
) -> Any:
|
||||
'''
|
||||
Generic request wrapper for Kucoin API
|
||||
|
||||
'''
|
||||
if self._config:
|
||||
headers = self._gen_auth_req_headers(
|
||||
action, endpoint, api_v)
|
||||
|
||||
api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}'
|
||||
|
||||
res = await asks.request(action, api_url, headers=headers)
|
||||
|
||||
if 'data' in res.json():
|
||||
return res.json()['data']
|
||||
else:
|
||||
log.error(
|
||||
f'Error making request to {api_url} -> {res.json()["msg"]}'
|
||||
)
|
||||
return res.json()['msg']
|
||||
|
||||
async def _get_ws_token(
|
||||
self,
|
||||
private: bool = False,
|
||||
) -> tuple[str, int] | None:
|
||||
'''
|
||||
Fetch ws token needed for sub access:
|
||||
https://docs.kucoin.com/#apply-connect-token
|
||||
returns a token and the interval we must ping
|
||||
the server at to keep the connection alive
|
||||
|
||||
'''
|
||||
token_type = 'private' if private else 'public'
|
||||
try:
|
||||
data: dict[str, Any] | None = await self._request(
|
||||
'POST', f'/bullet-{token_type}', 'v1'
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(
|
||||
f'Error making request for Kucoin ws token -> {str(e)}')
|
||||
return None
|
||||
|
||||
if data and 'token' in data:
|
||||
# ping_interval is in ms
|
||||
ping_interval: int = data['instanceServers'][0]['pingInterval']
|
||||
return data['token'], ping_interval
|
||||
elif data:
|
||||
log.error(
|
||||
'Error making request for Kucoin ws token'
|
||||
f'{data.json()["msg"]}'
|
||||
)
|
||||
|
||||
async def _get_pairs(
|
||||
self,
|
||||
) -> dict[str, KucoinMktPair]:
|
||||
entries = await self._request('GET', '/symbols')
|
||||
syms = {
|
||||
kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item)
|
||||
for item in entries
|
||||
}
|
||||
|
||||
log.info(f' {len(syms)} Kucoin market pairs fetched')
|
||||
return syms
|
||||
|
||||
async def cache_pairs(
|
||||
self,
|
||||
) -> dict[str, KucoinMktPair]:
|
||||
'''
|
||||
Get cached pairs and convert keyed symbols into fqsns if ya want
|
||||
|
||||
'''
|
||||
if not self._pairs:
|
||||
self._pairs = await self._get_pairs()
|
||||
|
||||
return self._pairs
|
||||
|
||||
async def search_symbols(
|
||||
self,
|
||||
pattern: str,
|
||||
limit: int = 30,
|
||||
) -> dict[str, KucoinMktPair]:
|
||||
data = await self.cache_pairs()
|
||||
|
||||
matches = fuzzy.extractBests(
|
||||
pattern, data, score_cutoff=35, limit=limit
|
||||
)
|
||||
# repack in dict form
|
||||
return {item[0].name: item[0] for item in matches}
|
||||
|
||||
async def last_trades(self, sym: str) -> list[AccountTrade]:
|
||||
trades = await self._request(
|
||||
'GET', f'/accounts/ledgers?currency={sym}', 'v1'
|
||||
)
|
||||
trades = AccountResponse(**trades)
|
||||
return trades.items
|
||||
|
||||
async def _get_bars(
|
||||
self,
|
||||
fqsn: str,
|
||||
start_dt: datetime | None = None,
|
||||
end_dt: datetime | None = None,
|
||||
limit: int = 1000,
|
||||
as_np: bool = True,
|
||||
type: str = '1min',
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Get OHLC data and convert to numpy array for perffff:
|
||||
https://docs.kucoin.com/#get-klines
|
||||
|
||||
Kucoin bar data format:
|
||||
[
|
||||
'1545904980', //Start time of the candle cycle 0
|
||||
'0.058', //opening price 1
|
||||
'0.049', //closing price 2
|
||||
'0.058', //highest price 3
|
||||
'0.049', //lowest price 4
|
||||
'0.018', //Transaction volume 5
|
||||
'0.000945' //Transaction amount 6
|
||||
],
|
||||
|
||||
piker ohlc numpy array format:
|
||||
[
|
||||
('index', int),
|
||||
('time', int),
|
||||
('open', float),
|
||||
('high', float),
|
||||
('low', float),
|
||||
('close', float),
|
||||
('volume', float),
|
||||
('bar_wap', float), # will be zeroed by sampler if not filled
|
||||
]
|
||||
|
||||
'''
|
||||
# Generate generic end and start time if values not passed
|
||||
# Currently gives us 12hrs of data
|
||||
if end_dt is None:
|
||||
end_dt = pendulum.now('UTC').add(minutes=1)
|
||||
|
||||
if start_dt is None:
|
||||
start_dt = end_dt.start_of(
|
||||
'minute').subtract(minutes=limit)
|
||||
|
||||
start_dt = int(start_dt.timestamp())
|
||||
end_dt = int(end_dt.timestamp())
|
||||
|
||||
kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs)
|
||||
|
||||
url = (
|
||||
f'/market/candles?type={type}'
|
||||
f'&symbol={kucoin_sym}'
|
||||
f'&startAt={start_dt}'
|
||||
f'&endAt={end_dt}'
|
||||
)
|
||||
|
||||
for i in range(10):
|
||||
data: list[list[str]] | dict = await self._request(
|
||||
'GET',
|
||||
url,
|
||||
api_v='v1',
|
||||
)
|
||||
|
||||
if not isinstance(data, list):
|
||||
# Do a gradual backoff if Kucoin is rate limiting us
|
||||
backoff_interval = i
|
||||
log.warn(
|
||||
f'History call failed, backing off for {backoff_interval}s'
|
||||
)
|
||||
await trio.sleep(backoff_interval)
|
||||
else:
|
||||
bars: list[list[str]] = data
|
||||
break
|
||||
|
||||
new_bars = []
|
||||
reversed_bars = bars[::-1]
|
||||
|
||||
# Convert from kucoin format to piker format
|
||||
for i, bar in enumerate(reversed_bars):
|
||||
new_bars.append(
|
||||
(
|
||||
# index
|
||||
i,
|
||||
# time
|
||||
int(bar[0]),
|
||||
# open
|
||||
float(bar[1]),
|
||||
# high
|
||||
float(bar[3]),
|
||||
# low
|
||||
float(bar[4]),
|
||||
# close
|
||||
float(bar[2]),
|
||||
# volume
|
||||
float(bar[5]),
|
||||
# bar_wap
|
||||
0.0,
|
||||
)
|
||||
)
|
||||
|
||||
array = np.array(
|
||||
new_bars, dtype=_ohlc_dtype) if as_np else bars
|
||||
return array
|
||||
|
||||
|
||||
def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str:
|
||||
pair_data = pairs[fqsn]
|
||||
return pair_data.baseCurrency + '-' + pair_data.quoteCurrency
|
||||
|
||||
|
||||
def kucoin_sym_to_fqsn(sym: str) -> str:
|
||||
return sym.lower().replace('-', '')
|
||||
|
||||
|
||||
@acm
|
||||
async def get_client() -> AsyncGenerator[Client, None]:
|
||||
client = Client()
|
||||
await client.cache_pairs()
|
||||
|
||||
yield client
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def open_symbol_search(
|
||||
ctx: tractor.Context,
|
||||
) -> None:
|
||||
async with open_cached_client('kucoin') as client:
|
||||
# load all symbols locally for fast search
|
||||
await client.cache_pairs()
|
||||
await ctx.started()
|
||||
|
||||
async with ctx.open_stream() as stream:
|
||||
async for pattern in stream:
|
||||
await stream.send(await client.search_symbols(pattern))
|
||||
log.info('Kucoin symbol search opened')
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ping_task(
|
||||
ws: wsproto.WSConnection,
|
||||
ping_interval, connect_id
|
||||
) -> AsyncGenerator[None, None]:
|
||||
'''
|
||||
Spawn a non-blocking task that pings the ws
|
||||
server every ping_interval so Kucoin doesn't drop
|
||||
our connection
|
||||
|
||||
'''
|
||||
async with trio.open_nursery() as n:
|
||||
# TODO: cache this task so it's only called once
|
||||
async def ping_server():
|
||||
while True:
|
||||
await trio.sleep((ping_interval - 1000) / 1000)
|
||||
await ws.send_msg({'id': connect_id, 'type': 'ping'})
|
||||
|
||||
log.info('Starting ping task for kucoin ws connection')
|
||||
n.start_soon(ping_server)
|
||||
|
||||
yield
|
||||
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
async def stream_quotes(
|
||||
send_chan: trio.abc.SendChannel,
|
||||
symbols: list[str],
|
||||
feed_is_live: trio.Event,
|
||||
loglevel: str = '',
|
||||
# startup sync
|
||||
task_status: TaskStatus[tuple[dict, dict]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
) -> None:
|
||||
'''
|
||||
Required piker api to stream real-time data.
|
||||
Where the rubber hits the road baby
|
||||
|
||||
'''
|
||||
async with open_cached_client('kucoin') as client:
|
||||
token, ping_interval = await client._get_ws_token()
|
||||
connect_id = str(uuid4())
|
||||
pairs = await client.cache_pairs()
|
||||
ws_url = (
|
||||
f'wss://ws-api-spot.kucoin.com/?'
|
||||
f'token={token}&[connectId={connect_id}]'
|
||||
)
|
||||
|
||||
# open ping task
|
||||
async with (
|
||||
open_autorecon_ws(ws_url) as ws,
|
||||
open_ping_task(ws, ping_interval, connect_id),
|
||||
):
|
||||
log.info('Starting up quote stream')
|
||||
# loop through symbols and sub to feedz
|
||||
for sym in symbols:
|
||||
pair: KucoinMktPair = pairs[sym]
|
||||
kucoin_sym = pair.symbol
|
||||
|
||||
init_msgs = {
|
||||
# pass back token, and bool, signalling if we're the writer
|
||||
# and that history has been written
|
||||
sym: {
|
||||
'symbol_info': {
|
||||
'asset_type': 'crypto',
|
||||
'price_tick_size': float(pair.baseIncrement),
|
||||
'lot_tick_size': float(pair.baseMinSize),
|
||||
},
|
||||
'shm_write_opts': {'sum_tick_vml': False},
|
||||
'fqsn': sym,
|
||||
}
|
||||
}
|
||||
|
||||
async with (
|
||||
subscribe(ws, connect_id, kucoin_sym),
|
||||
stream_messages(ws, sym) as msg_gen,
|
||||
):
|
||||
typ, quote = await anext(msg_gen)
|
||||
while typ != 'trade':
|
||||
# take care to not unblock here until we get a real
|
||||
# trade quote
|
||||
typ, quote = await anext(msg_gen)
|
||||
|
||||
task_status.started((init_msgs, quote))
|
||||
feed_is_live.set()
|
||||
|
||||
async for typ, msg in msg_gen:
|
||||
await send_chan.send({sym: msg})
|
||||
|
||||
|
||||
@acm
|
||||
async def subscribe(ws: wsproto.WSConnection, connect_id, sym) -> AsyncGenerator[None, None]:
|
||||
# level 2 sub
|
||||
await ws.send_msg(
|
||||
{
|
||||
'id': connect_id,
|
||||
'type': 'subscribe',
|
||||
'topic': f'/spotMarket/level2Depth5:{sym}',
|
||||
'privateChannel': False,
|
||||
'response': True,
|
||||
}
|
||||
)
|
||||
|
||||
# watch trades
|
||||
await ws.send_msg(
|
||||
{
|
||||
'id': connect_id,
|
||||
'type': 'subscribe',
|
||||
'topic': f'/market/ticker:{sym}',
|
||||
'privateChannel': False,
|
||||
'response': True,
|
||||
}
|
||||
)
|
||||
|
||||
yield
|
||||
|
||||
# unsub
|
||||
if ws.connected():
|
||||
log.info(f'Unsubscribing to {sym} feed')
|
||||
await ws.send_msg(
|
||||
{
|
||||
'id': connect_id,
|
||||
'type': 'unsubscribe',
|
||||
'topic': f'/market/ticker:{sym}',
|
||||
'privateChannel': False,
|
||||
'response': True,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@trio_async_generator
|
||||
async def stream_messages(
|
||||
ws: NoBsWs, sym: str
|
||||
) -> AsyncGenerator[tuple[str, dict], None]:
|
||||
timeouts = 0
|
||||
last_trade_ts = 0
|
||||
|
||||
while True:
|
||||
with trio.move_on_after(3) as cs:
|
||||
msg = await ws.recv_msg()
|
||||
if cs.cancelled_caught:
|
||||
timeouts += 1
|
||||
if timeouts > 2:
|
||||
log.error(
|
||||
'kucoin feed is sh**ing the bed... rebooting...')
|
||||
await ws._connect()
|
||||
|
||||
continue
|
||||
if msg.get('subject'):
|
||||
msg = KucoinMsg(**msg)
|
||||
match msg.subject:
|
||||
case 'trade.ticker':
|
||||
trade_data = KucoinTrade(**msg.data)
|
||||
|
||||
# XXX: Filter for duplicate messages as ws feed will
|
||||
# send duplicate market state
|
||||
# https://docs.kucoin.com/#level2-5-best-ask-bid-orders
|
||||
if trade_data.time == last_trade_ts:
|
||||
continue
|
||||
|
||||
last_trade_ts = trade_data.time
|
||||
|
||||
yield 'trade', {
|
||||
'symbol': sym,
|
||||
'last': trade_data.price,
|
||||
'brokerd_ts': last_trade_ts,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'trade',
|
||||
'price': float(trade_data.price),
|
||||
'size': float(trade_data.size),
|
||||
'broker_ts': last_trade_ts,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
case 'level2':
|
||||
l2_data = KucoinL2(**msg.data)
|
||||
first_ask = l2_data.asks[0]
|
||||
first_bid = l2_data.bids[0]
|
||||
yield 'l1', {
|
||||
'symbol': sym,
|
||||
'ticks': [
|
||||
{
|
||||
'type': 'bid',
|
||||
'price': float(first_bid[0]),
|
||||
'size': float(first_bid[1]),
|
||||
},
|
||||
{
|
||||
'type': 'bsize',
|
||||
'price': float(first_bid[0]),
|
||||
'size': float(first_bid[1]),
|
||||
},
|
||||
{
|
||||
'type': 'ask',
|
||||
'price': float(first_ask[0]),
|
||||
'size': float(first_ask[1]),
|
||||
},
|
||||
{
|
||||
'type': 'asize',
|
||||
'price': float(first_ask[0]),
|
||||
'size': float(first_ask[1]),
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
case _:
|
||||
log.warn(f'Unhandled message: {msg}')
|
||||
|
||||
|
||||
@acm
|
||||
async def open_history_client(
|
||||
symbol: str,
|
||||
) -> AsyncGenerator[Callable, None]:
|
||||
async with open_cached_client('kucoin') as client:
|
||||
log.info('Attempting to open kucoin history client')
|
||||
|
||||
async def get_ohlc_history(
|
||||
timeframe: float,
|
||||
end_dt: datetime | None = None,
|
||||
start_dt: datetime | None = None,
|
||||
) -> tuple[
|
||||
np.ndarray, datetime
|
||||
| None, datetime
|
||||
| None
|
||||
]: # start # end
|
||||
if timeframe != 60:
|
||||
raise DataUnavailable('Only 1m bars are supported')
|
||||
|
||||
array = await client._get_bars(
|
||||
symbol,
|
||||
start_dt=start_dt,
|
||||
end_dt=end_dt,
|
||||
)
|
||||
|
||||
times = array['time']
|
||||
|
||||
if end_dt is None:
|
||||
inow = round(time.time())
|
||||
|
||||
print(
|
||||
f'difference in time between load and processing'
|
||||
f'{inow - times[-1]}'
|
||||
)
|
||||
|
||||
start_dt = pendulum.from_timestamp(times[0])
|
||||
end_dt = pendulum.from_timestamp(times[-1])
|
||||
|
||||
log.info('History succesfully fetched baby')
|
||||
|
||||
return array, start_dt, end_dt
|
||||
|
||||
yield get_ohlc_history, {}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of pikers)
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -43,13 +43,10 @@ from ..calc import humanize, percent_change
|
|||
from .._cacheables import open_cached_client, async_lifo_cache
|
||||
from .. import config
|
||||
from ._util import resproc, BrokerError, SymbolNotFound
|
||||
from ..log import (
|
||||
colorize_json,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, colorize_json, get_console_log
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_use_practice_account = False
|
||||
_refresh_token_ep = 'https://{}login.questrade.com/oauth2/'
|
||||
|
|
|
|||
|
|
@ -27,13 +27,12 @@ from typing import List
|
|||
from async_generator import asynccontextmanager
|
||||
import asks
|
||||
|
||||
from ._util import (
|
||||
resproc,
|
||||
BrokerError,
|
||||
log,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._util import resproc, BrokerError
|
||||
from ..calc import percent_change
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_service_ep = 'https://api.robinhood.com'
|
||||
|
||||
|
||||
|
|
@ -66,10 +65,8 @@ class Client:
|
|||
self.api = _API(self._sess)
|
||||
|
||||
def _zip_in_order(self, symbols: [str], quotes: List[dict]):
|
||||
return {
|
||||
quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, quotes)
|
||||
}
|
||||
return {quote.get('symbol', sym) if quote else sym: quote
|
||||
for sym, quote in zip(symbols, results_dict)}
|
||||
|
||||
async def quote(self, symbols: [str]):
|
||||
"""Retrieve quotes for a list of ``symbols``.
|
||||
|
|
|
|||
|
|
@ -18,17 +18,3 @@
|
|||
Market machinery for order executions, book, management.
|
||||
|
||||
"""
|
||||
from ..log import get_logger
|
||||
from ._client import (
|
||||
open_ems,
|
||||
OrderClient,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'open_ems',
|
||||
'OrderClient',
|
||||
|
||||
]
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
|
|||
|
|
@ -22,10 +22,54 @@ from enum import Enum
|
|||
from typing import Optional
|
||||
|
||||
from bidict import bidict
|
||||
from pydantic import BaseModel, validator
|
||||
|
||||
from ._pos import Position
|
||||
from ._mktinfo import Symbol
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
from ._messages import BrokerdPosition, Status
|
||||
|
||||
|
||||
class Position(BaseModel):
|
||||
'''
|
||||
Basic pp (personal position) model with attached fills history.
|
||||
|
||||
This type should be IPC wire ready?
|
||||
|
||||
'''
|
||||
symbol: Symbol
|
||||
|
||||
# last size and avg entry price
|
||||
size: float
|
||||
avg_price: float # TODO: contextual pricing
|
||||
|
||||
# ordered record of known constituent trade messages
|
||||
fills: list[Status] = []
|
||||
|
||||
def update_from_msg(
|
||||
self,
|
||||
msg: BrokerdPosition,
|
||||
|
||||
) -> None:
|
||||
|
||||
# XXX: better place to do this?
|
||||
symbol = self.symbol
|
||||
|
||||
lot_size_digits = symbol.lot_size_digits
|
||||
avg_price, size = (
|
||||
round(msg['avg_price'], ndigits=symbol.tick_size_digits),
|
||||
round(msg['size'], ndigits=lot_size_digits),
|
||||
)
|
||||
|
||||
self.avg_price = avg_price
|
||||
self.size = size
|
||||
|
||||
@property
|
||||
def dsize(self) -> float:
|
||||
'''
|
||||
The "dollar" size of the pp, normally in trading (fiat) unit
|
||||
terms.
|
||||
|
||||
'''
|
||||
return self.avg_price * self.size
|
||||
|
||||
|
||||
_size_units = bidict({
|
||||
|
|
@ -40,9 +84,34 @@ SizeUnit = Enum(
|
|||
)
|
||||
|
||||
|
||||
class Allocator(Struct):
|
||||
class Allocator(BaseModel):
|
||||
|
||||
class Config:
|
||||
validate_assignment = True
|
||||
copy_on_model_validation = False
|
||||
arbitrary_types_allowed = True
|
||||
|
||||
# required to get the account validator lookup working?
|
||||
extra = 'allow'
|
||||
underscore_attrs_are_private = False
|
||||
|
||||
symbol: Symbol
|
||||
account: Optional[str] = 'paper'
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
size_unit: str = 'currency'
|
||||
_size_units: dict[str, Optional[str]] = _size_units
|
||||
|
||||
@validator('size_unit', pre=True)
|
||||
def maybe_lookup_key(cls, v):
|
||||
# apply the corresponding enum key for the text "description" value
|
||||
if v not in _size_units:
|
||||
return _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
return v
|
||||
|
||||
# TODO: if we ever want ot support non-uniform entry-slot-proportion
|
||||
# "sizes"
|
||||
|
|
@ -51,28 +120,6 @@ class Allocator(Struct):
|
|||
units_limit: float
|
||||
currency_limit: float
|
||||
slots: int
|
||||
account: Optional[str] = 'paper'
|
||||
|
||||
_size_units: bidict[str, Optional[str]] = _size_units
|
||||
|
||||
# TODO: for enums this clearly doesn't fucking work, you can't set
|
||||
# a default at startup by passing in a `dict` but yet you can set
|
||||
# that value through assignment..for wtv cucked reason.. honestly, pure
|
||||
# unintuitive garbage.
|
||||
_size_unit: str = 'currency'
|
||||
|
||||
@property
|
||||
def size_unit(self) -> str:
|
||||
return self._size_unit
|
||||
|
||||
@size_unit.setter
|
||||
def size_unit(self, v: str) -> Optional[str]:
|
||||
if v not in _size_units:
|
||||
v = _size_units.inverse[v]
|
||||
|
||||
assert v in _size_units
|
||||
self._size_unit = v
|
||||
return v
|
||||
|
||||
def step_sizes(
|
||||
self,
|
||||
|
|
@ -93,13 +140,10 @@ class Allocator(Struct):
|
|||
else:
|
||||
return self.units_limit
|
||||
|
||||
def limit_info(self) -> tuple[str, float]:
|
||||
return self.size_unit, self.limit()
|
||||
|
||||
def next_order_info(
|
||||
self,
|
||||
|
||||
# we only need a startup size for exit calcs, we can then
|
||||
# we only need a startup size for exit calcs, we can the
|
||||
# determine how large slots should be if the initial pp size was
|
||||
# larger then the current live one, and the live one is smaller
|
||||
# then the initial config settings.
|
||||
|
|
@ -129,7 +173,7 @@ class Allocator(Struct):
|
|||
l_sub_pp = self.units_limit - abs_live_size
|
||||
|
||||
elif size_unit == 'currency':
|
||||
live_cost_basis = abs_live_size * live_pp.ppu
|
||||
live_cost_basis = abs_live_size * live_pp.avg_price
|
||||
slot_size = currency_per_slot / price
|
||||
l_sub_pp = (self.currency_limit - live_cost_basis) / price
|
||||
|
||||
|
|
@ -140,14 +184,12 @@ class Allocator(Struct):
|
|||
|
||||
# an entry (adding-to or starting a pp)
|
||||
if (
|
||||
action == 'buy' and live_size > 0 or
|
||||
action == 'sell' and live_size < 0 or
|
||||
live_size == 0
|
||||
or (action == 'buy' and live_size > 0)
|
||||
or action == 'sell' and live_size < 0
|
||||
):
|
||||
order_size = min(
|
||||
slot_size,
|
||||
max(l_sub_pp, 0),
|
||||
)
|
||||
|
||||
order_size = min(slot_size, l_sub_pp)
|
||||
|
||||
# an exit (removing-from or going to net-zero pp)
|
||||
else:
|
||||
|
|
@ -163,7 +205,7 @@ class Allocator(Struct):
|
|||
if size_unit == 'currency':
|
||||
# compute the "projected" limit's worth of units at the
|
||||
# current pp (weighted) price:
|
||||
slot_size = currency_per_slot / live_pp.ppu
|
||||
slot_size = currency_per_slot / live_pp.avg_price
|
||||
|
||||
else:
|
||||
slot_size = u_per_slot
|
||||
|
|
@ -202,12 +244,7 @@ class Allocator(Struct):
|
|||
if order_size < slot_size:
|
||||
# compute a fractional slots size to display
|
||||
slots_used = self.slots_used(
|
||||
Position(
|
||||
symbol=sym,
|
||||
size=order_size,
|
||||
ppu=price,
|
||||
bs_mktid=sym,
|
||||
)
|
||||
Position(symbol=sym, size=order_size, avg_price=price)
|
||||
)
|
||||
|
||||
return {
|
||||
|
|
@ -234,8 +271,8 @@ class Allocator(Struct):
|
|||
abs_pp_size = abs(pp.size)
|
||||
|
||||
if self.size_unit == 'currency':
|
||||
# live_currency_size = size or (abs_pp_size * pp.ppu)
|
||||
live_currency_size = abs_pp_size * pp.ppu
|
||||
# live_currency_size = size or (abs_pp_size * pp.avg_price)
|
||||
live_currency_size = abs_pp_size * pp.avg_price
|
||||
prop = live_currency_size / self.currency_limit
|
||||
|
||||
else:
|
||||
|
|
@ -247,6 +284,14 @@ class Allocator(Struct):
|
|||
return round(prop * self.slots)
|
||||
|
||||
|
||||
_derivs = (
|
||||
'future',
|
||||
'continuous_future',
|
||||
'option',
|
||||
'futures_option',
|
||||
)
|
||||
|
||||
|
||||
def mk_allocator(
|
||||
|
||||
symbol: Symbol,
|
||||
|
|
@ -255,7 +300,7 @@ def mk_allocator(
|
|||
# default allocation settings
|
||||
defaults: dict[str, float] = {
|
||||
'account': None, # select paper by default
|
||||
# 'size_unit': 'currency',
|
||||
'size_unit': 'currency',
|
||||
'units_limit': 400,
|
||||
'currency_limit': 5e3,
|
||||
'slots': 4,
|
||||
|
|
@ -273,9 +318,42 @@ def mk_allocator(
|
|||
'currency_limit': 6e3,
|
||||
'slots': 6,
|
||||
}
|
||||
|
||||
defaults.update(user_def)
|
||||
|
||||
return Allocator(
|
||||
alloc = Allocator(
|
||||
symbol=symbol,
|
||||
**defaults,
|
||||
)
|
||||
|
||||
asset_type = symbol.type_key
|
||||
|
||||
# specific configs by asset class / type
|
||||
|
||||
if asset_type in _derivs:
|
||||
# since it's harder to know how currency "applies" in this case
|
||||
# given leverage properties
|
||||
alloc.size_unit = '# units'
|
||||
|
||||
# set units limit to slots size thus making make the next
|
||||
# entry step 1.0
|
||||
alloc.units_limit = alloc.slots
|
||||
|
||||
# if the current position is already greater then the limit
|
||||
# settings, increase the limit to the current position
|
||||
if alloc.size_unit == 'currency':
|
||||
startup_size = startup_pp.size * startup_pp.avg_price
|
||||
|
||||
if startup_size > alloc.currency_limit:
|
||||
alloc.currency_limit = round(startup_size, ndigits=2)
|
||||
|
||||
else:
|
||||
startup_size = abs(startup_pp.size)
|
||||
|
||||
if startup_size > alloc.units_limit:
|
||||
alloc.units_limit = startup_size
|
||||
|
||||
if asset_type in _derivs:
|
||||
alloc.slots = alloc.units_limit
|
||||
|
||||
return alloc
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,294 +18,211 @@
|
|||
Orders and execution client API.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from typing import Dict
|
||||
from pprint import pformat
|
||||
from typing import TYPE_CHECKING
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import trio
|
||||
import tractor
|
||||
from tractor.trionics import broadcast_receiver
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..accounting._mktinfo import unpack_fqme
|
||||
from ..data.types import Struct
|
||||
from ..service import maybe_open_emsd
|
||||
from ._messages import (
|
||||
Order,
|
||||
Cancel,
|
||||
BrokerdPosition,
|
||||
)
|
||||
from ..brokers import get_brokermod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._messages import (
|
||||
Status,
|
||||
)
|
||||
from ..log import get_logger
|
||||
from ._ems import _emsd_main
|
||||
from .._daemon import maybe_open_emsd
|
||||
from ._messages import Order, Cancel
|
||||
|
||||
|
||||
class OrderClient(Struct):
|
||||
'''
|
||||
EMS-client-side order book ctl and tracking.
|
||||
log = get_logger(__name__)
|
||||
|
||||
(A)sync API for submitting orders and alerts to the `emsd` service;
|
||||
this is the main control for execution management from client code.
|
||||
|
||||
@dataclass
|
||||
class OrderBook:
|
||||
'''EMS-client-side order book ctl and tracking.
|
||||
|
||||
A style similar to "model-view" is used here where this api is
|
||||
provided as a supervised control for an EMS actor which does all the
|
||||
hard/fast work of talking to brokers/exchanges to conduct
|
||||
executions.
|
||||
|
||||
Currently, this is mostly for keeping local state to match the EMS
|
||||
and use received events to trigger graphics updates.
|
||||
|
||||
'''
|
||||
# IPC stream to `emsd` actor
|
||||
_ems_stream: tractor.MsgStream
|
||||
|
||||
# mem channels used to relay order requests to the EMS daemon
|
||||
_to_relay_task: trio.abc.SendChannel
|
||||
_from_sync_order_client: trio.abc.ReceiveChannel
|
||||
_to_ems: trio.abc.SendChannel
|
||||
_from_order_book: trio.abc.ReceiveChannel
|
||||
|
||||
# history table
|
||||
_sent_orders: dict[str, Order] = {}
|
||||
_sent_orders: Dict[str, Order] = field(default_factory=dict)
|
||||
_ready_to_receive: trio.Event = trio.Event()
|
||||
|
||||
def send_nowait(
|
||||
def send(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
msg: Order,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Sync version of ``.send()``.
|
||||
|
||||
'''
|
||||
) -> dict:
|
||||
self._sent_orders[msg.oid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
return msg
|
||||
|
||||
async def send(
|
||||
def update(
|
||||
self,
|
||||
msg: Order | dict,
|
||||
|
||||
) -> dict | Order:
|
||||
'''
|
||||
Send a new order msg async to the `emsd` service.
|
||||
|
||||
'''
|
||||
self._sent_orders[msg.oid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def update_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Sync version of ``.update()``.
|
||||
|
||||
'''
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
self._to_relay_task.send_nowait(msg)
|
||||
return msg
|
||||
msg = cmd.dict()
|
||||
msg.update(data)
|
||||
self._sent_orders[uuid] = Order(**msg)
|
||||
self._to_ems.send_nowait(msg)
|
||||
return cmd
|
||||
|
||||
async def update(
|
||||
self,
|
||||
uuid: str,
|
||||
**data: dict,
|
||||
) -> dict:
|
||||
'''
|
||||
Update an existing order dialog with a msg updated from
|
||||
``update`` kwargs.
|
||||
def cancel(self, uuid: str) -> bool:
|
||||
"""Cancel an order (or alert) in the EMS.
|
||||
|
||||
'''
|
||||
"""
|
||||
cmd = self._sent_orders[uuid]
|
||||
msg = cmd.copy(update=data)
|
||||
self._sent_orders[uuid] = msg
|
||||
await self._ems_stream.send(msg)
|
||||
return msg
|
||||
|
||||
def _mk_cancel_msg(
|
||||
self,
|
||||
uuid: str,
|
||||
) -> Cancel:
|
||||
cmd = self._sent_orders.get(uuid)
|
||||
if not cmd:
|
||||
log.error(
|
||||
f'Unknown order {uuid}!?\n'
|
||||
f'Maybe there is a stale entry or line?\n'
|
||||
f'You should report this as a bug!'
|
||||
)
|
||||
fqme = str(cmd.symbol)
|
||||
return Cancel(
|
||||
msg = Cancel(
|
||||
oid=uuid,
|
||||
symbol=fqme,
|
||||
)
|
||||
|
||||
def cancel_nowait(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Sync version of ``.cancel()``.
|
||||
|
||||
'''
|
||||
self._to_relay_task.send_nowait(
|
||||
self._mk_cancel_msg(uuid)
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
uuid: str,
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Cancel an already existintg order (or alert) dialog.
|
||||
|
||||
'''
|
||||
await self._ems_stream.send(
|
||||
self._mk_cancel_msg(uuid)
|
||||
symbol=cmd.symbol,
|
||||
)
|
||||
self._to_ems.send_nowait(msg.dict())
|
||||
|
||||
|
||||
_client: OrderClient = None
|
||||
_orders: OrderBook = None
|
||||
|
||||
|
||||
async def relay_orders_from_sync_code(
|
||||
def get_orders(
|
||||
emsd_uid: tuple[str, str] = None
|
||||
) -> OrderBook:
|
||||
""""
|
||||
OrderBook singleton factory per actor.
|
||||
|
||||
client: OrderClient,
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
"""
|
||||
if emsd_uid is not None:
|
||||
# TODO: read in target emsd's active book on startup
|
||||
pass
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Order submission relay task: deliver orders sent from synchronous (UI)
|
||||
code to the EMS via ``OrderClient._from_sync_order_client``.
|
||||
global _orders
|
||||
|
||||
This is run in the UI actor (usually the one running Qt but could be
|
||||
any other client service code). This process simply delivers order
|
||||
messages to the above ``_to_relay_task`` send channel (from sync code using
|
||||
``.send_nowait()``), these values are pulled from the channel here
|
||||
and relayed to any consumer(s) that called this function using
|
||||
a ``tractor`` portal.
|
||||
|
||||
This effectively makes order messages look like they're being
|
||||
"pushed" from the parent to the EMS where local sync code is likely
|
||||
doing the pushing from some non-async UI handler.
|
||||
|
||||
'''
|
||||
async with (
|
||||
client._from_sync_order_client.subscribe() as sync_order_cmds
|
||||
):
|
||||
async for cmd in sync_order_cmds:
|
||||
sym = cmd.symbol
|
||||
msg = pformat(cmd.to_dict())
|
||||
|
||||
if sym == symbol_key:
|
||||
log.info(f'Send order cmd:\n{msg}')
|
||||
# send msg over IPC / wire
|
||||
await to_ems_stream.send(cmd)
|
||||
|
||||
else:
|
||||
log.warning(
|
||||
f'Ignoring unmatched order cmd for {sym} != {symbol_key}:'
|
||||
f'\n{msg}'
|
||||
)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ems(
|
||||
fqme: str,
|
||||
mode: str = 'live',
|
||||
loglevel: str = 'error',
|
||||
|
||||
) -> tuple[
|
||||
OrderClient,
|
||||
tractor.MsgStream,
|
||||
dict[
|
||||
# brokername, acctid
|
||||
tuple[str, str],
|
||||
dict[str, BrokerdPosition],
|
||||
],
|
||||
list[str],
|
||||
dict[str, Status],
|
||||
]:
|
||||
'''
|
||||
(Maybe) spawn an EMS-daemon (emsd), deliver an `OrderClient` for
|
||||
requesting orders/alerts and a `trades_stream` which delivers all
|
||||
response-msgs.
|
||||
|
||||
This is a "client side" entrypoint which may spawn the `emsd` service
|
||||
if it can't be discovered and generally speaking is the lowest level
|
||||
broker control client-API.
|
||||
|
||||
'''
|
||||
broker, mktep, venue, suffix = unpack_fqme(fqme)
|
||||
|
||||
async with maybe_open_emsd(
|
||||
broker,
|
||||
loglevel=loglevel,
|
||||
) as portal:
|
||||
|
||||
mod = get_brokermod(broker)
|
||||
if (
|
||||
not getattr(mod, 'trades_dialogue', None)
|
||||
or mode == 'paper'
|
||||
):
|
||||
mode = 'paper'
|
||||
|
||||
from ._ems import _emsd_main
|
||||
async with (
|
||||
# connect to emsd
|
||||
portal.open_context(
|
||||
_emsd_main,
|
||||
fqme=fqme,
|
||||
exec_mode=mode,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (
|
||||
ctx,
|
||||
(
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
),
|
||||
|
||||
# open 2-way trade command stream
|
||||
ctx.open_stream() as trades_stream,
|
||||
):
|
||||
# use any pre-existing actor singleton client.
|
||||
global _client
|
||||
if _client is None:
|
||||
if _orders is None:
|
||||
size = 100
|
||||
tx, rx = trio.open_memory_channel(size)
|
||||
brx = broadcast_receiver(rx, size)
|
||||
|
||||
# setup local ui event streaming channels for request/resp
|
||||
# streamging with EMS daemon
|
||||
_client = OrderClient(
|
||||
_ems_stream=trades_stream,
|
||||
_to_relay_task=tx,
|
||||
_from_sync_order_client=brx,
|
||||
_orders = OrderBook(
|
||||
_to_ems=tx,
|
||||
_from_order_book=brx,
|
||||
)
|
||||
|
||||
_client._ems_stream = trades_stream
|
||||
return _orders
|
||||
|
||||
# start sync code order msg delivery task
|
||||
|
||||
# TODO: we can get rid of this relay loop once we move
|
||||
# order_mode inputs to async code!
|
||||
async def relay_order_cmds_from_sync_code(
|
||||
|
||||
symbol_key: str,
|
||||
to_ems_stream: tractor.MsgStream,
|
||||
|
||||
) -> None:
|
||||
"""
|
||||
Order streaming task: deliver orders transmitted from UI
|
||||
to downstream consumers.
|
||||
|
||||
This is run in the UI actor (usually the one running Qt but could be
|
||||
any other client service code). This process simply delivers order
|
||||
messages to the above ``_to_ems`` send channel (from sync code using
|
||||
``.send_nowait()``), these values are pulled from the channel here
|
||||
and relayed to any consumer(s) that called this function using
|
||||
a ``tractor`` portal.
|
||||
|
||||
This effectively makes order messages look like they're being
|
||||
"pushed" from the parent to the EMS where local sync code is likely
|
||||
doing the pushing from some UI.
|
||||
|
||||
"""
|
||||
book = get_orders()
|
||||
async with book._from_order_book.subscribe() as orders_stream:
|
||||
async for cmd in orders_stream:
|
||||
if cmd['symbol'] == symbol_key:
|
||||
log.info(f'Send order cmd:\n{pformat(cmd)}')
|
||||
# send msg over IPC / wire
|
||||
await to_ems_stream.send(cmd)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_ems(
|
||||
fqsn: str,
|
||||
|
||||
) -> (
|
||||
OrderBook,
|
||||
tractor.MsgStream,
|
||||
dict,
|
||||
):
|
||||
'''
|
||||
Spawn an EMS daemon and begin sending orders and receiving
|
||||
alerts.
|
||||
|
||||
This EMS tries to reduce most broker's terrible order entry apis to
|
||||
a very simple protocol built on a few easy to grok and/or
|
||||
"rantsy" premises:
|
||||
|
||||
- most users will prefer "dark mode" where orders are not submitted
|
||||
to a broker until and execution condition is triggered
|
||||
(aka client-side "hidden orders")
|
||||
|
||||
- Brokers over-complicate their apis and generally speaking hire
|
||||
poor designers to create them. We're better off using creating a super
|
||||
minimal, schema-simple, request-event-stream protocol to unify all the
|
||||
existing piles of shit (and shocker, it'll probably just end up
|
||||
looking like a decent crypto exchange's api)
|
||||
|
||||
- all order types can be implemented with client-side limit orders
|
||||
|
||||
- we aren't reinventing a wheel in this case since none of these
|
||||
brokers are exposing FIX protocol; it is they doing the re-invention.
|
||||
|
||||
|
||||
TODO: make some fancy diagrams using mermaid.io
|
||||
|
||||
the possible set of responses from the stream is currently:
|
||||
- 'dark_submitted', 'broker_submitted'
|
||||
- 'dark_cancelled', 'broker_cancelled'
|
||||
- 'dark_executed', 'broker_executed'
|
||||
- 'broker_filled'
|
||||
|
||||
'''
|
||||
# wait for service to connect back to us signalling
|
||||
# ready for order commands
|
||||
book = get_orders()
|
||||
|
||||
from ..data._source import unpack_fqsn
|
||||
broker, symbol, suffix = unpack_fqsn(fqsn)
|
||||
|
||||
async with maybe_open_emsd(broker) as portal:
|
||||
|
||||
async with (
|
||||
# connect to emsd
|
||||
portal.open_context(
|
||||
|
||||
_emsd_main,
|
||||
fqsn=fqsn,
|
||||
|
||||
) as (ctx, (positions, accounts)),
|
||||
|
||||
# open 2-way trade command stream
|
||||
ctx.open_stream() as trades_stream,
|
||||
):
|
||||
async with trio.open_nursery() as n:
|
||||
n.start_soon(
|
||||
relay_orders_from_sync_code,
|
||||
_client,
|
||||
fqme,
|
||||
relay_order_cmds_from_sync_code,
|
||||
fqsn,
|
||||
trades_stream
|
||||
)
|
||||
|
||||
yield (
|
||||
_client,
|
||||
trades_stream,
|
||||
positions,
|
||||
accounts,
|
||||
dialogs,
|
||||
)
|
||||
|
||||
# stop the sync-msg-relay task on exit.
|
||||
n.cancel_scope.cancel()
|
||||
yield book, trades_stream, positions, accounts
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -15,162 +15,108 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Clearing sub-system message and protocols.
|
||||
Clearing system messagingn types and protocols.
|
||||
|
||||
"""
|
||||
# from collections import (
|
||||
# ChainMap,
|
||||
# deque,
|
||||
# )
|
||||
from typing import (
|
||||
Optional,
|
||||
Literal,
|
||||
)
|
||||
from typing import Optional, Union
|
||||
|
||||
from msgspec import field
|
||||
# TODO: try out just encoding/send direction for now?
|
||||
# import msgspec
|
||||
from pydantic import BaseModel
|
||||
|
||||
from ..data.types import Struct
|
||||
from ..data._source import Symbol
|
||||
|
||||
|
||||
# TODO: a composite for tracking msg flow on 2-legged
|
||||
# dialogs.
|
||||
# class Dialog(ChainMap):
|
||||
# '''
|
||||
# Msg collection abstraction to easily track the state changes of
|
||||
# a msg flow in one high level, query-able and immutable construct.
|
||||
|
||||
# The main use case is to query data from a (long-running)
|
||||
# msg-transaction-sequence
|
||||
|
||||
|
||||
# '''
|
||||
# def update(
|
||||
# self,
|
||||
# msg,
|
||||
# ) -> None:
|
||||
# self.maps.insert(0, msg.to_dict())
|
||||
|
||||
# def flatten(self) -> dict:
|
||||
# return dict(self)
|
||||
|
||||
|
||||
# TODO: ``msgspec`` stuff worth paying attention to:
|
||||
# - schema evolution:
|
||||
# https://jcristharif.com/msgspec/usage.html#schema-evolution
|
||||
# - for eg. ``BrokerdStatus``, instead just have separate messages?
|
||||
# - use literals for a common msg determined by diff keys?
|
||||
# - https://jcristharif.com/msgspec/usage.html#literal
|
||||
|
||||
# --------------
|
||||
# Client -> emsd
|
||||
# --------------
|
||||
|
||||
class Order(Struct):
|
||||
|
||||
# TODO: ideally we can combine these 2 fields into
|
||||
# 1 and just use the size polarity to determine a buy/sell.
|
||||
# i would like to see this become more like
|
||||
# https://jcristharif.com/msgspec/usage.html#literal
|
||||
# action: Literal[
|
||||
# 'live',
|
||||
# 'dark',
|
||||
# 'alert',
|
||||
# ]
|
||||
|
||||
action: Literal[
|
||||
'buy',
|
||||
'sell',
|
||||
'alert',
|
||||
]
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: Literal[
|
||||
'dark',
|
||||
'live',
|
||||
# 'paper', no right?
|
||||
]
|
||||
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
# TODO: figure out how to optionally typecast this to `MktPair`?
|
||||
symbol: str # | MktPair
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float # -ve is "sell", +ve is "buy"
|
||||
|
||||
brokers: list[str] = []
|
||||
|
||||
|
||||
class Cancel(Struct):
|
||||
'''
|
||||
Cancel msg for removing a dark (ems triggered) or
|
||||
class Cancel(BaseModel):
|
||||
'''Cancel msg for removing a dark (ems triggered) or
|
||||
broker-submitted (live) trigger/order.
|
||||
|
||||
'''
|
||||
action: str = 'cancel'
|
||||
oid: str # uuid4
|
||||
symbol: str
|
||||
action: str = 'cancel'
|
||||
|
||||
|
||||
# --------------
|
||||
class Order(BaseModel):
|
||||
|
||||
action: str # {'buy', 'sell', 'alert'}
|
||||
# internal ``emdsd`` unique "order id"
|
||||
oid: str # uuid4
|
||||
symbol: Union[str, Symbol]
|
||||
account: str # should we set a default as '' ?
|
||||
|
||||
price: float
|
||||
size: float
|
||||
brokers: list[str]
|
||||
|
||||
# Assigned once initial ack is received
|
||||
# ack_time_ns: Optional[int] = None
|
||||
|
||||
# determines whether the create execution
|
||||
# will be submitted to the ems or directly to
|
||||
# the backend broker
|
||||
exec_mode: str # {'dark', 'live', 'paper'}
|
||||
|
||||
class Config:
|
||||
# just for pre-loading a ``Symbol`` when used
|
||||
# in the order mode staging process
|
||||
arbitrary_types_allowed = True
|
||||
# don't copy this model instance when used in
|
||||
# a recursive model
|
||||
copy_on_model_validation = False
|
||||
|
||||
# Client <- emsd
|
||||
# --------------
|
||||
# update msgs from ems which relay state change info
|
||||
# from the active clearing engine.
|
||||
|
||||
class Status(Struct):
|
||||
|
||||
time_ns: int
|
||||
oid: str # uuid4 ems-order dialog id
|
||||
|
||||
resp: Literal[
|
||||
'pending', # acked by broker but not yet open
|
||||
'open',
|
||||
'dark_open', # dark/algo triggered order is open in ems clearing loop
|
||||
'triggered', # above triggered order sent to brokerd, or an alert closed
|
||||
'closed', # fully cleared all size/units
|
||||
'fill', # partial execution
|
||||
'canceled',
|
||||
'error',
|
||||
]
|
||||
class Status(BaseModel):
|
||||
|
||||
name: str = 'status'
|
||||
oid: str # uuid4
|
||||
time_ns: int
|
||||
|
||||
# {
|
||||
# 'dark_submitted',
|
||||
# 'dark_cancelled',
|
||||
# 'dark_triggered',
|
||||
|
||||
# 'broker_submitted',
|
||||
# 'broker_cancelled',
|
||||
# 'broker_executed',
|
||||
# 'broker_filled',
|
||||
# 'broker_errored',
|
||||
|
||||
# 'alert_submitted',
|
||||
# 'alert_triggered',
|
||||
|
||||
# }
|
||||
resp: str # "response", see above
|
||||
|
||||
# symbol: str
|
||||
|
||||
# trigger info
|
||||
trigger_price: Optional[float] = None
|
||||
# price: float
|
||||
|
||||
# broker: Optional[str] = None
|
||||
|
||||
# this maps normally to the ``BrokerdOrder.reqid`` below, an id
|
||||
# normally allocated internally by the backend broker routing system
|
||||
reqid: Optional[int | str] = None
|
||||
broker_reqid: Optional[Union[int, str]] = None
|
||||
|
||||
# the (last) source order/request msg if provided
|
||||
# (eg. the Order/Cancel which causes this msg) and
|
||||
# acts as a back-reference to the corresponding
|
||||
# request message which was the source of this msg.
|
||||
req: Order | None = None
|
||||
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
src: Optional[str] = None
|
||||
|
||||
# set when a cancel request msg was set for this order flow dialog
|
||||
# but the brokerd dialog isn't yet in a cancelled state.
|
||||
cancel_called: bool = False
|
||||
|
||||
# for relaying a boxed brokerd-dialog-side msg data "through" the
|
||||
# ems layer to clients.
|
||||
# for relaying backend msg data "through" the ems layer
|
||||
brokerd_msg: dict = {}
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd -> brokerd
|
||||
# ---------------
|
||||
# requests *sent* from ems to respective backend broker daemon
|
||||
|
||||
class BrokerdCancel(Struct):
|
||||
class BrokerdCancel(BaseModel):
|
||||
|
||||
action: str = 'cancel'
|
||||
oid: str # piker emsd order id
|
||||
time_ns: int
|
||||
|
||||
|
|
@ -181,39 +127,34 @@ class BrokerdCancel(Struct):
|
|||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: Optional[int | str] = None
|
||||
action: str = 'cancel'
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
|
||||
class BrokerdOrder(Struct):
|
||||
class BrokerdOrder(BaseModel):
|
||||
|
||||
action: str # {buy, sell}
|
||||
oid: str
|
||||
account: str
|
||||
time_ns: int
|
||||
|
||||
symbol: str # fqsn
|
||||
price: float
|
||||
size: float
|
||||
|
||||
# TODO: if we instead rely on a +ve/-ve size to determine
|
||||
# the action we more or less don't need this field right?
|
||||
action: str = '' # {buy, sell}
|
||||
|
||||
# "broker request id": broker specific/internal order id if this is
|
||||
# None, creates a new order otherwise if the id is valid the backend
|
||||
# api must modify the existing matching order. If the broker allows
|
||||
# for setting a unique order id then this value will be relayed back
|
||||
# on the emsd order request stream as the ``BrokerdOrderAck.reqid``
|
||||
# field
|
||||
reqid: Optional[int | str] = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
symbol: str # symbol.<providername> ?
|
||||
price: float
|
||||
size: float
|
||||
|
||||
|
||||
# ---------------
|
||||
# emsd <- brokerd
|
||||
# ---------------
|
||||
# requests *received* to ems from broker backend
|
||||
|
||||
class BrokerdOrderAck(Struct):
|
||||
|
||||
class BrokerdOrderAck(BaseModel):
|
||||
'''
|
||||
Immediate reponse to a brokerd order request providing the broker
|
||||
specific unique order id so that the EMS can associate this
|
||||
|
|
@ -221,93 +162,102 @@ class BrokerdOrderAck(Struct):
|
|||
``.oid`` (which is a uuid4).
|
||||
|
||||
'''
|
||||
name: str = 'ack'
|
||||
|
||||
# defined and provided by backend
|
||||
reqid: int | str
|
||||
reqid: Union[int, str]
|
||||
|
||||
# emsd id originally sent in matching request msg
|
||||
oid: str
|
||||
account: str = ''
|
||||
name: str = 'ack'
|
||||
|
||||
|
||||
class BrokerdStatus(Struct):
|
||||
class BrokerdStatus(BaseModel):
|
||||
|
||||
reqid: int | str
|
||||
time_ns: int
|
||||
status: Literal[
|
||||
'open',
|
||||
'canceled',
|
||||
'fill',
|
||||
'pending',
|
||||
'error',
|
||||
]
|
||||
|
||||
account: str
|
||||
name: str = 'status'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
|
||||
# XXX: should be best effort set for every update
|
||||
account: str = ''
|
||||
|
||||
# {
|
||||
# 'submitted',
|
||||
# 'cancelled',
|
||||
# 'filled',
|
||||
# }
|
||||
status: str
|
||||
|
||||
filled: float = 0.0
|
||||
reason: str = ''
|
||||
remaining: float = 0.0
|
||||
|
||||
# external: bool = False
|
||||
# XXX: better design/name here?
|
||||
# flag that can be set to indicate a message for an order
|
||||
# event that wasn't originated by piker's emsd (eg. some external
|
||||
# trading system which does it's own order control but that you
|
||||
# might want to "track" using piker UIs/systems).
|
||||
external: bool = False
|
||||
|
||||
# XXX: not required schema as of yet
|
||||
broker_details: dict = field(default_factory=lambda: {
|
||||
broker_details: dict = {
|
||||
'name': '',
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
class BrokerdFill(Struct):
|
||||
class BrokerdFill(BaseModel):
|
||||
'''
|
||||
A single message indicating a "fill-details" event from the broker
|
||||
if avaiable.
|
||||
|
||||
'''
|
||||
name: str = 'fill'
|
||||
reqid: Union[int, str]
|
||||
time_ns: int
|
||||
|
||||
# order exeuction related
|
||||
action: str
|
||||
size: float
|
||||
price: float
|
||||
|
||||
broker_details: dict = {} # meta-data (eg. commisions etc.)
|
||||
|
||||
# brokerd timestamp required for order mode arrow placement on x-axis
|
||||
|
||||
# TODO: maybe int if we force ns?
|
||||
# we need to normalize this somehow since backends will use their
|
||||
# own format and likely across many disparate epoch clocks...
|
||||
broker_time: float
|
||||
reqid: int | str
|
||||
time_ns: int
|
||||
|
||||
# order exeuction related
|
||||
size: float
|
||||
price: float
|
||||
|
||||
name: str = 'fill'
|
||||
action: Optional[str] = None
|
||||
broker_details: dict = {} # meta-data (eg. commisions etc.)
|
||||
|
||||
|
||||
class BrokerdError(Struct):
|
||||
class BrokerdError(BaseModel):
|
||||
'''
|
||||
Optional error type that can be relayed to emsd for error handling.
|
||||
|
||||
This is still a TODO thing since we're not sure how to employ it yet.
|
||||
|
||||
'''
|
||||
name: str = 'error'
|
||||
oid: str
|
||||
symbol: str
|
||||
reason: str
|
||||
|
||||
# if no brokerd order request was actually submitted (eg. we errored
|
||||
# at the ``pikerd`` layer) then there will be ``reqid`` allocated.
|
||||
reqid: Optional[int | str] = None
|
||||
reqid: Optional[Union[int, str]] = None
|
||||
|
||||
name: str = 'error'
|
||||
symbol: str
|
||||
reason: str
|
||||
broker_details: dict = {}
|
||||
|
||||
|
||||
class BrokerdPosition(Struct):
|
||||
'''
|
||||
Position update event from brokerd.
|
||||
class BrokerdPosition(BaseModel):
|
||||
'''Position update event from brokerd.
|
||||
|
||||
'''
|
||||
name: str = 'position'
|
||||
|
||||
broker: str
|
||||
account: str
|
||||
symbol: str
|
||||
currency: str
|
||||
size: float
|
||||
avg_price: float
|
||||
currency: str = ''
|
||||
name: str = 'position'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -14,84 +14,58 @@
|
|||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Fake trading: a full forward testing simulation engine.
|
||||
"""
|
||||
Fake trading for forward testing.
|
||||
|
||||
We can real-time emulate any mkt conditions you want bruddr B)
|
||||
Just slide us the model que quieres..
|
||||
|
||||
'''
|
||||
from collections import defaultdict
|
||||
from contextlib import asynccontextmanager as acm
|
||||
"""
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from operator import itemgetter
|
||||
import itertools
|
||||
import time
|
||||
from typing import (
|
||||
Callable,
|
||||
)
|
||||
from typing import Tuple, Optional, Callable
|
||||
import uuid
|
||||
|
||||
from bidict import bidict
|
||||
import pendulum
|
||||
import trio
|
||||
import tractor
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..brokers import get_brokermod
|
||||
from .. import data
|
||||
from ..data.types import Struct
|
||||
from ..accounting._mktinfo import (
|
||||
MktPair,
|
||||
)
|
||||
from ..accounting import (
|
||||
Position,
|
||||
PpTable,
|
||||
Transaction,
|
||||
TransactionLedger,
|
||||
open_trade_ledger,
|
||||
open_pps,
|
||||
)
|
||||
from ..data._normalize import iterticks
|
||||
from ..accounting._mktinfo import unpack_fqme
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ..data._source import unpack_fqsn
|
||||
from ..log import get_logger
|
||||
from ._messages import (
|
||||
BrokerdCancel,
|
||||
BrokerdOrder,
|
||||
BrokerdOrderAck,
|
||||
BrokerdStatus,
|
||||
BrokerdFill,
|
||||
BrokerdPosition,
|
||||
BrokerdError,
|
||||
BrokerdCancel, BrokerdOrder, BrokerdOrderAck, BrokerdStatus,
|
||||
BrokerdFill, BrokerdPosition, BrokerdError
|
||||
)
|
||||
|
||||
|
||||
class PaperBoi(Struct):
|
||||
'''
|
||||
Emulates a broker order client providing approximately the same API
|
||||
and delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements (eg open, closed, fill msgs).
|
||||
log = get_logger(__name__)
|
||||
|
||||
'''
|
||||
|
||||
@dataclass
|
||||
class PaperBoi:
|
||||
"""
|
||||
Emulates a broker order client providing the same API and
|
||||
delivering an order-event response stream but with methods for
|
||||
triggering desired events based on forward testing engine
|
||||
requirements.
|
||||
|
||||
"""
|
||||
broker: str
|
||||
|
||||
ems_trades_stream: tractor.MsgStream
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
|
||||
# map of paper "live" orders which be used
|
||||
# to simulate fills based on paper engine settings
|
||||
_buys: defaultdict[str, bidict]
|
||||
_sells: defaultdict[str, bidict]
|
||||
_buys: bidict
|
||||
_sells: bidict
|
||||
_reqids: bidict
|
||||
_mkts: dict[str, MktPair] = {}
|
||||
_positions: dict[str, BrokerdPosition]
|
||||
|
||||
# init edge case L1 spread
|
||||
last_ask: tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: tuple[float, float] = (0, 0)
|
||||
last_ask: Tuple[float, float] = (float('inf'), 0) # price, size
|
||||
last_bid: Tuple[float, float] = (0, 0)
|
||||
|
||||
async def submit_limit(
|
||||
self,
|
||||
|
|
@ -100,25 +74,28 @@ class PaperBoi(Struct):
|
|||
price: float,
|
||||
action: str,
|
||||
size: float,
|
||||
reqid: str | None,
|
||||
|
||||
reqid: Optional[str],
|
||||
) -> int:
|
||||
'''
|
||||
Place an order and return integer request id provided by client.
|
||||
"""Place an order and return integer request id provided by client.
|
||||
|
||||
"""
|
||||
is_modify: bool = False
|
||||
if reqid is None:
|
||||
reqid = str(uuid.uuid4())
|
||||
|
||||
else:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = self._reqids[reqid]
|
||||
assert old_price != price
|
||||
is_modify = True
|
||||
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
'''
|
||||
if action == 'alert':
|
||||
# bypass all fill simulation
|
||||
return reqid
|
||||
|
||||
entry = self._reqids.get(reqid)
|
||||
if entry:
|
||||
# order is already existing, this is a modify
|
||||
(oid, symbol, action, old_price) = entry
|
||||
else:
|
||||
# register order internally
|
||||
self._reqids[reqid] = (oid, symbol, action, price)
|
||||
|
||||
# TODO: net latency model
|
||||
# we checkpoint here quickly particulalry
|
||||
# for dark orders since we want the dark_executed
|
||||
|
|
@ -126,25 +103,19 @@ class PaperBoi(Struct):
|
|||
# in the broker trades event processing loop
|
||||
await trio.sleep(0.05)
|
||||
|
||||
if (
|
||||
action == 'sell'
|
||||
and size > 0
|
||||
):
|
||||
if action == 'sell':
|
||||
size = -size
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='open',
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='submitted',
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
filled=0.0,
|
||||
reason='paper_trigger',
|
||||
remaining=size,
|
||||
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
# if we're already a clearing price simulate an immediate fill
|
||||
if (
|
||||
|
|
@ -152,28 +123,28 @@ class PaperBoi(Struct):
|
|||
) or (
|
||||
action == 'sell' and (clear_price := self.last_bid[0]) >= price
|
||||
):
|
||||
await self.fake_fill(
|
||||
symbol,
|
||||
clear_price,
|
||||
size,
|
||||
action,
|
||||
reqid,
|
||||
oid,
|
||||
)
|
||||
await self.fake_fill(symbol, clear_price, size, action, reqid, oid)
|
||||
|
||||
# register this submissions as a paper live order
|
||||
else:
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
# register this submissions as a paper live order
|
||||
|
||||
# submit order to book simulation fill loop
|
||||
if action == 'buy':
|
||||
orders = self._buys
|
||||
|
||||
elif action == 'sell':
|
||||
orders = self._sells
|
||||
|
||||
# {symbol -> bidict[oid, (<price data>)]}
|
||||
orders[symbol][oid] = (price, size, reqid, action)
|
||||
# set the simulated order in the respective table for lookup
|
||||
# and trigger by the simulated clearing task normally
|
||||
# running ``simulate_fills()``.
|
||||
|
||||
if is_modify:
|
||||
# remove any existing order for the old price
|
||||
orders[symbol].pop((oid, old_price))
|
||||
|
||||
# buys/sells: (symbol -> (price -> order))
|
||||
orders.setdefault(symbol, {})[(oid, price)] = (size, reqid, action)
|
||||
|
||||
return reqid
|
||||
|
||||
|
|
@ -186,26 +157,26 @@ class PaperBoi(Struct):
|
|||
oid, symbol, action, price = self._reqids[reqid]
|
||||
|
||||
if action == 'buy':
|
||||
self._buys[symbol].pop(oid, None)
|
||||
self._buys[symbol].pop((oid, price))
|
||||
elif action == 'sell':
|
||||
self._sells[symbol].pop(oid, None)
|
||||
self._sells[symbol].pop((oid, price))
|
||||
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.05)
|
||||
|
||||
msg = BrokerdStatus(
|
||||
status='canceled',
|
||||
account='paper',
|
||||
status='cancelled',
|
||||
oid=oid,
|
||||
reqid=reqid,
|
||||
broker=self.broker,
|
||||
time_ns=time.time_ns(),
|
||||
broker_details={'name': 'paperboi'},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
async def fake_fill(
|
||||
self,
|
||||
|
||||
fqme: str,
|
||||
symbol: str,
|
||||
price: float,
|
||||
size: float,
|
||||
action: str, # one of {'buy', 'sell'}
|
||||
|
|
@ -219,21 +190,21 @@ class PaperBoi(Struct):
|
|||
remaining: float = 0,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Pretend to fill a broker order @ price and size.
|
||||
"""Pretend to fill a broker order @ price and size.
|
||||
|
||||
'''
|
||||
"""
|
||||
# TODO: net latency model
|
||||
await trio.sleep(0.05)
|
||||
fill_time_ns = time.time_ns()
|
||||
fill_time_s = time.time()
|
||||
|
||||
fill_msg = BrokerdFill(
|
||||
msg = BrokerdFill(
|
||||
|
||||
reqid=reqid,
|
||||
time_ns=fill_time_ns,
|
||||
time_ns=time.time_ns(),
|
||||
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
|
||||
broker_time=datetime.now().timestamp(),
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
|
|
@ -243,67 +214,79 @@ class PaperBoi(Struct):
|
|||
'name': self.broker + '_paper',
|
||||
},
|
||||
)
|
||||
log.info(f'Fake filling order:\n{fill_msg}')
|
||||
await self.ems_trades_stream.send(fill_msg)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
if order_complete:
|
||||
|
||||
msg = BrokerdStatus(
|
||||
|
||||
reqid=reqid,
|
||||
time_ns=time.time_ns(),
|
||||
# account=f'paper_{self.broker}',
|
||||
account='paper',
|
||||
status='closed',
|
||||
|
||||
status='filled',
|
||||
filled=size,
|
||||
remaining=0 if order_complete else remaining,
|
||||
)
|
||||
await self.ems_trades_stream.send(msg)
|
||||
|
||||
# NOTE: for paper we set the "bs_mktid" as just the fqme since
|
||||
# we don't actually have any unique backend symbol ourselves
|
||||
# other then this thing, our fqme address.
|
||||
bs_mktid: str = fqme
|
||||
t = Transaction(
|
||||
fqsn=fqme,
|
||||
sym=self._mkts[fqme],
|
||||
tid=oid,
|
||||
action=action,
|
||||
size=size,
|
||||
price=price,
|
||||
cost=0, # TODO: cost model
|
||||
dt=pendulum.from_timestamp(fill_time_s),
|
||||
bs_mktid=bs_mktid,
|
||||
|
||||
broker_details={
|
||||
'paper_info': {
|
||||
'oid': oid,
|
||||
},
|
||||
'name': self.broker,
|
||||
},
|
||||
)
|
||||
await self.ems_trades_stream.send(msg.dict())
|
||||
|
||||
# update in-mem ledger and pos table
|
||||
self.ledger.update_from_t(t)
|
||||
self.ppt.update_from_trans({oid: t})
|
||||
|
||||
# transmit pp msg to ems
|
||||
pp = self.ppt.pps[bs_mktid]
|
||||
pp_msg = BrokerdPosition(
|
||||
# lookup any existing position
|
||||
token = f'{symbol}.{self.broker}'
|
||||
pp_msg = self._positions.setdefault(
|
||||
token,
|
||||
BrokerdPosition(
|
||||
broker=self.broker,
|
||||
account='paper',
|
||||
symbol=fqme,
|
||||
|
||||
size=pp.size,
|
||||
avg_price=pp.ppu,
|
||||
|
||||
symbol=symbol,
|
||||
# TODO: we need to look up the asset currency from
|
||||
# broker info. i guess for crypto this can be
|
||||
# inferred from the pair?
|
||||
# currency=bs_mktid,
|
||||
currency='',
|
||||
size=0.0,
|
||||
avg_price=0,
|
||||
)
|
||||
)
|
||||
# write all updates to filesys immediately
|
||||
# (adds latency but that works for simulation anyway)
|
||||
self.ledger.write_config()
|
||||
self.ppt.write_config()
|
||||
|
||||
await self.ems_trades_stream.send(pp_msg)
|
||||
# "avg position price" calcs
|
||||
# TODO: eventually it'd be nice to have a small set of routines
|
||||
# to do this stuff from a sequence of cleared orders to enable
|
||||
# so called "contextual positions".
|
||||
new_size = size + pp_msg.size
|
||||
|
||||
# old size minus the new size gives us size differential with
|
||||
# +ve -> increase in pp size
|
||||
# -ve -> decrease in pp size
|
||||
size_diff = abs(new_size) - abs(pp_msg.size)
|
||||
|
||||
if new_size == 0:
|
||||
pp_msg.avg_price = 0
|
||||
|
||||
elif size_diff > 0:
|
||||
# only update the "average position price" when the position
|
||||
# size increases not when it decreases (i.e. the position is
|
||||
# being made smaller)
|
||||
pp_msg.avg_price = (
|
||||
abs(size) * price + pp_msg.avg_price * abs(pp_msg.size)
|
||||
) / abs(new_size)
|
||||
|
||||
pp_msg.size = new_size
|
||||
|
||||
await self.ems_trades_stream.send(pp_msg.dict())
|
||||
|
||||
|
||||
async def simulate_fills(
|
||||
quote_stream: tractor.MsgStream, # noqa
|
||||
quote_stream: 'tractor.ReceiveStream', # noqa
|
||||
client: PaperBoi,
|
||||
|
||||
) -> None:
|
||||
|
||||
# TODO: more machinery to better simulate real-world market things:
|
||||
|
|
@ -323,116 +306,61 @@ async def simulate_fills(
|
|||
|
||||
# this stream may eventually contain multiple symbols
|
||||
async for quotes in quote_stream:
|
||||
|
||||
for sym, quote in quotes.items():
|
||||
|
||||
for tick in iterticks(
|
||||
quote,
|
||||
# dark order price filter(s)
|
||||
types=('ask', 'bid', 'trade', 'last')
|
||||
):
|
||||
tick_price = tick['price']
|
||||
# print(tick)
|
||||
tick_price = tick.get('price')
|
||||
ttype = tick['type']
|
||||
|
||||
buys: bidict[str, tuple] = client._buys[sym]
|
||||
iter_buys = reversed(sorted(
|
||||
buys.values(),
|
||||
key=itemgetter(0),
|
||||
))
|
||||
if ttype in ('ask',):
|
||||
|
||||
def buy_on_ask(our_price):
|
||||
return tick_price <= our_price
|
||||
|
||||
sells: bidict[str, tuple] = client._sells[sym]
|
||||
iter_sells = sorted(
|
||||
sells.values(),
|
||||
key=itemgetter(0)
|
||||
)
|
||||
|
||||
def sell_on_bid(our_price):
|
||||
return tick_price >= our_price
|
||||
|
||||
match tick:
|
||||
|
||||
# on an ask queue tick, only clear buy entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'ask',
|
||||
}:
|
||||
client.last_ask = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_ask[1]),
|
||||
)
|
||||
|
||||
iter_entries = zip(
|
||||
iter_buys,
|
||||
itertools.repeat(buy_on_ask)
|
||||
)
|
||||
orders = client._buys.get(sym, {})
|
||||
|
||||
book_sequence = reversed(
|
||||
sorted(orders.keys(), key=itemgetter(1)))
|
||||
|
||||
def pred(our_price):
|
||||
return tick_price < our_price
|
||||
|
||||
elif ttype in ('bid',):
|
||||
|
||||
# on a bid queue tick, only clear sell entries
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': 'bid',
|
||||
}:
|
||||
client.last_bid = (
|
||||
tick_price,
|
||||
tick.get('size', client.last_bid[1]),
|
||||
)
|
||||
|
||||
iter_entries = zip(
|
||||
iter_sells,
|
||||
itertools.repeat(sell_on_bid)
|
||||
)
|
||||
orders = client._sells.get(sym, {})
|
||||
book_sequence = sorted(orders.keys(), key=itemgetter(1))
|
||||
|
||||
# TODO: fix this block, though it definitely
|
||||
# costs a lot more CPU-wise
|
||||
# - doesn't seem like clears are happening still on
|
||||
# "resting" limit orders?
|
||||
case {
|
||||
'price': tick_price,
|
||||
'type': ('trade' | 'last'),
|
||||
}:
|
||||
# in the clearing price / last price case we
|
||||
# want to iterate both sides of our book for
|
||||
# clears since we don't know which direction the
|
||||
# price is going to move (especially with HFT)
|
||||
# and thus we simply interleave both sides (buys
|
||||
# and sells) until one side clears and then
|
||||
# break until the next tick?
|
||||
def interleave():
|
||||
for pair in zip(
|
||||
iter_buys,
|
||||
iter_sells,
|
||||
):
|
||||
for order_info, pred in zip(
|
||||
pair,
|
||||
itertools.cycle([buy_on_ask, sell_on_bid]),
|
||||
):
|
||||
yield order_info, pred
|
||||
def pred(our_price):
|
||||
return tick_price > our_price
|
||||
|
||||
iter_entries = interleave()
|
||||
|
||||
# NOTE: all other (non-clearable) tick event types
|
||||
# - we don't want to sping the simulated clear loop
|
||||
# below unecessarily and further don't want to pop
|
||||
# simulated live orders prematurely.
|
||||
case _:
|
||||
elif ttype in ('trade', 'last'):
|
||||
# TODO: simulate actual book queues and our orders
|
||||
# place in it, might require full L2 data?
|
||||
continue
|
||||
|
||||
# iterate all potentially clearable book prices
|
||||
# in FIFO order per side.
|
||||
for order_info, pred in iter_entries:
|
||||
(our_price, size, reqid, action) = order_info
|
||||
# iterate book prices descending
|
||||
for oid, our_price in book_sequence:
|
||||
if pred(our_price):
|
||||
|
||||
# print(order_info)
|
||||
clearable = pred(our_price)
|
||||
if clearable:
|
||||
# pop and retreive order info
|
||||
oid = {
|
||||
'buy': buys,
|
||||
'sell': sells
|
||||
}[action].inverse.pop(order_info)
|
||||
# retreive order info
|
||||
(size, reqid, action) = orders.pop((oid, our_price))
|
||||
|
||||
# clearing price would have filled entirely
|
||||
await client.fake_fill(
|
||||
fqme=sym,
|
||||
symbol=sym,
|
||||
# todo slippage to determine fill price
|
||||
price=tick_price,
|
||||
size=size,
|
||||
|
|
@ -440,6 +368,9 @@ async def simulate_fills(
|
|||
reqid=reqid,
|
||||
oid=oid,
|
||||
)
|
||||
else:
|
||||
# prices are iterated in sorted order so we're done
|
||||
break
|
||||
|
||||
|
||||
async def handle_order_requests(
|
||||
|
|
@ -449,235 +380,119 @@ async def handle_order_requests(
|
|||
|
||||
) -> None:
|
||||
|
||||
request_msg: dict
|
||||
# order_request: dict
|
||||
async for request_msg in ems_order_stream:
|
||||
match request_msg:
|
||||
case {'action': ('buy' | 'sell')}:
|
||||
order = BrokerdOrder(**request_msg)
|
||||
account = order.account
|
||||
|
||||
# error on bad inputs
|
||||
reason = None
|
||||
action = request_msg['action']
|
||||
|
||||
if action in {'buy', 'sell'}:
|
||||
|
||||
account = request_msg['account']
|
||||
if account != 'paper':
|
||||
reason = f'No account found:`{account}` (paper only)?'
|
||||
|
||||
elif order.size == 0:
|
||||
reason = 'Invalid size: 0'
|
||||
|
||||
if reason:
|
||||
log.error(reason)
|
||||
log.error(
|
||||
'This is a paper account, only a `paper` selection is valid'
|
||||
)
|
||||
await ems_order_stream.send(BrokerdError(
|
||||
oid=order.oid,
|
||||
symbol=order.symbol,
|
||||
reason=reason,
|
||||
))
|
||||
oid=request_msg['oid'],
|
||||
symbol=request_msg['symbol'],
|
||||
reason=f'Paper only. No account found: `{account}` ?',
|
||||
).dict())
|
||||
continue
|
||||
|
||||
reqid = order.reqid or str(uuid.uuid4())
|
||||
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
oid=order.oid,
|
||||
reqid=reqid,
|
||||
account='paper'
|
||||
)
|
||||
)
|
||||
# validate
|
||||
order = BrokerdOrder(**request_msg)
|
||||
|
||||
# call our client api to submit the order
|
||||
reqid = await client.submit_limit(
|
||||
|
||||
oid=order.oid,
|
||||
symbol=f'{order.symbol}.{client.broker}',
|
||||
symbol=order.symbol,
|
||||
price=order.price,
|
||||
action=order.action,
|
||||
size=order.size,
|
||||
|
||||
# XXX: by default 0 tells ``ib_insync`` methods that
|
||||
# there is no existing order so ask the client to create
|
||||
# a new one (which it seems to do by allocating an int
|
||||
# counter - collision prone..)
|
||||
reqid=reqid,
|
||||
reqid=order.reqid,
|
||||
)
|
||||
log.info(f'Submitted paper LIMIT {reqid}:\n{order}')
|
||||
|
||||
case {'action': 'cancel'}:
|
||||
# deliver ack that order has been submitted to broker routing
|
||||
await ems_order_stream.send(
|
||||
BrokerdOrderAck(
|
||||
|
||||
# ems order request id
|
||||
oid=order.oid,
|
||||
|
||||
# broker specific request id
|
||||
reqid=reqid,
|
||||
|
||||
).dict()
|
||||
)
|
||||
|
||||
elif action == 'cancel':
|
||||
msg = BrokerdCancel(**request_msg)
|
||||
|
||||
await client.submit_cancel(
|
||||
reqid=msg.reqid
|
||||
)
|
||||
|
||||
case _:
|
||||
else:
|
||||
log.error(f'Unknown order command: {request_msg}')
|
||||
|
||||
|
||||
_reqids: bidict[str, tuple] = {}
|
||||
_buys: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
_sells: defaultdict[
|
||||
str, # symbol
|
||||
bidict[
|
||||
str, # oid
|
||||
tuple[float, float, str, str], # order info
|
||||
]
|
||||
] = defaultdict(bidict)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def trades_dialogue(
|
||||
|
||||
ctx: tractor.Context,
|
||||
broker: str,
|
||||
fqme: str | None = None, # if empty, we only boot broker mode
|
||||
loglevel: str = 'warning',
|
||||
fqsn: str,
|
||||
loglevel: str = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
tractor.log.get_console_log(loglevel)
|
||||
|
||||
ppt: PpTable
|
||||
ledger: TransactionLedger
|
||||
with (
|
||||
open_pps(
|
||||
broker,
|
||||
'paper',
|
||||
write_on_exit=True,
|
||||
) as ppt,
|
||||
|
||||
open_trade_ledger(
|
||||
broker,
|
||||
'paper',
|
||||
) as ledger
|
||||
):
|
||||
# NOTE: retreive market(pair) info from the backend broker
|
||||
# since ledger entries (in their backend native format) often
|
||||
# don't contain necessary market info per trade record entry..
|
||||
# - if no fqme was passed in, we presume we're running in
|
||||
# "ledger-sync-only mode" and thus we load mkt info for
|
||||
# each symbol found in the ledger to a ppt table manually.
|
||||
|
||||
# TODO: how to process ledger info from backends?
|
||||
# - should we be rolling our own actor-cached version of these
|
||||
# client API refs or using portal IPC to send requests to the
|
||||
# existing brokerd daemon?
|
||||
# - alternatively we can possibly expect and use
|
||||
# a `.broker.norm_trade_records()` ep?
|
||||
brokermod = get_brokermod(broker)
|
||||
gmi = getattr(brokermod, 'get_mkt_info', None)
|
||||
|
||||
# update all transactions with mkt info before
|
||||
# loading any pps
|
||||
mkt_by_fqme: dict[str, MktPair] = {}
|
||||
if fqme:
|
||||
mkt, _ = await brokermod.get_mkt_info(fqme.rstrip(f'.{broker}'))
|
||||
mkt_by_fqme[fqme] = mkt
|
||||
|
||||
# for each sym in the ledger load it's `MktPair` info
|
||||
for tid, tdict in ledger.data.items():
|
||||
# TODO: switch this to fqme
|
||||
l_fqme = tdict['fqsn']
|
||||
|
||||
if (
|
||||
gmi
|
||||
and l_fqme not in mkt_by_fqme
|
||||
):
|
||||
mkt, pair = await brokermod.get_mkt_info(
|
||||
l_fqme.rstrip(f'.{broker}'),
|
||||
)
|
||||
mkt_by_fqme[l_fqme] = mkt
|
||||
|
||||
# if an ``fqme: str`` input was provided we only
|
||||
# need a ``MktPair`` for that one market, since we're
|
||||
# running in real simulated-clearing mode, not just ledger
|
||||
# syncing.
|
||||
if (
|
||||
fqme is not None
|
||||
and fqme in mkt_by_fqme
|
||||
):
|
||||
break
|
||||
|
||||
# update pos table from ledger history and provide a ``MktPair``
|
||||
# lookup for internal position accounting calcs.
|
||||
ppt.update_from_trans(ledger.to_trans(mkt_by_fqme=mkt_by_fqme))
|
||||
|
||||
pp_msgs: list[BrokerdPosition] = []
|
||||
pos: Position
|
||||
token: str # f'{symbol}.{self.broker}'
|
||||
for token, pos in ppt.pps.items():
|
||||
pp_msgs.append(BrokerdPosition(
|
||||
broker=broker,
|
||||
account='paper',
|
||||
symbol=pos.symbol.fqme,
|
||||
size=pos.size,
|
||||
avg_price=pos.ppu,
|
||||
))
|
||||
|
||||
await ctx.started((
|
||||
pp_msgs,
|
||||
['paper'],
|
||||
))
|
||||
|
||||
# write new positions state in case ledger was
|
||||
# newer then that tracked in pps.toml
|
||||
ppt.write_config()
|
||||
|
||||
# exit early since no fqme was passed,
|
||||
# normally this case is just to load
|
||||
# positions "offline".
|
||||
if fqme is None:
|
||||
log.warning(
|
||||
'Paper engine only running in position delivery mode!\n'
|
||||
'NO SIMULATED CLEARING LOOP IS ACTIVE!'
|
||||
)
|
||||
await trio.sleep_forever()
|
||||
return
|
||||
|
||||
async with (
|
||||
|
||||
data.open_feed(
|
||||
[fqme],
|
||||
[fqsn],
|
||||
loglevel=loglevel,
|
||||
) as feed,
|
||||
|
||||
):
|
||||
# sanity check all the mkt infos
|
||||
for fqme, flume in feed.flumes.items():
|
||||
assert mkt_by_fqme[fqme] == flume.mkt
|
||||
# TODO: load paper positions per broker from .toml config file
|
||||
# and pass as symbol to position data mapping: ``dict[str, dict]``
|
||||
# await ctx.started(all_positions)
|
||||
await ctx.started(({}, {'paper',}))
|
||||
|
||||
async with (
|
||||
ctx.open_stream() as ems_stream,
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
|
||||
client = PaperBoi(
|
||||
broker=broker,
|
||||
ems_trades_stream=ems_stream,
|
||||
ppt=ppt,
|
||||
ledger=ledger,
|
||||
|
||||
_buys=_buys,
|
||||
_sells=_sells,
|
||||
_reqids=_reqids,
|
||||
|
||||
_mkts=mkt_by_fqme,
|
||||
|
||||
)
|
||||
|
||||
n.start_soon(
|
||||
handle_order_requests,
|
||||
client,
|
||||
broker,
|
||||
ems_stream,
|
||||
_buys={},
|
||||
_sells={},
|
||||
|
||||
_reqids={},
|
||||
|
||||
# TODO: load paper positions from ``positions.toml``
|
||||
_positions={},
|
||||
)
|
||||
|
||||
n.start_soon(handle_order_requests, client, ems_stream)
|
||||
|
||||
# paper engine simulator clearing task
|
||||
await simulate_fills(feed.streams[broker], client)
|
||||
await simulate_fills(feed.stream, client)
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_paperboi(
|
||||
fqme: str | None = None,
|
||||
broker: str | None = None,
|
||||
loglevel: str | None = None,
|
||||
fqsn: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> Callable:
|
||||
'''
|
||||
|
|
@ -685,39 +500,28 @@ async def open_paperboi(
|
|||
its context.
|
||||
|
||||
'''
|
||||
if not fqme:
|
||||
assert broker, 'One of `broker` or `fqme` is required siss..!'
|
||||
else:
|
||||
broker, _, _, _ = unpack_fqme(fqme)
|
||||
|
||||
we_spawned: bool = False
|
||||
broker, symbol, expiry = unpack_fqsn(fqsn)
|
||||
service_name = f'paperboi.{broker}'
|
||||
|
||||
async with (
|
||||
tractor.find_actor(service_name) as portal,
|
||||
tractor.open_nursery() as tn,
|
||||
):
|
||||
# NOTE: only spawn if no paperboi already is up since we likely
|
||||
# don't need more then one actor for simulated order clearing
|
||||
# per broker-backend.
|
||||
# only spawn if no paperboi already is up
|
||||
# (we likely don't need more then one proc for basic
|
||||
# simulated order clearing)
|
||||
if portal is None:
|
||||
log.info('Starting new paper-engine actor')
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
we_spawned = True
|
||||
|
||||
async with portal.open_context(
|
||||
trades_dialogue,
|
||||
broker=broker,
|
||||
fqme=fqme,
|
||||
fqsn=fqsn,
|
||||
loglevel=loglevel,
|
||||
|
||||
) as (ctx, first):
|
||||
yield ctx, first
|
||||
|
||||
# tear down connection and any spawned actor on exit
|
||||
await ctx.cancel()
|
||||
if we_spawned:
|
||||
await portal.cancel_actor()
|
||||
yield ctx, first
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Sub-sys module commons.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.clearing'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -19,58 +19,38 @@ CLI commons.
|
|||
|
||||
'''
|
||||
import os
|
||||
from pprint import pformat
|
||||
|
||||
import click
|
||||
import trio
|
||||
import tractor
|
||||
|
||||
from ..log import (
|
||||
get_console_log,
|
||||
get_logger,
|
||||
colorize_json,
|
||||
)
|
||||
from ..log import get_console_log, get_logger, colorize_json
|
||||
from ..brokers import get_brokermod
|
||||
from ..service import (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
from .._daemon import _tractor_kwargs
|
||||
from .. import config
|
||||
|
||||
|
||||
log = get_logger('cli')
|
||||
DEFAULT_BROKER = 'questrade'
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--pdb', is_flag=True, help='Enable tractor debug mode')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.option('--host', '-h', default='127.0.0.1', help='Host address to bind')
|
||||
@click.option(
|
||||
'--tsdb',
|
||||
is_flag=True,
|
||||
help='Enable local ``marketstore`` instance'
|
||||
)
|
||||
@click.option(
|
||||
'--es',
|
||||
is_flag=True,
|
||||
help='Enable local ``elasticsearch`` instance'
|
||||
)
|
||||
def pikerd(
|
||||
loglevel: str,
|
||||
host: str,
|
||||
port: int,
|
||||
tl: bool,
|
||||
pdb: bool,
|
||||
tsdb: bool,
|
||||
es: bool,
|
||||
):
|
||||
def pikerd(loglevel, host, tl, pdb, tsdb):
|
||||
'''
|
||||
Spawn the piker broker-daemon.
|
||||
|
||||
'''
|
||||
|
||||
from ..service import open_pikerd
|
||||
from .._daemon import open_pikerd
|
||||
log = get_console_log(loglevel)
|
||||
|
||||
if pdb:
|
||||
|
|
@ -82,25 +62,32 @@ def pikerd(
|
|||
"\n"
|
||||
))
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
|
||||
async def main():
|
||||
|
||||
async with (
|
||||
open_pikerd(
|
||||
tsdb=tsdb,
|
||||
es=es,
|
||||
loglevel=loglevel,
|
||||
debug_mode=pdb,
|
||||
registry_addr=reg_addr,
|
||||
|
||||
), # normally delivers a ``Services`` handle
|
||||
trio.open_nursery() as n,
|
||||
):
|
||||
if tsdb:
|
||||
from piker.data._ahab import start_ahab
|
||||
from piker.data.marketstore import start_marketstore
|
||||
|
||||
log.info('Spawning `marketstore` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await n.start(
|
||||
start_ahab,
|
||||
'marketstored',
|
||||
start_marketstore,
|
||||
|
||||
)
|
||||
log.info(
|
||||
f'`marketstore` up!\n'
|
||||
f'`marketstored` pid: {pid}\n'
|
||||
f'docker container id: {cid}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
|
@ -110,46 +97,25 @@ def pikerd(
|
|||
@click.group(context_settings=config._context_defaults)
|
||||
@click.option(
|
||||
'--brokers', '-b',
|
||||
default=None,
|
||||
default=[DEFAULT_BROKER],
|
||||
multiple=True,
|
||||
help='Broker backend to use'
|
||||
)
|
||||
@click.option('--loglevel', '-l', default='warning', help='Logging level')
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.option('--configdir', '-c', help='Configuration directory')
|
||||
@click.option('--host', '-h', default=None, help='Host addr to bind')
|
||||
@click.option('--port', '-p', default=None, help='Port number to bind')
|
||||
@click.pass_context
|
||||
def cli(
|
||||
ctx: click.Context,
|
||||
brokers: list[str],
|
||||
loglevel: str,
|
||||
tl: bool,
|
||||
configdir: str,
|
||||
host: str,
|
||||
port: int,
|
||||
|
||||
) -> None:
|
||||
def cli(ctx, brokers, loglevel, tl, configdir):
|
||||
if configdir is not None:
|
||||
assert os.path.isdir(configdir), f"`{configdir}` is not a valid path"
|
||||
config._override_config_dir(configdir)
|
||||
|
||||
ctx.ensure_object(dict)
|
||||
|
||||
if not brokers:
|
||||
# (try to) load all (supposedly) supported data/broker backends
|
||||
from piker.brokers import __brokers__
|
||||
brokers = __brokers__
|
||||
|
||||
if len(brokers) == 1:
|
||||
brokermods = [get_brokermod(brokers[0])]
|
||||
else:
|
||||
brokermods = [get_brokermod(broker) for broker in brokers]
|
||||
assert brokermods
|
||||
|
||||
reg_addr: None | tuple[str, int] = None
|
||||
if host or port:
|
||||
reg_addr = (
|
||||
host or _default_registry_host,
|
||||
int(port) or _default_registry_port,
|
||||
)
|
||||
|
||||
ctx.obj.update({
|
||||
'brokers': brokers,
|
||||
|
|
@ -159,7 +125,6 @@ def cli(
|
|||
'log': get_console_log(loglevel),
|
||||
'confdir': config._config_dir,
|
||||
'wl_path': config._watchlists_data_path,
|
||||
'registry_addr': reg_addr,
|
||||
})
|
||||
|
||||
# allow enabling same loglevel in ``tractor`` machinery
|
||||
|
|
@ -169,45 +134,33 @@ def cli(
|
|||
|
||||
@cli.command()
|
||||
@click.option('--tl', is_flag=True, help='Enable tractor logging')
|
||||
@click.argument('ports', nargs=-1, required=False)
|
||||
@click.argument('names', nargs=-1, required=False)
|
||||
@click.pass_obj
|
||||
def services(config, tl, ports):
|
||||
|
||||
from ..service import (
|
||||
open_piker_runtime,
|
||||
_default_registry_port,
|
||||
_default_registry_host,
|
||||
)
|
||||
|
||||
host = _default_registry_host
|
||||
if not ports:
|
||||
ports = [_default_registry_port]
|
||||
def services(config, tl, names):
|
||||
|
||||
async def list_services():
|
||||
nonlocal host
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
),
|
||||
tractor.get_arbiter(
|
||||
host=host,
|
||||
port=ports[0]
|
||||
) as portal
|
||||
):
|
||||
|
||||
async with tractor.get_arbiter(
|
||||
*_tractor_kwargs['arbiter_addr']
|
||||
) as portal:
|
||||
registry = await portal.run_from_ns('self', 'get_registry')
|
||||
json_d = {}
|
||||
for key, socket in registry.items():
|
||||
# name, uuid = uid
|
||||
host, port = socket
|
||||
json_d[key] = f'{host}:{port}'
|
||||
click.echo(f"{colorize_json(json_d)}")
|
||||
|
||||
trio.run(list_services)
|
||||
tractor.run(
|
||||
list_services,
|
||||
name='service_query',
|
||||
loglevel=config['loglevel'] if tl else None,
|
||||
arbiter_addr=_tractor_kwargs['arbiter_addr'],
|
||||
)
|
||||
|
||||
|
||||
def _load_clis() -> None:
|
||||
from ..service import marketstore # noqa
|
||||
from ..service import elastic
|
||||
from ..data import marketstore # noqa
|
||||
from ..data import cli # noqa
|
||||
from ..brokers import cli # noqa
|
||||
from ..ui import cli # noqa
|
||||
|
|
|
|||
170
piker/config.py
170
piker/config.py
|
|
@ -15,15 +15,15 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Platform configuration (files) mgmt.
|
||||
Broker configuration mgmt.
|
||||
|
||||
"""
|
||||
import platform
|
||||
import sys
|
||||
import os
|
||||
from os.path import dirname
|
||||
import shutil
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
|
||||
from bidict import bidict
|
||||
import toml
|
||||
|
|
@ -33,16 +33,9 @@ from .log import get_logger
|
|||
log = get_logger('broker-config')
|
||||
|
||||
|
||||
# XXX NOTE: taken from ``click`` since apparently they have some
|
||||
# taken from ``click`` since apparently they have some
|
||||
# super weirdness with sigint and sudo..no clue
|
||||
# we're probably going to slowly just modify it to our own version over
|
||||
# time..
|
||||
def get_app_dir(
|
||||
app_name: str,
|
||||
roaming: bool = True,
|
||||
force_posix: bool = False,
|
||||
|
||||
) -> str:
|
||||
def get_app_dir(app_name, roaming=True, force_posix=False):
|
||||
r"""Returns the config folder for the application. The default behavior
|
||||
is to return whatever is most appropriate for the operating system.
|
||||
|
||||
|
|
@ -81,30 +74,7 @@ def get_app_dir(
|
|||
def _posixify(name):
|
||||
return "-".join(name.split()).lower()
|
||||
|
||||
# NOTE: for testing with `pytest` we leverage the `tmp_dir`
|
||||
# fixture to generate (and clean up) a test-request-specific
|
||||
# directory for isolated configuration files such that,
|
||||
# - multiple tests can run (possibly in parallel) without data races
|
||||
# on the config state,
|
||||
# - we don't need to ever worry about leaking configs into the
|
||||
# system thus avoiding needing to manage config cleaup fixtures or
|
||||
# other bothers (since obviously `tmp_dir` cleans up after itself).
|
||||
#
|
||||
# In order to "pass down" the test dir path to all (sub-)actors in
|
||||
# the actor tree we preload the root actor's runtime vars state (an
|
||||
# internal mechanism for inheriting state down an actor tree in
|
||||
# `tractor`) with the testing dir and check for it whenever we
|
||||
# detect `pytest` is being used (which it isn't under normal
|
||||
# operation).
|
||||
if "pytest" in sys.modules:
|
||||
import tractor
|
||||
actor = tractor.current_actor(err_on_no_runtime=False)
|
||||
if actor: # runtime is up
|
||||
rvs = tractor._state._runtime_vars
|
||||
testdirpath = Path(rvs['piker_vars']['piker_test_dir'])
|
||||
assert testdirpath.exists(), 'piker test harness might be borked!?'
|
||||
app_name = str(testdirpath)
|
||||
|
||||
# if WIN:
|
||||
if platform.system() == 'Windows':
|
||||
key = "APPDATA" if roaming else "LOCALAPPDATA"
|
||||
folder = os.environ.get(key)
|
||||
|
|
@ -124,35 +94,28 @@ def get_app_dir(
|
|||
)
|
||||
|
||||
|
||||
_click_config_dir: Path = Path(get_app_dir('piker'))
|
||||
_config_dir: Path = _click_config_dir
|
||||
_parent_user: str = os.environ.get('SUDO_USER')
|
||||
_config_dir = _click_config_dir = get_app_dir('piker')
|
||||
_parent_user = os.environ.get('SUDO_USER')
|
||||
|
||||
if _parent_user:
|
||||
non_root_user_dir = Path(
|
||||
os.path.expanduser(f'~{_parent_user}')
|
||||
non_root_user_dir = os.path.expanduser(
|
||||
f'~{_parent_user}'
|
||||
)
|
||||
root: str = 'root'
|
||||
_ccds: str = str(_click_config_dir) # click config dir string
|
||||
i_tail: int = int(_ccds.rfind(root) + len(root))
|
||||
root = 'root'
|
||||
_config_dir = (
|
||||
non_root_user_dir
|
||||
/
|
||||
Path(_ccds[i_tail+1:]) # +1 to capture trailing '/'
|
||||
non_root_user_dir +
|
||||
_click_config_dir[
|
||||
_click_config_dir.rfind(root) + len(root):
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_conf_names: set[str] = {
|
||||
'brokers',
|
||||
# 'pps',
|
||||
'trades',
|
||||
'watchlists',
|
||||
'paper_trades'
|
||||
}
|
||||
|
||||
# TODO: probably drop all this super legacy, questrade specific,
|
||||
# config stuff XD ?
|
||||
_watchlists_data_path: Path = _config_dir / Path('watchlists.json')
|
||||
_watchlists_data_path = os.path.join(_config_dir, 'watchlists.json')
|
||||
_context_defaults = dict(
|
||||
default_map={
|
||||
# Questrade specific quote poll rates
|
||||
|
|
@ -183,85 +146,72 @@ def _conf_fn_w_ext(
|
|||
def get_conf_path(
|
||||
conf_name: str = 'brokers',
|
||||
|
||||
) -> Path:
|
||||
'''
|
||||
Return the top-level default config path normally under
|
||||
``~/.config/piker`` on linux for a given ``conf_name``, the config
|
||||
name.
|
||||
) -> str:
|
||||
"""Return the default config path normally under
|
||||
``~/.config/piker`` on linux.
|
||||
|
||||
Contains files such as:
|
||||
- brokers.toml
|
||||
- pp.toml
|
||||
- watchlists.toml
|
||||
- trades.toml
|
||||
|
||||
# maybe coming soon ;)
|
||||
- signals.toml
|
||||
- strats.toml
|
||||
|
||||
'''
|
||||
if 'pps.' not in conf_name:
|
||||
assert str(conf_name) in _conf_names
|
||||
|
||||
"""
|
||||
assert conf_name in _conf_names
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
return _config_dir / Path(fn)
|
||||
return os.path.join(
|
||||
_config_dir,
|
||||
fn,
|
||||
)
|
||||
|
||||
|
||||
def repodir() -> Path:
|
||||
def repodir():
|
||||
'''
|
||||
Return the abspath as ``Path`` to the git repo's root dir.
|
||||
Return the abspath to the repo directory.
|
||||
|
||||
'''
|
||||
return Path(__file__).absolute().parent.parent
|
||||
dirpath = os.path.abspath(
|
||||
# we're 3 levels down in **this** module file
|
||||
dirname(dirname(os.path.realpath(__file__)))
|
||||
)
|
||||
return dirpath
|
||||
|
||||
|
||||
def load(
|
||||
conf_name: str = 'brokers',
|
||||
path: Path | None = None,
|
||||
path: str = None
|
||||
|
||||
**tomlkws,
|
||||
|
||||
) -> tuple[dict, str]:
|
||||
) -> (dict, str):
|
||||
'''
|
||||
Load config file by name.
|
||||
|
||||
'''
|
||||
path: Path = path or get_conf_path(conf_name)
|
||||
path = path or get_conf_path(conf_name)
|
||||
if not os.path.isfile(path):
|
||||
fn = _conf_fn_w_ext(conf_name)
|
||||
|
||||
if not _config_dir.is_dir():
|
||||
_config_dir.mkdir(
|
||||
parents=True,
|
||||
exist_ok=True,
|
||||
template = os.path.join(
|
||||
repodir(),
|
||||
'config',
|
||||
fn
|
||||
)
|
||||
|
||||
if not path.is_file():
|
||||
fn: str = _conf_fn_w_ext(conf_name)
|
||||
|
||||
# try to copy in a template config to the user's directory if
|
||||
# one exists.
|
||||
template: Path = repodir() / 'config' / fn
|
||||
if template.is_file():
|
||||
# try to copy in a template config to the user's directory
|
||||
# if one exists.
|
||||
if os.path.isfile(template):
|
||||
shutil.copyfile(template, path)
|
||||
else:
|
||||
# create empty file
|
||||
with path.open(mode='x'):
|
||||
pass
|
||||
else:
|
||||
with path.open(mode='r'):
|
||||
pass # touch it
|
||||
|
||||
config: dict = toml.load(str(path), **tomlkws)
|
||||
config = toml.load(path)
|
||||
log.debug(f"Read config file {path}")
|
||||
return config, path
|
||||
|
||||
|
||||
def write(
|
||||
config: dict, # toml config as dict
|
||||
|
||||
name: str | None = None,
|
||||
path: Path | None = None,
|
||||
fail_empty: bool = True,
|
||||
|
||||
**toml_kwargs,
|
||||
name: str = 'brokers',
|
||||
path: str = None,
|
||||
|
||||
) -> None:
|
||||
''''
|
||||
|
|
@ -270,34 +220,26 @@ def write(
|
|||
Create a ``brokers.ini`` file if one does not exist.
|
||||
|
||||
'''
|
||||
if name:
|
||||
path: Path = path or get_conf_path(name)
|
||||
dirname: Path = path.parent
|
||||
if not dirname.is_dir():
|
||||
path = path or get_conf_path(name)
|
||||
dirname = os.path.dirname(path)
|
||||
if not os.path.isdir(dirname):
|
||||
log.debug(f"Creating config dir {_config_dir}")
|
||||
dirname.mkdir()
|
||||
os.makedirs(dirname)
|
||||
|
||||
if (
|
||||
not config
|
||||
and fail_empty
|
||||
):
|
||||
if not config:
|
||||
raise ValueError(
|
||||
"Watch out you're trying to write a blank config!"
|
||||
)
|
||||
"Watch out you're trying to write a blank config!")
|
||||
|
||||
log.debug(
|
||||
f"Writing config `{name}` file to:\n"
|
||||
f"{path}"
|
||||
)
|
||||
with path.open(mode='w') as cf:
|
||||
return toml.dump(
|
||||
config,
|
||||
cf,
|
||||
**toml_kwargs,
|
||||
)
|
||||
with open(path, 'w') as cf:
|
||||
return toml.dump(config, cf)
|
||||
|
||||
|
||||
def load_accounts(
|
||||
|
||||
providers: Optional[list[str]] = None
|
||||
|
||||
) -> bidict[str, Optional[str]]:
|
||||
|
|
|
|||
|
|
@ -22,12 +22,6 @@ and storing data from your brokers as well as
|
|||
sharing live streams over a network.
|
||||
|
||||
"""
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from ._util import (
|
||||
get_console_log,
|
||||
)
|
||||
from ._normalize import iterticks
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
|
|
@ -38,6 +32,7 @@ from ._sharedmem import (
|
|||
)
|
||||
from .feed import (
|
||||
open_feed,
|
||||
_setup_persistent_brokerd,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -49,4 +44,5 @@ __all__ = [
|
|||
'attach_shm_array',
|
||||
'open_shm_array',
|
||||
'get_shm_token',
|
||||
'_setup_persistent_brokerd',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -15,12 +15,9 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Supervisor for ``docker`` with included async and SC wrapping
|
||||
to ensure a cancellable container lifetime system.
|
||||
Supervisor for docker with included specific-image service helpers.
|
||||
|
||||
'''
|
||||
from collections import ChainMap
|
||||
from functools import partial
|
||||
import os
|
||||
import time
|
||||
from typing import (
|
||||
|
|
@ -40,27 +37,21 @@ from docker.models.containers import Container as DockerContainer
|
|||
from docker.errors import (
|
||||
DockerException,
|
||||
APIError,
|
||||
# ContainerError,
|
||||
)
|
||||
import requests
|
||||
from requests.exceptions import (
|
||||
ConnectionError,
|
||||
ReadTimeout,
|
||||
)
|
||||
from requests.exceptions import ConnectionError, ReadTimeout
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ..log import get_logger, get_console_log
|
||||
from .. import config
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DockerNotStarted(Exception):
|
||||
'Prolly you dint start da daemon bruh'
|
||||
|
||||
|
||||
class ApplicationLogError(Exception):
|
||||
'App in container reported an error in logs'
|
||||
class ContainerError(RuntimeError):
|
||||
'Error reported via app-container logging level'
|
||||
|
||||
|
||||
@acm
|
||||
|
|
@ -105,9 +96,9 @@ async def open_docker(
|
|||
# not perms?
|
||||
raise
|
||||
|
||||
# finally:
|
||||
# if client:
|
||||
# client.close()
|
||||
finally:
|
||||
if client:
|
||||
client.close()
|
||||
|
||||
|
||||
class Container:
|
||||
|
|
@ -128,19 +119,8 @@ class Container:
|
|||
|
||||
async def process_logs_until(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
|
||||
# this is a predicate func for matching log msgs emitted by the
|
||||
# underlying containerized app
|
||||
patt_matcher: Callable[[str], bool],
|
||||
|
||||
# XXX WARNING XXX: do not touch this sleep value unless
|
||||
# you know what you are doing! the value is critical to
|
||||
# making sure the caller code inside the startup context
|
||||
# does not timeout BEFORE we receive a match on the
|
||||
# ``patt_matcher()`` predicate above.
|
||||
checkpoint_period: float = 0.001,
|
||||
|
||||
patt: str,
|
||||
bp_on_msg: bool = False,
|
||||
) -> bool:
|
||||
'''
|
||||
Attempt to capture container log messages and relay through our
|
||||
|
|
@ -151,15 +131,6 @@ class Container:
|
|||
|
||||
while True:
|
||||
logs = self.cntr.logs()
|
||||
try:
|
||||
logs = self.cntr.logs()
|
||||
except (
|
||||
docker.errors.NotFound,
|
||||
docker.errors.APIError
|
||||
):
|
||||
log.exception('Failed to parse logs?')
|
||||
return False
|
||||
|
||||
entries = logs.decode().split('\n')
|
||||
for entry in entries:
|
||||
|
||||
|
|
@ -167,44 +138,34 @@ class Container:
|
|||
if not entry:
|
||||
continue
|
||||
|
||||
entry = entry.strip()
|
||||
try:
|
||||
record = json.loads(entry)
|
||||
msg = record[log_msg_key]
|
||||
level = record['level']
|
||||
|
||||
record = json.loads(entry.strip())
|
||||
except json.JSONDecodeError:
|
||||
msg = entry
|
||||
level = 'error'
|
||||
if 'Error' in entry:
|
||||
raise RuntimeError(entry)
|
||||
raise
|
||||
|
||||
# TODO: do we need a more general mechanism
|
||||
# for these kinda of "log record entries"?
|
||||
# if 'Error' in entry:
|
||||
# raise RuntimeError(entry)
|
||||
|
||||
if (
|
||||
msg
|
||||
and entry not in seen_so_far
|
||||
):
|
||||
msg = record['msg']
|
||||
level = record['level']
|
||||
if msg and entry not in seen_so_far:
|
||||
seen_so_far.add(entry)
|
||||
getattr(log, level.lower(), log.error)(f'{msg}')
|
||||
if bp_on_msg:
|
||||
await tractor.breakpoint()
|
||||
|
||||
if level == 'fatal':
|
||||
raise ApplicationLogError(msg)
|
||||
getattr(log, level, log.error)(f'{msg}')
|
||||
|
||||
if await patt_matcher(msg):
|
||||
# print(f'level: {level}')
|
||||
if level in ('error', 'fatal'):
|
||||
raise ContainerError(msg)
|
||||
|
||||
if patt in msg:
|
||||
return True
|
||||
|
||||
# do a checkpoint so we don't block if cancelled B)
|
||||
await trio.sleep(checkpoint_period)
|
||||
await trio.sleep(0.01)
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def cuid(self) -> str:
|
||||
fqcn: str = self.cntr.attrs['Config']['Image']
|
||||
return f'{fqcn}[{self.cntr.short_id}]'
|
||||
|
||||
def try_signal(
|
||||
self,
|
||||
signal: str = 'SIGINT',
|
||||
|
|
@ -224,66 +185,30 @@ class Container:
|
|||
if 'is not running' in err.explanation:
|
||||
return False
|
||||
|
||||
def hard_kill(self, start: float) -> None:
|
||||
delay = time.time() - start
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
log.error(
|
||||
f'SIGKILL-ing: {self.cntr.id} after {delay}s\n'
|
||||
)
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
log_msg_key: str,
|
||||
stop_predicate: Callable[[str], bool],
|
||||
|
||||
hard_kill: bool = False,
|
||||
|
||||
stop_msg: str,
|
||||
) -> None:
|
||||
'''
|
||||
Attempt to cancel this container gracefully, fail over to
|
||||
a hard kill on timeout.
|
||||
|
||||
'''
|
||||
cid = self.cntr.id
|
||||
|
||||
# first try a graceful cancel
|
||||
log.cancel(
|
||||
f'SIGINT cancelling container: {self.cuid}\n'
|
||||
'waiting on stop predicate...'
|
||||
f'SIGINT cancelling container: {cid}\n'
|
||||
f'waiting on stop msg: "{stop_msg}"'
|
||||
)
|
||||
self.try_signal('SIGINT')
|
||||
|
||||
start = time.time()
|
||||
for _ in range(6):
|
||||
for _ in range(30):
|
||||
|
||||
with trio.move_on_after(0.5) as cs:
|
||||
log.cancel('polling for CNTR logs...')
|
||||
cs.shield = True
|
||||
await self.process_logs_until(stop_msg)
|
||||
|
||||
try:
|
||||
await self.process_logs_until(
|
||||
log_msg_key,
|
||||
stop_predicate,
|
||||
)
|
||||
except ApplicationLogError:
|
||||
hard_kill = True
|
||||
else:
|
||||
# if we aren't cancelled on above checkpoint then we
|
||||
# assume we read the expected stop msg and
|
||||
# terminated.
|
||||
# assume we read the expected stop msg and terminated.
|
||||
break
|
||||
|
||||
if cs.cancelled_caught:
|
||||
# on timeout just try a hard kill after
|
||||
# a quick container sync-wait.
|
||||
hard_kill = True
|
||||
|
||||
try:
|
||||
log.info(f'Polling for container shutdown:\n{cid}')
|
||||
|
||||
|
|
@ -293,7 +218,6 @@ class Container:
|
|||
condition='not-running',
|
||||
)
|
||||
|
||||
# graceful exit if we didn't time out
|
||||
break
|
||||
|
||||
except (
|
||||
|
|
@ -305,22 +229,24 @@ class Container:
|
|||
except (
|
||||
docker.errors.APIError,
|
||||
ConnectionError,
|
||||
requests.exceptions.ConnectionError,
|
||||
trio.Cancelled,
|
||||
):
|
||||
log.exception('Docker connection failure')
|
||||
self.hard_kill(start)
|
||||
raise
|
||||
|
||||
except trio.Cancelled:
|
||||
log.exception('trio cancelled...')
|
||||
self.hard_kill(start)
|
||||
break
|
||||
else:
|
||||
hard_kill = True
|
||||
delay = time.time() - start
|
||||
log.error(
|
||||
f'Failed to kill container {cid} after {delay}s\n'
|
||||
'sending SIGKILL..'
|
||||
)
|
||||
# get out the big guns, bc apparently marketstore
|
||||
# doesn't actually know how to terminate gracefully
|
||||
# :eyeroll:...
|
||||
self.try_signal('SIGKILL')
|
||||
self.cntr.wait(
|
||||
timeout=3,
|
||||
condition='not-running',
|
||||
)
|
||||
|
||||
if hard_kill:
|
||||
self.hard_kill(start)
|
||||
else:
|
||||
log.cancel(f'Container stopped: {cid}')
|
||||
|
||||
|
||||
|
|
@ -328,16 +254,11 @@ class Container:
|
|||
async def open_ahabd(
|
||||
ctx: tractor.Context,
|
||||
endpoint: str, # ns-pointer str-msg-type
|
||||
loglevel: str | None = 'cancel',
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> None:
|
||||
|
||||
log = get_console_log(
|
||||
loglevel,
|
||||
name=__name__,
|
||||
)
|
||||
get_console_log('info', name=__name__)
|
||||
|
||||
async with open_docker() as client:
|
||||
|
||||
|
|
@ -348,84 +269,26 @@ async def open_ahabd(
|
|||
(
|
||||
dcntr,
|
||||
cntr_config,
|
||||
start_pred,
|
||||
stop_pred,
|
||||
start_msg,
|
||||
stop_msg,
|
||||
) = ep_func(client)
|
||||
cntr = Container(dcntr)
|
||||
|
||||
conf: ChainMap[str, Any] = ChainMap(
|
||||
with trio.move_on_after(1):
|
||||
found = await cntr.process_logs_until(start_msg)
|
||||
|
||||
# container specific
|
||||
cntr_config,
|
||||
|
||||
# defaults
|
||||
{
|
||||
# startup time limit which is the max the supervisor
|
||||
# will wait for the container to be registered in
|
||||
# ``client.containers.list()``
|
||||
'startup_timeout': 1.0,
|
||||
|
||||
# how fast to poll for the starup predicate by sleeping
|
||||
# this amount incrementally thus yielding to the
|
||||
# ``trio`` scheduler on during sync polling execution.
|
||||
'startup_query_period': 0.001,
|
||||
|
||||
# str-key value expected to contain log message body-contents
|
||||
# when read using:
|
||||
# ``json.loads(entry for entry in DockerContainer.logs())``
|
||||
'log_msg_key': 'msg',
|
||||
|
||||
|
||||
# startup sync func, like `Nursery.started()`
|
||||
'started_afunc': None,
|
||||
},
|
||||
if not found and cntr not in client.containers.list():
|
||||
raise RuntimeError(
|
||||
'Failed to start `marketstore` check logs deats'
|
||||
)
|
||||
|
||||
try:
|
||||
with trio.move_on_after(conf['startup_timeout']) as cs:
|
||||
async with trio.open_nursery() as tn:
|
||||
tn.start_soon(
|
||||
partial(
|
||||
cntr.process_logs_until,
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
patt_matcher=start_pred,
|
||||
checkpoint_period=conf['startup_query_period'],
|
||||
)
|
||||
)
|
||||
|
||||
# optional blocking routine
|
||||
started = conf['started_afunc']
|
||||
if started:
|
||||
await started()
|
||||
|
||||
# poll for container startup or timeout
|
||||
while not cs.cancel_called:
|
||||
if dcntr in client.containers.list():
|
||||
break
|
||||
|
||||
await trio.sleep(conf['startup_query_period'])
|
||||
|
||||
# sync with remote caller actor-task but allow log
|
||||
# processing to continue running in bg.
|
||||
await ctx.started((
|
||||
cntr.cntr.id,
|
||||
os.getpid(),
|
||||
cntr_config,
|
||||
))
|
||||
|
||||
# XXX: if we timeout on finding the "startup msg" we
|
||||
# expect then we want to FOR SURE raise an error
|
||||
# upwards!
|
||||
if cs.cancelled_caught:
|
||||
# if dcntr not in client.containers.list():
|
||||
for entry in cntr.seen_so_far:
|
||||
log.info(entry)
|
||||
|
||||
raise DockerNotStarted(
|
||||
f'Failed to start container: {cntr.cuid}\n'
|
||||
f'due to timeout={conf["startup_timeout"]}s\n\n'
|
||||
"check ur container's logs!"
|
||||
)
|
||||
try:
|
||||
|
||||
# TODO: we might eventually want a proxy-style msg-prot here
|
||||
# to allow remote control of containers without needing
|
||||
|
|
@ -433,25 +296,13 @@ async def open_ahabd(
|
|||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
# TODO: ensure loglevel can be set and teardown logs are
|
||||
# reported if possible on error or cancel..
|
||||
# XXX WARNING: currently shielding here can result in hangs
|
||||
# on ctl-c from user.. ideally we can avoid a cancel getting
|
||||
# consumed and not propagating whilst still doing teardown
|
||||
# logging..
|
||||
with trio.CancelScope(shield=True):
|
||||
await cntr.cancel(
|
||||
log_msg_key=conf['log_msg_key'],
|
||||
stop_predicate=stop_pred,
|
||||
)
|
||||
await cntr.cancel(stop_msg)
|
||||
|
||||
|
||||
async def start_ahab(
|
||||
service_name: str,
|
||||
endpoint: Callable[docker.DockerClient, DockerContainer],
|
||||
loglevel: str | None = 'cancel',
|
||||
drop_root_perms: bool = True,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.Event,
|
||||
|
|
@ -472,12 +323,13 @@ async def start_ahab(
|
|||
'''
|
||||
cn_ready = trio.Event()
|
||||
try:
|
||||
async with tractor.open_nursery() as an:
|
||||
async with tractor.open_nursery(
|
||||
loglevel='runtime',
|
||||
) as tn:
|
||||
|
||||
portal = await an.start_actor(
|
||||
portal = await tn.start_actor(
|
||||
service_name,
|
||||
enable_modules=[__name__],
|
||||
loglevel=loglevel,
|
||||
enable_modules=[__name__]
|
||||
)
|
||||
|
||||
# TODO: we have issues with this on teardown
|
||||
|
|
@ -487,10 +339,7 @@ async def start_ahab(
|
|||
|
||||
# de-escalate root perms to the original user
|
||||
# after the docker supervisor actor is spawned.
|
||||
if (
|
||||
drop_root_perms
|
||||
and config._parent_user
|
||||
):
|
||||
if config._parent_user:
|
||||
import pwd
|
||||
os.setuid(
|
||||
pwd.getpwnam(
|
||||
|
|
@ -501,7 +350,6 @@ async def start_ahab(
|
|||
async with portal.open_context(
|
||||
open_ahabd,
|
||||
endpoint=str(NamespacePath.from_ref(endpoint)),
|
||||
loglevel='cancel',
|
||||
) as (ctx, first):
|
||||
|
||||
cid, pid, cntr_config = first
|
||||
|
|
@ -1,827 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Pre-(path)-graphics formatted x/y nd/1d rendering subsystem.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import msgspec
|
||||
from msgspec import field
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ._sharedmem import (
|
||||
ShmArray,
|
||||
)
|
||||
from ._pathops import (
|
||||
path_arrays_from_ohlc,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._dataviz import (
|
||||
Viz,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
|
||||
class IncrementalFormatter(msgspec.Struct):
|
||||
'''
|
||||
Incrementally updating, pre-path-graphics tracking, formatter.
|
||||
|
||||
Allows tracking source data state in an updateable pre-graphics
|
||||
``np.ndarray`` format (in local process memory) as well as
|
||||
incrementally rendering from that format **to** 1d x/y for path
|
||||
generation using ``pg.functions.arrayToQPath()``.
|
||||
|
||||
'''
|
||||
shm: ShmArray
|
||||
viz: Viz
|
||||
|
||||
# the value to be multiplied any any index into the x/y_1d arrays
|
||||
# given the input index is based on the original source data array.
|
||||
flat_index_ratio: float = 1
|
||||
|
||||
@property
|
||||
def index_field(self) -> 'str':
|
||||
'''
|
||||
Value (``str``) used to look up the "index series" from the
|
||||
underlying source ``numpy`` struct-array; delegate directly to
|
||||
the managing ``Viz``.
|
||||
|
||||
'''
|
||||
return self.viz.index_field
|
||||
|
||||
# Incrementally updated xy ndarray formatted data, a pre-1d
|
||||
# format which is updated and cached independently of the final
|
||||
# pre-graphics-path 1d format.
|
||||
x_nd: Optional[np.ndarray] = None
|
||||
y_nd: Optional[np.ndarray] = None
|
||||
|
||||
@property
|
||||
def xy_nd(self) -> tuple[np.ndarray, np.ndarray]:
|
||||
return (
|
||||
self.x_nd[self.xy_slice],
|
||||
self.y_nd[self.xy_slice],
|
||||
)
|
||||
|
||||
@property
|
||||
def xy_slice(self) -> slice:
|
||||
return slice(
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
)
|
||||
|
||||
# indexes which slice into the above arrays (which are allocated
|
||||
# based on source data shm input size) and allow retrieving
|
||||
# incrementally updated data.
|
||||
xy_nd_start: int | None = None
|
||||
xy_nd_stop: int | None = None
|
||||
|
||||
# TODO: eventually incrementally update 1d-pre-graphics path data?
|
||||
x_1d: np.ndarray | None = None
|
||||
y_1d: np.ndarray | None = None
|
||||
|
||||
# incremental view-change state(s) tracking
|
||||
_last_vr: tuple[float, float] | None = None
|
||||
_last_ivdr: tuple[float, float] | None = None
|
||||
|
||||
@property
|
||||
def index_step_size(self) -> float:
|
||||
'''
|
||||
Readonly value computed on first ``.diff()`` call.
|
||||
|
||||
'''
|
||||
return self.viz.index_step()
|
||||
|
||||
def diff(
|
||||
self,
|
||||
new_read: tuple[np.ndarray],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
# TODO:
|
||||
# - can the renderer just call ``Viz.read()`` directly? unpack
|
||||
# latest source data read
|
||||
# - eventually maybe we can implement some kind of
|
||||
# transform on the ``QPainterPath`` that will more or less
|
||||
# detect the diff in "elements" terms? update diff state since
|
||||
# we've now rendered paths.
|
||||
(
|
||||
xfirst,
|
||||
xlast,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
) = new_read
|
||||
|
||||
index = array['index']
|
||||
|
||||
# if the first index in the read array is 0 then
|
||||
# it means the source buffer has bee completely backfilled to
|
||||
# available space.
|
||||
src_start = index[0]
|
||||
src_stop = index[-1] + 1
|
||||
|
||||
# these are the "formatted output data" indices
|
||||
# for the pre-graphics arrays.
|
||||
nd_start = self.xy_nd_start
|
||||
nd_stop = self.xy_nd_stop
|
||||
|
||||
if (
|
||||
nd_start is None
|
||||
):
|
||||
assert nd_stop is None
|
||||
|
||||
# setup to do a prepend of all existing src history
|
||||
nd_start = self.xy_nd_start = src_stop
|
||||
# set us in a zero-to-append state
|
||||
nd_stop = self.xy_nd_stop = src_stop
|
||||
|
||||
# compute the length diffs between the first/last index entry in
|
||||
# the input data and the last indexes we have on record from the
|
||||
# last time we updated the curve index.
|
||||
prepend_length = int(nd_start - src_start)
|
||||
append_length = int(src_stop - nd_stop)
|
||||
|
||||
# blah blah blah
|
||||
# do diffing for prepend, append and last entry
|
||||
return (
|
||||
slice(src_start, nd_start),
|
||||
prepend_length,
|
||||
append_length,
|
||||
slice(nd_stop, src_stop),
|
||||
)
|
||||
|
||||
def _track_inview_range(
|
||||
self,
|
||||
view_range: tuple[int, int],
|
||||
|
||||
) -> bool:
|
||||
# if a view range is passed, plan to draw the
|
||||
# source ouput that's "in view" of the chart.
|
||||
vl, vr = view_range
|
||||
zoom_or_append = False
|
||||
last_vr = self._last_vr
|
||||
|
||||
# incremental in-view data update.
|
||||
if last_vr:
|
||||
lvl, lvr = last_vr # relative slice indices
|
||||
|
||||
# TODO: detecting more specifically the interaction changes
|
||||
# last_ivr = self._last_ivdr or (vl, vr)
|
||||
# al, ar = last_ivr # abs slice indices
|
||||
# left_change = abs(x_iv[0] - al) >= 1
|
||||
# right_change = abs(x_iv[-1] - ar) >= 1
|
||||
|
||||
# likely a zoom/pan view change or data append update
|
||||
if (
|
||||
(vr - lvr) > 2
|
||||
or vl < lvl
|
||||
|
||||
# append / prepend update
|
||||
# we had an append update where the view range
|
||||
# didn't change but the data-viewed (shifted)
|
||||
# underneath, so we need to redraw.
|
||||
# or left_change and right_change and last_vr == view_range
|
||||
|
||||
# not (left_change and right_change) and ivr
|
||||
# (
|
||||
# or abs(x_iv[ivr] - livr) > 1
|
||||
):
|
||||
zoom_or_append = True
|
||||
|
||||
self._last_vr = view_range
|
||||
|
||||
return zoom_or_append
|
||||
|
||||
def format_to_1d(
|
||||
self,
|
||||
new_read: tuple,
|
||||
array_key: str,
|
||||
profiler: Profiler,
|
||||
|
||||
slice_to_inview: bool = True,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
shm = self.shm
|
||||
|
||||
(
|
||||
_,
|
||||
_,
|
||||
array,
|
||||
ivl,
|
||||
ivr,
|
||||
in_view,
|
||||
|
||||
) = new_read
|
||||
|
||||
(
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
append_len,
|
||||
post_slice,
|
||||
) = self.diff(new_read)
|
||||
|
||||
# we first need to allocate xy data arrays
|
||||
# from the source data.
|
||||
if self.y_nd is None:
|
||||
self.xy_nd_start = shm._first.value
|
||||
self.xy_nd_stop = shm._last.value
|
||||
self.x_nd, self.y_nd = self.allocate_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
)
|
||||
profiler('allocated xy history')
|
||||
|
||||
# once allocated we do incremental pre/append
|
||||
# updates from the diff with the source buffer.
|
||||
else:
|
||||
if prepend_len:
|
||||
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
# this is the pre-sliced, "normally expected"
|
||||
# new data that an updater would normally be
|
||||
# expected to process, however in some cases (like
|
||||
# step curves) the updater routine may want to do
|
||||
# the source history-data reading itself, so we pass
|
||||
# both here.
|
||||
shm._array[pre_slice],
|
||||
pre_slice,
|
||||
prepend_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=False,
|
||||
)
|
||||
|
||||
self.xy_nd_start -= prepend_len
|
||||
profiler('prepended xy history: {prepend_length}')
|
||||
|
||||
if append_len:
|
||||
self.incr_update_xy_nd(
|
||||
shm,
|
||||
array_key,
|
||||
|
||||
shm._array[post_slice],
|
||||
post_slice,
|
||||
append_len,
|
||||
|
||||
self.xy_nd_start,
|
||||
self.xy_nd_stop,
|
||||
is_append=True,
|
||||
)
|
||||
self.xy_nd_stop += append_len
|
||||
profiler('appened xy history: {append_length}')
|
||||
# sanity
|
||||
# slice_ln = post_slice.stop - post_slice.start
|
||||
# assert append_len == slice_ln
|
||||
|
||||
view_changed: bool = False
|
||||
view_range: tuple[int, int] = (ivl, ivr)
|
||||
if slice_to_inview:
|
||||
view_changed = self._track_inview_range(view_range)
|
||||
array = in_view
|
||||
profiler(f'{self.viz.name} view range slice {view_range}')
|
||||
|
||||
# TODO: we need to check if the last-datum-in-view is true and
|
||||
# if so only slice to the 2nd last datumonly slice to the 2nd
|
||||
# last datum.
|
||||
# hist = array[:slice_to_head]
|
||||
|
||||
# XXX: WOA WTF TRACTOR DEBUGGING BUGGG
|
||||
# assert 0
|
||||
|
||||
# xy-path data transform: convert source data to a format
|
||||
# able to be passed to a `QPainterPath` rendering routine.
|
||||
if not len(array):
|
||||
# XXX: this might be why the profiler only has exits?
|
||||
return
|
||||
|
||||
# TODO: hist here should be the pre-sliced
|
||||
# x/y_data in the case where allocate_xy is
|
||||
# defined?
|
||||
x_1d, y_1d, connect = self.format_xy_nd_to_1d(
|
||||
array,
|
||||
array_key,
|
||||
view_range,
|
||||
)
|
||||
# cache/save last 1d outputs for use by other
|
||||
# readers (eg. `Viz.draw_last_datum()` in the
|
||||
# only-draw-last-uppx case).
|
||||
self.x_1d = x_1d
|
||||
self.y_1d = y_1d
|
||||
|
||||
# app_tres = None
|
||||
# if append_len:
|
||||
# appended = array[-append_len-1:slice_to_head]
|
||||
# app_tres = self.format_xy_nd_to_1d(
|
||||
# appended,
|
||||
# array_key,
|
||||
# (
|
||||
# view_range[1] - append_len + slice_to_head,
|
||||
# view_range[1]
|
||||
# ),
|
||||
# )
|
||||
# # assert (len(appended) - 1) == append_len
|
||||
# # assert len(appended) == append_len
|
||||
# print(
|
||||
# f'{self.viz.name} APPEND LEN: {append_len}\n'
|
||||
# f'{self.viz.name} APPENDED: {appended}\n'
|
||||
# f'{self.viz.name} app_tres: {app_tres}\n'
|
||||
# )
|
||||
|
||||
# update the last "in view data range"
|
||||
if len(x_1d):
|
||||
self._last_ivdr = x_1d[0], x_1d[-1]
|
||||
|
||||
profiler('.format_to_1d()')
|
||||
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
connect,
|
||||
prepend_len,
|
||||
append_len,
|
||||
view_changed,
|
||||
# app_tres,
|
||||
)
|
||||
|
||||
###############################
|
||||
# Sub-type override interface #
|
||||
###############################
|
||||
|
||||
x_offset: np.ndarray = np.array([0])
|
||||
|
||||
# optional pre-graphics xy formatted data which
|
||||
# is incrementally updated in sync with the source data.
|
||||
# XXX: was ``.allocate_xy()``
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert the structured-array ``src_shm`` format to
|
||||
a equivalently shaped (and field-less) ``np.ndarray``.
|
||||
|
||||
Eg. a 4 field x N struct-array => (N, 4)
|
||||
|
||||
'''
|
||||
y_nd = src_shm._array[data_field].copy()
|
||||
x_nd = (
|
||||
src_shm._array[self.index_field].copy()
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
return x_nd, y_nd
|
||||
|
||||
# XXX: was ``.update_xy()``
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write pushed data to flattened copy
|
||||
y_nd_new = new_from_src[data_field]
|
||||
self.y_nd[read_slc] = y_nd_new
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name.upper()}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{list(x_nd[-s:])}\n'
|
||||
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{list(y_nd[-s:])}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new[0]}\n'
|
||||
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
# XXX: was ``.format_xy()``
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # 1d x
|
||||
np.ndarray, # 1d y
|
||||
np.ndarray | str, # connection array/style
|
||||
]:
|
||||
'''
|
||||
Default xy-nd array to 1d pre-graphics-path render routine.
|
||||
|
||||
Return single field column data verbatim
|
||||
|
||||
'''
|
||||
# NOTE: we don't include the very last datum which is filled in
|
||||
# normally by another graphics object.
|
||||
x_1d = array[self.index_field][:-1]
|
||||
y_1d = array[array_key][:-1]
|
||||
|
||||
# name = self.viz.name
|
||||
# if 'trade_rate' == name:
|
||||
# s = 4
|
||||
# x_nd = list(self.x_nd[self.xy_slice][-s:-1])
|
||||
# y_nd = list(self.y_nd[self.xy_slice][-s:-1])
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# f'XY data:\n'
|
||||
# f'x: {x_nd}\n'
|
||||
# f'y: {y_nd}\n\n'
|
||||
# f'x_1d: {list(x_1d[-s:])}\n'
|
||||
# f'y_1d: {list(y_1d[-s:])}\n\n'
|
||||
|
||||
# )
|
||||
return (
|
||||
x_1d,
|
||||
y_1d,
|
||||
|
||||
# 1d connection array or style-key to
|
||||
# ``pg.functions.arrayToQPath()``
|
||||
'all',
|
||||
)
|
||||
|
||||
|
||||
class OHLCBarsFmtr(IncrementalFormatter):
|
||||
x_offset: np.ndarray = np.array([
|
||||
-0.5,
|
||||
0,
|
||||
0,
|
||||
0.5,
|
||||
])
|
||||
|
||||
fields: list[str] = field(
|
||||
default_factory=lambda: ['open', 'high', 'low', 'close']
|
||||
)
|
||||
flat_index_ratio: float = 4
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
ohlc_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input struct-array holding OHLC samples into a pair of
|
||||
flattened x, y arrays with the same size (datums wise) as the source
|
||||
data.
|
||||
|
||||
'''
|
||||
y_nd = ohlc_shm.ustruct(self.fields)
|
||||
|
||||
# generate an flat-interpolated x-domain
|
||||
x_nd = (
|
||||
np.broadcast_to(
|
||||
ohlc_shm._array[self.index_field][:, None],
|
||||
(
|
||||
ohlc_shm._array.size,
|
||||
# 4, # only ohlc
|
||||
y_nd.shape[1],
|
||||
),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
assert y_nd.any()
|
||||
|
||||
# write pushed data to flattened copy
|
||||
return (
|
||||
x_nd,
|
||||
y_nd,
|
||||
)
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> None:
|
||||
# write newly pushed data to flattened copy
|
||||
# a struct-arr is always passed in.
|
||||
new_y_nd = rfn.structured_to_unstructured(
|
||||
new_from_src[self.fields]
|
||||
)
|
||||
self.y_nd[read_slc] = new_y_nd
|
||||
|
||||
# generate same-valued-per-row x support based on y shape
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = np.broadcast_to(
|
||||
new_from_src[self.index_field][:, None],
|
||||
new_y_nd.shape,
|
||||
) + self.x_offset
|
||||
|
||||
# TODO: can we drop this frame and just use the above?
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
start: int = 0, # XXX: do we need this?
|
||||
# 0.5 is no overlap between arms, 1.0 is full overlap
|
||||
w: float = 0.16,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
More or less direct proxy to the ``numba``-fied
|
||||
``path_arrays_from_ohlc()`` (above) but with closed in kwargs
|
||||
for line spacing.
|
||||
|
||||
'''
|
||||
x, y, c = path_arrays_from_ohlc(
|
||||
array[:-1],
|
||||
start,
|
||||
bar_w=self.index_step_size,
|
||||
bar_gap=w * self.index_step_size,
|
||||
|
||||
# XXX: don't ask, due to a ``numba`` bug..
|
||||
use_time_index=(self.index_field == 'time'),
|
||||
)
|
||||
return x, y, c
|
||||
|
||||
|
||||
class OHLCBarsAsCurveFmtr(OHLCBarsFmtr):
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
# TODO: in the case of an existing ``.update_xy()``
|
||||
# should we be passing in array as an xy arrays tuple?
|
||||
|
||||
# 2 more datum-indexes to capture zero at end
|
||||
x_flat = self.x_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
y_flat = self.y_nd[self.xy_nd_start:self.xy_nd_stop-1]
|
||||
|
||||
# slice to view
|
||||
ivl, ivr = vr
|
||||
x_iv_flat = x_flat[ivl:ivr]
|
||||
y_iv_flat = y_flat[ivl:ivr]
|
||||
|
||||
# reshape to 1d for graphics rendering
|
||||
y_iv = y_iv_flat.reshape(-1)
|
||||
x_iv = x_iv_flat.reshape(-1)
|
||||
|
||||
return x_iv, y_iv, 'all'
|
||||
|
||||
|
||||
class StepCurveFmtr(IncrementalFormatter):
|
||||
|
||||
x_offset: np.ndarray = np.array([
|
||||
0,
|
||||
1,
|
||||
])
|
||||
|
||||
def allocate_xy_nd(
|
||||
self,
|
||||
|
||||
shm: ShmArray,
|
||||
data_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # x
|
||||
np.nd.array # y
|
||||
]:
|
||||
'''
|
||||
Convert an input 1d shm array to a "step array" format
|
||||
for use by path graphics generation.
|
||||
|
||||
'''
|
||||
i = shm._array[self.index_field].copy()
|
||||
out = shm._array[data_field].copy()
|
||||
|
||||
x_out = (
|
||||
np.broadcast_to(
|
||||
i[:, None],
|
||||
(i.size, 2),
|
||||
)
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# fill out Nx2 array to hold each step's left + right vertices.
|
||||
y_out = np.empty(
|
||||
x_out.shape,
|
||||
dtype=out.dtype,
|
||||
)
|
||||
# fill in (current) values from source shm buffer
|
||||
y_out[:] = out[:, np.newaxis]
|
||||
|
||||
# TODO: pretty sure we can drop this?
|
||||
# start y at origin level
|
||||
# y_out[0, 0] = 0
|
||||
# y_out[self.xy_nd_start] = 0
|
||||
return x_out, y_out
|
||||
|
||||
def incr_update_xy_nd(
|
||||
self,
|
||||
|
||||
src_shm: ShmArray,
|
||||
array_key: str,
|
||||
|
||||
new_from_src: np.ndarray, # portion of source that was updated
|
||||
read_slc: slice,
|
||||
ln: int, # len of updated
|
||||
|
||||
nd_start: int,
|
||||
nd_stop: int,
|
||||
|
||||
is_append: bool,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
slice,
|
||||
]:
|
||||
# NOTE: for a step curve we slice from one datum prior
|
||||
# to the current "update slice" to get the previous
|
||||
# "level".
|
||||
#
|
||||
# why this is needed,
|
||||
# - the current new append slice will often have a zero
|
||||
# value in the latest datum-step (at least for zero-on-new
|
||||
# cases like vlm in the) as per configuration of the FSP
|
||||
# engine.
|
||||
# - we need to look back a datum to get the last level which
|
||||
# will be used to terminate/complete the last step x-width
|
||||
# which will be set to pair with the last x-index THIS MEANS
|
||||
#
|
||||
# XXX: this means WE CAN'T USE the append slice since we need to
|
||||
# "look backward" one step to get the needed back-to-zero level
|
||||
# and the update data in ``new_from_src`` will only contain the
|
||||
# latest new data.
|
||||
back_1 = slice(
|
||||
read_slc.start - 1,
|
||||
read_slc.stop,
|
||||
)
|
||||
|
||||
to_write = src_shm._array[back_1]
|
||||
y_nd_new = self.y_nd[back_1]
|
||||
y_nd_new[:] = to_write[array_key][:, None]
|
||||
|
||||
x_nd_new = self.x_nd[read_slc]
|
||||
x_nd_new[:] = (
|
||||
new_from_src[self.index_field][:, None]
|
||||
+
|
||||
self.x_offset
|
||||
)
|
||||
|
||||
# XXX: uncomment for debugging
|
||||
# x_nd = self.x_nd[self.xy_slice]
|
||||
# y_nd = self.y_nd[self.xy_slice]
|
||||
# name = self.viz.name
|
||||
# if 'dolla_vlm' in name:
|
||||
# s = 4
|
||||
# print(
|
||||
# f'{name}:\n'
|
||||
# 'NEW_FROM_SRC:\n'
|
||||
# f'new_from_src: {new_from_src}\n\n'
|
||||
|
||||
# f'PRE self.x_nd:'
|
||||
# f'\n{x_nd[-s:]}\n'
|
||||
# f'PRE self.y_nd:\n'
|
||||
# f'{y_nd[-s:]}\n\n'
|
||||
|
||||
# f'TO WRITE:\n'
|
||||
# f'x_nd_new:\n'
|
||||
# f'{x_nd_new}\n'
|
||||
# f'y_nd_new:\n'
|
||||
# f'{y_nd_new}\n'
|
||||
# )
|
||||
|
||||
def format_xy_nd_to_1d(
|
||||
self,
|
||||
|
||||
array: np.ndarray,
|
||||
array_key: str,
|
||||
vr: tuple[int, int],
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
str,
|
||||
]:
|
||||
last_t, last = array[-1][[self.index_field, array_key]]
|
||||
|
||||
start = self.xy_nd_start
|
||||
stop = self.xy_nd_stop
|
||||
|
||||
x_step = self.x_nd[start:stop]
|
||||
y_step = self.y_nd[start:stop]
|
||||
|
||||
# slice out in-view data
|
||||
ivl, ivr = vr
|
||||
|
||||
# NOTE: add an extra step to get the vertical-line-down-to-zero
|
||||
# adjacent to the last-datum graphic (filled rect).
|
||||
x_step_iv = x_step[ivl:ivr+1]
|
||||
y_step_iv = y_step[ivl:ivr+1]
|
||||
|
||||
# flatten to 1d
|
||||
x_1d = x_step_iv.reshape(x_step_iv.size)
|
||||
y_1d = y_step_iv.reshape(y_step_iv.size)
|
||||
|
||||
# debugging
|
||||
# if y_1d.any():
|
||||
# s = 6
|
||||
# print(
|
||||
# f'x_step_iv:\n{x_step_iv[-s:]}\n'
|
||||
# f'y_step_iv:\n{y_step_iv[-s:]}\n\n'
|
||||
# f'x_1d:\n{x_1d[-s:]}\n'
|
||||
# f'y_1d:\n{y_1d[-s:]}\n'
|
||||
# )
|
||||
|
||||
return x_1d, y_1d, 'all'
|
||||
|
|
@ -56,7 +56,7 @@ def iterticks(
|
|||
sig = (
|
||||
time,
|
||||
tick['price'],
|
||||
tick.get('size')
|
||||
tick['size']
|
||||
)
|
||||
|
||||
if ttype == 'dark_trade':
|
||||
|
|
|
|||
|
|
@ -1,446 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Super fast ``QPainterPath`` generation related operator routines.
|
||||
|
||||
"""
|
||||
from math import (
|
||||
ceil,
|
||||
floor,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
from numba import (
|
||||
# types,
|
||||
njit,
|
||||
float64,
|
||||
int64,
|
||||
# optional,
|
||||
)
|
||||
|
||||
# TODO: for ``numba`` typing..
|
||||
# from ._source import numba_ohlc_dtype
|
||||
from ._m4 import ds_m4
|
||||
from .._profile import (
|
||||
Profiler,
|
||||
pg_profile_enabled,
|
||||
ms_slower_then,
|
||||
)
|
||||
|
||||
|
||||
def xy_downsample(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
|
||||
x_spacer: float = 0.5,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
float,
|
||||
float,
|
||||
]:
|
||||
'''
|
||||
Downsample 1D (flat ``numpy.ndarray``) arrays using M4 given an input
|
||||
``uppx`` (units-per-pixel) and add space between discreet datums.
|
||||
|
||||
'''
|
||||
# downsample whenever more then 1 pixels per datum can be shown.
|
||||
# always refresh data bounds until we get diffing
|
||||
# working properly, see above..
|
||||
m4_out = ds_m4(
|
||||
x,
|
||||
y,
|
||||
uppx,
|
||||
)
|
||||
|
||||
if m4_out is not None:
|
||||
bins, x, y, ymn, ymx = m4_out
|
||||
# flatten output to 1d arrays suitable for path-graphics generation.
|
||||
x = np.broadcast_to(x[:, None], y.shape)
|
||||
x = (x + np.array(
|
||||
[-x_spacer, 0, 0, x_spacer]
|
||||
)).flatten()
|
||||
y = y.flatten()
|
||||
|
||||
return x, y, ymn, ymx
|
||||
|
||||
# XXX: we accept a None output for the case where the input range
|
||||
# to ``ds_m4()`` is bad (-ve) and we want to catch and debug
|
||||
# that (seemingly super rare) circumstance..
|
||||
return None
|
||||
|
||||
|
||||
@njit(
|
||||
# NOTE: need to construct this manually for readonly
|
||||
# arrays, see https://github.com/numba/numba/issues/4511
|
||||
# (
|
||||
# types.Array(
|
||||
# numba_ohlc_dtype,
|
||||
# 1,
|
||||
# 'C',
|
||||
# readonly=True,
|
||||
# ),
|
||||
# int64,
|
||||
# types.unicode_type,
|
||||
# optional(float64),
|
||||
# ),
|
||||
nogil=True
|
||||
)
|
||||
def path_arrays_from_ohlc(
|
||||
data: np.ndarray,
|
||||
start: int64,
|
||||
bar_w: float64,
|
||||
bar_gap: float64 = 0.16,
|
||||
use_time_index: bool = True,
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index_field: str,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
]:
|
||||
'''
|
||||
Generate an array of lines objects from input ohlc data.
|
||||
|
||||
'''
|
||||
size = int(data.shape[0] * 6)
|
||||
|
||||
# XXX: see this for why the dtype might have to be defined outside
|
||||
# the routine.
|
||||
# https://github.com/numba/numba/issues/4098#issuecomment-493914533
|
||||
x = np.zeros(
|
||||
shape=size,
|
||||
dtype=float64,
|
||||
)
|
||||
y, c = x.copy(), x.copy()
|
||||
|
||||
half_w: float = bar_w/2
|
||||
|
||||
# TODO: report bug for assert @
|
||||
# /home/goodboy/repos/piker/env/lib/python3.8/site-packages/numba/core/typing/builtins.py:991
|
||||
for i, q in enumerate(data[start:], start):
|
||||
|
||||
open = q['open']
|
||||
high = q['high']
|
||||
low = q['low']
|
||||
close = q['close']
|
||||
|
||||
if use_time_index:
|
||||
index = float64(q['time'])
|
||||
else:
|
||||
index = float64(q['index'])
|
||||
|
||||
# XXX: ``numba`` issue: https://github.com/numba/numba/issues/8622
|
||||
# index = float64(q[index_field])
|
||||
# AND this (probably)
|
||||
# open, high, low, close, index = q[
|
||||
# ['open', 'high', 'low', 'close', 'index']]
|
||||
|
||||
istart = i * 6
|
||||
istop = istart + 6
|
||||
|
||||
# x,y detail the 6 points which connect all vertexes of a ohlc bar
|
||||
mid: float = index + half_w
|
||||
x[istart:istop] = (
|
||||
index + bar_gap,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
mid,
|
||||
index + bar_w - bar_gap,
|
||||
)
|
||||
y[istart:istop] = (
|
||||
open,
|
||||
open,
|
||||
low,
|
||||
high,
|
||||
close,
|
||||
close,
|
||||
)
|
||||
|
||||
# specifies that the first edge is never connected to the
|
||||
# prior bars last edge thus providing a small "gap"/"space"
|
||||
# between bars determined by ``bar_gap``.
|
||||
c[istart:istop] = (1, 1, 1, 1, 1, 0)
|
||||
|
||||
return x, y, c
|
||||
|
||||
|
||||
def hl2mxmn(
|
||||
ohlc: np.ndarray,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> np.ndarray:
|
||||
'''
|
||||
Convert a OHLC struct-array containing 'high'/'low' columns
|
||||
to a "joined" max/min 1-d array.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
hls = ohlc[[
|
||||
'low',
|
||||
'high',
|
||||
]]
|
||||
|
||||
mxmn = np.empty(2*hls.size, dtype=np.float64)
|
||||
x = np.empty(2*hls.size, dtype=np.float64)
|
||||
trace_hl(hls, mxmn, x, index[0])
|
||||
x = x + index[0]
|
||||
|
||||
return mxmn, x
|
||||
|
||||
|
||||
@njit(
|
||||
# TODO: the type annots..
|
||||
# float64[:](float64[:],),
|
||||
)
|
||||
def trace_hl(
|
||||
hl: 'np.ndarray',
|
||||
out: np.ndarray,
|
||||
x: np.ndarray,
|
||||
start: int,
|
||||
|
||||
# the "offset" values in the x-domain which
|
||||
# place the 2 output points around each ``int``
|
||||
# master index.
|
||||
margin: float = 0.43,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
"Trace" the outline of the high-low values of an ohlc sequence
|
||||
as a line such that the maximum deviation (aka disperaion) between
|
||||
bars if preserved.
|
||||
|
||||
This routine is expected to modify input arrays in-place.
|
||||
|
||||
'''
|
||||
last_l = hl['low'][0]
|
||||
last_h = hl['high'][0]
|
||||
|
||||
for i in range(hl.size):
|
||||
row = hl[i]
|
||||
l, h = row['low'], row['high']
|
||||
|
||||
up_diff = h - last_l
|
||||
down_diff = last_h - l
|
||||
|
||||
if up_diff > down_diff:
|
||||
out[2*i + 1] = h
|
||||
out[2*i] = last_l
|
||||
else:
|
||||
out[2*i + 1] = l
|
||||
out[2*i] = last_h
|
||||
|
||||
last_l = l
|
||||
last_h = h
|
||||
|
||||
x[2*i] = int(i) - margin
|
||||
x[2*i + 1] = int(i) + margin
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def ohlc_flatten(
|
||||
ohlc: np.ndarray,
|
||||
use_mxmn: bool = True,
|
||||
index_field: str = 'index',
|
||||
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
'''
|
||||
Convert an OHLCV struct-array into a flat ready-for-line-plotting
|
||||
1-d array that is 4 times the size with x-domain values distributed
|
||||
evenly (by 0.5 steps) over each index.
|
||||
|
||||
'''
|
||||
index = ohlc[index_field]
|
||||
|
||||
if use_mxmn:
|
||||
# traces a line optimally over highs to lows
|
||||
# using numba. NOTE: pretty sure this is faster
|
||||
# and looks about the same as the below output.
|
||||
flat, x = hl2mxmn(ohlc)
|
||||
|
||||
else:
|
||||
flat = rfn.structured_to_unstructured(
|
||||
ohlc[['open', 'high', 'low', 'close']]
|
||||
).flatten()
|
||||
|
||||
x = np.linspace(
|
||||
start=index[0] - 0.5,
|
||||
stop=index[-1] + 0.5,
|
||||
num=len(flat),
|
||||
)
|
||||
return x, flat
|
||||
|
||||
|
||||
def slice_from_time(
|
||||
arr: np.ndarray,
|
||||
start_t: float,
|
||||
stop_t: float,
|
||||
step: float, # sampler period step-diff
|
||||
|
||||
) -> slice:
|
||||
'''
|
||||
Calculate array indices mapped from a time range and return them in
|
||||
a slice.
|
||||
|
||||
Given an input array with an epoch `'time'` series entry, calculate
|
||||
the indices which span the time range and return in a slice. Presume
|
||||
each `'time'` step increment is uniform and when the time stamp
|
||||
series contains gaps (the uniform presumption is untrue) use
|
||||
``np.searchsorted()`` binary search to look up the appropriate
|
||||
index.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
msg='slice_from_time()',
|
||||
disabled=not pg_profile_enabled(),
|
||||
ms_threshold=ms_slower_then,
|
||||
)
|
||||
|
||||
times = arr['time']
|
||||
t_first = floor(times[0])
|
||||
t_last = ceil(times[-1])
|
||||
|
||||
# the greatest index we can return which slices to the
|
||||
# end of the input array.
|
||||
read_i_max = arr.shape[0]
|
||||
|
||||
# compute (presumed) uniform-time-step index offsets
|
||||
i_start_t = floor(start_t)
|
||||
read_i_start = floor(((i_start_t - t_first) // step)) - 1
|
||||
|
||||
i_stop_t = ceil(stop_t)
|
||||
|
||||
# XXX: edge case -> always set stop index to last in array whenever
|
||||
# the input stop time is detected to be greater then the equiv time
|
||||
# stamp at that last entry.
|
||||
if i_stop_t >= t_last:
|
||||
read_i_stop = read_i_max
|
||||
else:
|
||||
read_i_stop = ceil((i_stop_t - t_first) // step) + 1
|
||||
|
||||
# always clip outputs to array support
|
||||
# for read start:
|
||||
# - never allow a start < the 0 index
|
||||
# - never allow an end index > the read array len
|
||||
read_i_start = min(
|
||||
max(0, read_i_start),
|
||||
read_i_max - 1,
|
||||
)
|
||||
read_i_stop = max(
|
||||
0,
|
||||
min(read_i_stop, read_i_max),
|
||||
)
|
||||
|
||||
# check for larger-then-latest calculated index for given start
|
||||
# time, in which case we do a binary search for the correct index.
|
||||
# NOTE: this is usually the result of a time series with time gaps
|
||||
# where it is expected that each index step maps to a uniform step
|
||||
# in the time stamp series.
|
||||
t_iv_start = times[read_i_start]
|
||||
if (
|
||||
t_iv_start > i_start_t
|
||||
):
|
||||
# do a binary search for the best index mapping to ``start_t``
|
||||
# given we measured an overshoot using the uniform-time-step
|
||||
# calculation from above.
|
||||
|
||||
# TODO: once we start caching these per source-array,
|
||||
# we can just overwrite ``read_i_start`` directly.
|
||||
new_read_i_start = np.searchsorted(
|
||||
times,
|
||||
i_start_t,
|
||||
side='left',
|
||||
)
|
||||
|
||||
# TODO: minimize binary search work as much as possible:
|
||||
# - cache these remap values which compensate for gaps in the
|
||||
# uniform time step basis where we calc a later start
|
||||
# index for the given input ``start_t``.
|
||||
# - can we shorten the input search sequence by heuristic?
|
||||
# up_to_arith_start = index[:read_i_start]
|
||||
|
||||
if (
|
||||
new_read_i_start <= read_i_start
|
||||
):
|
||||
# t_diff = t_iv_start - start_t
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'start_t:{start_t} -> 0index start_t:{t_iv_start}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# f'REMAPPED START i: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
read_i_start = new_read_i_start
|
||||
|
||||
t_iv_stop = times[read_i_stop - 1]
|
||||
if (
|
||||
t_iv_stop > i_stop_t
|
||||
):
|
||||
# t_diff = stop_t - t_iv_stop
|
||||
# print(
|
||||
# f"WE'RE CUTTING OUT TIME - STEP:{step}\n"
|
||||
# f'calced iv stop:{t_iv_stop} -> stop_t:{stop_t}\n'
|
||||
# f'diff: {t_diff}\n'
|
||||
# # f'SHOULD REMAP STOP: {read_i_start} -> {new_read_i_start}\n'
|
||||
# )
|
||||
new_read_i_stop = np.searchsorted(
|
||||
times[read_i_start:],
|
||||
# times,
|
||||
i_stop_t,
|
||||
side='right',
|
||||
)
|
||||
|
||||
if (
|
||||
new_read_i_stop <= read_i_stop
|
||||
):
|
||||
read_i_stop = read_i_start + new_read_i_stop + 1
|
||||
|
||||
# sanity checks for range size
|
||||
# samples = (i_stop_t - i_start_t) // step
|
||||
# index_diff = read_i_stop - read_i_start + 1
|
||||
# if index_diff > (samples + 3):
|
||||
# breakpoint()
|
||||
|
||||
# read-relative indexes: gives a slice where `shm.array[read_slc]`
|
||||
# will be the data spanning the input time range `start_t` ->
|
||||
# `stop_t`
|
||||
read_slc = slice(
|
||||
int(read_i_start),
|
||||
int(read_i_stop),
|
||||
)
|
||||
|
||||
profiler(
|
||||
'slicing complete'
|
||||
# f'{start_t} -> {abs_slc.start} | {read_slc.start}\n'
|
||||
# f'{stop_t} -> {abs_slc.stop} | {read_slc.stop}\n'
|
||||
)
|
||||
|
||||
# NOTE: if caller needs absolute buffer indices they can
|
||||
# slice the buffer abs index like so:
|
||||
# index = arr['index']
|
||||
# abs_indx = index[read_slc]
|
||||
# abs_slc = slice(
|
||||
# int(abs_indx[0]),
|
||||
# int(abs_indx[-1]),
|
||||
# )
|
||||
|
||||
return read_slc
|
||||
|
|
@ -20,93 +20,53 @@ financial data flows.
|
|||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import (
|
||||
Counter,
|
||||
defaultdict,
|
||||
)
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from collections import Counter
|
||||
import time
|
||||
from typing import (
|
||||
AsyncIterator,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
import tractor
|
||||
from tractor.trionics import (
|
||||
maybe_open_nursery,
|
||||
)
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
get_console_log,
|
||||
)
|
||||
from ..service import maybe_spawn_daemon
|
||||
from ..log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._sharedmem import (
|
||||
ShmArray,
|
||||
)
|
||||
from ._sharedmem import ShmArray
|
||||
from .feed import _FeedsBus
|
||||
|
||||
|
||||
# highest frequency sample step is 1 second by default, though in
|
||||
# the future we may want to support shorter periods or a dynamic style
|
||||
# tick-event stream.
|
||||
_default_delay_s: float = 1.0
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class Sampler:
|
||||
class sampler:
|
||||
'''
|
||||
Global sampling engine registry.
|
||||
|
||||
Manages state for sampling events, shm incrementing and
|
||||
sample period logic.
|
||||
|
||||
This non-instantiated type is meant to be a singleton within
|
||||
a `samplerd` actor-service spawned once by the user wishing to
|
||||
time-step-sample (real-time) quote feeds, see
|
||||
``.service.maybe_open_samplerd()`` and the below
|
||||
``register_with_sampler()``.
|
||||
|
||||
'''
|
||||
service_nursery: None | trio.Nursery = None
|
||||
|
||||
# TODO: we could stick these in a composed type to avoid
|
||||
# angering the "i hate module scoped variables crowd" (yawn).
|
||||
ohlcv_shms: dict[float, list[ShmArray]] = {}
|
||||
ohlcv_shms: dict[int, list[ShmArray]] = {}
|
||||
|
||||
# holds one-task-per-sample-period tasks which are spawned as-needed by
|
||||
# data feed requests with a given detected time step usually from
|
||||
# history loading.
|
||||
incr_task_cs: trio.CancelScope | None = None
|
||||
incrementers: dict[int, trio.CancelScope] = {}
|
||||
|
||||
# holds all the ``tractor.Context`` remote subscriptions for
|
||||
# a particular sample period increment event: all subscribers are
|
||||
# notified on a step.
|
||||
subscribers: defaultdict[
|
||||
float,
|
||||
list[
|
||||
float,
|
||||
set[tractor.MsgStream]
|
||||
],
|
||||
] = defaultdict(
|
||||
lambda: [
|
||||
round(time.time()),
|
||||
set(),
|
||||
]
|
||||
)
|
||||
subscribers: dict[int, tractor.Context] = {}
|
||||
|
||||
@classmethod
|
||||
async def increment_ohlc_buffer(
|
||||
self,
|
||||
period_s: float,
|
||||
|
||||
async def increment_ohlc_buffer(
|
||||
delay_s: int,
|
||||
task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED,
|
||||
):
|
||||
):
|
||||
'''
|
||||
Task which inserts new bars into the provide shared memory array
|
||||
every ``period_s`` seconds.
|
||||
every ``delay_s`` seconds.
|
||||
|
||||
This task fulfills 2 purposes:
|
||||
- it takes the subscribed set of shm arrays and increments them
|
||||
|
|
@ -118,152 +78,101 @@ class Sampler:
|
|||
the underlying buffers will actually be incremented.
|
||||
|
||||
'''
|
||||
# # wait for brokerd to signal we should start sampling
|
||||
# await shm_incrementing(shm_token['shm_name']).wait()
|
||||
|
||||
# TODO: right now we'll spin printing bars if the last time stamp is
|
||||
# before a large period of no market activity. Likely the best way
|
||||
# to solve this is to make this task aware of the instrument's
|
||||
# tradable hours?
|
||||
|
||||
total_s: float = 0 # total seconds counted
|
||||
ad = period_s - 0.001 # compensate for trio processing time
|
||||
# adjust delay to compensate for trio processing time
|
||||
ad = min(sampler.ohlcv_shms.keys()) - 0.001
|
||||
|
||||
total_s = 0 # total seconds counted
|
||||
lowest = min(sampler.ohlcv_shms.keys())
|
||||
lowest_shm = sampler.ohlcv_shms[lowest][0]
|
||||
ad = lowest - 0.001
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
|
||||
# register this time period step as active
|
||||
sampler.incrementers[delay_s] = cs
|
||||
task_status.started(cs)
|
||||
|
||||
# sample step loop:
|
||||
# includes broadcasting to all connected consumers on every
|
||||
# new sample step as well incrementing any registered
|
||||
# buffers by registered sample period.
|
||||
while True:
|
||||
# TODO: do we want to support dynamically
|
||||
# adding a "lower" lowest increment period?
|
||||
await trio.sleep(ad)
|
||||
total_s += period_s
|
||||
total_s += lowest
|
||||
|
||||
# increment all subscribed shm arrays
|
||||
# TODO:
|
||||
# - this in ``numba``
|
||||
# - just lookup shms for this step instead of iterating?
|
||||
|
||||
i_epoch = round(time.time())
|
||||
broadcasted: set[float] = set()
|
||||
|
||||
# print(f'epoch: {i_epoch} -> REGISTRY {self.ohlcv_shms}')
|
||||
for shm_period_s, shms in self.ohlcv_shms.items():
|
||||
|
||||
# short-circuit on any not-ready because slower sample
|
||||
# rate consuming shm buffers.
|
||||
if total_s % shm_period_s != 0:
|
||||
# print(f'skipping `{shm_period_s}s` sample update')
|
||||
for delay_s, shms in sampler.ohlcv_shms.items():
|
||||
if total_s % delay_s != 0:
|
||||
continue
|
||||
|
||||
# update last epoch stamp for this period group
|
||||
if shm_period_s not in broadcasted:
|
||||
sub_pair = self.subscribers[shm_period_s]
|
||||
sub_pair[0] = i_epoch
|
||||
broadcasted.add(shm_period_s)
|
||||
|
||||
# TODO: ``numba`` this!
|
||||
for shm in shms:
|
||||
# print(f'UPDATE {shm_period_s}s STEP for {shm.token}')
|
||||
# TODO: in theory we could make this faster by copying the
|
||||
# "last" readable value into the underlying larger buffer's
|
||||
# next value and then incrementing the counter instead of
|
||||
# using ``.push()``?
|
||||
|
||||
# append new entry to buffer thus "incrementing"
|
||||
# the bar
|
||||
# append new entry to buffer thus "incrementing" the bar
|
||||
array = shm.array
|
||||
last = array[-1:][shm._write_fields].copy()
|
||||
# (index, t, close) = last[0][['index', 'time', 'close']]
|
||||
(t, close) = last[0][['time', 'close']]
|
||||
|
||||
# guard against startup backfilling races where
|
||||
# the buffer has not yet been filled.
|
||||
if not last.size:
|
||||
continue
|
||||
|
||||
(t, close) = last[0][[
|
||||
'time',
|
||||
'close',
|
||||
]]
|
||||
|
||||
next_t = t + shm_period_s
|
||||
|
||||
if shm_period_s <= 1:
|
||||
next_t = i_epoch
|
||||
|
||||
# this copies non-std fields (eg. vwap) from the
|
||||
# last datum
|
||||
last[[
|
||||
'time',
|
||||
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
|
||||
'volume',
|
||||
]][0] = (
|
||||
# epoch timestamp
|
||||
next_t,
|
||||
|
||||
# OHLC
|
||||
close,
|
||||
close,
|
||||
close,
|
||||
close,
|
||||
|
||||
0, # vlm
|
||||
)
|
||||
|
||||
# TODO: in theory we could make this faster by
|
||||
# copying the "last" readable value into the
|
||||
# underlying larger buffer's next value and then
|
||||
# incrementing the counter instead of using
|
||||
# ``.push()``?
|
||||
# this copies non-std fields (eg. vwap) from the last datum
|
||||
last[
|
||||
['time', 'volume', 'open', 'high', 'low', 'close']
|
||||
][0] = (t + delay_s, 0, close, close, close, close)
|
||||
|
||||
# write to the buffer
|
||||
shm.push(last)
|
||||
|
||||
# broadcast increment msg to all updated subs per period
|
||||
for shm_period_s in broadcasted:
|
||||
await self.broadcast(
|
||||
period_s=shm_period_s,
|
||||
time_stamp=i_epoch,
|
||||
)
|
||||
await broadcast(delay_s, shm=lowest_shm)
|
||||
|
||||
@classmethod
|
||||
async def broadcast(
|
||||
self,
|
||||
period_s: float,
|
||||
time_stamp: float | None = None,
|
||||
|
||||
) -> None:
|
||||
async def broadcast(
|
||||
delay_s: int,
|
||||
shm: Optional[ShmArray] = None,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Broadcast the period size and last index step value to all
|
||||
Broadcast the given ``shm: ShmArray``'s buffer index step to any
|
||||
subscribers for a given sample period.
|
||||
|
||||
The sent msg will include the first and last index which slice into
|
||||
the buffer's non-empty data.
|
||||
|
||||
'''
|
||||
pair: list[float, set]
|
||||
pair = self.subscribers[period_s]
|
||||
subs = sampler.subscribers.get(delay_s, ())
|
||||
|
||||
last_ts: float
|
||||
subs: set
|
||||
last_ts, subs = pair
|
||||
first = last = -1
|
||||
|
||||
task = trio.lowlevel.current_task()
|
||||
log.debug(
|
||||
f'SUBS {self.subscribers}\n'
|
||||
f'PAIR {pair}\n'
|
||||
f'TASK: {task}: {id(task)}\n'
|
||||
f'broadcasting {period_s} -> {last_ts}\n'
|
||||
# f'consumers: {subs}'
|
||||
)
|
||||
borked: set[tractor.MsgStream] = set()
|
||||
sent: set[tractor.MsgStream] = set()
|
||||
while True:
|
||||
try:
|
||||
for stream in (subs - sent):
|
||||
if shm is None:
|
||||
periods = sampler.ohlcv_shms.keys()
|
||||
# if this is an update triggered by a history update there
|
||||
# might not actually be any sampling bus setup since there's
|
||||
# no "live feed" active yet.
|
||||
if periods:
|
||||
lowest = min(periods)
|
||||
shm = sampler.ohlcv_shms[lowest][0]
|
||||
first = shm._first.value
|
||||
last = shm._last.value
|
||||
|
||||
for stream in subs:
|
||||
try:
|
||||
await stream.send({
|
||||
'index': time_stamp or last_ts,
|
||||
'period': period_s,
|
||||
'first': first,
|
||||
'last': last,
|
||||
'index': last,
|
||||
})
|
||||
sent.add(stream)
|
||||
|
||||
except (
|
||||
trio.BrokenResourceError,
|
||||
trio.ClosedResourceError
|
||||
|
|
@ -271,270 +180,60 @@ class Sampler:
|
|||
log.error(
|
||||
f'{stream._ctx.chan.uid} dropped connection'
|
||||
)
|
||||
borked.add(stream)
|
||||
else:
|
||||
break
|
||||
except RuntimeError:
|
||||
log.warning(f'Client subs {subs} changed while broadcasting')
|
||||
continue
|
||||
|
||||
for stream in borked:
|
||||
try:
|
||||
subs.remove(stream)
|
||||
except KeyError:
|
||||
except ValueError:
|
||||
log.warning(
|
||||
f'{stream._ctx.chan.uid} sub already removed!?'
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def broadcast_all(self) -> None:
|
||||
for period_s in self.subscribers:
|
||||
await self.broadcast(period_s)
|
||||
|
||||
|
||||
@tractor.context
|
||||
async def register_with_sampler(
|
||||
async def iter_ohlc_periods(
|
||||
ctx: tractor.Context,
|
||||
period_s: float,
|
||||
shms_by_period: dict[float, dict] | None = None,
|
||||
|
||||
open_index_stream: bool = True, # open a 2way stream for sample step msgs?
|
||||
sub_for_broadcasts: bool = True, # sampler side to send step updates?
|
||||
delay_s: int,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Subscribe to OHLC sampling "step" events: when the time
|
||||
aggregation period increments, this event stream emits an index
|
||||
event.
|
||||
|
||||
get_console_log(tractor.current_actor().loglevel)
|
||||
incr_was_started: bool = False
|
||||
|
||||
try:
|
||||
async with maybe_open_nursery(
|
||||
Sampler.service_nursery
|
||||
) as service_nursery:
|
||||
|
||||
# init startup, create (actor-)local service nursery and start
|
||||
# increment task
|
||||
Sampler.service_nursery = service_nursery
|
||||
|
||||
# always ensure a period subs entry exists
|
||||
last_ts, subs = Sampler.subscribers[float(period_s)]
|
||||
|
||||
async with trio.Lock():
|
||||
if Sampler.incr_task_cs is None:
|
||||
Sampler.incr_task_cs = await service_nursery.start(
|
||||
Sampler.increment_ohlc_buffer,
|
||||
1.,
|
||||
)
|
||||
incr_was_started = True
|
||||
|
||||
# insert the base 1s period (for OHLC style sampling) into
|
||||
# the increment buffer set to update and shift every second.
|
||||
if shms_by_period is not None:
|
||||
from ._sharedmem import (
|
||||
attach_shm_array,
|
||||
_Token,
|
||||
)
|
||||
for period in shms_by_period:
|
||||
|
||||
# load and register shm handles
|
||||
shm_token_msg = shms_by_period[period]
|
||||
shm = attach_shm_array(
|
||||
_Token.from_msg(shm_token_msg),
|
||||
readonly=False,
|
||||
)
|
||||
shms_by_period[period] = shm
|
||||
Sampler.ohlcv_shms.setdefault(period, []).append(shm)
|
||||
|
||||
assert Sampler.ohlcv_shms
|
||||
|
||||
# unblock caller
|
||||
await ctx.started(set(Sampler.ohlcv_shms.keys()))
|
||||
|
||||
if open_index_stream:
|
||||
try:
|
||||
'''
|
||||
# add our subscription
|
||||
subs = sampler.subscribers.setdefault(delay_s, [])
|
||||
await ctx.started()
|
||||
async with ctx.open_stream() as stream:
|
||||
if sub_for_broadcasts:
|
||||
subs.add(stream)
|
||||
subs.append(stream)
|
||||
|
||||
# except broadcast requests from the subscriber
|
||||
async for msg in stream:
|
||||
if msg == 'broadcast_all':
|
||||
await Sampler.broadcast_all()
|
||||
finally:
|
||||
if sub_for_broadcasts:
|
||||
subs.remove(stream)
|
||||
else:
|
||||
# if no shms are passed in we just wait until cancelled
|
||||
# by caller.
|
||||
try:
|
||||
# stream and block until cancelled
|
||||
await trio.sleep_forever()
|
||||
|
||||
finally:
|
||||
# TODO: why tf isn't this working?
|
||||
if shms_by_period is not None:
|
||||
for period, shm in shms_by_period.items():
|
||||
Sampler.ohlcv_shms[period].remove(shm)
|
||||
|
||||
if incr_was_started:
|
||||
Sampler.incr_task_cs.cancel()
|
||||
Sampler.incr_task_cs = None
|
||||
|
||||
|
||||
async def spawn_samplerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
'''
|
||||
Daemon-side service task: start a sampling daemon for common step
|
||||
update and increment count write and stream broadcasting.
|
||||
|
||||
'''
|
||||
from piker.service import Services
|
||||
|
||||
dname = 'samplerd'
|
||||
log.info(f'Spawning `{dname}`')
|
||||
|
||||
# singleton lock creation of ``samplerd`` since we only ever want
|
||||
# one daemon per ``pikerd`` proc tree.
|
||||
# TODO: make this built-into the service api?
|
||||
async with Services.locks[dname + '_singleton']:
|
||||
|
||||
if dname not in Services.service_tasks:
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
dname,
|
||||
enable_modules=[
|
||||
'piker.data._sampling',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
try:
|
||||
subs.remove(stream)
|
||||
except ValueError:
|
||||
log.error(
|
||||
f'iOHLC step stream was already dropped {ctx.chan.uid}?'
|
||||
)
|
||||
|
||||
await Services.start_service_task(
|
||||
dname,
|
||||
portal,
|
||||
register_with_sampler,
|
||||
period_s=1,
|
||||
sub_for_broadcasts=False,
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_samplerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal: # noqa
|
||||
'''
|
||||
Client-side helper to maybe startup the ``samplerd`` service
|
||||
under the ``pikerd`` tree.
|
||||
|
||||
'''
|
||||
dname = 'samplerd'
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
dname,
|
||||
service_task_target=spawn_samplerd,
|
||||
spawn_args={},
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
||||
|
||||
@acm
|
||||
async def open_sample_stream(
|
||||
period_s: float,
|
||||
shms_by_period: dict[float, dict] | None = None,
|
||||
open_index_stream: bool = True,
|
||||
sub_for_broadcasts: bool = True,
|
||||
|
||||
cache_key: str | None = None,
|
||||
allow_new_sampler: bool = True,
|
||||
|
||||
) -> AsyncIterator[dict[str, float]]:
|
||||
'''
|
||||
Subscribe to OHLC sampling "step" events: when the time aggregation
|
||||
period increments, this event stream emits an index event.
|
||||
|
||||
This is a client-side endpoint that does all the work of ensuring
|
||||
the `samplerd` actor is up and that mult-consumer-tasks are given
|
||||
a broadcast stream when possible.
|
||||
|
||||
'''
|
||||
# TODO: wrap this manager with the following to make it cached
|
||||
# per client-multitasks entry.
|
||||
# maybe_open_context(
|
||||
# acm_func=partial(
|
||||
# portal.open_context,
|
||||
# register_with_sampler,
|
||||
# ),
|
||||
# key=cache_key or period_s,
|
||||
# )
|
||||
# if cache_hit:
|
||||
# # add a new broadcast subscription for the quote stream
|
||||
# # if this feed is likely already in use
|
||||
# async with istream.subscribe() as bistream:
|
||||
# yield bistream
|
||||
# else:
|
||||
|
||||
async with (
|
||||
# XXX: this should be singleton on a host,
|
||||
# a lone broker-daemon per provider should be
|
||||
# created for all practical purposes
|
||||
maybe_open_samplerd() as portal,
|
||||
|
||||
portal.open_context(
|
||||
register_with_sampler,
|
||||
**{
|
||||
'period_s': period_s,
|
||||
'shms_by_period': shms_by_period,
|
||||
'open_index_stream': open_index_stream,
|
||||
'sub_for_broadcasts': sub_for_broadcasts,
|
||||
},
|
||||
) as (ctx, first)
|
||||
):
|
||||
async with (
|
||||
ctx.open_stream() as istream,
|
||||
|
||||
# TODO: we don't need this task-bcasting right?
|
||||
# istream.subscribe() as istream,
|
||||
):
|
||||
yield istream
|
||||
|
||||
|
||||
async def sample_and_broadcast(
|
||||
|
||||
bus: _FeedsBus, # noqa
|
||||
rt_shm: ShmArray,
|
||||
hist_shm: ShmArray,
|
||||
shm: ShmArray,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
brokername: str,
|
||||
sum_tick_vlm: bool = True,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
`brokerd`-side task which writes latest datum sampled data.
|
||||
|
||||
This task is meant to run in the same actor (mem space) as the
|
||||
`brokerd` real-time quote feed which is being sampled to
|
||||
a ``ShmArray`` buffer.
|
||||
|
||||
'''
|
||||
log.info("Started shared mem bar writer")
|
||||
|
||||
overruns = Counter()
|
||||
|
||||
# iterate stream delivered by broker
|
||||
async for quotes in quote_stream:
|
||||
# print(quotes)
|
||||
|
||||
# TODO: ``numba`` this!
|
||||
for broker_symbol, quote in quotes.items():
|
||||
# TODO: in theory you can send the IPC msg *before* writing
|
||||
|
|
@ -558,9 +257,6 @@ async def sample_and_broadcast(
|
|||
|
||||
last = tick['price']
|
||||
|
||||
# more compact inline-way to do this assignment
|
||||
# to both buffers?
|
||||
for shm in [rt_shm, hist_shm]:
|
||||
# update last entry
|
||||
# benchmarked in the 4-5 us range
|
||||
o, high, low, v = shm.array[-1][
|
||||
|
|
@ -597,37 +293,29 @@ async def sample_and_broadcast(
|
|||
volume,
|
||||
)
|
||||
|
||||
# TODO: PUT THIS IN A ``_FeedsBus.broadcast()`` method!
|
||||
# XXX: we need to be very cautious here that no
|
||||
# context-channel is left lingering which doesn't have
|
||||
# a far end receiver actor-task. In such a case you can
|
||||
# end up triggering backpressure which which will
|
||||
# eventually block this producer end of the feed and
|
||||
# thus other consumers still attached.
|
||||
sub_key: str = broker_symbol.lower()
|
||||
subs: list[
|
||||
tuple[
|
||||
tractor.MsgStream | trio.MemorySendChannel,
|
||||
float | None, # tick throttle in Hz
|
||||
Union[tractor.MsgStream, trio.MemorySendChannel],
|
||||
tractor.Context,
|
||||
Optional[float], # tick throttle in Hz
|
||||
]
|
||||
] = bus.get_subs(sub_key)
|
||||
] = bus._subscribers[broker_symbol.lower()]
|
||||
|
||||
# NOTE: by default the broker backend doesn't append
|
||||
# it's own "name" into the fqsn schema (but maybe it
|
||||
# should?) so we have to manually generate the correct
|
||||
# key here.
|
||||
fqsn = f'{broker_symbol}.{brokername}'
|
||||
bsym = f'{broker_symbol}.{brokername}'
|
||||
lags: int = 0
|
||||
|
||||
# TODO: speed up this loop in an AOT compiled lang (like
|
||||
# rust or nim or zig) and/or instead of doing a fan out to
|
||||
# TCP sockets here, we add a shm-style tick queue which
|
||||
# readers can pull from instead of placing the burden of
|
||||
# broadcast on solely on this `brokerd` actor. see issues:
|
||||
# - https://github.com/pikers/piker/issues/98
|
||||
# - https://github.com/pikers/piker/issues/107
|
||||
for (stream, ctx, tick_throttle) in subs:
|
||||
|
||||
for (stream, tick_throttle) in subs.copy():
|
||||
try:
|
||||
with trio.move_on_after(0.2) as cs:
|
||||
if tick_throttle:
|
||||
|
|
@ -635,39 +323,47 @@ async def sample_and_broadcast(
|
|||
# pushes to the ``uniform_rate_send()`` below.
|
||||
try:
|
||||
stream.send_nowait(
|
||||
(fqsn, quote)
|
||||
(bsym, quote)
|
||||
)
|
||||
except trio.WouldBlock:
|
||||
overruns[sub_key] += 1
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
|
||||
if ctx:
|
||||
log.warning(
|
||||
f'Feed OVERRUN {sub_key}'
|
||||
'@{bus.brokername} -> \n'
|
||||
f'feed @ {chan.uid}\n'
|
||||
f'throttle = {tick_throttle} Hz'
|
||||
f'Feed overrun {bus.brokername} ->'
|
||||
f'{chan.uid} !!!'
|
||||
)
|
||||
|
||||
if overruns[sub_key] > 6:
|
||||
else:
|
||||
key = id(stream)
|
||||
overruns[key] += 1
|
||||
log.warning(
|
||||
f'Feed overrun {broker_symbol}'
|
||||
'@{bus.brokername} -> '
|
||||
f'feed @ {tick_throttle} Hz'
|
||||
)
|
||||
if overruns[key] > 6:
|
||||
# TODO: should we check for the
|
||||
# context being cancelled? this
|
||||
# could happen but the
|
||||
# channel-ipc-pipe is still up.
|
||||
if (
|
||||
not chan.connected()
|
||||
or ctx._cancel_called
|
||||
):
|
||||
if not chan.connected():
|
||||
log.warning(
|
||||
'Dropping broken consumer:\n'
|
||||
f'{sub_key}:'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
await stream.aclose()
|
||||
raise trio.BrokenResourceError
|
||||
else:
|
||||
log.warning(
|
||||
'Feed getting overrun bro!\n'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
continue
|
||||
|
||||
else:
|
||||
await stream.send(
|
||||
{fqsn: quote}
|
||||
{bsym: quote}
|
||||
)
|
||||
|
||||
if cs.cancelled_caught:
|
||||
|
|
@ -680,7 +376,6 @@ async def sample_and_broadcast(
|
|||
trio.ClosedResourceError,
|
||||
trio.EndOfChannel,
|
||||
):
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
if ctx:
|
||||
log.warning(
|
||||
|
|
@ -696,65 +391,19 @@ async def sample_and_broadcast(
|
|||
# so far seems like no since this should all
|
||||
# be single-threaded. Doing it anyway though
|
||||
# since there seems to be some kinda race..
|
||||
bus.remove_subs(
|
||||
sub_key,
|
||||
{(stream, tick_throttle)},
|
||||
try:
|
||||
subs.remove((stream, tick_throttle))
|
||||
except ValueError:
|
||||
log.error(
|
||||
f'Stream was already removed from subs!?\n'
|
||||
f'{broker_symbol}:'
|
||||
f'{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
|
||||
|
||||
# a working tick-type-classes template
|
||||
_tick_groups = {
|
||||
'clears': {'trade', 'dark_trade', 'last'},
|
||||
'bids': {'bid', 'bsize'},
|
||||
'asks': {'ask', 'asize'},
|
||||
}
|
||||
|
||||
|
||||
def frame_ticks(
|
||||
first_quote: dict,
|
||||
last_quote: dict,
|
||||
ticks_by_type: dict,
|
||||
) -> None:
|
||||
# append quotes since last iteration into the last quote's
|
||||
# tick array/buffer.
|
||||
ticks = last_quote.get('ticks')
|
||||
|
||||
# TODO: once we decide to get fancy really we should
|
||||
# have a shared mem tick buffer that is just
|
||||
# continually filled and the UI just ready from it
|
||||
# at it's display rate.
|
||||
if ticks:
|
||||
# TODO: do we need this any more or can we just
|
||||
# expect the receiver to unwind the below
|
||||
# `ticks_by_type: dict`?
|
||||
# => undwinding would potentially require a
|
||||
# `dict[str, set | list]` instead with an
|
||||
# included `'types' field which is an (ordered)
|
||||
# set of tick type fields in the order which
|
||||
# types arrived?
|
||||
first_quote['ticks'].extend(ticks)
|
||||
|
||||
# XXX: build a tick-by-type table of lists
|
||||
# of tick messages. This allows for less
|
||||
# iteration on the receiver side by allowing for
|
||||
# a single "latest tick event" look up by
|
||||
# indexing the last entry in each sub-list.
|
||||
# tbt = {
|
||||
# 'types': ['bid', 'asize', 'last', .. '<type_n>'],
|
||||
|
||||
# 'bid': [tick0, tick1, tick2, .., tickn],
|
||||
# 'asize': [tick0, tick1, tick2, .., tickn],
|
||||
# 'last': [tick0, tick1, tick2, .., tickn],
|
||||
# ...
|
||||
# '<type_n>': [tick0, tick1, tick2, .., tickn],
|
||||
# }
|
||||
|
||||
# append in reverse FIFO order for in-order iteration on
|
||||
# receiver side.
|
||||
for tick in ticks:
|
||||
ttype = tick['type']
|
||||
ticks_by_type[ttype].append(tick)
|
||||
|
||||
# TODO: a less naive throttler, here's some snippets:
|
||||
# token bucket by njs:
|
||||
# https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||
|
||||
async def uniform_rate_send(
|
||||
|
||||
|
|
@ -765,21 +414,7 @@ async def uniform_rate_send(
|
|||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Throttle a real-time (presumably tick event) stream to a uniform
|
||||
transmissiom rate, normally for the purposes of throttling a data
|
||||
flow being consumed by a graphics rendering actor which itself is limited
|
||||
by a fixed maximum display rate.
|
||||
|
||||
Though this function isn't documented (nor was intentially written
|
||||
to be) a token-bucket style algo, it effectively operates as one (we
|
||||
think?).
|
||||
|
||||
TODO: a less naive throttler, here's some snippets:
|
||||
token bucket by njs:
|
||||
https://gist.github.com/njsmith/7ea44ec07e901cb78ebe1dd8dd846cb9
|
||||
|
||||
'''
|
||||
# TODO: compute the approx overhead latency per cycle
|
||||
left_to_sleep = throttle_period = 1/rate - 0.000616
|
||||
|
||||
|
|
@ -789,12 +424,6 @@ async def uniform_rate_send(
|
|||
diff = 0
|
||||
|
||||
task_status.started()
|
||||
ticks_by_type: defaultdict[
|
||||
str,
|
||||
list[dict],
|
||||
] = defaultdict(list)
|
||||
|
||||
clear_types = _tick_groups['clears']
|
||||
|
||||
while True:
|
||||
|
||||
|
|
@ -813,17 +442,34 @@ async def uniform_rate_send(
|
|||
|
||||
if not first_quote:
|
||||
first_quote = last_quote
|
||||
# first_quote['tbt'] = ticks_by_type
|
||||
|
||||
if (throttle_period - diff) > 0:
|
||||
# received a quote but the send cycle period hasn't yet
|
||||
# expired we aren't supposed to send yet so append
|
||||
# to the tick frame.
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
last_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# append quotes since last iteration into the last quote's
|
||||
# tick array/buffer.
|
||||
ticks = last_quote.get('ticks')
|
||||
|
||||
# XXX: idea for frame type data structure we could
|
||||
# use on the wire instead of a simple list?
|
||||
# frames = {
|
||||
# 'index': ['type_a', 'type_c', 'type_n', 'type_n'],
|
||||
|
||||
# 'type_a': [tick0, tick1, tick2, .., tickn],
|
||||
# 'type_b': [tick0, tick1, tick2, .., tickn],
|
||||
# 'type_c': [tick0, tick1, tick2, .., tickn],
|
||||
# ...
|
||||
# 'type_n': [tick0, tick1, tick2, .., tickn],
|
||||
# }
|
||||
|
||||
# TODO: once we decide to get fancy really we should
|
||||
# have a shared mem tick buffer that is just
|
||||
# continually filled and the UI just ready from it
|
||||
# at it's display rate.
|
||||
if ticks:
|
||||
first_quote['ticks'].extend(ticks)
|
||||
|
||||
# send cycle isn't due yet so continue waiting
|
||||
continue
|
||||
|
|
@ -840,50 +486,17 @@ async def uniform_rate_send(
|
|||
# received quote ASAP.
|
||||
sym, first_quote = await quote_stream.receive()
|
||||
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
first_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# we have a quote already so send it now.
|
||||
|
||||
with trio.move_on_after(throttle_period) as cs:
|
||||
while (
|
||||
not set(ticks_by_type).intersection(clear_types)
|
||||
):
|
||||
try:
|
||||
sym, last_quote = await quote_stream.receive()
|
||||
except trio.EndOfChannel:
|
||||
log.exception(f"feed for {stream} ended?")
|
||||
break
|
||||
|
||||
frame_ticks(
|
||||
first_quote,
|
||||
last_quote,
|
||||
ticks_by_type,
|
||||
)
|
||||
|
||||
# measured_rate = 1 / (time.time() - last_send)
|
||||
# log.info(
|
||||
# f'`{sym}` throttled send hz: {round(measured_rate, ndigits=1)}'
|
||||
# )
|
||||
first_quote['tbt'] = ticks_by_type
|
||||
|
||||
# TODO: now if only we could sync this to the display
|
||||
# rate timing exactly lul
|
||||
try:
|
||||
await stream.send({sym: first_quote})
|
||||
except tractor.RemoteActorError as rme:
|
||||
if rme.type is not tractor._exceptions.StreamOverrun:
|
||||
raise
|
||||
ctx = stream._ctx
|
||||
chan = ctx.chan
|
||||
log.warning(
|
||||
'Throttled quote-stream overrun!\n'
|
||||
f'{sym}:{ctx.cid}@{chan.uid}'
|
||||
)
|
||||
|
||||
except (
|
||||
# NOTE: any of these can be raised by ``tractor``'s IPC
|
||||
# transport-layer and we want to be highly resilient
|
||||
|
|
@ -904,4 +517,3 @@ async def uniform_rate_send(
|
|||
first_quote = last_quote = None
|
||||
diff = 0
|
||||
last_send = time.time()
|
||||
ticks_by_type.clear()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -27,14 +27,16 @@ from multiprocessing.shared_memory import SharedMemory, _USE_POSIX
|
|||
if _USE_POSIX:
|
||||
from _posixshmem import shm_unlink
|
||||
|
||||
# import msgspec
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
import tractor
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
from numpy.lib import recfunctions as rfn
|
||||
|
||||
from ._util import log
|
||||
from ..log import get_logger
|
||||
from ._source import base_iohlc_dtype
|
||||
from .types import Struct
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# how much is probably dependent on lifestyle
|
||||
|
|
@ -47,11 +49,7 @@ _rt_buffer_start = int((_days_worth - 1) * _secs_in_day)
|
|||
|
||||
|
||||
def cuckoff_mantracker():
|
||||
'''
|
||||
Disable all ``multiprocessing``` "resource tracking" machinery since
|
||||
it's an absolute multi-threaded mess of non-SC madness.
|
||||
|
||||
'''
|
||||
from multiprocessing import resource_tracker as mantracker
|
||||
|
||||
# Tell the "resource tracker" thing to fuck off.
|
||||
|
|
@ -109,39 +107,36 @@ class SharedInt:
|
|||
log.warning(f'Shm for {name} already unlinked?')
|
||||
|
||||
|
||||
class _Token(Struct, frozen=True):
|
||||
class _Token(BaseModel):
|
||||
'''
|
||||
Internal represenation of a shared memory "token"
|
||||
which can be used to key a system wide post shm entry.
|
||||
|
||||
'''
|
||||
class Config:
|
||||
frozen = True
|
||||
|
||||
shm_name: str # this servers as a "key" value
|
||||
shm_first_index_name: str
|
||||
shm_last_index_name: str
|
||||
dtype_descr: tuple
|
||||
size: int # in struct-array index / row terms
|
||||
|
||||
@property
|
||||
def dtype(self) -> np.dtype:
|
||||
return np.dtype(list(map(tuple, self.dtype_descr))).descr
|
||||
|
||||
def as_msg(self):
|
||||
return self.to_dict()
|
||||
return self.dict()
|
||||
|
||||
@classmethod
|
||||
def from_msg(cls, msg: dict) -> _Token:
|
||||
if isinstance(msg, _Token):
|
||||
return msg
|
||||
|
||||
# TODO: native struct decoding
|
||||
# return _token_dec.decode(msg)
|
||||
|
||||
msg['dtype_descr'] = tuple(map(tuple, msg['dtype_descr']))
|
||||
return _Token(**msg)
|
||||
|
||||
|
||||
# _token_dec = msgspec.msgpack.Decoder(_Token)
|
||||
|
||||
# TODO: this api?
|
||||
# _known_tokens = tractor.ActorVar('_shm_tokens', {})
|
||||
# _known_tokens = tractor.ContextStack('_known_tokens', )
|
||||
|
|
@ -160,7 +155,6 @@ def get_shm_token(key: str) -> _Token:
|
|||
|
||||
def _make_token(
|
||||
key: str,
|
||||
size: int,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
) -> _Token:
|
||||
'''
|
||||
|
|
@ -173,8 +167,7 @@ def _make_token(
|
|||
shm_name=key,
|
||||
shm_first_index_name=key + "_first",
|
||||
shm_last_index_name=key + "_last",
|
||||
dtype_descr=tuple(np.dtype(dtype).descr),
|
||||
size=size,
|
||||
dtype_descr=np.dtype(dtype).descr
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -226,7 +219,6 @@ class ShmArray:
|
|||
shm_first_index_name=self._first._shm.name,
|
||||
shm_last_index_name=self._last._shm.name,
|
||||
dtype_descr=tuple(self._array.dtype.descr),
|
||||
size=self._len,
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
@ -268,7 +260,7 @@ class ShmArray:
|
|||
|
||||
# type that all field values will be cast to
|
||||
# in the returned view.
|
||||
common_dtype: np.dtype = float,
|
||||
common_dtype: np.dtype = np.float,
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
|
|
@ -441,7 +433,7 @@ class ShmArray:
|
|||
def open_shm_array(
|
||||
|
||||
key: Optional[str] = None,
|
||||
size: int = _default_size, # see above
|
||||
size: int = _default_size,
|
||||
dtype: Optional[np.dtype] = None,
|
||||
readonly: bool = False,
|
||||
|
||||
|
|
@ -472,8 +464,7 @@ def open_shm_array(
|
|||
|
||||
token = _make_token(
|
||||
key=key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
dtype=dtype
|
||||
)
|
||||
|
||||
# create single entry arrays for storing an first and last indices
|
||||
|
|
@ -525,15 +516,15 @@ def open_shm_array(
|
|||
# "unlink" created shm on process teardown by
|
||||
# pushing teardown calls onto actor context stack
|
||||
|
||||
stack = tractor.current_actor().lifetime_stack
|
||||
stack.callback(shmarr.close)
|
||||
stack.callback(shmarr.destroy)
|
||||
tractor._actor._lifetime_stack.callback(shmarr.close)
|
||||
tractor._actor._lifetime_stack.callback(shmarr.destroy)
|
||||
|
||||
return shmarr
|
||||
|
||||
|
||||
def attach_shm_array(
|
||||
token: tuple[str, str, tuple[str, str]],
|
||||
size: int = _default_size,
|
||||
readonly: bool = True,
|
||||
|
||||
) -> ShmArray:
|
||||
|
|
@ -572,7 +563,7 @@ def attach_shm_array(
|
|||
raise _err
|
||||
|
||||
shmarr = np.ndarray(
|
||||
(token.size,),
|
||||
(size,),
|
||||
dtype=token.dtype,
|
||||
buffer=shm.buf
|
||||
)
|
||||
|
|
@ -611,8 +602,8 @@ def attach_shm_array(
|
|||
if key not in _known_tokens:
|
||||
_known_tokens[key] = token
|
||||
|
||||
# "close" attached shm on actor teardown
|
||||
tractor.current_actor().lifetime_stack.callback(sha.close)
|
||||
# "close" attached shm on process teardown
|
||||
tractor._actor._lifetime_stack.callback(sha.close)
|
||||
|
||||
return sha
|
||||
|
||||
|
|
@ -640,23 +631,18 @@ def maybe_open_shm_array(
|
|||
use ``attach_shm_array``.
|
||||
|
||||
'''
|
||||
size = kwargs.pop('size', _default_size)
|
||||
try:
|
||||
# see if we already know this key
|
||||
token = _known_tokens[key]
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except KeyError:
|
||||
log.debug(f"Could not find {key} in shms cache")
|
||||
log.warning(f"Could not find {key} in shms cache")
|
||||
if dtype:
|
||||
token = _make_token(
|
||||
key,
|
||||
size=size,
|
||||
dtype=dtype,
|
||||
)
|
||||
token = _make_token(key, dtype)
|
||||
try:
|
||||
return attach_shm_array(token=token, **kwargs), False
|
||||
except FileNotFoundError:
|
||||
log.debug(f"Could not attach to shm with token {token}")
|
||||
log.warning(f"Could not attach to shm with token {token}")
|
||||
|
||||
# This actor does not know about memory
|
||||
# associated with the provided "key".
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) 2018-present Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,9 +18,13 @@
|
|||
numpy data source coversion helpers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import Any
|
||||
import decimal
|
||||
|
||||
from bidict import bidict
|
||||
import numpy as np
|
||||
from pydantic import BaseModel
|
||||
# from numba import from_dtype
|
||||
|
||||
|
||||
ohlc_fields = [
|
||||
|
|
@ -42,7 +46,6 @@ base_ohlc_dtype = np.dtype(ohlc_fields)
|
|||
|
||||
# TODO: for now need to construct this manually for readonly arrays, see
|
||||
# https://github.com/numba/numba/issues/4511
|
||||
# from numba import from_dtype
|
||||
# numba_ohlc_dtype = from_dtype(base_ohlc_dtype)
|
||||
|
||||
# map time frame "keys" to seconds values
|
||||
|
|
@ -57,6 +60,28 @@ tf_in_1s = bidict({
|
|||
})
|
||||
|
||||
|
||||
def mk_fqsn(
|
||||
provider: str,
|
||||
symbol: str,
|
||||
|
||||
) -> str:
|
||||
'''
|
||||
Generate a "fully qualified symbol name" which is
|
||||
a reverse-hierarchical cross broker/provider symbol
|
||||
|
||||
'''
|
||||
return '.'.join([symbol, provider]).lower()
|
||||
|
||||
|
||||
def float_digits(
|
||||
value: float,
|
||||
) -> int:
|
||||
if value == 0:
|
||||
return 0
|
||||
|
||||
return int(-decimal.Decimal(str(value)).as_tuple().exponent)
|
||||
|
||||
|
||||
def ohlc_zeros(length: int) -> np.ndarray:
|
||||
"""Construct an OHLC field formatted structarray.
|
||||
|
||||
|
|
@ -67,6 +92,168 @@ def ohlc_zeros(length: int) -> np.ndarray:
|
|||
return np.zeros(length, dtype=base_ohlc_dtype)
|
||||
|
||||
|
||||
def unpack_fqsn(fqsn: str) -> tuple[str, str, str]:
|
||||
'''
|
||||
Unpack a fully-qualified-symbol-name to ``tuple``.
|
||||
|
||||
'''
|
||||
venue = ''
|
||||
suffix = ''
|
||||
|
||||
# TODO: probably reverse the order of all this XD
|
||||
tokens = fqsn.split('.')
|
||||
if len(tokens) < 3:
|
||||
# probably crypto
|
||||
symbol, broker = tokens
|
||||
return (
|
||||
broker,
|
||||
symbol,
|
||||
'',
|
||||
)
|
||||
|
||||
elif len(tokens) > 3:
|
||||
symbol, venue, suffix, broker = tokens
|
||||
else:
|
||||
symbol, venue, broker = tokens
|
||||
suffix = ''
|
||||
|
||||
# head, _, broker = fqsn.rpartition('.')
|
||||
# symbol, _, suffix = head.rpartition('.')
|
||||
return (
|
||||
broker,
|
||||
'.'.join([symbol, venue]),
|
||||
suffix,
|
||||
)
|
||||
|
||||
|
||||
class Symbol(BaseModel):
|
||||
'''
|
||||
I guess this is some kinda container thing for dealing with
|
||||
all the different meta-data formats from brokers?
|
||||
|
||||
'''
|
||||
key: str
|
||||
tick_size: float = 0.01
|
||||
lot_tick_size: float = 0.0 # "volume" precision as min step value
|
||||
tick_size_digits: int = 2
|
||||
lot_size_digits: int = 0
|
||||
suffix: str = ''
|
||||
broker_info: dict[str, dict[str, Any]] = {}
|
||||
|
||||
# specifies a "class" of financial instrument
|
||||
# ex. stock, futer, option, bond etc.
|
||||
|
||||
# @validate_arguments
|
||||
@classmethod
|
||||
def from_broker_info(
|
||||
cls,
|
||||
broker: str,
|
||||
symbol: str,
|
||||
info: dict[str, Any],
|
||||
suffix: str = '',
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
|
||||
tick_size = info.get('price_tick_size', 0.01)
|
||||
lot_tick_size = info.get('lot_tick_size', 0.0)
|
||||
|
||||
return Symbol(
|
||||
key=symbol,
|
||||
tick_size=tick_size,
|
||||
lot_tick_size=lot_tick_size,
|
||||
tick_size_digits=float_digits(tick_size),
|
||||
lot_size_digits=float_digits(lot_tick_size),
|
||||
suffix=suffix,
|
||||
broker_info={broker: info},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_fqsn(
|
||||
cls,
|
||||
fqsn: str,
|
||||
info: dict[str, Any],
|
||||
|
||||
# XXX: like wtf..
|
||||
# ) -> 'Symbol':
|
||||
) -> None:
|
||||
broker, key, suffix = unpack_fqsn(fqsn)
|
||||
return cls.from_broker_info(
|
||||
broker,
|
||||
key,
|
||||
info=info,
|
||||
suffix=suffix,
|
||||
)
|
||||
|
||||
@property
|
||||
def type_key(self) -> str:
|
||||
return list(self.broker_info.values())[0]['asset_type']
|
||||
|
||||
@property
|
||||
def brokers(self) -> list[str]:
|
||||
return list(self.broker_info.keys())
|
||||
|
||||
def nearest_tick(self, value: float) -> float:
|
||||
'''
|
||||
Return the nearest tick value based on mininum increment.
|
||||
|
||||
'''
|
||||
mult = 1 / self.tick_size
|
||||
return round(value * mult) / mult
|
||||
|
||||
def front_feed(self) -> tuple[str, str]:
|
||||
'''
|
||||
Return the "current" feed key for this symbol.
|
||||
|
||||
(i.e. the broker + symbol key in a tuple).
|
||||
|
||||
'''
|
||||
return (
|
||||
list(self.broker_info.keys())[0],
|
||||
self.key,
|
||||
)
|
||||
|
||||
def tokens(self) -> tuple[str]:
|
||||
broker, key = self.front_feed()
|
||||
if self.suffix:
|
||||
return (key, self.suffix, broker)
|
||||
else:
|
||||
return (key, broker)
|
||||
|
||||
def front_fqsn(self) -> str:
|
||||
'''
|
||||
fqsn = "fully qualified symbol name"
|
||||
|
||||
Basically the idea here is for all client-ish code (aka programs/actors
|
||||
that ask the provider agnostic layers in the stack for data) should be
|
||||
able to tell which backend / venue / derivative each data feed/flow is
|
||||
from by an explicit string key of the current form:
|
||||
|
||||
<instrumentname>.<venue>.<suffixwithmetadata>.<brokerbackendname>
|
||||
|
||||
TODO: I have thoughts that we should actually change this to be
|
||||
more like an "attr lookup" (like how the web should have done
|
||||
urls, but marketting peeps ruined it etc. etc.):
|
||||
|
||||
<broker>.<venue>.<instrumentname>.<suffixwithmetadata>
|
||||
|
||||
'''
|
||||
tokens = self.tokens()
|
||||
fqsn = '.'.join(tokens)
|
||||
return fqsn
|
||||
|
||||
def iterfqsns(self) -> list[str]:
|
||||
keys = []
|
||||
for broker in self.broker_info.keys():
|
||||
fqsn = mk_fqsn(self.key, broker)
|
||||
if self.suffix:
|
||||
fqsn += f'.{self.suffix}'
|
||||
keys.append(fqsn)
|
||||
|
||||
return keys
|
||||
|
||||
|
||||
def _nan_to_closest_num(array: np.ndarray):
|
||||
"""Return interpolated values instead of NaN.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Data layer module commons.
|
||||
|
||||
'''
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.data'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,30 +18,13 @@
|
|||
ToOlS fOr CoPInG wITh "tHE wEB" protocols.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from itertools import count
|
||||
from functools import partial
|
||||
from contextlib import asynccontextmanager, AsyncExitStack
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
Callable,
|
||||
AsyncContextManager,
|
||||
AsyncGenerator,
|
||||
Iterable,
|
||||
)
|
||||
from typing import Any, Callable, AsyncGenerator
|
||||
import json
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
from trio_websocket import (
|
||||
WebSocketConnection,
|
||||
open_websocket_url,
|
||||
)
|
||||
from wsproto.utilities import LocalProtocolError
|
||||
import trio_websocket
|
||||
from trio_websocket._impl import (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
|
|
@ -50,71 +33,81 @@ from trio_websocket._impl import (
|
|||
ConnectionTimeout,
|
||||
)
|
||||
|
||||
from ._util import log
|
||||
from .types import Struct
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class NoBsWs:
|
||||
'''
|
||||
Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
"""Make ``trio_websocket`` sockets stay up no matter the bs.
|
||||
|
||||
A shim interface that allows client code to stream from some
|
||||
``WebSocketConnection`` but where any connectivy bs is handled
|
||||
automatcially and entirely in the background.
|
||||
|
||||
NOTE: this type should never be created directly but instead is
|
||||
provided via the ``open_autorecon_ws()`` factor below.
|
||||
|
||||
'''
|
||||
# apparently we can QoS for all sorts of reasons..so catch em.
|
||||
"""
|
||||
recon_errors = (
|
||||
ConnectionClosed,
|
||||
DisconnectionTimeout,
|
||||
ConnectionRejected,
|
||||
HandshakeError,
|
||||
ConnectionTimeout,
|
||||
LocalProtocolError,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
rxchan: trio.MemoryReceiveChannel,
|
||||
msg_recv_timeout: float,
|
||||
|
||||
serializer: ModuleType = json
|
||||
token: str,
|
||||
stack: AsyncExitStack,
|
||||
fixture: Callable,
|
||||
serializer: ModuleType = json,
|
||||
):
|
||||
self.url = url
|
||||
self._rx = rxchan
|
||||
self._timeout = msg_recv_timeout
|
||||
self.token = token
|
||||
self.fixture = fixture
|
||||
self._stack = stack
|
||||
self._ws: 'WebSocketConnection' = None # noqa
|
||||
|
||||
# signaling between caller and relay task which determines when
|
||||
# socket is connected (and subscribed).
|
||||
self._connected: trio.Event = trio.Event()
|
||||
async def _connect(
|
||||
self,
|
||||
tries: int = 1000,
|
||||
) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._stack.aclose()
|
||||
except (DisconnectionTimeout, RuntimeError):
|
||||
await trio.sleep(0.5)
|
||||
else:
|
||||
break
|
||||
|
||||
# dynamically reset by the bg relay task
|
||||
self._ws: WebSocketConnection | None = None
|
||||
self._cs: trio.CancelScope | None = None
|
||||
last_err = None
|
||||
for i in range(tries):
|
||||
try:
|
||||
self._ws = await self._stack.enter_async_context(
|
||||
trio_websocket.open_websocket_url(self.url)
|
||||
)
|
||||
# rerun user code fixture
|
||||
if self.token == '':
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self)
|
||||
)
|
||||
else:
|
||||
ret = await self._stack.enter_async_context(
|
||||
self.fixture(self, self.token)
|
||||
)
|
||||
|
||||
# interchange codec methods
|
||||
# TODO: obviously the method API here may be different
|
||||
# for another interchange format..
|
||||
self._dumps: Callable = serializer.dumps
|
||||
self._loads: Callable = serializer.loads
|
||||
assert ret is None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return self._connected.is_set()
|
||||
log.info(f'Connection success: {self.url}')
|
||||
return self._ws
|
||||
|
||||
async def reset(self) -> None:
|
||||
'''
|
||||
Reset the underlying ws connection by cancelling
|
||||
the bg relay task and waiting for it to signal
|
||||
a new connection.
|
||||
|
||||
'''
|
||||
self._connected = trio.Event()
|
||||
self._cs.cancel()
|
||||
await self._connected.wait()
|
||||
except self.recon_errors as err:
|
||||
last_err = err
|
||||
log.error(
|
||||
f'{self} connection bail with '
|
||||
f'{type(err)}...retry attempt {i}'
|
||||
)
|
||||
await trio.sleep(0.5)
|
||||
continue
|
||||
else:
|
||||
log.exception('ws connection fail...')
|
||||
raise last_err
|
||||
|
||||
async def send_msg(
|
||||
self,
|
||||
|
|
@ -122,318 +115,38 @@ class NoBsWs:
|
|||
) -> None:
|
||||
while True:
|
||||
try:
|
||||
msg: Any = self._dumps(data)
|
||||
return await self._ws.send_message(msg)
|
||||
return await self._ws.send_message(json.dumps(data))
|
||||
except self.recon_errors:
|
||||
await self.reset()
|
||||
await self._connect()
|
||||
|
||||
async def recv_msg(self) -> Any:
|
||||
msg: Any = await self._rx.receive()
|
||||
data = self._loads(msg)
|
||||
return data
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
return await self.recv_msg()
|
||||
|
||||
def set_recv_timeout(
|
||||
async def recv_msg(
|
||||
self,
|
||||
timeout: float,
|
||||
) -> None:
|
||||
self._timeout = timeout
|
||||
|
||||
|
||||
async def _reconnect_forever(
|
||||
url: str,
|
||||
snd: trio.MemorySendChannel,
|
||||
nobsws: NoBsWs,
|
||||
reset_after: int, # msg recv timeout before reset attempt
|
||||
|
||||
fixture: AsyncContextManager | None = None,
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
||||
async def proxy_msgs(
|
||||
ws: WebSocketConnection,
|
||||
pcs: trio.CancelScope, # parent cancel scope
|
||||
):
|
||||
'''
|
||||
Receive (under `timeout` deadline) all msgs from from underlying
|
||||
websocket and relay them to (calling) parent task via ``trio``
|
||||
mem chan.
|
||||
|
||||
'''
|
||||
# after so many msg recv timeouts, reset the connection
|
||||
timeouts: int = 0
|
||||
|
||||
) -> Any:
|
||||
while True:
|
||||
with trio.move_on_after(
|
||||
# can be dynamically changed by user code
|
||||
nobsws._timeout,
|
||||
) as cs:
|
||||
try:
|
||||
msg: Any = await ws.get_message()
|
||||
await snd.send(msg)
|
||||
except nobsws.recon_errors:
|
||||
log.exception(
|
||||
f'{url} connection bail with:'
|
||||
)
|
||||
await trio.sleep(0.5)
|
||||
pcs.cancel()
|
||||
|
||||
# go back to reonnect loop in parent task
|
||||
return
|
||||
|
||||
if cs.cancelled_caught:
|
||||
timeouts += 1
|
||||
if timeouts > reset_after:
|
||||
log.error(
|
||||
'WS feed seems down and slow af? .. resetting\n'
|
||||
)
|
||||
pcs.cancel()
|
||||
|
||||
# go back to reonnect loop in parent task
|
||||
return
|
||||
|
||||
async def open_fixture(
|
||||
fixture: AsyncContextManager,
|
||||
nobsws: NoBsWs,
|
||||
task_status: TaskStatus = trio.TASK_STATUS_IGNORED,
|
||||
):
|
||||
'''
|
||||
Open user provided `@acm` and sleep until any connection
|
||||
reset occurs.
|
||||
|
||||
'''
|
||||
async with fixture(nobsws) as ret:
|
||||
assert ret is None
|
||||
task_status.started()
|
||||
await trio.sleep_forever()
|
||||
|
||||
# last_err = None
|
||||
nobsws._connected = trio.Event()
|
||||
task_status.started()
|
||||
|
||||
while not snd._closed:
|
||||
log.info(f'{url} trying (RE)CONNECT')
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
cs = nobsws._cs = n.cancel_scope
|
||||
ws: WebSocketConnection
|
||||
async with open_websocket_url(url) as ws:
|
||||
nobsws._ws = ws
|
||||
log.info(f'Connection success: {url}')
|
||||
|
||||
# begin relay loop to forward msgs
|
||||
n.start_soon(
|
||||
proxy_msgs,
|
||||
ws,
|
||||
cs,
|
||||
)
|
||||
|
||||
if fixture is not None:
|
||||
log.info(f'Entering fixture: {fixture}')
|
||||
|
||||
# TODO: should we return an explicit sub-cs
|
||||
# from this fixture task?
|
||||
await n.start(
|
||||
open_fixture,
|
||||
fixture,
|
||||
nobsws,
|
||||
)
|
||||
|
||||
# indicate to wrapper / opener that we are up and block
|
||||
# to let tasks run **inside** the ws open block above.
|
||||
nobsws._connected.set()
|
||||
await trio.sleep_forever()
|
||||
|
||||
# ws open block end
|
||||
# nursery block end
|
||||
nobsws._connected = trio.Event()
|
||||
if cs.cancelled_caught:
|
||||
log.cancel(
|
||||
f'{url} connection cancelled!'
|
||||
)
|
||||
# if wrapper cancelled us, we expect it to also
|
||||
# have re-assigned a new event
|
||||
assert (
|
||||
nobsws._connected
|
||||
and not nobsws._connected.is_set()
|
||||
)
|
||||
|
||||
# -> from here, move to next reconnect attempt
|
||||
|
||||
else:
|
||||
log.exception('ws connection closed by client...')
|
||||
return json.loads(await self._ws.get_message())
|
||||
except self.recon_errors:
|
||||
await self._connect()
|
||||
|
||||
|
||||
@acm
|
||||
@asynccontextmanager
|
||||
async def open_autorecon_ws(
|
||||
url: str,
|
||||
|
||||
fixture: AsyncContextManager | None = None,
|
||||
|
||||
# time in sec
|
||||
msg_recv_timeout: float = 3,
|
||||
|
||||
# count of the number of above timeouts before connection reset
|
||||
reset_after: int = 3,
|
||||
|
||||
# TODO: proper type annot smh
|
||||
fixture: Callable,
|
||||
# used for authenticated websockets
|
||||
token: str = '',
|
||||
) -> AsyncGenerator[tuple[...], NoBsWs]:
|
||||
'''
|
||||
An auto-reconnect websocket (wrapper API) around
|
||||
``trio_websocket.open_websocket_url()`` providing automatic
|
||||
re-connection on network errors, msg latency and thus roaming.
|
||||
"""Apparently we can QoS for all sorts of reasons..so catch em.
|
||||
|
||||
Here we implement a re-connect websocket interface where a bg
|
||||
nursery runs ``WebSocketConnection.receive_message()``s in a loop
|
||||
and restarts the full http(s) handshake on catches of certain
|
||||
connetivity errors, or some user defined recv timeout.
|
||||
|
||||
You can provide a ``fixture`` async-context-manager which will be
|
||||
entered/exitted around each connection reset; eg. for (re)requesting
|
||||
subscriptions without requiring streaming setup code to rerun.
|
||||
|
||||
'''
|
||||
snd: trio.MemorySendChannel
|
||||
rcv: trio.MemoryReceiveChannel
|
||||
snd, rcv = trio.open_memory_channel(616)
|
||||
|
||||
async with trio.open_nursery() as n:
|
||||
nobsws = NoBsWs(
|
||||
url,
|
||||
rcv,
|
||||
msg_recv_timeout=msg_recv_timeout,
|
||||
)
|
||||
await n.start(
|
||||
partial(
|
||||
_reconnect_forever,
|
||||
url,
|
||||
snd,
|
||||
nobsws,
|
||||
fixture=fixture,
|
||||
reset_after=reset_after,
|
||||
)
|
||||
)
|
||||
await nobsws._connected.wait()
|
||||
assert nobsws._cs
|
||||
assert nobsws.connected()
|
||||
"""
|
||||
async with AsyncExitStack() as stack:
|
||||
ws = NoBsWs(url, token, stack, fixture=fixture)
|
||||
await ws._connect()
|
||||
|
||||
try:
|
||||
yield nobsws
|
||||
yield ws
|
||||
|
||||
finally:
|
||||
n.cancel_scope.cancel()
|
||||
|
||||
|
||||
'''
|
||||
JSONRPC response-request style machinery for transparent multiplexing of msgs
|
||||
over a NoBsWs.
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class JSONRPCResult(Struct):
|
||||
id: int
|
||||
jsonrpc: str = '2.0'
|
||||
result: Optional[dict] = None
|
||||
error: Optional[dict] = None
|
||||
|
||||
|
||||
@acm
|
||||
async def open_jsonrpc_session(
|
||||
url: str,
|
||||
start_id: int = 0,
|
||||
response_type: type = JSONRPCResult,
|
||||
request_type: Optional[type] = None,
|
||||
request_hook: Optional[Callable] = None,
|
||||
error_hook: Optional[Callable] = None,
|
||||
) -> Callable[[str, dict], dict]:
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
open_autorecon_ws(url) as ws
|
||||
):
|
||||
rpc_id: Iterable = count(start_id)
|
||||
rpc_results: dict[int, dict] = {}
|
||||
|
||||
async def json_rpc(method: str, params: dict) -> dict:
|
||||
'''
|
||||
perform a json rpc call and wait for the result, raise exception in
|
||||
case of error field present on response
|
||||
'''
|
||||
msg = {
|
||||
'jsonrpc': '2.0',
|
||||
'id': next(rpc_id),
|
||||
'method': method,
|
||||
'params': params
|
||||
}
|
||||
_id = msg['id']
|
||||
|
||||
rpc_results[_id] = {
|
||||
'result': None,
|
||||
'event': trio.Event()
|
||||
}
|
||||
|
||||
await ws.send_msg(msg)
|
||||
|
||||
await rpc_results[_id]['event'].wait()
|
||||
|
||||
ret = rpc_results[_id]['result']
|
||||
|
||||
del rpc_results[_id]
|
||||
|
||||
if ret.error is not None:
|
||||
raise Exception(json.dumps(ret.error, indent=4))
|
||||
|
||||
return ret
|
||||
|
||||
async def recv_task():
|
||||
'''
|
||||
receives every ws message and stores it in its corresponding
|
||||
result field, then sets the event to wakeup original sender
|
||||
tasks. also recieves responses to requests originated from
|
||||
the server side.
|
||||
|
||||
'''
|
||||
async for msg in ws:
|
||||
match msg:
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if res_entry := rpc_results.get(mid):
|
||||
|
||||
res_entry['result'] = response_type(**msg)
|
||||
res_entry['event'].set()
|
||||
|
||||
case {
|
||||
'result': _,
|
||||
'id': mid,
|
||||
} if not rpc_results.get(mid):
|
||||
log.warning(
|
||||
f'Unexpected ws msg: {json.dumps(msg, indent=4)}'
|
||||
)
|
||||
|
||||
case {
|
||||
'method': _,
|
||||
'params': _,
|
||||
}:
|
||||
log.debug(f'Recieved\n{msg}')
|
||||
if request_hook:
|
||||
await request_hook(request_type(**msg))
|
||||
|
||||
case {
|
||||
'error': error
|
||||
}:
|
||||
log.warning(f'Recieved\n{error}')
|
||||
if error_hook:
|
||||
await error_hook(response_type(**msg))
|
||||
|
||||
case _:
|
||||
log.warning(f'Unhandled JSON-RPC msg!?\n{msg}')
|
||||
|
||||
n.start_soon(recv_task)
|
||||
yield json_rpc
|
||||
n.cancel_scope.cancel()
|
||||
await stack.aclose()
|
||||
|
|
|
|||
|
|
@ -18,23 +18,35 @@
|
|||
marketstore cli.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
from pprint import pformat
|
||||
|
||||
from anyio_marketstore import open_marketstore_client
|
||||
import trio
|
||||
import tractor
|
||||
import click
|
||||
import numpy as np
|
||||
|
||||
from ..service.marketstore import (
|
||||
# get_client,
|
||||
from .marketstore import (
|
||||
get_client,
|
||||
# stream_quotes,
|
||||
ingest_quote_stream,
|
||||
# _url,
|
||||
# _tick_tbk_ids,
|
||||
# mk_tbk,
|
||||
_tick_tbk_ids,
|
||||
mk_tbk,
|
||||
)
|
||||
from ..cli import cli
|
||||
from .. import watchlists as wl
|
||||
from ._util import (
|
||||
log,
|
||||
from ..log import get_logger
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
)
|
||||
from ._source import (
|
||||
base_iohlc_dtype,
|
||||
)
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
@cli.command()
|
||||
|
|
@ -101,11 +113,15 @@ def ms_stream(
|
|||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--tsdb_host',
|
||||
'--tl',
|
||||
is_flag=True,
|
||||
help='Enable tractor logging')
|
||||
@click.option(
|
||||
'--host',
|
||||
default='localhost'
|
||||
)
|
||||
@click.option(
|
||||
'--tsdb_port',
|
||||
'--port',
|
||||
default=5993
|
||||
)
|
||||
@click.argument('symbols', nargs=-1)
|
||||
|
|
@ -121,93 +137,18 @@ def storesh(
|
|||
Start an IPython shell ready to query the local marketstore db.
|
||||
|
||||
'''
|
||||
from piker.data.marketstore import open_tsdb_client
|
||||
from piker.service import open_piker_runtime
|
||||
from piker.data.marketstore import tsdb_history_update
|
||||
from piker._daemon import open_piker_runtime
|
||||
|
||||
async def main():
|
||||
nonlocal symbols
|
||||
|
||||
async with open_piker_runtime(
|
||||
'storesh',
|
||||
enable_modules=['piker.service._ahab'],
|
||||
enable_modules=['piker.data._ahab'],
|
||||
):
|
||||
symbol = symbols[0]
|
||||
|
||||
async with open_tsdb_client(symbol):
|
||||
# TODO: ask if user wants to write history for detected
|
||||
# available shm buffers?
|
||||
from tractor.trionics import ipython_embed
|
||||
await ipython_embed()
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option(
|
||||
'--host',
|
||||
default='localhost'
|
||||
)
|
||||
@click.option(
|
||||
'--port',
|
||||
default=5993
|
||||
)
|
||||
@click.option(
|
||||
'--delete',
|
||||
'-d',
|
||||
is_flag=True,
|
||||
help='Delete history (1 Min) for symbol(s)',
|
||||
)
|
||||
@click.argument('symbols', nargs=-1)
|
||||
@click.pass_obj
|
||||
def storage(
|
||||
config,
|
||||
host,
|
||||
port,
|
||||
symbols: list[str],
|
||||
delete: bool,
|
||||
|
||||
):
|
||||
'''
|
||||
Start an IPython shell ready to query the local marketstore db.
|
||||
|
||||
'''
|
||||
from piker.service.marketstore import open_tsdb_client
|
||||
from piker.service import open_piker_runtime
|
||||
|
||||
async def main():
|
||||
nonlocal symbols
|
||||
|
||||
async with open_piker_runtime(
|
||||
'tsdb_storage',
|
||||
enable_modules=['piker.service._ahab'],
|
||||
):
|
||||
symbol = symbols[0]
|
||||
async with open_tsdb_client(symbol) as storage:
|
||||
if delete:
|
||||
for fqsn in symbols:
|
||||
syms = await storage.client.list_symbols()
|
||||
|
||||
resp60s = await storage.delete_ts(fqsn, 60)
|
||||
|
||||
msgish = resp60s.ListFields()[0][1]
|
||||
if 'error' in str(msgish):
|
||||
|
||||
# TODO: MEGA LOL, apparently the symbols don't
|
||||
# flush out until you refresh something or other
|
||||
# (maybe the WALFILE)... #lelandorlulzone, classic
|
||||
# alpaca(Rtm) design here ..
|
||||
# well, if we ever can make this work we
|
||||
# probably want to dogsplain the real reason
|
||||
# for the delete errurz..llululu
|
||||
if fqsn not in syms:
|
||||
log.error(f'Pair {fqsn} dne in DB')
|
||||
|
||||
log.error(f'Deletion error: {fqsn}\n{msgish}')
|
||||
|
||||
resp1s = await storage.delete_ts(fqsn, 1)
|
||||
msgish = resp1s.ListFields()[0][1]
|
||||
if 'error' in str(msgish):
|
||||
log.error(f'Deletion error: {fqsn}\n{msgish}')
|
||||
await tsdb_history_update(symbol)
|
||||
|
||||
trio.run(main)
|
||||
|
||||
|
|
|
|||
1589
piker/data/feed.py
1589
piker/data/feed.py
File diff suppressed because it is too large
Load Diff
|
|
@ -1,242 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Public abstractions for organizing, managing and generally operating-on
|
||||
real-time data processing data-structures.
|
||||
|
||||
"Streams, flumes, cascades and flows.."
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
# from decimal import Decimal
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import pendulum
|
||||
import numpy as np
|
||||
|
||||
from ..accounting._mktinfo import (
|
||||
MktPair,
|
||||
Symbol,
|
||||
)
|
||||
from ._util import (
|
||||
log,
|
||||
)
|
||||
from .types import Struct
|
||||
from ._sharedmem import (
|
||||
attach_shm_array,
|
||||
ShmArray,
|
||||
_Token,
|
||||
)
|
||||
# from .._profile import (
|
||||
# Profiler,
|
||||
# pg_profile_enabled,
|
||||
# )
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# from pyqtgraph import PlotItem
|
||||
from .feed import Feed
|
||||
|
||||
|
||||
# TODO: ideas for further abstractions as per
|
||||
# https://github.com/pikers/piker/issues/216 and
|
||||
# https://github.com/pikers/piker/issues/270:
|
||||
# - a ``Cascade`` would be the minimal "connection" of 2 ``Flumes``
|
||||
# as per circuit parlance:
|
||||
# https://en.wikipedia.org/wiki/Two-port_network#Cascade_connection
|
||||
# - could cover the combination of our `FspAdmin` and the
|
||||
# backend `.fsp._engine` related machinery to "connect" one flume
|
||||
# to another?
|
||||
# - a (financial signal) ``Flow`` would be the a "collection" of such
|
||||
# minmial cascades. Some engineering based jargon concepts:
|
||||
# - https://en.wikipedia.org/wiki/Signal_chain
|
||||
# - https://en.wikipedia.org/wiki/Daisy_chain_(electrical_engineering)
|
||||
# - https://en.wikipedia.org/wiki/Audio_signal_flow
|
||||
# - https://en.wikipedia.org/wiki/Digital_signal_processing#Implementation
|
||||
# - https://en.wikipedia.org/wiki/Dataflow_programming
|
||||
# - https://en.wikipedia.org/wiki/Signal_programming
|
||||
# - https://en.wikipedia.org/wiki/Incremental_computing
|
||||
|
||||
|
||||
class Flume(Struct):
|
||||
'''
|
||||
Composite reference type which points to all the addressing handles
|
||||
and other meta-data necessary for the read, measure and management
|
||||
of a set of real-time updated data flows.
|
||||
|
||||
Can be thought of as a "flow descriptor" or "flow frame" which
|
||||
describes the high level properties of a set of data flows that can
|
||||
be used seamlessly across process-memory boundaries.
|
||||
|
||||
Each instance's sub-components normally includes:
|
||||
- a msg oriented quote stream provided via an IPC transport
|
||||
- history and real-time shm buffers which are both real-time
|
||||
updated and backfilled.
|
||||
- associated startup indexing information related to both buffer
|
||||
real-time-append and historical prepend addresses.
|
||||
- low level APIs to read and measure the updated data and manage
|
||||
queuing properties.
|
||||
|
||||
'''
|
||||
mkt: MktPair | Symbol
|
||||
first_quote: dict
|
||||
_rt_shm_token: _Token
|
||||
|
||||
@property
|
||||
def symbol(self) -> MktPair | Symbol:
|
||||
log.warning(
|
||||
'`Flume.symbol` is deprecated!\n'
|
||||
'Use `.mkt: MktPair` instead!'
|
||||
)
|
||||
return self.mkt
|
||||
|
||||
# optional since some data flows won't have a "downsampled" history
|
||||
# buffer/stream (eg. FSPs).
|
||||
_hist_shm_token: _Token | None = None
|
||||
|
||||
# private shm refs loaded dynamically from tokens
|
||||
_hist_shm: ShmArray | None = None
|
||||
_rt_shm: ShmArray | None = None
|
||||
|
||||
stream: tractor.MsgStream | None = None
|
||||
izero_hist: int = 0
|
||||
izero_rt: int = 0
|
||||
throttle_rate: int | None = None
|
||||
|
||||
# TODO: do we need this really if we can pull the `Portal` from
|
||||
# ``tractor``'s internals?
|
||||
feed: Feed | None = None
|
||||
|
||||
@property
|
||||
def rt_shm(self) -> ShmArray:
|
||||
|
||||
if self._rt_shm is None:
|
||||
self._rt_shm = attach_shm_array(
|
||||
token=self._rt_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._rt_shm
|
||||
|
||||
@property
|
||||
def hist_shm(self) -> ShmArray:
|
||||
|
||||
if self._hist_shm_token is None:
|
||||
raise RuntimeError(
|
||||
'No shm token has been set for the history buffer?'
|
||||
)
|
||||
|
||||
if (
|
||||
self._hist_shm is None
|
||||
):
|
||||
self._hist_shm = attach_shm_array(
|
||||
token=self._hist_shm_token,
|
||||
readonly=True,
|
||||
)
|
||||
|
||||
return self._hist_shm
|
||||
|
||||
async def receive(self) -> dict:
|
||||
return await self.stream.receive()
|
||||
|
||||
def get_ds_info(
|
||||
self,
|
||||
) -> tuple[float, float, float]:
|
||||
'''
|
||||
Compute the "downsampling" ratio info between the historical shm
|
||||
buffer and the real-time (HFT) one.
|
||||
|
||||
Return a tuple of the fast sample period, historical sample
|
||||
period and ratio between them.
|
||||
|
||||
'''
|
||||
times = self.hist_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
hist_step_size_s = (end - start).seconds
|
||||
|
||||
times = self.rt_shm.array['time']
|
||||
end = pendulum.from_timestamp(times[-1])
|
||||
start = pendulum.from_timestamp(times[times != times[-1]][-1])
|
||||
rt_step_size_s = (end - start).seconds
|
||||
|
||||
ratio = hist_step_size_s / rt_step_size_s
|
||||
return (
|
||||
rt_step_size_s,
|
||||
hist_step_size_s,
|
||||
ratio,
|
||||
)
|
||||
|
||||
# TODO: get native msgspec decoding for these workinn
|
||||
def to_msg(self) -> dict:
|
||||
|
||||
msg = self.to_dict()
|
||||
msg['mkt'] = self.mkt.to_dict()
|
||||
|
||||
# can't serialize the stream or feed objects, it's expected
|
||||
# you'll have a ref to it since this msg should be rxed on
|
||||
# a stream on whatever far end IPC..
|
||||
msg.pop('stream')
|
||||
msg.pop('feed')
|
||||
return msg
|
||||
|
||||
@classmethod
|
||||
def from_msg(
|
||||
cls,
|
||||
msg: dict,
|
||||
|
||||
) -> dict:
|
||||
'''
|
||||
Load from an IPC msg presumably in either `dict` or
|
||||
`msgspec.Struct` form.
|
||||
|
||||
'''
|
||||
mkt_msg = msg.pop('mkt')
|
||||
|
||||
if 'dst' in mkt_msg:
|
||||
mkt = MktPair.from_msg(mkt_msg)
|
||||
|
||||
else:
|
||||
# XXX NOTE: ``msgspec`` can encode `Decimal`
|
||||
# but it doesn't decide to it by default since
|
||||
# we aren't spec-cing these msgs as structs, SO
|
||||
# we have to ensure we do a struct type case (which `.copy()`
|
||||
# does) to ensure we get the right type!
|
||||
mkt = Symbol(**mkt_msg).copy()
|
||||
|
||||
return cls(mkt=mkt, **msg)
|
||||
|
||||
def get_index(
|
||||
self,
|
||||
time_s: float,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> int | float:
|
||||
'''
|
||||
Return array shm-buffer index for for epoch time.
|
||||
|
||||
'''
|
||||
times = array['time']
|
||||
first = np.searchsorted(
|
||||
times,
|
||||
time_s,
|
||||
side='left',
|
||||
)
|
||||
imx = times.shape[0] - 1
|
||||
return min(first, imx)
|
||||
|
|
@ -1,770 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Historical data business logic for load, backfill and tsdb storage.
|
||||
|
||||
'''
|
||||
from __future__ import annotations
|
||||
from collections import (
|
||||
Counter,
|
||||
)
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
import time
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Callable,
|
||||
Optional,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
import pendulum
|
||||
import numpy as np
|
||||
|
||||
from ._util import (
|
||||
log,
|
||||
)
|
||||
from ..service import (
|
||||
check_for_service,
|
||||
)
|
||||
from ._sharedmem import (
|
||||
maybe_open_shm_array,
|
||||
ShmArray,
|
||||
_secs_in_day,
|
||||
)
|
||||
from ..accounting._mktinfo import (
|
||||
unpack_fqme,
|
||||
)
|
||||
from ._source import base_iohlc_dtype
|
||||
from ._sampling import (
|
||||
open_sample_stream,
|
||||
)
|
||||
from ..brokers._util import (
|
||||
DataUnavailable,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..service.marketstore import Storage
|
||||
from .feed import _FeedsBus
|
||||
|
||||
|
||||
def diff_history(
|
||||
array: np.ndarray,
|
||||
timeframe: int,
|
||||
start_dt: datetime,
|
||||
end_dt: datetime,
|
||||
last_tsdb_dt: datetime | None = None
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
# no diffing with tsdb dt index possible..
|
||||
if last_tsdb_dt is None:
|
||||
return array
|
||||
|
||||
time = array['time']
|
||||
return array[time > last_tsdb_dt.timestamp()]
|
||||
|
||||
|
||||
async def start_backfill(
|
||||
mod: ModuleType,
|
||||
bfqsn: str,
|
||||
shm: ShmArray,
|
||||
timeframe: float,
|
||||
sampler_stream: tractor.MsgStream,
|
||||
feed_is_live: trio.Event,
|
||||
|
||||
last_tsdb_dt: Optional[datetime] = None,
|
||||
storage: Optional[Storage] = None,
|
||||
write_tsdb: bool = True,
|
||||
tsdb_is_up: bool = False,
|
||||
|
||||
task_status: TaskStatus[tuple] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> int:
|
||||
|
||||
hist: Callable[
|
||||
[int, datetime, datetime],
|
||||
tuple[np.ndarray, str]
|
||||
]
|
||||
config: dict[str, int]
|
||||
async with mod.open_history_client(bfqsn) as (hist, config):
|
||||
|
||||
# get latest query's worth of history all the way
|
||||
# back to what is recorded in the tsdb
|
||||
array, start_dt, end_dt = await hist(
|
||||
timeframe,
|
||||
end_dt=None,
|
||||
)
|
||||
times = array['time']
|
||||
|
||||
# sample period step size in seconds
|
||||
step_size_s = (
|
||||
pendulum.from_timestamp(times[-1])
|
||||
- pendulum.from_timestamp(times[-2])
|
||||
).seconds
|
||||
|
||||
# if the market is open (aka we have a live feed) but the
|
||||
# history sample step index seems off we report the surrounding
|
||||
# data and drop into a bp. this case shouldn't really ever
|
||||
# happen if we're doing history retrieval correctly.
|
||||
if (
|
||||
step_size_s == 60
|
||||
and feed_is_live.is_set()
|
||||
):
|
||||
inow = round(time.time())
|
||||
diff = inow - times[-1]
|
||||
if abs(diff) > 60:
|
||||
surr = array[-6:]
|
||||
diff_in_mins = round(diff/60., ndigits=2)
|
||||
log.warning(
|
||||
f'STEP ERROR `{bfqsn}` for period {step_size_s}s:\n'
|
||||
f'Off by `{diff}` seconds (or `{diff_in_mins}` mins)\n'
|
||||
'Surrounding 6 time stamps:\n'
|
||||
f'{list(surr["time"])}\n'
|
||||
'Here is surrounding 6 samples:\n'
|
||||
f'{surr}\nn'
|
||||
)
|
||||
|
||||
# uncomment this for a hacker who wants to investigate
|
||||
# this case manually..
|
||||
# await tractor.breakpoint()
|
||||
|
||||
# frame's worth of sample-period-steps, in seconds
|
||||
frame_size_s = len(array) * step_size_s
|
||||
|
||||
to_push = diff_history(
|
||||
array,
|
||||
timeframe,
|
||||
start_dt,
|
||||
end_dt,
|
||||
last_tsdb_dt=last_tsdb_dt,
|
||||
)
|
||||
|
||||
log.info(f'Pushing {to_push.size} to shm!')
|
||||
shm.push(to_push, prepend=True)
|
||||
|
||||
# TODO: *** THIS IS A BUG ***
|
||||
# we need to only broadcast to subscribers for this fqsn..
|
||||
# otherwise all fsps get reset on every chart..
|
||||
await sampler_stream.send('broadcast_all')
|
||||
|
||||
# signal that backfilling to tsdb's end datum is complete
|
||||
bf_done = trio.Event()
|
||||
|
||||
# let caller unblock and deliver latest history frame
|
||||
task_status.started((
|
||||
start_dt,
|
||||
end_dt,
|
||||
bf_done,
|
||||
))
|
||||
|
||||
# based on the sample step size, maybe load a certain amount history
|
||||
if last_tsdb_dt is None:
|
||||
if step_size_s not in (1, 60):
|
||||
raise ValueError(
|
||||
'`piker` only needs to support 1m and 1s sampling '
|
||||
'but ur api is trying to deliver a longer '
|
||||
f'timeframe of {step_size_s} seconds..\n'
|
||||
'So yuh.. dun do dat brudder.'
|
||||
)
|
||||
|
||||
# when no tsdb "last datum" is provided, we just load
|
||||
# some near-term history.
|
||||
periods = {
|
||||
1: {'days': 1},
|
||||
60: {'days': 14},
|
||||
}
|
||||
|
||||
if tsdb_is_up:
|
||||
# do a decently sized backfill and load it into storage.
|
||||
periods = {
|
||||
1: {'days': 6},
|
||||
60: {'years': 6},
|
||||
}
|
||||
|
||||
period_duration = periods[step_size_s]
|
||||
|
||||
# NOTE: manually set the "latest" datetime which we intend to
|
||||
# backfill history "until" so as to adhere to the history
|
||||
# settings above when the tsdb is detected as being empty.
|
||||
last_tsdb_dt = start_dt.subtract(**period_duration)
|
||||
|
||||
# configure async query throttling
|
||||
# rate = config.get('rate', 1)
|
||||
# XXX: legacy from ``trimeter`` code but unsupported now.
|
||||
# erlangs = config.get('erlangs', 1)
|
||||
|
||||
# avoid duplicate history frames with a set of datetime frame
|
||||
# starts and associated counts of how many duplicates we see
|
||||
# per time stamp.
|
||||
starts: Counter[datetime] = Counter()
|
||||
|
||||
# inline sequential loop where we simply pass the
|
||||
# last retrieved start dt to the next request as
|
||||
# it's end dt.
|
||||
while end_dt > last_tsdb_dt:
|
||||
log.debug(
|
||||
f'Requesting {step_size_s}s frame ending in {start_dt}'
|
||||
)
|
||||
|
||||
try:
|
||||
array, next_start_dt, end_dt = await hist(
|
||||
timeframe,
|
||||
end_dt=start_dt,
|
||||
)
|
||||
|
||||
# broker says there never was or is no more history to pull
|
||||
except DataUnavailable:
|
||||
log.warning(
|
||||
f'NO-MORE-DATA: backend {mod.name} halted history!?'
|
||||
)
|
||||
|
||||
# ugh, what's a better way?
|
||||
# TODO: fwiw, we probably want a way to signal a throttle
|
||||
# condition (eg. with ib) so that we can halt the
|
||||
# request loop until the condition is resolved?
|
||||
return
|
||||
|
||||
if (
|
||||
next_start_dt in starts
|
||||
and starts[next_start_dt] <= 6
|
||||
):
|
||||
start_dt = min(starts)
|
||||
log.warning(
|
||||
f"{bfqsn}: skipping duplicate frame @ {next_start_dt}"
|
||||
)
|
||||
starts[start_dt] += 1
|
||||
continue
|
||||
|
||||
elif starts[next_start_dt] > 6:
|
||||
log.warning(
|
||||
f'NO-MORE-DATA: backend {mod.name} before {next_start_dt}?'
|
||||
)
|
||||
return
|
||||
|
||||
# only update new start point if not-yet-seen
|
||||
start_dt = next_start_dt
|
||||
starts[start_dt] += 1
|
||||
|
||||
assert array['time'][0] == start_dt.timestamp()
|
||||
|
||||
diff = end_dt - start_dt
|
||||
frame_time_diff_s = diff.seconds
|
||||
expected_frame_size_s = frame_size_s + step_size_s
|
||||
|
||||
if frame_time_diff_s > expected_frame_size_s:
|
||||
|
||||
# XXX: query result includes a start point prior to our
|
||||
# expected "frame size" and thus is likely some kind of
|
||||
# history gap (eg. market closed period, outage, etc.)
|
||||
# so just report it to console for now.
|
||||
log.warning(
|
||||
f'History frame ending @ {end_dt} appears to have a gap:\n'
|
||||
f'{diff} ~= {frame_time_diff_s} seconds'
|
||||
)
|
||||
|
||||
to_push = diff_history(
|
||||
array,
|
||||
timeframe,
|
||||
start_dt,
|
||||
end_dt,
|
||||
last_tsdb_dt=last_tsdb_dt,
|
||||
)
|
||||
ln = len(to_push)
|
||||
if ln:
|
||||
log.info(f'{ln} bars for {start_dt} -> {end_dt}')
|
||||
|
||||
else:
|
||||
log.warning(
|
||||
f'{ln} BARS TO PUSH after diff?!: {start_dt} -> {end_dt}'
|
||||
)
|
||||
|
||||
# bail gracefully on shm allocation overrun/full condition
|
||||
try:
|
||||
shm.push(to_push, prepend=True)
|
||||
except ValueError:
|
||||
log.info(
|
||||
f'Shm buffer overrun on: {start_dt} -> {end_dt}?'
|
||||
)
|
||||
# can't push the entire frame? so
|
||||
# push only the amount that can fit..
|
||||
break
|
||||
|
||||
log.info(
|
||||
f'Shm pushed {ln} frame:\n'
|
||||
f'{start_dt} -> {end_dt}'
|
||||
)
|
||||
|
||||
if (
|
||||
storage is not None
|
||||
and write_tsdb
|
||||
):
|
||||
log.info(
|
||||
f'Writing {ln} frame to storage:\n'
|
||||
f'{start_dt} -> {end_dt}'
|
||||
)
|
||||
await storage.write_ohlcv(
|
||||
f'{bfqsn}.{mod.name}', # lul..
|
||||
to_push,
|
||||
timeframe,
|
||||
)
|
||||
|
||||
# TODO: can we only trigger this if the respective
|
||||
# history in "in view"?!?
|
||||
|
||||
# XXX: extremely important, there can be no checkpoints
|
||||
# in the block above to avoid entering new ``frames``
|
||||
# values while we're pipelining the current ones to
|
||||
# memory...
|
||||
await sampler_stream.send('broadcast_all')
|
||||
|
||||
# short-circuit (for now)
|
||||
bf_done.set()
|
||||
|
||||
|
||||
async def basic_backfill(
|
||||
bus: _FeedsBus,
|
||||
mod: ModuleType,
|
||||
bfqsn: str,
|
||||
shms: dict[int, ShmArray],
|
||||
sampler_stream: tractor.MsgStream,
|
||||
feed_is_live: trio.Event,
|
||||
|
||||
) -> None:
|
||||
|
||||
# do a legacy incremental backfill from the provider.
|
||||
log.info('No TSDB (marketstored) found, doing basic backfill..')
|
||||
|
||||
# start history backfill task ``backfill_bars()`` is
|
||||
# a required backend func this must block until shm is
|
||||
# filled with first set of ohlc bars
|
||||
for timeframe, shm in shms.items():
|
||||
try:
|
||||
await bus.nursery.start(
|
||||
partial(
|
||||
start_backfill,
|
||||
mod,
|
||||
bfqsn,
|
||||
shm,
|
||||
timeframe,
|
||||
sampler_stream,
|
||||
feed_is_live,
|
||||
)
|
||||
)
|
||||
except DataUnavailable:
|
||||
# XXX: timeframe not supported for backend
|
||||
continue
|
||||
|
||||
|
||||
async def tsdb_backfill(
|
||||
mod: ModuleType,
|
||||
marketstore: ModuleType,
|
||||
bus: _FeedsBus,
|
||||
storage: Storage,
|
||||
fqsn: str,
|
||||
bfqsn: str,
|
||||
shms: dict[int, ShmArray],
|
||||
sampler_stream: tractor.MsgStream,
|
||||
feed_is_live: trio.Event,
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[ShmArray, ShmArray]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
|
||||
# TODO: this should be used verbatim for the pure
|
||||
# shm backfiller approach below.
|
||||
dts_per_tf: dict[int, datetime] = {}
|
||||
|
||||
# start history anal and load missing new data via backend.
|
||||
for timeframe, shm in shms.items():
|
||||
# loads a (large) frame of data from the tsdb depending
|
||||
# on the db's query size limit.
|
||||
tsdb_history, first_tsdb_dt, last_tsdb_dt = await storage.load(
|
||||
fqsn,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
|
||||
broker, *_ = unpack_fqme(fqsn)
|
||||
try:
|
||||
(
|
||||
latest_start_dt,
|
||||
latest_end_dt,
|
||||
bf_done,
|
||||
) = await bus.nursery.start(
|
||||
partial(
|
||||
start_backfill,
|
||||
mod,
|
||||
bfqsn,
|
||||
shm,
|
||||
timeframe,
|
||||
sampler_stream,
|
||||
feed_is_live,
|
||||
|
||||
last_tsdb_dt=last_tsdb_dt,
|
||||
tsdb_is_up=True,
|
||||
storage=storage,
|
||||
)
|
||||
)
|
||||
except DataUnavailable:
|
||||
# XXX: timeframe not supported for backend
|
||||
dts_per_tf[timeframe] = (
|
||||
tsdb_history,
|
||||
last_tsdb_dt,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
continue
|
||||
|
||||
# tsdb_history = series.get(timeframe)
|
||||
dts_per_tf[timeframe] = (
|
||||
tsdb_history,
|
||||
last_tsdb_dt,
|
||||
latest_start_dt,
|
||||
latest_end_dt,
|
||||
bf_done,
|
||||
)
|
||||
|
||||
# if len(hist_shm.array) < 2:
|
||||
# TODO: there's an edge case here to solve where if the last
|
||||
# frame before market close (at least on ib) was pushed and
|
||||
# there was only "1 new" row pushed from the first backfill
|
||||
# query-iteration, then the sample step sizing calcs will
|
||||
# break upstream from here since you can't diff on at least
|
||||
# 2 steps... probably should also add logic to compute from
|
||||
# the tsdb series and stash that somewhere as meta data on
|
||||
# the shm buffer?.. no se.
|
||||
|
||||
# unblock the feed bus management task
|
||||
# assert len(shms[1].array)
|
||||
task_status.started()
|
||||
|
||||
async def back_load_from_tsdb(
|
||||
timeframe: int,
|
||||
shm: ShmArray,
|
||||
):
|
||||
(
|
||||
tsdb_history,
|
||||
last_tsdb_dt,
|
||||
latest_start_dt,
|
||||
latest_end_dt,
|
||||
bf_done,
|
||||
) = dts_per_tf[timeframe]
|
||||
|
||||
# sync to backend history task's query/load completion
|
||||
if bf_done:
|
||||
await bf_done.wait()
|
||||
|
||||
# TODO: eventually it'd be nice to not require a shm array/buffer
|
||||
# to accomplish this.. maybe we can do some kind of tsdb direct to
|
||||
# graphics format eventually in a child-actor?
|
||||
|
||||
# TODO: see if there's faster multi-field reads:
|
||||
# https://numpy.org/doc/stable/user/basics.rec.html#accessing-multiple-fields
|
||||
# re-index with a `time` and index field
|
||||
prepend_start = shm._first.value
|
||||
array = shm.array
|
||||
if len(array):
|
||||
shm_last_dt = pendulum.from_timestamp(shm.array[0]['time'])
|
||||
else:
|
||||
shm_last_dt = None
|
||||
|
||||
if last_tsdb_dt:
|
||||
assert shm_last_dt >= last_tsdb_dt
|
||||
|
||||
# do diff against start index of last frame of history and only
|
||||
# fill in an amount of datums from tsdb allows for most recent
|
||||
# to be loaded into mem *before* tsdb data.
|
||||
if (
|
||||
last_tsdb_dt
|
||||
and latest_start_dt
|
||||
):
|
||||
backfilled_size_s = (
|
||||
latest_start_dt - last_tsdb_dt
|
||||
).seconds
|
||||
# if the shm buffer len is not large enough to contain
|
||||
# all missing data between the most recent backend-queried frame
|
||||
# and the most recent dt-index in the db we warn that we only
|
||||
# want to load a portion of the next tsdb query to fill that
|
||||
# space.
|
||||
log.info(
|
||||
f'{backfilled_size_s} seconds worth of {timeframe}s loaded'
|
||||
)
|
||||
|
||||
# Load TSDB history into shm buffer (for display) if there is
|
||||
# remaining buffer space.
|
||||
|
||||
if (
|
||||
len(tsdb_history)
|
||||
):
|
||||
# load the first (smaller) bit of history originally loaded
|
||||
# above from ``Storage.load()``.
|
||||
to_push = tsdb_history[-prepend_start:]
|
||||
shm.push(
|
||||
to_push,
|
||||
|
||||
# insert the history pre a "days worth" of samples
|
||||
# to leave some real-time buffer space at the end.
|
||||
prepend=True,
|
||||
# update_first=False,
|
||||
# start=prepend_start,
|
||||
field_map=marketstore.ohlc_key_map,
|
||||
)
|
||||
|
||||
tsdb_last_frame_start = tsdb_history['Epoch'][0]
|
||||
|
||||
if timeframe == 1:
|
||||
times = shm.array['time']
|
||||
assert (times[1] - times[0]) == 1
|
||||
|
||||
# load as much from storage into shm possible (depends on
|
||||
# user's shm size settings).
|
||||
while shm._first.value > 0:
|
||||
|
||||
tsdb_history = await storage.read_ohlcv(
|
||||
fqsn,
|
||||
timeframe=timeframe,
|
||||
end=tsdb_last_frame_start,
|
||||
)
|
||||
|
||||
# empty query
|
||||
if not len(tsdb_history):
|
||||
break
|
||||
|
||||
next_start = tsdb_history['Epoch'][0]
|
||||
if next_start >= tsdb_last_frame_start:
|
||||
# no earlier data detected
|
||||
break
|
||||
else:
|
||||
tsdb_last_frame_start = next_start
|
||||
|
||||
prepend_start = shm._first.value
|
||||
to_push = tsdb_history[-prepend_start:]
|
||||
|
||||
# insert the history pre a "days worth" of samples
|
||||
# to leave some real-time buffer space at the end.
|
||||
shm.push(
|
||||
to_push,
|
||||
prepend=True,
|
||||
field_map=marketstore.ohlc_key_map,
|
||||
)
|
||||
log.info(f'Loaded {to_push.shape} datums from storage')
|
||||
|
||||
# manually trigger step update to update charts/fsps
|
||||
# which need an incremental update.
|
||||
# NOTE: the way this works is super duper
|
||||
# un-intuitive right now:
|
||||
# - the broadcaster fires a msg to the fsp subsystem.
|
||||
# - fsp subsys then checks for a sample step diff and
|
||||
# possibly recomputes prepended history.
|
||||
# - the fsp then sends back to the parent actor
|
||||
# (usually a chart showing graphics for said fsp)
|
||||
# which tells the chart to conduct a manual full
|
||||
# graphics loop cycle.
|
||||
await sampler_stream.send('broadcast_all')
|
||||
|
||||
# TODO: write new data to tsdb to be ready to for next read.
|
||||
|
||||
# backload from db (concurrently per timeframe) once backfilling of
|
||||
# recent dat a loaded from the backend provider (see
|
||||
# ``bf_done.wait()`` call).
|
||||
async with trio.open_nursery() as nurse:
|
||||
for timeframe, shm in shms.items():
|
||||
nurse.start_soon(
|
||||
back_load_from_tsdb,
|
||||
timeframe,
|
||||
shm,
|
||||
)
|
||||
|
||||
|
||||
async def manage_history(
|
||||
mod: ModuleType,
|
||||
bus: _FeedsBus,
|
||||
fqsn: str,
|
||||
some_data_ready: trio.Event,
|
||||
feed_is_live: trio.Event,
|
||||
timeframe: float = 60, # in seconds
|
||||
|
||||
task_status: TaskStatus[
|
||||
tuple[ShmArray, ShmArray]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Load and manage historical data including the loading of any
|
||||
available series from `marketstore` as well as conducting real-time
|
||||
update of both that existing db and the allocated shared memory
|
||||
buffer.
|
||||
|
||||
'''
|
||||
|
||||
# TODO: is there a way to make each shm file key
|
||||
# actor-tree-discovery-addr unique so we avoid collisions
|
||||
# when doing tests which also allocate shms for certain instruments
|
||||
# that may be in use on the system by some other running daemons?
|
||||
# from tractor._state import _runtime_vars
|
||||
# port = _runtime_vars['_root_mailbox'][1]
|
||||
|
||||
uid = tractor.current_actor().uid
|
||||
name, uuid = uid
|
||||
service = name.rstrip(f'.{mod.name}')
|
||||
|
||||
# (maybe) allocate shm array for this broker/symbol which will
|
||||
# be used for fast near-term history capture and processing.
|
||||
hist_shm, opened = maybe_open_shm_array(
|
||||
# key=f'{fqsn}_hist_p{port}',
|
||||
key=f'piker.{service}[{uuid[:16]}.{fqsn}.hist',
|
||||
|
||||
# use any broker defined ohlc dtype:
|
||||
dtype=getattr(mod, '_ohlc_dtype', base_iohlc_dtype),
|
||||
|
||||
# we expect the sub-actor to write
|
||||
readonly=False,
|
||||
)
|
||||
hist_zero_index = hist_shm.index - 1
|
||||
|
||||
# TODO: history validation
|
||||
if not opened:
|
||||
raise RuntimeError(
|
||||
"Persistent shm for sym was already open?!"
|
||||
)
|
||||
|
||||
rt_shm, opened = maybe_open_shm_array(
|
||||
# key=f'{fqsn}_rt_p{port}',
|
||||
# key=f'piker.{service}.{fqsn}_rt.{uuid}',
|
||||
key=f'piker.{service}[{uuid[:16]}.{fqsn}.rt',
|
||||
|
||||
# use any broker defined ohlc dtype:
|
||||
dtype=getattr(mod, '_ohlc_dtype', base_iohlc_dtype),
|
||||
|
||||
# we expect the sub-actor to write
|
||||
readonly=False,
|
||||
size=3*_secs_in_day,
|
||||
)
|
||||
|
||||
# (for now) set the rt (hft) shm array with space to prepend
|
||||
# only a few days worth of 1s history.
|
||||
days = 2
|
||||
start_index = days*_secs_in_day
|
||||
rt_shm._first.value = start_index
|
||||
rt_shm._last.value = start_index
|
||||
rt_zero_index = rt_shm.index - 1
|
||||
|
||||
if not opened:
|
||||
raise RuntimeError(
|
||||
"Persistent shm for sym was already open?!"
|
||||
)
|
||||
|
||||
# register 1s and 1m buffers with the global incrementer task
|
||||
async with open_sample_stream(
|
||||
period_s=1.,
|
||||
shms_by_period={
|
||||
1.: rt_shm.token,
|
||||
60.: hist_shm.token,
|
||||
},
|
||||
|
||||
# NOTE: we want to only open a stream for doing broadcasts on
|
||||
# backfill operations, not receive the sample index-stream
|
||||
# (since there's no code in this data feed layer that needs to
|
||||
# consume it).
|
||||
open_index_stream=True,
|
||||
sub_for_broadcasts=False,
|
||||
|
||||
) as sample_stream:
|
||||
|
||||
log.info('Scanning for existing `marketstored`')
|
||||
tsdb_is_up = await check_for_service('marketstored')
|
||||
|
||||
bfqsn = fqsn.replace('.' + mod.name, '')
|
||||
open_history_client = getattr(mod, 'open_history_client', None)
|
||||
assert open_history_client
|
||||
|
||||
if (
|
||||
tsdb_is_up
|
||||
and opened
|
||||
and open_history_client
|
||||
):
|
||||
log.info('Found existing `marketstored`')
|
||||
|
||||
from ..service import marketstore
|
||||
async with (
|
||||
marketstore.open_storage_client(fqsn)as storage,
|
||||
):
|
||||
# TODO: drop returning the output that we pass in?
|
||||
await bus.nursery.start(
|
||||
tsdb_backfill,
|
||||
mod,
|
||||
marketstore,
|
||||
bus,
|
||||
storage,
|
||||
fqsn,
|
||||
bfqsn,
|
||||
{
|
||||
1: rt_shm,
|
||||
60: hist_shm,
|
||||
},
|
||||
sample_stream,
|
||||
feed_is_live,
|
||||
)
|
||||
|
||||
# yield back after client connect with filled shm
|
||||
task_status.started((
|
||||
hist_zero_index,
|
||||
hist_shm,
|
||||
rt_zero_index,
|
||||
rt_shm,
|
||||
))
|
||||
|
||||
# indicate to caller that feed can be delivered to
|
||||
# remote requesting client since we've loaded history
|
||||
# data that can be used.
|
||||
some_data_ready.set()
|
||||
|
||||
# history retreival loop depending on user interaction
|
||||
# and thus a small RPC-prot for remotely controllinlg
|
||||
# what data is loaded for viewing.
|
||||
await trio.sleep_forever()
|
||||
|
||||
# load less history if no tsdb can be found
|
||||
elif (
|
||||
not tsdb_is_up
|
||||
and opened
|
||||
):
|
||||
await basic_backfill(
|
||||
bus,
|
||||
mod,
|
||||
bfqsn,
|
||||
{
|
||||
1: rt_shm,
|
||||
60: hist_shm,
|
||||
},
|
||||
sample_stream,
|
||||
feed_is_live,
|
||||
)
|
||||
task_status.started((
|
||||
hist_zero_index,
|
||||
hist_shm,
|
||||
rt_zero_index,
|
||||
rt_shm,
|
||||
))
|
||||
some_data_ready.set()
|
||||
await trio.sleep_forever()
|
||||
|
|
@ -23,7 +23,7 @@ Api layer likely in here...
|
|||
from types import ModuleType
|
||||
from importlib import import_module
|
||||
|
||||
from ._util import get_logger
|
||||
from ..log import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
from __future__ import annotations
|
||||
from contextlib import asynccontextmanager as acm
|
||||
from datetime import datetime
|
||||
from pprint import pformat
|
||||
from typing import (
|
||||
Any,
|
||||
Optional,
|
||||
|
|
@ -34,11 +35,10 @@ from typing import (
|
|||
)
|
||||
import time
|
||||
from math import isnan
|
||||
from pathlib import Path
|
||||
|
||||
from bidict import bidict
|
||||
from msgspec.msgpack import encode, decode
|
||||
# import pyqtgraph as pg
|
||||
import msgpack
|
||||
import pyqtgraph as pg
|
||||
import numpy as np
|
||||
import tractor
|
||||
from trio_websocket import open_websocket_url
|
||||
|
|
@ -54,20 +54,18 @@ if TYPE_CHECKING:
|
|||
import docker
|
||||
from ._ahab import DockerContainer
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ..data.feed import maybe_open_feed
|
||||
from .._profile import Profiler
|
||||
from .feed import maybe_open_feed
|
||||
from ..log import get_logger, get_console_log
|
||||
|
||||
|
||||
# ahabd-supervisor and container level config
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
# container level config
|
||||
_config = {
|
||||
'grpc_listen_port': 5995,
|
||||
'ws_listen_port': 5993,
|
||||
'log_level': 'debug',
|
||||
'startup_timeout': 2,
|
||||
}
|
||||
|
||||
_yaml_config = '''
|
||||
|
|
@ -133,10 +131,7 @@ def start_marketstore(
|
|||
|
||||
mktsdir = os.path.join(config._config_dir, 'marketstore')
|
||||
|
||||
# create dirs when dne
|
||||
if not os.path.isdir(config._config_dir):
|
||||
Path(config._config_dir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# create when dne
|
||||
if not os.path.isdir(mktsdir):
|
||||
os.mkdir(mktsdir)
|
||||
|
||||
|
|
@ -185,29 +180,18 @@ def start_marketstore(
|
|||
config_dir_mnt,
|
||||
data_dir_mnt,
|
||||
],
|
||||
|
||||
# XXX: this must be set to allow backgrounding/non-blocking
|
||||
# usage interaction with the container's process.
|
||||
detach=True,
|
||||
|
||||
# stop_signal='SIGINT',
|
||||
init=True,
|
||||
# remove=True,
|
||||
)
|
||||
|
||||
async def start_matcher(msg: str):
|
||||
return "launching tcp listener for all services..." in msg
|
||||
|
||||
async def stop_matcher(msg: str):
|
||||
return "exiting..." in msg
|
||||
|
||||
return (
|
||||
dcntr,
|
||||
_config,
|
||||
|
||||
# expected startup and stop msgs
|
||||
start_matcher,
|
||||
stop_matcher,
|
||||
"launching tcp listener for all services...",
|
||||
"exiting...",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -328,7 +312,7 @@ def quote_to_marketstore_structarray(
|
|||
@acm
|
||||
async def get_client(
|
||||
host: str = 'localhost',
|
||||
port: int = _config['grpc_listen_port'],
|
||||
port: int = 5995
|
||||
|
||||
) -> MarketstoreClient:
|
||||
'''
|
||||
|
|
@ -403,54 +387,50 @@ class Storage:
|
|||
async def load(
|
||||
self,
|
||||
fqsn: str,
|
||||
timeframe: int,
|
||||
|
||||
) -> tuple[
|
||||
np.ndarray, # timeframe sampled array-series
|
||||
dict[int, np.ndarray], # timeframe (in secs) to series
|
||||
Optional[datetime], # first dt
|
||||
Optional[datetime], # last dt
|
||||
]:
|
||||
|
||||
first_tsdb_dt, last_tsdb_dt = None, None
|
||||
hist = await self.read_ohlcv(
|
||||
tsdb_arrays = await self.read_ohlcv(
|
||||
fqsn,
|
||||
# on first load we don't need to pull the max
|
||||
# history per request size worth.
|
||||
limit=3000,
|
||||
timeframe=timeframe,
|
||||
)
|
||||
log.info(f'Loaded tsdb history {hist}')
|
||||
log.info(f'Loaded tsdb history {tsdb_arrays}')
|
||||
|
||||
if len(hist):
|
||||
times = hist['Epoch']
|
||||
if tsdb_arrays:
|
||||
fastest = list(tsdb_arrays.values())[0]
|
||||
times = fastest['Epoch']
|
||||
first, last = times[0], times[-1]
|
||||
first_tsdb_dt, last_tsdb_dt = map(
|
||||
pendulum.from_timestamp, [first, last]
|
||||
)
|
||||
|
||||
return (
|
||||
hist, # array-data
|
||||
first_tsdb_dt, # start of query-frame
|
||||
last_tsdb_dt, # most recent
|
||||
)
|
||||
return tsdb_arrays, first_tsdb_dt, last_tsdb_dt
|
||||
|
||||
async def read_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
timeframe: int | str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
end: Optional[int] = None,
|
||||
limit: int = int(800e3),
|
||||
|
||||
) -> np.ndarray:
|
||||
|
||||
) -> tuple[
|
||||
MarketstoreClient,
|
||||
Union[dict, np.ndarray]
|
||||
]:
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
|
||||
if fqsn not in syms:
|
||||
return {}
|
||||
|
||||
# use the provided timeframe or 1s by default
|
||||
tfstr = tf_in_1s.get(timeframe, tf_in_1s[1])
|
||||
tfstr = tf_in_1s[1]
|
||||
|
||||
params = Params(
|
||||
symbols=fqsn,
|
||||
|
|
@ -464,71 +444,58 @@ class Storage:
|
|||
limit=limit,
|
||||
)
|
||||
|
||||
if timeframe is None:
|
||||
log.info(f'starting {fqsn} tsdb granularity scan..')
|
||||
# loop through and try to find highest granularity
|
||||
for tfstr in tf_in_1s.values():
|
||||
try:
|
||||
log.info(f'querying for {tfstr}@{fqsn}')
|
||||
params.set('timeframe', tfstr)
|
||||
result = await client.query(params)
|
||||
except purerpc.grpclib.exceptions.UnknownError as err:
|
||||
# indicate there is no history for this timeframe
|
||||
log.exception(
|
||||
f'Unknown mkts QUERY error: {params}\n'
|
||||
f'{err.args}'
|
||||
)
|
||||
break
|
||||
|
||||
except purerpc.grpclib.exceptions.UnknownError:
|
||||
# XXX: this is already logged by the container and
|
||||
# thus shows up through `marketstored` logs relay.
|
||||
# log.warning(f'{tfstr}@{fqsn} not found')
|
||||
continue
|
||||
else:
|
||||
return {}
|
||||
|
||||
else:
|
||||
result = await client.query(params)
|
||||
|
||||
# TODO: it turns out column access on recarrays is actually slower:
|
||||
# https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist
|
||||
# it might make sense to make these structured arrays?
|
||||
data_set = result.by_symbols()[fqsn]
|
||||
array = data_set.array
|
||||
# Fill out a `numpy` array-results map
|
||||
arrays = {}
|
||||
for fqsn, data_set in result.by_symbols().items():
|
||||
arrays.setdefault(fqsn, {})[
|
||||
tf_in_1s.inverse[data_set.timeframe]
|
||||
] = data_set.array
|
||||
|
||||
# XXX: ensure sample rate is as expected
|
||||
time = data_set.array['Epoch']
|
||||
if len(time) > 1:
|
||||
time_step = time[-1] - time[-2]
|
||||
ts = tf_in_1s.inverse[data_set.timeframe]
|
||||
|
||||
if time_step != ts:
|
||||
log.warning(
|
||||
f'MKTS BUG: wrong timeframe loaded: {time_step}'
|
||||
'YOUR DATABASE LIKELY CONTAINS BAD DATA FROM AN OLD BUG'
|
||||
f'WIPING HISTORY FOR {ts}s'
|
||||
)
|
||||
await self.delete_ts(fqsn, timeframe)
|
||||
|
||||
# try reading again..
|
||||
return await self.read_ohlcv(
|
||||
fqsn,
|
||||
timeframe,
|
||||
end,
|
||||
limit,
|
||||
)
|
||||
|
||||
return array
|
||||
return arrays[fqsn][timeframe] if timeframe else arrays[fqsn]
|
||||
|
||||
async def delete_ts(
|
||||
self,
|
||||
key: str,
|
||||
timeframe: Optional[Union[int, str]] = None,
|
||||
fmt: str = 'OHLCV',
|
||||
|
||||
) -> bool:
|
||||
|
||||
client = self.client
|
||||
syms = await client.list_symbols()
|
||||
if key not in syms:
|
||||
raise KeyError(f'`{key}` table key not found in\n{syms}?')
|
||||
print(syms)
|
||||
# if key not in syms:
|
||||
# raise KeyError(f'`{fqsn}` table key not found?')
|
||||
|
||||
tbk = mk_tbk((
|
||||
key,
|
||||
tf_in_1s.get(timeframe, tf_in_1s[60]),
|
||||
fmt,
|
||||
))
|
||||
return await client.destroy(tbk=tbk)
|
||||
return await client.destroy(tbk=key)
|
||||
|
||||
async def write_ohlcv(
|
||||
self,
|
||||
fqsn: str,
|
||||
ohlcv: np.ndarray,
|
||||
timeframe: int,
|
||||
append_and_duplicate: bool = True,
|
||||
limit: int = int(800e3),
|
||||
|
||||
|
|
@ -552,18 +519,17 @@ class Storage:
|
|||
|
||||
m, r = divmod(len(mkts_array), limit)
|
||||
|
||||
tfkey = tf_in_1s[timeframe]
|
||||
for i in range(m, 1):
|
||||
to_push = mkts_array[i-1:i*limit]
|
||||
|
||||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/{tfkey}/OHLCV',
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
# TODO: pre-deduplicate?
|
||||
# TODO: pre deduplicate?
|
||||
isvariablelength=append_and_duplicate,
|
||||
)
|
||||
|
||||
|
|
@ -582,7 +548,7 @@ class Storage:
|
|||
# write to db
|
||||
resp = await self.client.write(
|
||||
to_push,
|
||||
tbk=f'{fqsn}/{tfkey}/OHLCV',
|
||||
tbk=f'{fqsn}/1Sec/OHLCV',
|
||||
|
||||
# NOTE: will will append duplicates
|
||||
# for the same timestamp-index.
|
||||
|
|
@ -611,7 +577,6 @@ class Storage:
|
|||
# def delete_range(self, start_dt, end_dt) -> None:
|
||||
# ...
|
||||
|
||||
|
||||
@acm
|
||||
async def open_storage_client(
|
||||
fqsn: str,
|
||||
|
|
@ -630,10 +595,10 @@ async def open_storage_client(
|
|||
yield Storage(client)
|
||||
|
||||
|
||||
@acm
|
||||
async def open_tsdb_client(
|
||||
fqsn: str,
|
||||
) -> Storage:
|
||||
async def tsdb_history_update(
|
||||
fqsn: Optional[str] = None,
|
||||
|
||||
) -> list[str]:
|
||||
|
||||
# TODO: real-time dedicated task for ensuring
|
||||
# history consistency between the tsdb, shm and real-time feed..
|
||||
|
|
@ -661,8 +626,8 @@ async def open_tsdb_client(
|
|||
# * the original data feed arch blurb:
|
||||
# - https://github.com/pikers/piker/issues/98
|
||||
#
|
||||
profiler = Profiler(
|
||||
disabled=True, # not pg_profile_enabled(),
|
||||
profiler = pg.debug.Profiler(
|
||||
disabled=False, # not pg_profile_enabled(),
|
||||
delayed=False,
|
||||
)
|
||||
|
||||
|
|
@ -673,40 +638,43 @@ async def open_tsdb_client(
|
|||
[fqsn],
|
||||
start_stream=False,
|
||||
|
||||
) as feed,
|
||||
) as (feed, stream),
|
||||
):
|
||||
profiler(f'opened feed for {fqsn}')
|
||||
|
||||
# to_append = feed.hist_shm.array
|
||||
# to_prepend = None
|
||||
to_append = feed.shm.array
|
||||
to_prepend = None
|
||||
|
||||
if fqsn:
|
||||
flume = feed.flumes[fqsn]
|
||||
symbol = flume.symbol
|
||||
symbol = feed.symbols.get(fqsn)
|
||||
if symbol:
|
||||
fqsn = symbol.fqsn
|
||||
fqsn = symbol.front_fqsn()
|
||||
|
||||
# diff db history with shm and only write the missing portions
|
||||
# ohlcv = flume.hist_shm.array
|
||||
ohlcv = feed.shm.array
|
||||
|
||||
# TODO: use pg profiler
|
||||
# for secs in (1, 60):
|
||||
# tsdb_array = await storage.read_ohlcv(
|
||||
# fqsn,
|
||||
# timeframe=timeframe,
|
||||
# )
|
||||
# # hist diffing:
|
||||
# # these aren't currently used but can be referenced from
|
||||
# # within the embedded ipython shell below.
|
||||
# to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
|
||||
# to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
|
||||
tsdb_arrays = await storage.read_ohlcv(fqsn)
|
||||
# hist diffing
|
||||
if tsdb_arrays:
|
||||
for secs in (1, 60):
|
||||
ts = tsdb_arrays.get(secs)
|
||||
if ts is not None and len(ts):
|
||||
# these aren't currently used but can be referenced from
|
||||
# within the embedded ipython shell below.
|
||||
to_append = ohlcv[ohlcv['time'] > ts['Epoch'][-1]]
|
||||
to_prepend = ohlcv[ohlcv['time'] < ts['Epoch'][0]]
|
||||
|
||||
# profiler('Finished db arrays diffs')
|
||||
profiler('Finished db arrays diffs')
|
||||
|
||||
_ = await storage.client.list_symbols()
|
||||
# log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
|
||||
# profiler(f'listed symbols {syms}')
|
||||
yield storage
|
||||
syms = await storage.client.list_symbols()
|
||||
log.info(f'Existing tsdb symbol set:\n{pformat(syms)}')
|
||||
profiler(f'listed symbols {syms}')
|
||||
|
||||
# TODO: ask if user wants to write history for detected
|
||||
# available shm buffers?
|
||||
from tractor.trionics import ipython_embed
|
||||
await ipython_embed()
|
||||
|
||||
# for array in [to_append, to_prepend]:
|
||||
# if array is None:
|
||||
|
|
@ -806,13 +774,12 @@ async def stream_quotes(
|
|||
async with open_websocket_url(f'ws://{host}:{port}/ws') as ws:
|
||||
# send subs topics to server
|
||||
resp = await ws.send_message(
|
||||
|
||||
encode({'streams': list(tbks.values())})
|
||||
msgpack.dumps({'streams': list(tbks.values())})
|
||||
)
|
||||
log.info(resp)
|
||||
|
||||
async def recv() -> dict[str, Any]:
|
||||
return decode((await ws.get_message()), encoding='utf-8')
|
||||
return msgpack.loads((await ws.get_message()), encoding='utf-8')
|
||||
|
||||
streams = (await recv())['streams']
|
||||
log.info(f"Subscribed to {streams}")
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Guillermo Rodriguez (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Built-in (extension) types.
|
||||
|
||||
"""
|
||||
import sys
|
||||
from pprint import pformat
|
||||
|
||||
import msgspec
|
||||
|
||||
|
||||
class Struct(
|
||||
msgspec.Struct,
|
||||
|
||||
# https://jcristharif.com/msgspec/structs.html#tagged-unions
|
||||
# tag='pikerstruct',
|
||||
# tag=True,
|
||||
):
|
||||
'''
|
||||
A "human friendlier" (aka repl buddy) struct subtype.
|
||||
|
||||
'''
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
f: getattr(self, f)
|
||||
for f in self.__struct_fields__
|
||||
}
|
||||
|
||||
# Lul, doesn't seem to work that well..
|
||||
# def __repr__(self):
|
||||
# # only turn on pprint when we detect a python REPL
|
||||
# # at runtime B)
|
||||
# if (
|
||||
# hasattr(sys, 'ps1')
|
||||
# # TODO: check if we're in pdb
|
||||
# ):
|
||||
# return self.pformat()
|
||||
|
||||
# return super().__repr__()
|
||||
|
||||
def pformat(self) -> str:
|
||||
return f'Struct({pformat(self.to_dict())})'
|
||||
|
||||
def copy(
|
||||
self,
|
||||
update: dict | None = None,
|
||||
|
||||
) -> msgspec.Struct:
|
||||
'''
|
||||
Validate-typecast all self defined fields, return a copy of us
|
||||
with all such fields.
|
||||
|
||||
This is kinda like the default behaviour in `pydantic.BaseModel`.
|
||||
|
||||
'''
|
||||
if update:
|
||||
for k, v in update.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
# roundtrip serialize to validate
|
||||
return msgspec.msgpack.Decoder(
|
||||
type=type(self)
|
||||
).decode(
|
||||
msgspec.msgpack.Encoder().encode(self)
|
||||
)
|
||||
|
||||
# NOTE XXX: this won't work on frozen types!
|
||||
# use ``.copy()`` above in such cases.
|
||||
def typecast(
|
||||
self,
|
||||
# fields: list[str] | None = None,
|
||||
) -> None:
|
||||
for fname, ftype in self.__annotations__.items():
|
||||
setattr(self, fname, ftype(getattr(self, fname)))
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
'''
|
||||
Data feed synchronization protocols, init msgs, and general
|
||||
data-provider-backend-agnostic schema definitions.
|
||||
|
||||
'''
|
||||
from decimal import Decimal
|
||||
from pprint import pformat
|
||||
from types import ModuleType
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
from .types import Struct
|
||||
from ..accounting import (
|
||||
Asset,
|
||||
MktPair,
|
||||
)
|
||||
from ._util import log
|
||||
|
||||
|
||||
class FeedInitializationError(ValueError):
|
||||
'''
|
||||
Live data feed setup failed due to API / msg incompatiblity!
|
||||
|
||||
'''
|
||||
|
||||
|
||||
class FeedInit(Struct, frozen=True):
|
||||
'''
|
||||
A stringent data provider startup msg schema validator.
|
||||
|
||||
The fields defined here are matched with those absolutely required
|
||||
from each backend broker/data provider.
|
||||
|
||||
'''
|
||||
mkt_info: MktPair
|
||||
shm_write_opts: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def validate_backend(
|
||||
mod: ModuleType,
|
||||
syms: list[str],
|
||||
init_msgs: list[FeedInit] | dict[str, dict[str, Any]],
|
||||
|
||||
# TODO: do a module method scan and report mismatches.
|
||||
check_eps: bool = False,
|
||||
|
||||
api_log_msg_level: str = 'critical'
|
||||
|
||||
) -> FeedInit:
|
||||
'''
|
||||
Fail on malformed live quotes feed config/init or warn on changes
|
||||
that haven't been implemented by this backend yet.
|
||||
|
||||
'''
|
||||
inits: list[
|
||||
FeedInit | dict[str, Any]
|
||||
] = init_msgs
|
||||
|
||||
# convert to list if from old dict-style
|
||||
if isinstance(init_msgs, dict):
|
||||
inits = list(init_msgs.values())
|
||||
|
||||
init: FeedInit | dict[str, Any]
|
||||
for i, init in enumerate(inits):
|
||||
|
||||
# XXX: eventually this WILL NOT necessarily be true.
|
||||
if i > 0:
|
||||
assert not len(init_msgs) == 1
|
||||
if isinstance(init_msgs, dict):
|
||||
keys: set = set(init_msgs.keys()) - set(syms)
|
||||
raise FeedInitializationError(
|
||||
'TOO MANY INIT MSGS!\n'
|
||||
f'Unexpected keys: {keys}\n'
|
||||
'ALL MSGS:\n'
|
||||
f'{pformat(init_msgs)}\n'
|
||||
)
|
||||
else:
|
||||
raise FeedInitializationError(
|
||||
'TOO MANY INIT MSGS!\n'
|
||||
f'{pformat(init_msgs)}\n'
|
||||
)
|
||||
|
||||
# TODO: once all backends are updated we can remove this branching.
|
||||
rx_msg: bool = False
|
||||
warn_msg: str = ''
|
||||
if not isinstance(init, FeedInit):
|
||||
warn_msg += (
|
||||
'\n'
|
||||
'--------------------------\n'
|
||||
':::DEPRECATED API STYLE:::\n'
|
||||
'--------------------------\n'
|
||||
f'`{mod.name}.stream_quotes()` should deliver '
|
||||
'`.started(FeedInit)`\n'
|
||||
f'|-> CURRENTLY it is using DEPRECATED `.started(dict)` style!\n'
|
||||
f'|-> SEE `FeedInit` in `piker.data.validate`\n'
|
||||
'--------------------------------------------\n'
|
||||
)
|
||||
else:
|
||||
rx_msg = True
|
||||
|
||||
# verify feed init state / schema
|
||||
bs_fqme: str # backend specific fqme
|
||||
mkt: MktPair
|
||||
|
||||
match init:
|
||||
case {
|
||||
'symbol_info': dict(symbol_info),
|
||||
'fqsn': bs_fqme,
|
||||
} | {
|
||||
'mkt_info': dict(symbol_info),
|
||||
'fqsn': bs_fqme,
|
||||
}:
|
||||
symbol_info: dict
|
||||
warn_msg += (
|
||||
'It may also be still using the legacy `Symbol` style API\n'
|
||||
'IT SHOULD BE PORTED TO THE NEW '
|
||||
'`.accounting._mktinfo.MktPair`\n'
|
||||
'STATTTTT!!!\n'
|
||||
)
|
||||
|
||||
# XXX use default legacy (aka discrete precision) mkt
|
||||
# price/size_ticks if none delivered.
|
||||
price_tick = symbol_info.get(
|
||||
'price_tick_size',
|
||||
Decimal('0.01'),
|
||||
)
|
||||
size_tick = symbol_info.get(
|
||||
'lot_tick_size',
|
||||
Decimal('1'),
|
||||
)
|
||||
mkt = MktPair.from_fqme(
|
||||
fqme=f'{bs_fqme}.{mod.name}',
|
||||
|
||||
price_tick=price_tick,
|
||||
size_tick=size_tick,
|
||||
|
||||
bs_mktid=str(init['bs_mktid']),
|
||||
_atype=symbol_info['asset_type']
|
||||
)
|
||||
|
||||
case {
|
||||
'mkt_info': MktPair(
|
||||
dst=Asset(),
|
||||
) as mkt,
|
||||
'fqsn': bs_fqme,
|
||||
}:
|
||||
warn_msg += (
|
||||
f'{mod.name} in API compat transition?\n'
|
||||
"It's half dict, half man..\n"
|
||||
'-------------------------------------\n'
|
||||
)
|
||||
|
||||
case FeedInit(
|
||||
mkt_info=MktPair(dst=Asset()) as mkt,
|
||||
shm_write_opts=dict(shm_opts),
|
||||
) as init:
|
||||
name: str = mod.name
|
||||
log.info(
|
||||
f'NICE JOB {name} BACKEND being fully up to API spec B)\n'
|
||||
f"{name}'s `MktPair` info:\n"
|
||||
f'{pformat(mkt.to_dict())}\n'
|
||||
f'shm conf: {pformat(shm_opts)}\n'
|
||||
)
|
||||
|
||||
case _:
|
||||
raise FeedInitializationError(init)
|
||||
|
||||
# build a msg if we received a dict for input.
|
||||
if not rx_msg:
|
||||
assert bs_fqme in mkt.fqme
|
||||
init = FeedInit(
|
||||
mkt_info=mkt,
|
||||
shm_write_opts=init.get('shm_write_opts'),
|
||||
)
|
||||
|
||||
# `MktPair` value audits
|
||||
mkt = init.mkt_info
|
||||
assert mkt.type_key
|
||||
|
||||
# `MktPair` wish list
|
||||
if not isinstance(mkt.src, Asset):
|
||||
warn_msg += (
|
||||
f'ALSO, {mod.name.upper()} should try to deliver\n'
|
||||
'the new `MktPair.src: Asset` field!\n'
|
||||
'-----------------------------------------------\n'
|
||||
)
|
||||
|
||||
# complain about any non-idealities
|
||||
if warn_msg:
|
||||
# TODO: would be nice to register an API_COMPAT or something in
|
||||
# maybe cyan for this in general throughput piker no?
|
||||
logmeth = getattr(log, api_log_msg_level)
|
||||
logmeth(warn_msg)
|
||||
|
||||
return init.copy()
|
||||
|
|
@ -78,8 +78,7 @@ class Fsp:
|
|||
# + the consuming fsp *to* the consumers output
|
||||
# shm flow.
|
||||
_flow_registry: dict[
|
||||
tuple[_Token, str],
|
||||
tuple[_Token, Optional[ShmArray]],
|
||||
tuple[_Token, str], _Token,
|
||||
] = {}
|
||||
|
||||
def __init__(
|
||||
|
|
@ -121,6 +120,7 @@ class Fsp:
|
|||
):
|
||||
return self.func(*args, **kwargs)
|
||||
|
||||
# TODO: lru_cache this? prettty sure it'll work?
|
||||
def get_shm(
|
||||
self,
|
||||
src_shm: ShmArray,
|
||||
|
|
@ -131,27 +131,12 @@ class Fsp:
|
|||
for this "instance" of a signal processor for
|
||||
the given ``key``.
|
||||
|
||||
The destination shm "token" and array are cached if possible to
|
||||
minimize multiple stdlib/system calls.
|
||||
|
||||
'''
|
||||
dst_token, maybe_array = self._flow_registry[
|
||||
dst_token = self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
]
|
||||
if maybe_array is None:
|
||||
self._flow_registry[
|
||||
(src_shm._token, self.name)
|
||||
] = (
|
||||
dst_token,
|
||||
# "cache" the ``ShmArray`` such that
|
||||
# we call the underlying "attach" code as few
|
||||
# times as possible as per:
|
||||
# - https://github.com/pikers/piker/issues/359
|
||||
# - https://github.com/pikers/piker/issues/332
|
||||
maybe_array := attach_shm_array(dst_token)
|
||||
)
|
||||
|
||||
return maybe_array
|
||||
shm = attach_shm_array(dst_token)
|
||||
return shm
|
||||
|
||||
|
||||
def fsp(
|
||||
|
|
@ -179,9 +164,8 @@ def mk_fsp_shm_key(
|
|||
target: Fsp
|
||||
|
||||
) -> str:
|
||||
actor_name, uuid = tractor.current_actor().uid
|
||||
uuid_snip: str = uuid[:16]
|
||||
return f'piker.{actor_name}[{uuid_snip}].{sym}.{target.name}'
|
||||
uid = tractor.current_actor().uid
|
||||
return f'{sym}.fsp.{target.name}.{".".join(uid)}'
|
||||
|
||||
|
||||
def maybe_mk_fsp_shm(
|
||||
|
|
@ -200,10 +184,7 @@ def maybe_mk_fsp_shm(
|
|||
# TODO: load output types from `Fsp`
|
||||
# - should `index` be a required internal field?
|
||||
fsp_dtype = np.dtype(
|
||||
[('index', int)]
|
||||
+
|
||||
[('time', float)]
|
||||
+
|
||||
[('index', int)] +
|
||||
[(field_name, float) for field_name in target.outputs]
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ core task logic for processing chains
|
|||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import (
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Optional,
|
||||
AsyncIterator, Callable, Optional,
|
||||
Union,
|
||||
)
|
||||
|
||||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
|
@ -36,22 +35,14 @@ from tractor.msg import NamespacePath
|
|||
from ..log import get_logger, get_console_log
|
||||
from .. import data
|
||||
from ..data import attach_shm_array
|
||||
from ..data.feed import (
|
||||
Flume,
|
||||
Feed,
|
||||
)
|
||||
from ..data.feed import Feed
|
||||
from ..data._sharedmem import ShmArray
|
||||
from ..data._sampling import (
|
||||
_default_delay_s,
|
||||
open_sample_stream,
|
||||
)
|
||||
from ..accounting._mktinfo import Symbol
|
||||
from ..data._source import Symbol
|
||||
from ._api import (
|
||||
Fsp,
|
||||
_load_builtins,
|
||||
_Token,
|
||||
)
|
||||
from .._profile import Profiler
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -86,7 +77,7 @@ async def filter_quotes_by_sym(
|
|||
async def fsp_compute(
|
||||
|
||||
symbol: Symbol,
|
||||
flume: Flume,
|
||||
feed: Feed,
|
||||
quote_stream: trio.abc.ReceiveChannel,
|
||||
|
||||
src: ShmArray,
|
||||
|
|
@ -99,12 +90,12 @@ async def fsp_compute(
|
|||
|
||||
) -> None:
|
||||
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=True
|
||||
)
|
||||
|
||||
fqsn = symbol.fqme
|
||||
fqsn = symbol.front_fqsn()
|
||||
out_stream = func(
|
||||
|
||||
# TODO: do we even need this if we do the feed api right?
|
||||
|
|
@ -114,17 +105,16 @@ async def fsp_compute(
|
|||
filter_quotes_by_sym(fqsn, quote_stream),
|
||||
|
||||
# XXX: currently the ``ohlcv`` arg
|
||||
flume.rt_shm,
|
||||
feed.shm,
|
||||
)
|
||||
|
||||
# HISTORY COMPUTE PHASE
|
||||
# conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output.
|
||||
# Conduct a single iteration of fsp with historical bars input
|
||||
# and get historical output
|
||||
history_output: Union[
|
||||
dict[str, np.ndarray], # multi-output case
|
||||
np.ndarray, # single output case
|
||||
]
|
||||
history_output = await anext(out_stream)
|
||||
history_output = await out_stream.__anext__()
|
||||
|
||||
func_name = func.__name__
|
||||
profiler(f'{func_name} generated history')
|
||||
|
|
@ -136,13 +126,9 @@ async def fsp_compute(
|
|||
# each respective field.
|
||||
fields = getattr(dst.array.dtype, 'fields', None).copy()
|
||||
fields.pop('index')
|
||||
history_by_field: Optional[np.ndarray] = None
|
||||
src_time = src.array['time']
|
||||
history: Optional[np.ndarray] = None # TODO: nptyping here!
|
||||
|
||||
if (
|
||||
fields and
|
||||
len(fields) > 1
|
||||
):
|
||||
if fields and len(fields) > 1 and fields:
|
||||
if not isinstance(history_output, dict):
|
||||
raise ValueError(
|
||||
f'`{func_name}` is a multi-output FSP and should yield a '
|
||||
|
|
@ -153,7 +139,7 @@ async def fsp_compute(
|
|||
if key in history_output:
|
||||
output = history_output[key]
|
||||
|
||||
if history_by_field is None:
|
||||
if history is None:
|
||||
|
||||
if output is None:
|
||||
length = len(src.array)
|
||||
|
|
@ -163,7 +149,7 @@ async def fsp_compute(
|
|||
# using the first output, determine
|
||||
# the length of the struct-array that
|
||||
# will be pushed to shm.
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
length,
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
|
|
@ -171,7 +157,7 @@ async def fsp_compute(
|
|||
if output is None:
|
||||
continue
|
||||
|
||||
history_by_field[key] = output
|
||||
history[key] = output
|
||||
|
||||
# single-key output stream
|
||||
else:
|
||||
|
|
@ -180,15 +166,11 @@ async def fsp_compute(
|
|||
f'`{func_name}` is a single output FSP and should yield an '
|
||||
'`np.ndarray` for history'
|
||||
)
|
||||
history_by_field = np.zeros(
|
||||
history = np.zeros(
|
||||
len(history_output),
|
||||
dtype=dst.array.dtype
|
||||
)
|
||||
history_by_field[func_name] = history_output
|
||||
|
||||
history_by_field['time'] = src_time[-len(history_by_field):]
|
||||
|
||||
history_output['time'] = src.array['time']
|
||||
history[func_name] = history_output
|
||||
|
||||
# TODO: XXX:
|
||||
# THERE'S A BIG BUG HERE WITH THE `index` field since we're
|
||||
|
|
@ -205,10 +187,7 @@ async def fsp_compute(
|
|||
|
||||
# TODO: can we use this `start` flag instead of the manual
|
||||
# setting above?
|
||||
index = dst.push(
|
||||
history_by_field,
|
||||
start=first,
|
||||
)
|
||||
index = dst.push(history, start=first)
|
||||
|
||||
profiler(f'{func_name} pushed history')
|
||||
profiler.finish()
|
||||
|
|
@ -234,14 +213,8 @@ async def fsp_compute(
|
|||
|
||||
log.debug(f"{func_name}: {processed}")
|
||||
key, output = processed
|
||||
# dst.array[-1][key] = output
|
||||
dst.array[[key, 'time']][-1] = (
|
||||
output,
|
||||
# TODO: what about pushing ``time.time_ns()``
|
||||
# in which case we'll need to round at the graphics
|
||||
# processing / sampling layer?
|
||||
src.array[-1]['time']
|
||||
)
|
||||
index = src.index
|
||||
dst.array[-1][key] = output
|
||||
|
||||
# NOTE: for now we aren't streaming this to the consumer
|
||||
# stream latest array index entry which basically just acts
|
||||
|
|
@ -252,7 +225,6 @@ async def fsp_compute(
|
|||
# N-consumers who subscribe for the real-time output,
|
||||
# which we'll likely want to implement using local-mem
|
||||
# chans for the fan out?
|
||||
# index = src.index
|
||||
# if attach_stream:
|
||||
# await client_stream.send(index)
|
||||
|
||||
|
|
@ -289,7 +261,7 @@ async def cascade(
|
|||
destination shm array buffer.
|
||||
|
||||
'''
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
delayed=False,
|
||||
disabled=False
|
||||
)
|
||||
|
|
@ -312,10 +284,9 @@ async def cascade(
|
|||
# TODO: ugh i hate this wind/unwind to list over the wire
|
||||
# but not sure how else to do it.
|
||||
for (token, fsp_name, dst_token) in shm_registry:
|
||||
Fsp._flow_registry[(
|
||||
_Token.from_msg(token),
|
||||
fsp_name,
|
||||
)] = _Token.from_msg(dst_token), None
|
||||
Fsp._flow_registry[
|
||||
(_Token.from_msg(token), fsp_name)
|
||||
] = _Token.from_msg(dst_token)
|
||||
|
||||
fsp: Fsp = reg.get(
|
||||
NamespacePath(ns_path)
|
||||
|
|
@ -327,7 +298,6 @@ async def cascade(
|
|||
raise ValueError(f'Unknown fsp target: {ns_path}')
|
||||
|
||||
# open a data feed stream with requested broker
|
||||
feed: Feed
|
||||
async with data.feed.maybe_open_feed(
|
||||
[fqsn],
|
||||
|
||||
|
|
@ -337,13 +307,14 @@ async def cascade(
|
|||
# needs to get throttled the ticks we generate.
|
||||
# tick_throttle=60,
|
||||
|
||||
) as feed:
|
||||
) as (feed, quote_stream):
|
||||
symbol = feed.symbols[fqsn]
|
||||
|
||||
flume = feed.flumes[fqsn]
|
||||
symbol = flume.symbol
|
||||
assert src.token == flume.rt_shm.token
|
||||
profiler(f'{func}: feed up')
|
||||
|
||||
assert src.token == feed.shm.token
|
||||
# last_len = new_len = len(src.array)
|
||||
|
||||
func_name = func.__name__
|
||||
async with (
|
||||
trio.open_nursery() as n,
|
||||
|
|
@ -353,8 +324,8 @@ async def cascade(
|
|||
|
||||
fsp_compute,
|
||||
symbol=symbol,
|
||||
flume=flume,
|
||||
quote_stream=flume.stream,
|
||||
feed=feed,
|
||||
quote_stream=quote_stream,
|
||||
|
||||
# shm
|
||||
src=src,
|
||||
|
|
@ -390,7 +361,7 @@ async def cascade(
|
|||
) -> tuple[TaskTracker, int]:
|
||||
# TODO: adopt an incremental update engine/approach
|
||||
# where possible here eventually!
|
||||
log.info(f're-syncing fsp {func_name} to source')
|
||||
log.debug(f're-syncing fsp {func_name} to source')
|
||||
tracker.cs.cancel()
|
||||
await tracker.complete.wait()
|
||||
tracker, index = await n.start(fsp_target)
|
||||
|
|
@ -403,16 +374,14 @@ async def cascade(
|
|||
'key': dst_shm_token,
|
||||
'first': dst._first.value,
|
||||
'last': dst._last.value,
|
||||
}
|
||||
})
|
||||
}})
|
||||
return tracker, index
|
||||
|
||||
def is_synced(
|
||||
src: ShmArray,
|
||||
dst: ShmArray
|
||||
) -> tuple[bool, int, int]:
|
||||
'''
|
||||
Predicate to dertmine if a destination FSP
|
||||
'''Predicate to dertmine if a destination FSP
|
||||
output array is aligned to its source array.
|
||||
|
||||
'''
|
||||
|
|
@ -421,15 +390,16 @@ async def cascade(
|
|||
return not (
|
||||
# the source is likely backfilling and we must
|
||||
# sync history calculations
|
||||
len_diff > 2
|
||||
len_diff > 2 or
|
||||
|
||||
# we aren't step synced to the source and may be
|
||||
# leading/lagging by a step
|
||||
or step_diff > 1
|
||||
or step_diff < 0
|
||||
step_diff > 1 or
|
||||
step_diff < 0
|
||||
), step_diff, len_diff
|
||||
|
||||
async def poll_and_sync_to_step(
|
||||
|
||||
tracker: TaskTracker,
|
||||
src: ShmArray,
|
||||
dst: ShmArray,
|
||||
|
|
@ -448,23 +418,18 @@ async def cascade(
|
|||
# detect sample period step for subscription to increment
|
||||
# signal
|
||||
times = src.array['time']
|
||||
if len(times) > 1:
|
||||
last_ts = times[-1]
|
||||
delay_s = float(last_ts - times[times != last_ts][-1])
|
||||
else:
|
||||
# our default "HFT" sample rate.
|
||||
delay_s = _default_delay_s
|
||||
delay_s = times[-1] - times[times != times[-1]][-1]
|
||||
|
||||
# sub and increment the underlying shared memory buffer
|
||||
# on every step msg received from the global `samplerd`
|
||||
# service.
|
||||
async with open_sample_stream(float(delay_s)) as istream:
|
||||
# Increment the underlying shared memory buffer on every
|
||||
# "increment" msg received from the underlying data feed.
|
||||
async with feed.index_stream(
|
||||
int(delay_s)
|
||||
) as istream:
|
||||
|
||||
profiler(f'{func_name}: sample stream up')
|
||||
profiler.finish()
|
||||
|
||||
async for i in istream:
|
||||
# print(f'FSP incrementing {i}')
|
||||
async for _ in istream:
|
||||
|
||||
# respawn the compute task if the source
|
||||
# array has been updated such that we compute
|
||||
|
|
@ -493,23 +458,3 @@ async def cascade(
|
|||
last = array[-1:].copy()
|
||||
|
||||
dst.push(last)
|
||||
|
||||
# sync with source buffer's time step
|
||||
src_l2 = src.array[-2:]
|
||||
src_li, src_lt = src_l2[-1][['index', 'time']]
|
||||
src_2li, src_2lt = src_l2[-2][['index', 'time']]
|
||||
dst._array['time'][src_li] = src_lt
|
||||
dst._array['time'][src_2li] = src_2lt
|
||||
|
||||
# last2 = dst.array[-2:]
|
||||
# if (
|
||||
# last2[-1]['index'] != src_li
|
||||
# or last2[-2]['index'] != src_2li
|
||||
# ):
|
||||
# dstl2 = list(last2)
|
||||
# srcl2 = list(src_l2)
|
||||
# print(
|
||||
# # f'{dst.token}\n'
|
||||
# f'src: {srcl2}\n'
|
||||
# f'dst: {dstl2}\n'
|
||||
# )
|
||||
|
|
|
|||
|
|
@ -234,7 +234,7 @@ async def flow_rates(
|
|||
# FSPs, user input, and possibly any general event stream in
|
||||
# real-time. Hint: ideally implemented with caching until mutated
|
||||
# ;)
|
||||
period: 'Param[int]' = 1, # noqa
|
||||
period: 'Param[int]' = 6, # noqa
|
||||
|
||||
# TODO: support other means by providing a map
|
||||
# to weights `partial()`-ed with `wma()`?
|
||||
|
|
@ -268,7 +268,8 @@ async def flow_rates(
|
|||
'dark_dvlm_rate': None,
|
||||
}
|
||||
|
||||
quote = await anext(source)
|
||||
# TODO: 3.10 do ``anext()``
|
||||
quote = await source.__anext__()
|
||||
|
||||
# ltr = 0
|
||||
# lvr = 0
|
||||
|
|
|
|||
46
piker/log.py
46
piker/log.py
|
|
@ -21,11 +21,7 @@ import logging
|
|||
import json
|
||||
|
||||
import tractor
|
||||
from pygments import (
|
||||
highlight,
|
||||
lexers,
|
||||
formatters,
|
||||
)
|
||||
from pygments import highlight, lexers, formatters
|
||||
|
||||
# Makes it so we only see the full module name when using ``__name__``
|
||||
# without the extra "piker." prefix.
|
||||
|
|
@ -36,48 +32,26 @@ def get_logger(
|
|||
name: str = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''
|
||||
Return the package log or a sub-log for `name` if provided.
|
||||
|
||||
'''Return the package log or a sub-log for `name` if provided.
|
||||
'''
|
||||
return tractor.log.get_logger(name=name, _root_name=_proj_name)
|
||||
|
||||
|
||||
def get_console_log(
|
||||
level: str | None = None,
|
||||
name: str | None = None,
|
||||
|
||||
) -> logging.Logger:
|
||||
'''
|
||||
Get the package logger and enable a handler which writes to stderr.
|
||||
def get_console_log(level: str = None, name: str = None) -> logging.Logger:
|
||||
'''Get the package logger and enable a handler which writes to stderr.
|
||||
|
||||
Yeah yeah, i know we can use ``DictConfig``. You do it...
|
||||
|
||||
'''
|
||||
return tractor.log.get_console_log(
|
||||
level,
|
||||
name=name,
|
||||
_root_name=_proj_name,
|
||||
) # our root logger
|
||||
level, name=name, _root_name=_proj_name) # our root logger
|
||||
|
||||
|
||||
def colorize_json(
|
||||
data: dict,
|
||||
style='algol_nu',
|
||||
):
|
||||
'''
|
||||
Colorize json output using ``pygments``.
|
||||
|
||||
'''
|
||||
formatted_json = json.dumps(
|
||||
data,
|
||||
sort_keys=True,
|
||||
indent=4,
|
||||
)
|
||||
def colorize_json(data, style='algol_nu'):
|
||||
"""Colorize json output using ``pygments``.
|
||||
"""
|
||||
formatted_json = json.dumps(data, sort_keys=True, indent=4)
|
||||
return highlight(
|
||||
formatted_json,
|
||||
lexers.JsonLexer(),
|
||||
|
||||
formatted_json, lexers.JsonLexer(),
|
||||
# likeable styles: algol_nu, tango, monokai
|
||||
formatters.TerminalTrueColorFormatter(style=style)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,62 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Actor-runtime service orchestration machinery.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from ._mngr import Services
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
open_registry,
|
||||
find_service,
|
||||
check_for_service,
|
||||
)
|
||||
from ._daemon import ( # noqa
|
||||
maybe_spawn_daemon,
|
||||
spawn_emsd,
|
||||
maybe_open_emsd,
|
||||
)
|
||||
from ._actor_runtime import (
|
||||
open_piker_runtime,
|
||||
maybe_open_pikerd,
|
||||
open_pikerd,
|
||||
get_tractor_runtime_kwargs,
|
||||
)
|
||||
from ..brokers._daemon import (
|
||||
spawn_brokerd,
|
||||
maybe_spawn_brokerd,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
'check_for_service',
|
||||
'Services',
|
||||
'maybe_spawn_daemon',
|
||||
'spawn_brokerd',
|
||||
'maybe_spawn_brokerd',
|
||||
'spawn_emsd',
|
||||
'maybe_open_emsd',
|
||||
'open_piker_runtime',
|
||||
'maybe_open_pikerd',
|
||||
'open_pikerd',
|
||||
'get_tractor_runtime_kwargs',
|
||||
]
|
||||
|
|
@ -1,351 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
``tractor`` wrapping + default config to bootstrap the `pikerd`.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from pprint import pformat
|
||||
from functools import partial
|
||||
import os
|
||||
from typing import (
|
||||
Optional,
|
||||
Any,
|
||||
ClassVar,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
import trio
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
get_console_log,
|
||||
)
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._registry import ( # noqa
|
||||
_tractor_kwargs,
|
||||
_default_reg_addr,
|
||||
open_registry,
|
||||
)
|
||||
|
||||
|
||||
def get_tractor_runtime_kwargs() -> dict[str, Any]:
|
||||
'''
|
||||
Deliver ``tractor`` related runtime variables in a `dict`.
|
||||
|
||||
'''
|
||||
return _tractor_kwargs
|
||||
|
||||
|
||||
@acm
|
||||
async def open_piker_runtime(
|
||||
name: str,
|
||||
enable_modules: list[str] = [],
|
||||
loglevel: Optional[str] = None,
|
||||
|
||||
# XXX NOTE XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
# TODO: once we have `rsyscall` support we will read a config
|
||||
# and spawn the service tree distributed per that.
|
||||
start_method: str = 'trio',
|
||||
|
||||
tractor_runtime_overrides: dict | None = None,
|
||||
**tractor_kwargs,
|
||||
|
||||
) -> tuple[
|
||||
tractor.Actor,
|
||||
tuple[str, int],
|
||||
]:
|
||||
'''
|
||||
Start a piker actor who's runtime will automatically sync with
|
||||
existing piker actors on the local link based on configuration.
|
||||
|
||||
Can be called from a subactor or any program that needs to start
|
||||
a root actor.
|
||||
|
||||
'''
|
||||
try:
|
||||
# check for existing runtime
|
||||
actor = tractor.current_actor().uid
|
||||
|
||||
except tractor._exceptions.NoRuntime:
|
||||
tractor._state._runtime_vars[
|
||||
'piker_vars'] = tractor_runtime_overrides
|
||||
|
||||
registry_addr = registry_addr or _default_reg_addr
|
||||
|
||||
async with (
|
||||
tractor.open_root_actor(
|
||||
|
||||
# passed through to ``open_root_actor``
|
||||
arbiter_addr=registry_addr,
|
||||
name=name,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
start_method=start_method,
|
||||
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=enable_modules,
|
||||
|
||||
**tractor_kwargs,
|
||||
) as _,
|
||||
|
||||
open_registry(registry_addr, ensure_exists=False) as addr,
|
||||
):
|
||||
yield (
|
||||
tractor.current_actor(),
|
||||
addr,
|
||||
)
|
||||
else:
|
||||
async with open_registry(registry_addr) as addr:
|
||||
yield (
|
||||
actor,
|
||||
addr,
|
||||
)
|
||||
|
||||
|
||||
_root_dname = 'pikerd'
|
||||
_root_modules = [
|
||||
__name__,
|
||||
'piker.service._daemon',
|
||||
'piker.brokers._daemon',
|
||||
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
|
||||
'piker.data._sampling',
|
||||
]
|
||||
|
||||
|
||||
@acm
|
||||
async def open_pikerd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
|
||||
# XXX: you should pretty much never want debug mode
|
||||
# for data daemons when running in production.
|
||||
debug_mode: bool = False,
|
||||
registry_addr: None | tuple[str, int] = None,
|
||||
|
||||
# db init flags
|
||||
tsdb: bool = False,
|
||||
es: bool = False,
|
||||
drop_root_perms_for_ahab: bool = True,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> Services:
|
||||
'''
|
||||
Start a root piker daemon with an indefinite lifetime.
|
||||
|
||||
A root actor nursery is created which can be used to create and keep
|
||||
alive underling services (see below).
|
||||
|
||||
'''
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
|
||||
name=_root_dname,
|
||||
# TODO: eventually we should be able to avoid
|
||||
# having the root have more then permissions to
|
||||
# spawn other specialized daemons I think?
|
||||
enable_modules=_root_modules,
|
||||
loglevel=loglevel,
|
||||
debug_mode=debug_mode,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) as (root_actor, reg_addr),
|
||||
tractor.open_nursery() as actor_nursery,
|
||||
trio.open_nursery() as service_nursery,
|
||||
):
|
||||
if root_actor.accept_addr != reg_addr:
|
||||
raise RuntimeError(
|
||||
f'`pikerd` failed to bind on {reg_addr}!\n'
|
||||
'Maybe you have another daemon already running?'
|
||||
)
|
||||
|
||||
# assign globally for future daemon/task creation
|
||||
Services.actor_n = actor_nursery
|
||||
Services.service_n = service_nursery
|
||||
Services.debug_mode = debug_mode
|
||||
|
||||
if tsdb:
|
||||
from ._ahab import start_ahab
|
||||
from .marketstore import start_marketstore
|
||||
|
||||
log.info('Spawning `marketstore` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await service_nursery.start(
|
||||
partial(
|
||||
start_ahab,
|
||||
'marketstored',
|
||||
start_marketstore,
|
||||
loglevel=loglevel,
|
||||
drop_root_perms=drop_root_perms_for_ahab,
|
||||
)
|
||||
|
||||
)
|
||||
log.info(
|
||||
f'`marketstored` up!\n'
|
||||
f'pid: {pid}\n'
|
||||
f'container id: {cid[:12]}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
if es:
|
||||
from ._ahab import start_ahab
|
||||
from .elastic import start_elasticsearch
|
||||
|
||||
log.info('Spawning `elasticsearch` supervisor')
|
||||
ctn_ready, config, (cid, pid) = await service_nursery.start(
|
||||
partial(
|
||||
start_ahab,
|
||||
'elasticsearch',
|
||||
start_elasticsearch,
|
||||
loglevel=loglevel,
|
||||
drop_root_perms=drop_root_perms_for_ahab,
|
||||
)
|
||||
)
|
||||
|
||||
log.info(
|
||||
f'`elasticsearch` up!\n'
|
||||
f'pid: {pid}\n'
|
||||
f'container id: {cid[:12]}\n'
|
||||
f'config: {pformat(config)}'
|
||||
)
|
||||
|
||||
try:
|
||||
yield Services
|
||||
|
||||
finally:
|
||||
# TODO: is this more clever/efficient?
|
||||
# if 'samplerd' in Services.service_tasks:
|
||||
# await Services.cancel_service('samplerd')
|
||||
service_nursery.cancel_scope.cancel()
|
||||
|
||||
|
||||
# TODO: do we even need this?
|
||||
# @acm
|
||||
# async def maybe_open_runtime(
|
||||
# loglevel: Optional[str] = None,
|
||||
# **kwargs,
|
||||
|
||||
# ) -> None:
|
||||
# '''
|
||||
# Start the ``tractor`` runtime (a root actor) if none exists.
|
||||
|
||||
# '''
|
||||
# name = kwargs.pop('name')
|
||||
|
||||
# if not tractor.current_actor(err_on_no_runtime=False):
|
||||
# async with open_piker_runtime(
|
||||
# name,
|
||||
# loglevel=loglevel,
|
||||
# **kwargs,
|
||||
# ) as (_, addr):
|
||||
# yield addr,
|
||||
# else:
|
||||
# async with open_registry() as addr:
|
||||
# yield addr
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_pikerd(
|
||||
loglevel: Optional[str] = None,
|
||||
registry_addr: None | tuple = None,
|
||||
tsdb: bool = False,
|
||||
es: bool = False,
|
||||
drop_root_perms_for_ahab: bool = True,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tractor._portal.Portal | ClassVar[Services]:
|
||||
'''
|
||||
If no ``pikerd`` daemon-root-actor can be found start it and
|
||||
yield up (we should probably figure out returning a portal to self
|
||||
though).
|
||||
|
||||
'''
|
||||
if loglevel:
|
||||
get_console_log(loglevel)
|
||||
|
||||
# subtle, we must have the runtime up here or portal lookup will fail
|
||||
query_name = kwargs.pop(
|
||||
'name',
|
||||
f'piker_query_{os.getpid()}',
|
||||
)
|
||||
|
||||
# TODO: if we need to make the query part faster we could not init
|
||||
# an actor runtime and instead just hit the socket?
|
||||
# from tractor._ipc import _connect_chan, Channel
|
||||
# async with _connect_chan(host, port) as chan:
|
||||
# async with open_portal(chan) as arb_portal:
|
||||
# yield arb_portal
|
||||
|
||||
async with (
|
||||
open_piker_runtime(
|
||||
name=query_name,
|
||||
registry_addr=registry_addr,
|
||||
loglevel=loglevel,
|
||||
**kwargs,
|
||||
) as _,
|
||||
|
||||
tractor.find_actor(
|
||||
_root_dname,
|
||||
arbiter_sockaddr=registry_addr,
|
||||
) as portal
|
||||
):
|
||||
# connect to any existing daemon presuming
|
||||
# its registry socket was selected.
|
||||
if (
|
||||
portal is not None
|
||||
):
|
||||
yield portal
|
||||
return
|
||||
|
||||
# presume pikerd role since no daemon could be found at
|
||||
# configured address
|
||||
async with open_pikerd(
|
||||
loglevel=loglevel,
|
||||
registry_addr=registry_addr,
|
||||
|
||||
# ahabd (docker super) specific controls
|
||||
tsdb=tsdb,
|
||||
es=es,
|
||||
drop_root_perms_for_ahab=drop_root_perms_for_ahab,
|
||||
|
||||
# passthrough to ``tractor`` init
|
||||
**kwargs,
|
||||
|
||||
) as service_manager:
|
||||
# in the case where we're starting up the
|
||||
# tractor-piker runtime stack in **this** process
|
||||
# we return no portal to self.
|
||||
assert service_manager
|
||||
yield service_manager
|
||||
|
|
@ -1,190 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Daemon-actor spawning "endpoint-hooks".
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
from ._mngr import (
|
||||
Services,
|
||||
)
|
||||
from ._actor_runtime import maybe_open_pikerd
|
||||
from ._registry import find_service
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_spawn_daemon(
|
||||
|
||||
service_name: str,
|
||||
service_task_target: Callable,
|
||||
|
||||
spawn_args: dict[str, Any],
|
||||
|
||||
loglevel: str | None = None,
|
||||
singleton: bool = False,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal:
|
||||
'''
|
||||
If no ``service_name`` daemon-actor can be found,
|
||||
spawn one in a local subactor and return a portal to it.
|
||||
|
||||
If this function is called from a non-pikerd actor, the
|
||||
spawned service will persist as long as pikerd does or
|
||||
it is requested to be cancelled.
|
||||
|
||||
This can be seen as a service starting api for remote-actor
|
||||
clients.
|
||||
|
||||
'''
|
||||
# serialize access to this section to avoid
|
||||
# 2 or more tasks racing to create a daemon
|
||||
lock = Services.locks[service_name]
|
||||
await lock.acquire()
|
||||
|
||||
async with find_service(service_name) as portal:
|
||||
if portal is not None:
|
||||
lock.release()
|
||||
yield portal
|
||||
return
|
||||
|
||||
log.warning(
|
||||
f"Couldn't find any existing {service_name}\n"
|
||||
'Attempting to spawn new daemon-service..'
|
||||
)
|
||||
|
||||
# ask root ``pikerd`` daemon to spawn the daemon we need if
|
||||
# pikerd is not live we now become the root of the
|
||||
# process tree
|
||||
async with maybe_open_pikerd(
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as pikerd_portal:
|
||||
|
||||
# we are the root and thus are `pikerd`
|
||||
# so spawn the target service directly by calling
|
||||
# the provided target routine.
|
||||
# XXX: this assumes that the target is well formed and will
|
||||
# do the right things to setup both a sub-actor **and** call
|
||||
# the ``_Services`` api from above to start the top level
|
||||
# service task for that actor.
|
||||
started: bool
|
||||
if pikerd_portal is None:
|
||||
started = await service_task_target(
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
else:
|
||||
# request a remote `pikerd` (service manager) to start the
|
||||
# target daemon-task, the target can't return
|
||||
# a non-serializable value since it is expected that service
|
||||
# starting is non-blocking and the target task will persist
|
||||
# running "under" or "within" the `pikerd` actor tree after
|
||||
# the questing client disconnects. in other words this
|
||||
# spawns a persistent daemon actor that continues to live
|
||||
# for the lifespan of whatever the service manager inside
|
||||
# `pikerd` says it should.
|
||||
started = await pikerd_portal.run(
|
||||
service_task_target,
|
||||
loglevel=loglevel,
|
||||
**spawn_args,
|
||||
)
|
||||
|
||||
if started:
|
||||
log.info(f'Service {service_name} started!')
|
||||
|
||||
# block until we can discover (by IPC connection) to the newly
|
||||
# spawned daemon-actor and then deliver the portal to the
|
||||
# caller.
|
||||
async with tractor.wait_for_actor(service_name) as portal:
|
||||
lock.release()
|
||||
yield portal
|
||||
await portal.cancel_actor()
|
||||
|
||||
|
||||
async def spawn_emsd(
|
||||
|
||||
loglevel: str | None = None,
|
||||
**extra_tractor_kwargs
|
||||
|
||||
) -> bool:
|
||||
"""
|
||||
Start the clearing engine under ``pikerd``.
|
||||
|
||||
"""
|
||||
log.info('Spawning emsd')
|
||||
|
||||
portal = await Services.actor_n.start_actor(
|
||||
'emsd',
|
||||
enable_modules=[
|
||||
'piker.clearing._ems',
|
||||
'piker.clearing._client',
|
||||
],
|
||||
loglevel=loglevel,
|
||||
debug_mode=Services.debug_mode, # set by pikerd flag
|
||||
**extra_tractor_kwargs
|
||||
)
|
||||
|
||||
# non-blocking setup of clearing service
|
||||
from ..clearing._ems import _setup_persistent_emsd
|
||||
|
||||
await Services.start_service_task(
|
||||
'emsd',
|
||||
portal,
|
||||
|
||||
# signature of target root-task endpoint
|
||||
_setup_persistent_emsd,
|
||||
loglevel=loglevel,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
@acm
|
||||
async def maybe_open_emsd(
|
||||
|
||||
brokername: str,
|
||||
loglevel: str | None = None,
|
||||
|
||||
**pikerd_kwargs,
|
||||
|
||||
) -> tractor.Portal: # noqa
|
||||
|
||||
async with maybe_spawn_daemon(
|
||||
'emsd',
|
||||
service_task_target=spawn_emsd,
|
||||
spawn_args={},
|
||||
loglevel=loglevel,
|
||||
**pikerd_kwargs,
|
||||
|
||||
) as portal:
|
||||
yield portal
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
daemon-service management API.
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from typing import (
|
||||
Callable,
|
||||
Any,
|
||||
)
|
||||
|
||||
import trio
|
||||
from trio_typing import TaskStatus
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
|
||||
# TODO: factor this into a ``tractor.highlevel`` extension
|
||||
# pack for the library.
|
||||
class Services:
|
||||
|
||||
actor_n: tractor._supervise.ActorNursery
|
||||
service_n: trio.Nursery
|
||||
debug_mode: bool # tractor sub-actor debug mode flag
|
||||
service_tasks: dict[
|
||||
str,
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
tractor.Portal,
|
||||
trio.Event,
|
||||
]
|
||||
] = {}
|
||||
locks = defaultdict(trio.Lock)
|
||||
|
||||
@classmethod
|
||||
async def start_service_task(
|
||||
self,
|
||||
name: str,
|
||||
portal: tractor.Portal,
|
||||
target: Callable,
|
||||
**ctx_kwargs,
|
||||
|
||||
) -> (trio.CancelScope, tractor.Context):
|
||||
'''
|
||||
Open a context in a service sub-actor, add to a stack
|
||||
that gets unwound at ``pikerd`` teardown.
|
||||
|
||||
This allows for allocating long-running sub-services in our main
|
||||
daemon and explicitly controlling their lifetimes.
|
||||
|
||||
'''
|
||||
async def open_context_in_task(
|
||||
task_status: TaskStatus[
|
||||
tuple[
|
||||
trio.CancelScope,
|
||||
trio.Event,
|
||||
Any,
|
||||
]
|
||||
] = trio.TASK_STATUS_IGNORED,
|
||||
|
||||
) -> Any:
|
||||
|
||||
with trio.CancelScope() as cs:
|
||||
async with portal.open_context(
|
||||
target,
|
||||
**ctx_kwargs,
|
||||
|
||||
) as (ctx, first):
|
||||
|
||||
# unblock once the remote context has started
|
||||
complete = trio.Event()
|
||||
task_status.started((cs, complete, first))
|
||||
log.info(
|
||||
f'`pikerd` service {name} started with value {first}'
|
||||
)
|
||||
try:
|
||||
# wait on any context's return value
|
||||
# and any final portal result from the
|
||||
# sub-actor.
|
||||
ctx_res = await ctx.result()
|
||||
|
||||
# NOTE: blocks indefinitely until cancelled
|
||||
# either by error from the target context
|
||||
# function or by being cancelled here by the
|
||||
# surrounding cancel scope.
|
||||
return (await portal.result(), ctx_res)
|
||||
|
||||
finally:
|
||||
await portal.cancel_actor()
|
||||
complete.set()
|
||||
self.service_tasks.pop(name)
|
||||
|
||||
cs, complete, first = await self.service_n.start(open_context_in_task)
|
||||
|
||||
# store the cancel scope and portal for later cancellation or
|
||||
# retstart if needed.
|
||||
self.service_tasks[name] = (cs, portal, complete)
|
||||
|
||||
return cs, first
|
||||
|
||||
@classmethod
|
||||
async def cancel_service(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
) -> Any:
|
||||
'''
|
||||
Cancel the service task and actor for the given ``name``.
|
||||
|
||||
'''
|
||||
log.info(f'Cancelling `pikerd` service {name}')
|
||||
cs, portal, complete = self.service_tasks[name]
|
||||
cs.cancel()
|
||||
await complete.wait()
|
||||
assert name not in self.service_tasks, \
|
||||
f'Serice task for {name} not terminated?'
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
"""
|
||||
Inter-actor "discovery" (protocol) layer.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from contextlib import (
|
||||
asynccontextmanager as acm,
|
||||
)
|
||||
from typing import (
|
||||
Any,
|
||||
)
|
||||
|
||||
import tractor
|
||||
|
||||
from ._util import (
|
||||
log, # sub-sys logger
|
||||
)
|
||||
|
||||
_default_registry_host: str = '127.0.0.1'
|
||||
_default_registry_port: int = 6116
|
||||
_default_reg_addr: tuple[str, int] = (
|
||||
_default_registry_host,
|
||||
_default_registry_port,
|
||||
)
|
||||
|
||||
|
||||
# NOTE: this value is set as an actor-global once the first endpoint
|
||||
# who is capable, spawns a `pikerd` service tree.
|
||||
_registry: Registry | None = None
|
||||
|
||||
|
||||
class Registry:
|
||||
addr: None | tuple[str, int] = None
|
||||
|
||||
# TODO: table of uids to sockaddrs
|
||||
peers: dict[
|
||||
tuple[str, str],
|
||||
tuple[str, int],
|
||||
] = {}
|
||||
|
||||
|
||||
_tractor_kwargs: dict[str, Any] = {}
|
||||
|
||||
|
||||
@acm
|
||||
async def open_registry(
|
||||
addr: None | tuple[str, int] = None,
|
||||
ensure_exists: bool = True,
|
||||
|
||||
) -> tuple[str, int]:
|
||||
|
||||
global _tractor_kwargs
|
||||
actor = tractor.current_actor()
|
||||
uid = actor.uid
|
||||
if (
|
||||
Registry.addr is not None
|
||||
and addr
|
||||
):
|
||||
raise RuntimeError(
|
||||
f'`{uid}` registry addr already bound @ {_registry.sockaddr}'
|
||||
)
|
||||
|
||||
was_set: bool = False
|
||||
|
||||
if (
|
||||
not tractor.is_root_process()
|
||||
and Registry.addr is None
|
||||
):
|
||||
Registry.addr = actor._arb_addr
|
||||
|
||||
if (
|
||||
ensure_exists
|
||||
and Registry.addr is None
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"`{uid}` registry should already exist bug doesn't?"
|
||||
)
|
||||
|
||||
if (
|
||||
Registry.addr is None
|
||||
):
|
||||
was_set = True
|
||||
Registry.addr = addr or _default_reg_addr
|
||||
|
||||
_tractor_kwargs['arbiter_addr'] = Registry.addr
|
||||
|
||||
try:
|
||||
yield Registry.addr
|
||||
finally:
|
||||
# XXX: always clear the global addr if we set it so that the
|
||||
# next (set of) calls will apply whatever new one is passed
|
||||
# in.
|
||||
if was_set:
|
||||
Registry.addr = None
|
||||
|
||||
|
||||
@acm
|
||||
async def find_service(
|
||||
service_name: str,
|
||||
) -> tractor.Portal | None:
|
||||
|
||||
async with open_registry() as reg_addr:
|
||||
log.info(f'Scanning for service `{service_name}`')
|
||||
# attach to existing daemon by name if possible
|
||||
async with tractor.find_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as maybe_portal:
|
||||
yield maybe_portal
|
||||
|
||||
|
||||
async def check_for_service(
|
||||
service_name: str,
|
||||
|
||||
) -> None | tuple[str, int]:
|
||||
'''
|
||||
Service daemon "liveness" predicate.
|
||||
|
||||
'''
|
||||
async with open_registry(ensure_exists=False) as reg_addr:
|
||||
async with tractor.query_actor(
|
||||
service_name,
|
||||
arbiter_sockaddr=reg_addr,
|
||||
) as sockaddr:
|
||||
return sockaddr
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
"""
|
||||
Sub-sys module commons.
|
||||
|
||||
"""
|
||||
from functools import partial
|
||||
|
||||
from ..log import (
|
||||
get_logger,
|
||||
get_console_log,
|
||||
)
|
||||
subsys: str = 'piker.service'
|
||||
|
||||
log = get_logger(subsys)
|
||||
|
||||
get_console_log = partial(
|
||||
get_console_log,
|
||||
name=subsys,
|
||||
)
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import (
|
||||
Any,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
|
||||
import asks
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import docker
|
||||
from ._ahab import DockerContainer
|
||||
|
||||
from ._util import log # sub-sys logger
|
||||
from ._util import (
|
||||
get_console_log,
|
||||
)
|
||||
|
||||
|
||||
# container level config
|
||||
_config = {
|
||||
'port': 19200,
|
||||
'log_level': 'debug',
|
||||
|
||||
# hardcoded to our image version
|
||||
'version': '7.17.4',
|
||||
}
|
||||
|
||||
|
||||
def start_elasticsearch(
|
||||
client: docker.DockerClient,
|
||||
|
||||
**kwargs,
|
||||
|
||||
) -> tuple[DockerContainer, dict[str, Any]]:
|
||||
'''
|
||||
Start and supervise an elasticsearch instance with its config bind-mounted
|
||||
in from the piker config directory on the system.
|
||||
|
||||
The equivalent cli cmd to this code is:
|
||||
|
||||
sudo docker run \
|
||||
-itd \
|
||||
--rm \
|
||||
--network=host \
|
||||
--mount type=bind,source="$(pwd)"/elastic,\
|
||||
target=/usr/share/elasticsearch/data \
|
||||
--env "elastic_username=elastic" \
|
||||
--env "elastic_password=password" \
|
||||
--env "xpack.security.enabled=false" \
|
||||
elastic
|
||||
|
||||
'''
|
||||
get_console_log('info', name=__name__)
|
||||
|
||||
dcntr: DockerContainer = client.containers.run(
|
||||
'piker:elastic',
|
||||
name='piker-elastic',
|
||||
network='host',
|
||||
detach=True,
|
||||
remove=True
|
||||
)
|
||||
|
||||
async def health_query(msg: str | None = None):
|
||||
if (
|
||||
msg
|
||||
and _config['version'] in msg
|
||||
):
|
||||
return True
|
||||
|
||||
try:
|
||||
health = (await asks.get(
|
||||
'http://localhost:19200/_cat/health',
|
||||
params={'format': 'json'}
|
||||
)).json()
|
||||
log.info(
|
||||
'ElasticSearch cntr health:\n'
|
||||
f'{health}'
|
||||
)
|
||||
|
||||
except OSError:
|
||||
log.exception('couldnt reach elastic container')
|
||||
return False
|
||||
|
||||
log.info(health)
|
||||
return health[0]['status'] == 'green'
|
||||
|
||||
async def chk_for_closed_msg(msg: str):
|
||||
return msg == 'closed'
|
||||
|
||||
return (
|
||||
dcntr,
|
||||
{
|
||||
# apparently we're REALLY tolerant of startup latency
|
||||
# for CI XD
|
||||
'startup_timeout': 240.0,
|
||||
|
||||
# XXX: decrease http poll period bc docker
|
||||
# is shite at handling fast poll rates..
|
||||
'startup_query_period': 0.1,
|
||||
|
||||
'log_msg_key': 'message',
|
||||
|
||||
# 'started_afunc': health_query,
|
||||
},
|
||||
# expected startup and stop msgs
|
||||
health_query,
|
||||
chk_for_closed_msg,
|
||||
)
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
Annotations for ur faces.
|
||||
|
||||
"""
|
||||
from typing import Callable
|
||||
from typing import Callable, Optional
|
||||
|
||||
from PyQt5 import QtCore, QtGui, QtWidgets
|
||||
from PyQt5.QtCore import QPointF, QRectF
|
||||
|
|
@ -32,22 +32,16 @@ def mk_marker_path(
|
|||
style: str,
|
||||
|
||||
) -> QGraphicsPathItem:
|
||||
'''
|
||||
Add a marker to be displayed on the line wrapped in
|
||||
a ``QGraphicsPathItem`` ready to be placed using scene coordinates
|
||||
(not view).
|
||||
"""Add a marker to be displayed on the line wrapped in a ``QGraphicsPathItem``
|
||||
ready to be placed using scene coordinates (not view).
|
||||
|
||||
**Arguments**
|
||||
style String indicating the style of marker to add:
|
||||
``'<|'``, ``'|>'``, ``'>|'``, ``'|<'``, ``'<|>'``,
|
||||
``'>|<'``, ``'^'``, ``'v'``, ``'o'``
|
||||
size Size of the marker in pixels.
|
||||
|
||||
This code is taken nearly verbatim from the
|
||||
`InfiniteLine.addMarker()` method but does not attempt do be aware
|
||||
of low(er) level graphics controls and expects for the output
|
||||
polygon to be applied to a ``QGraphicsPathItem``.
|
||||
|
||||
'''
|
||||
"""
|
||||
path = QtGui.QPainterPath()
|
||||
|
||||
if style == 'o':
|
||||
|
|
@ -93,8 +87,7 @@ def mk_marker_path(
|
|||
|
||||
|
||||
class LevelMarker(QGraphicsPathItem):
|
||||
'''
|
||||
An arrow marker path graphich which redraws itself
|
||||
'''An arrow marker path graphich which redraws itself
|
||||
to the specified view coordinate level on each paint cycle.
|
||||
|
||||
'''
|
||||
|
|
@ -105,14 +98,13 @@ class LevelMarker(QGraphicsPathItem):
|
|||
get_level: Callable[..., float],
|
||||
size: float = 20,
|
||||
keep_in_view: bool = True,
|
||||
on_paint: Callable | None = None,
|
||||
on_paint: Optional[Callable] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
# get polygon and scale
|
||||
super().__init__()
|
||||
# self.setScale(size, size)
|
||||
self.setScale(size)
|
||||
self.scale(size, size)
|
||||
|
||||
# interally generates path
|
||||
self._style = None
|
||||
|
|
@ -122,7 +114,6 @@ class LevelMarker(QGraphicsPathItem):
|
|||
|
||||
self.get_level = get_level
|
||||
self._on_paint = on_paint
|
||||
|
||||
self.scene_x = lambda: chart.marker_right_points()[1]
|
||||
self.level: float = 0
|
||||
self.keep_in_view = keep_in_view
|
||||
|
|
@ -158,9 +149,12 @@ class LevelMarker(QGraphicsPathItem):
|
|||
def w(self) -> float:
|
||||
return self.path_br().width()
|
||||
|
||||
def position_in_view(self) -> None:
|
||||
'''
|
||||
Show a pp off-screen indicator for a level label.
|
||||
def position_in_view(
|
||||
self,
|
||||
# level: float,
|
||||
|
||||
) -> None:
|
||||
'''Show a pp off-screen indicator for a level label.
|
||||
|
||||
This is like in fps games where you have a gps "nav" indicator
|
||||
but your teammate is outside the range of view, except in 2D, on
|
||||
|
|
@ -168,6 +162,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
|
||||
'''
|
||||
level = self.get_level()
|
||||
|
||||
view = self.chart.getViewBox()
|
||||
vr = view.state['viewRange']
|
||||
ymn, ymx = vr[1]
|
||||
|
|
@ -191,6 +186,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
)
|
||||
|
||||
elif level < ymn: # pin to bottom of view
|
||||
|
||||
self.setPos(
|
||||
QPointF(
|
||||
x,
|
||||
|
|
@ -215,8 +211,7 @@ class LevelMarker(QGraphicsPathItem):
|
|||
w: QtWidgets.QWidget
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Core paint which we override to always update
|
||||
'''Core paint which we override to always update
|
||||
our marker position in scene coordinates from a
|
||||
view cooridnate "level".
|
||||
|
||||
|
|
@ -240,12 +235,11 @@ def qgo_draw_markers(
|
|||
right_offset: float,
|
||||
|
||||
) -> float:
|
||||
'''
|
||||
Paint markers in ``pg.GraphicsItem`` style by first
|
||||
"""Paint markers in ``pg.GraphicsItem`` style by first
|
||||
removing the view transform for the painter, drawing the markers
|
||||
in scene coords, then restoring the view coords.
|
||||
|
||||
'''
|
||||
"""
|
||||
# paint markers in native coordinate system
|
||||
orig_tr = p.transform()
|
||||
|
||||
|
|
|
|||
|
|
@ -19,16 +19,15 @@ Main app startup and run.
|
|||
|
||||
'''
|
||||
from functools import partial
|
||||
from types import ModuleType
|
||||
|
||||
from PyQt5.QtCore import QEvent
|
||||
import trio
|
||||
|
||||
from ..service import maybe_spawn_brokerd
|
||||
from .._daemon import maybe_spawn_brokerd
|
||||
from ..brokers import get_brokermod
|
||||
from . import _event
|
||||
from ._exec import run_qtractor
|
||||
from ..data.feed import install_brokerd_search
|
||||
from ..accounting._mktinfo import unpack_fqme
|
||||
from . import _search
|
||||
from ._chart import GodWidget
|
||||
from ..log import get_logger
|
||||
|
|
@ -37,26 +36,27 @@ log = get_logger(__name__)
|
|||
|
||||
|
||||
async def load_provider_search(
|
||||
brokermod: str,
|
||||
|
||||
broker: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> None:
|
||||
|
||||
name = brokermod.name
|
||||
log.info(f'loading brokerd for {name}..')
|
||||
log.info(f'loading brokerd for {broker}..')
|
||||
|
||||
async with (
|
||||
|
||||
maybe_spawn_brokerd(
|
||||
name,
|
||||
broker,
|
||||
loglevel=loglevel
|
||||
) as portal,
|
||||
|
||||
install_brokerd_search(
|
||||
portal,
|
||||
brokermod,
|
||||
get_brokermod(broker),
|
||||
),
|
||||
):
|
||||
|
||||
# keep search engine stream up until cancelled
|
||||
await trio.sleep_forever()
|
||||
|
||||
|
|
@ -66,8 +66,8 @@ async def _async_main(
|
|||
# implicit required argument provided by ``qtractor_run()``
|
||||
main_widget: GodWidget,
|
||||
|
||||
syms: list[str],
|
||||
brokers: dict[str, ModuleType],
|
||||
sym: str,
|
||||
brokernames: str,
|
||||
loglevel: str,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -78,8 +78,6 @@ async def _async_main(
|
|||
|
||||
"""
|
||||
from . import _display
|
||||
from ._pg_overrides import _do_overrides
|
||||
_do_overrides()
|
||||
|
||||
godwidget = main_widget
|
||||
|
||||
|
|
@ -99,11 +97,6 @@ async def _async_main(
|
|||
sbar = godwidget.window.status_bar
|
||||
starting_done = sbar.open_status('starting ze sexy chartz')
|
||||
|
||||
needed_brokermods: dict[str, ModuleType] = {}
|
||||
for fqsn in syms:
|
||||
brokername, *_ = unpack_fqme(fqsn)
|
||||
needed_brokermods[brokername] = brokers[brokername]
|
||||
|
||||
async with (
|
||||
trio.open_nursery() as root_n,
|
||||
):
|
||||
|
|
@ -114,14 +107,18 @@ async def _async_main(
|
|||
# setup search widget and focus main chart view at startup
|
||||
# search widget is a singleton alongside the godwidget
|
||||
search = _search.SearchWidget(godwidget=godwidget)
|
||||
# search.bar.unfocus()
|
||||
# godwidget.hbox.addWidget(search)
|
||||
search.bar.unfocus()
|
||||
|
||||
godwidget.hbox.addWidget(search)
|
||||
godwidget.search = search
|
||||
|
||||
symbol, _, provider = sym.rpartition('.')
|
||||
|
||||
# this internally starts a ``display_symbol_data()`` task above
|
||||
order_mode_ready = await godwidget.load_symbols(
|
||||
fqsns=syms,
|
||||
loglevel=loglevel,
|
||||
order_mode_ready = await godwidget.load_symbol(
|
||||
provider,
|
||||
symbol,
|
||||
loglevel
|
||||
)
|
||||
|
||||
# spin up a search engine for the local cached symbol set
|
||||
|
|
@ -138,12 +135,8 @@ async def _async_main(
|
|||
):
|
||||
# load other providers into search **after**
|
||||
# the chart's select cache
|
||||
for brokername, mod in needed_brokermods.items():
|
||||
root_n.start_soon(
|
||||
load_provider_search,
|
||||
mod,
|
||||
loglevel,
|
||||
)
|
||||
for broker in brokernames:
|
||||
root_n.start_soon(load_provider_search, broker, loglevel)
|
||||
|
||||
await order_mode_ready.wait()
|
||||
|
||||
|
|
@ -172,22 +165,19 @@ async def _async_main(
|
|||
|
||||
|
||||
def _main(
|
||||
syms: list[str],
|
||||
brokermods: list[ModuleType],
|
||||
sym: str,
|
||||
brokernames: [str],
|
||||
piker_loglevel: str,
|
||||
tractor_kwargs,
|
||||
) -> None:
|
||||
'''
|
||||
Sync entry point to start a chart: a ``tractor`` + Qt runtime.
|
||||
Sync entry point to start a chart: a ``tractor`` + Qt runtime
|
||||
entry point
|
||||
|
||||
'''
|
||||
run_qtractor(
|
||||
func=_async_main,
|
||||
args=(
|
||||
syms,
|
||||
{mod.name: mod for mod in brokermods},
|
||||
piker_loglevel,
|
||||
),
|
||||
main_widget_type=GodWidget,
|
||||
args=(sym, brokernames, piker_loglevel),
|
||||
main_widget=GodWidget,
|
||||
tractor_kwargs=tractor_kwargs,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,9 +18,8 @@
|
|||
Chart axes graphics and behavior.
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from functools import lru_cache
|
||||
from typing import Callable
|
||||
from typing import Optional, Callable
|
||||
from math import floor
|
||||
|
||||
import numpy as np
|
||||
|
|
@ -28,8 +27,7 @@ import pyqtgraph as pg
|
|||
from PyQt5 import QtCore, QtGui, QtWidgets
|
||||
from PyQt5.QtCore import QPointF
|
||||
|
||||
from . import _pg_overrides as pgo
|
||||
from ..accounting._mktinfo import float_digits
|
||||
from ..data._source import float_digits
|
||||
from ._label import Label
|
||||
from ._style import DpiAwareFont, hcolor, _font
|
||||
from ._interaction import ChartView
|
||||
|
|
@ -41,17 +39,12 @@ class Axis(pg.AxisItem):
|
|||
'''
|
||||
A better axis that sizes tick contents considering font size.
|
||||
|
||||
Also includes tick values lru caching originally proposed in but never
|
||||
accepted upstream:
|
||||
https://github.com/pyqtgraph/pyqtgraph/pull/2160
|
||||
|
||||
'''
|
||||
def __init__(
|
||||
self,
|
||||
plotitem: pgo.PlotItem,
|
||||
typical_max_str: str = '100 000.000 ',
|
||||
linkedsplits,
|
||||
typical_max_str: str = '100 000.000',
|
||||
text_color: str = 'bracket',
|
||||
lru_cache_tick_strings: bool = True,
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
|
@ -60,82 +53,44 @@ class Axis(pg.AxisItem):
|
|||
**kwargs
|
||||
)
|
||||
|
||||
# XXX: pretty sure this makes things slower!
|
||||
# no idea why given we only move labels for the most part?
|
||||
# XXX: pretty sure this makes things slower
|
||||
# self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
self.pi = plotitem
|
||||
self.linkedsplits = linkedsplits
|
||||
self._dpi_font = _font
|
||||
|
||||
self.setTickFont(_font.font)
|
||||
font_size = self._dpi_font.font.pixelSize()
|
||||
|
||||
style_conf = {
|
||||
'textFillLimits': [(0, 0.5)],
|
||||
'tickFont': self._dpi_font.font,
|
||||
|
||||
}
|
||||
text_offset = None
|
||||
if self.orientation in ('bottom',):
|
||||
text_offset = floor(0.25 * font_size)
|
||||
|
||||
elif self.orientation in ('left', 'right'):
|
||||
text_offset = floor(font_size / 2)
|
||||
|
||||
if text_offset:
|
||||
style_conf.update({
|
||||
self.setStyle(**{
|
||||
'textFillLimits': [(0, 0.5)],
|
||||
'tickFont': self._dpi_font.font,
|
||||
|
||||
# offset of text *away from* axis line in px
|
||||
# use approx. half the font pixel size (height)
|
||||
'tickTextOffset': text_offset,
|
||||
})
|
||||
|
||||
self.setStyle(**style_conf)
|
||||
self.setTickFont(_font.font)
|
||||
|
||||
# NOTE: this is for surrounding "border"
|
||||
self.setPen(_axis_pen)
|
||||
|
||||
# this is the text color
|
||||
# self.setTextPen(pg.mkPen(hcolor(text_color)))
|
||||
self.text_color = text_color
|
||||
|
||||
# generate a bounding rect based on sizing to a "typical"
|
||||
# maximum length-ed string defined as init default.
|
||||
self.typical_br = _font._qfm.boundingRect(typical_max_str)
|
||||
|
||||
# size the pertinent axis dimension to a "typical value"
|
||||
self.size_to_values()
|
||||
|
||||
# NOTE: requires override ``.tickValues()`` method seen below.
|
||||
if lru_cache_tick_strings:
|
||||
self.tickStrings = lru_cache(
|
||||
maxsize=2**20
|
||||
)(self.tickStrings)
|
||||
|
||||
# axis "sticky" labels
|
||||
self._stickies: dict[str, YAxisLabel] = {}
|
||||
|
||||
# NOTE: only overriden to cast tick values entries into tuples
|
||||
# for use with the lru caching.
|
||||
def tickValues(
|
||||
self,
|
||||
minVal: float,
|
||||
maxVal: float,
|
||||
size: int,
|
||||
|
||||
) -> list[tuple[float, tuple[str]]]:
|
||||
'''
|
||||
Repack tick values into tuples for lru caching.
|
||||
|
||||
'''
|
||||
ticks = []
|
||||
for scalar, values in super().tickValues(minVal, maxVal, size):
|
||||
ticks.append((
|
||||
scalar,
|
||||
tuple(values), # this
|
||||
))
|
||||
|
||||
return ticks
|
||||
|
||||
@property
|
||||
def text_color(self) -> str:
|
||||
return self._text_color
|
||||
|
|
@ -151,38 +106,6 @@ class Axis(pg.AxisItem):
|
|||
def txt_offsets(self) -> tuple[int, int]:
|
||||
return tuple(self.style['tickTextOffset'])
|
||||
|
||||
def add_sticky(
|
||||
self,
|
||||
pi: pgo.PlotItem,
|
||||
name: None | str = None,
|
||||
digits: None | int = 2,
|
||||
bg_color='default',
|
||||
fg_color='black',
|
||||
|
||||
) -> YAxisLabel:
|
||||
|
||||
# if the sticky is for our symbol
|
||||
# use the tick size precision for display
|
||||
name = name or pi.name
|
||||
digits = digits or 2
|
||||
|
||||
# TODO: ``._ysticks`` should really be an attr on each
|
||||
# ``PlotItem`` now instead of the containing widget (because of
|
||||
# overlays) ?
|
||||
|
||||
# add y-axis "last" value label
|
||||
sticky = self._stickies[name] = YAxisLabel(
|
||||
pi=pi,
|
||||
parent=self,
|
||||
digits=digits, # TODO: pass this from symbol data
|
||||
opacity=0.9, # slight see-through
|
||||
bg_color=bg_color,
|
||||
fg_color=fg_color,
|
||||
)
|
||||
|
||||
pi.sigRangeChanged.connect(sticky.update_on_resize)
|
||||
return sticky
|
||||
|
||||
|
||||
class PriceAxis(Axis):
|
||||
|
||||
|
|
@ -191,7 +114,7 @@ class PriceAxis(Axis):
|
|||
*args,
|
||||
min_tick: int = 2,
|
||||
title: str = '',
|
||||
formatter: Callable[[float], str] | None = None,
|
||||
formatter: Optional[Callable[[float], str]] = None,
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
|
@ -203,8 +126,8 @@ class PriceAxis(Axis):
|
|||
def set_title(
|
||||
self,
|
||||
title: str,
|
||||
view: ChartView | None = None,
|
||||
color: str | None = None,
|
||||
view: Optional[ChartView] = None,
|
||||
color: Optional[str] = None,
|
||||
|
||||
) -> Label:
|
||||
'''
|
||||
|
|
@ -244,6 +167,7 @@ class PriceAxis(Axis):
|
|||
self._min_tick = size
|
||||
|
||||
def size_to_values(self) -> None:
|
||||
# self.typical_br = _font._qfm.boundingRect(typical_max_str)
|
||||
self.setWidth(self.typical_br.width())
|
||||
|
||||
# XXX: drop for now since it just eats up h space
|
||||
|
|
@ -298,53 +222,28 @@ class DynamicDateAxis(Axis):
|
|||
|
||||
) -> list[str]:
|
||||
|
||||
# XX: ARGGGGG AG:LKSKDJF:LKJSDFD
|
||||
chart = self.pi.chart_widget
|
||||
|
||||
viz = chart._vizs[chart.name]
|
||||
shm = viz.shm
|
||||
array = shm.array
|
||||
ifield = viz.index_field
|
||||
index = array[ifield]
|
||||
i_0, i_l = index[0], index[-1]
|
||||
|
||||
# edge cases
|
||||
if (
|
||||
not indexes
|
||||
or
|
||||
(indexes[0] < i_0
|
||||
and indexes[-1] < i_l)
|
||||
or
|
||||
(indexes[0] > i_0
|
||||
and indexes[-1] > i_l)
|
||||
):
|
||||
# print(f"x-label indexes empty edge case: {indexes}")
|
||||
return []
|
||||
|
||||
if ifield == 'index':
|
||||
arr_len = index.shape[0]
|
||||
chart = self.linkedsplits.chart
|
||||
flow = chart._flows[chart.name]
|
||||
shm = flow.shm
|
||||
bars = shm.array
|
||||
first = shm._first.value
|
||||
times = array['time']
|
||||
epochs = times[
|
||||
list(
|
||||
|
||||
bars_len = len(bars)
|
||||
times = bars['time']
|
||||
|
||||
epochs = times[list(
|
||||
map(
|
||||
int,
|
||||
filter(
|
||||
lambda i: i > 0 and i < arr_len,
|
||||
(i - first for i in indexes)
|
||||
lambda i: i > 0 and i < bars_len,
|
||||
(i-first for i in indexes)
|
||||
)
|
||||
)
|
||||
)
|
||||
]
|
||||
else:
|
||||
epochs = list(map(int, indexes))
|
||||
)]
|
||||
|
||||
# TODO: **don't** have this hard coded shift to EST
|
||||
# delay = times[-1] - times[-2]
|
||||
dts = np.array(
|
||||
epochs,
|
||||
dtype='datetime64[s]',
|
||||
)
|
||||
dts = np.array(epochs, dtype='datetime64[s]')
|
||||
|
||||
# see units listing:
|
||||
# https://numpy.org/devdocs/reference/arrays.datetime.html#datetime-units
|
||||
|
|
@ -362,39 +261,24 @@ class DynamicDateAxis(Axis):
|
|||
spacing: float,
|
||||
|
||||
) -> list[str]:
|
||||
|
||||
return self._indexes_to_timestrs(values)
|
||||
|
||||
# NOTE: handy for debugging the lru cache
|
||||
# info = self.tickStrings.cache_info()
|
||||
# print(info)
|
||||
return self._indexes_to_timestrs(values)
|
||||
|
||||
|
||||
class AxisLabel(pg.GraphicsObject):
|
||||
|
||||
# relative offsets *OF* the bounding rect relative
|
||||
# to parent graphics object.
|
||||
# eg. <parent>| => <_x_br_offset> => | <text> |
|
||||
_x_br_offset: float = 0
|
||||
_y_br_offset: float = 0
|
||||
|
||||
# relative offsets of text *within* bounding rect
|
||||
# eg. | <_x_margin> => <text> |
|
||||
_x_margin: float = 0
|
||||
_y_margin: float = 0
|
||||
|
||||
# multiplier of the text content's height in order
|
||||
# to force a larger (y-dimension) bounding rect.
|
||||
_y_txt_h_scaling: float = 1
|
||||
_x_margin = 0
|
||||
_y_margin = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
parent: pg.GraphicsItem,
|
||||
digits: int = 2,
|
||||
|
||||
bg_color: str = 'default',
|
||||
bg_color: str = 'bracket',
|
||||
fg_color: str = 'black',
|
||||
opacity: int = .8, # XXX: seriously don't set this to 0
|
||||
opacity: int = 1, # XXX: seriously don't set this to 0
|
||||
font_size: str = 'default',
|
||||
|
||||
use_arrow: bool = True,
|
||||
|
|
@ -405,7 +289,6 @@ class AxisLabel(pg.GraphicsObject):
|
|||
self.setParentItem(parent)
|
||||
|
||||
self.setFlag(self.ItemIgnoresTransformations)
|
||||
self.setZValue(100)
|
||||
|
||||
# XXX: pretty sure this is faster
|
||||
self.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
|
@ -437,14 +320,14 @@ class AxisLabel(pg.GraphicsObject):
|
|||
p: QtGui.QPainter,
|
||||
opt: QtWidgets.QStyleOptionGraphicsItem,
|
||||
w: QtWidgets.QWidget
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
Draw a filled rectangle based on the size of ``.label_str`` text.
|
||||
"""Draw a filled rectangle based on the size of ``.label_str`` text.
|
||||
|
||||
Subtypes can customize further by overloading ``.draw()``.
|
||||
|
||||
'''
|
||||
"""
|
||||
# p.setCompositionMode(QtWidgets.QPainter.CompositionMode_SourceOver)
|
||||
|
||||
if self.label_str:
|
||||
|
||||
# if not self.rect:
|
||||
|
|
@ -455,11 +338,7 @@ class AxisLabel(pg.GraphicsObject):
|
|||
|
||||
p.setFont(self._dpifont.font)
|
||||
p.setPen(self.fg_color)
|
||||
p.drawText(
|
||||
self.rect,
|
||||
self.text_flags,
|
||||
self.label_str,
|
||||
)
|
||||
p.drawText(self.rect, self.text_flags, self.label_str)
|
||||
|
||||
def draw(
|
||||
self,
|
||||
|
|
@ -467,8 +346,6 @@ class AxisLabel(pg.GraphicsObject):
|
|||
rect: QtCore.QRectF
|
||||
) -> None:
|
||||
|
||||
p.setOpacity(self.opacity)
|
||||
|
||||
if self._use_arrow:
|
||||
if not self.path:
|
||||
self._draw_arrow_path()
|
||||
|
|
@ -476,13 +353,15 @@ class AxisLabel(pg.GraphicsObject):
|
|||
p.drawPath(self.path)
|
||||
p.fillPath(self.path, pg.mkBrush(self.bg_color))
|
||||
|
||||
# this adds a nice black outline around the label for some odd
|
||||
# reason; ok by us
|
||||
p.setOpacity(self.opacity)
|
||||
|
||||
# this cause the L1 labels to glitch out if used in the subtype
|
||||
# and it will leave a small black strip with the arrow path if
|
||||
# done before the above
|
||||
p.fillRect(
|
||||
self.rect,
|
||||
self.bg_color,
|
||||
)
|
||||
p.fillRect(self.rect, self.bg_color)
|
||||
|
||||
|
||||
def boundingRect(self): # noqa
|
||||
'''
|
||||
|
|
@ -526,18 +405,15 @@ class AxisLabel(pg.GraphicsObject):
|
|||
txt_h, txt_w = txt_br.height(), txt_br.width()
|
||||
# print(f'wsw: {self._dpifont.boundingRect(" ")}')
|
||||
|
||||
# allow subtypes to override width and height
|
||||
# allow subtypes to specify a static width and height
|
||||
h, w = self.size_hint()
|
||||
# print(f'axis size: {self._parent.size()}')
|
||||
# print(f'axis geo: {self._parent.geometry()}')
|
||||
|
||||
self.rect = QtCore.QRectF(
|
||||
|
||||
# relative bounds offsets
|
||||
self._x_br_offset,
|
||||
self._y_br_offset,
|
||||
|
||||
0, 0,
|
||||
(w or txt_w) + self._x_margin / 2,
|
||||
|
||||
(h or txt_h) * self._y_txt_h_scaling + (self._y_margin / 2),
|
||||
(h or txt_h) + self._y_margin / 2,
|
||||
)
|
||||
# print(self.rect)
|
||||
# hb = self.path.controlPointRect()
|
||||
|
|
@ -613,7 +489,7 @@ class XAxisLabel(AxisLabel):
|
|||
|
||||
|
||||
class YAxisLabel(AxisLabel):
|
||||
_y_margin: int = 4
|
||||
_y_margin = 4
|
||||
|
||||
text_flags = (
|
||||
QtCore.Qt.AlignLeft
|
||||
|
|
@ -624,19 +500,19 @@ class YAxisLabel(AxisLabel):
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
pi: pgo.PlotItem,
|
||||
chart,
|
||||
*args,
|
||||
**kwargs
|
||||
) -> None:
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self._pi = pi
|
||||
pi.sigRangeChanged.connect(self.update_on_resize)
|
||||
self._chart = chart
|
||||
|
||||
chart.sigRangeChanged.connect(self.update_on_resize)
|
||||
|
||||
self._last_datum = (None, None)
|
||||
|
||||
self.x_offset = 0
|
||||
# pull text offset from axis from parent axis
|
||||
if getattr(self._parent, 'txt_offsets', False):
|
||||
self.x_offset, y_offset = self._parent.txt_offsets()
|
||||
|
|
@ -655,8 +531,7 @@ class YAxisLabel(AxisLabel):
|
|||
value: float, # data for text
|
||||
|
||||
# on odd dimension and/or adds nice black line
|
||||
x_offset: int = 0,
|
||||
|
||||
x_offset: Optional[int] = None
|
||||
) -> None:
|
||||
|
||||
# this is read inside ``.paint()``
|
||||
|
|
@ -702,7 +577,7 @@ class YAxisLabel(AxisLabel):
|
|||
self._last_datum = (index, value)
|
||||
|
||||
self.update_label(
|
||||
self._pi.mapFromView(QPointF(index, value)),
|
||||
self._chart.mapFromView(QPointF(index, value)),
|
||||
value
|
||||
)
|
||||
|
||||
|
|
|
|||
1016
piker/ui/_chart.py
1016
piker/ui/_chart.py
File diff suppressed because it is too large
Load Diff
|
|
@ -15,34 +15,127 @@
|
|||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
'''
|
||||
Graphics downsampling using the infamous M4 algorithm.
|
||||
|
||||
This is one of ``piker``'s secret weapons allowing us to boss all other
|
||||
charting platforms B)
|
||||
|
||||
(AND DON'T YOU DARE TAKE THIS CODE WITHOUT CREDIT OR WE'LL SUE UR F#&@* ASS).
|
||||
|
||||
NOTES: this method is a so called "visualization driven data
|
||||
aggregation" approach. It gives error-free line chart
|
||||
downsampling, see
|
||||
further scientific paper resources:
|
||||
- http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
- http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
|
||||
|
||||
Details on implementation of this algo are based in,
|
||||
https://github.com/pikers/piker/issues/109
|
||||
Graphics related downsampling routines for compressing to pixel
|
||||
limits on the display device.
|
||||
|
||||
'''
|
||||
import math
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
from numpy.lib import recfunctions as rfn
|
||||
from numba import (
|
||||
njit,
|
||||
jit,
|
||||
# float64, optional, int64,
|
||||
)
|
||||
|
||||
from ._util import log
|
||||
from ..log import get_logger
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def hl2mxmn(ohlc: np.ndarray) -> np.ndarray:
|
||||
'''
|
||||
Convert a OHLC struct-array containing 'high'/'low' columns
|
||||
to a "joined" max/min 1-d array.
|
||||
|
||||
'''
|
||||
index = ohlc['index']
|
||||
hls = ohlc[[
|
||||
'low',
|
||||
'high',
|
||||
]]
|
||||
|
||||
mxmn = np.empty(2*hls.size, dtype=np.float64)
|
||||
x = np.empty(2*hls.size, dtype=np.float64)
|
||||
trace_hl(hls, mxmn, x, index[0])
|
||||
x = x + index[0]
|
||||
|
||||
return mxmn, x
|
||||
|
||||
|
||||
@jit(
|
||||
# TODO: the type annots..
|
||||
# float64[:](float64[:],),
|
||||
nopython=True,
|
||||
)
|
||||
def trace_hl(
|
||||
hl: 'np.ndarray',
|
||||
out: np.ndarray,
|
||||
x: np.ndarray,
|
||||
start: int,
|
||||
|
||||
# the "offset" values in the x-domain which
|
||||
# place the 2 output points around each ``int``
|
||||
# master index.
|
||||
margin: float = 0.43,
|
||||
|
||||
) -> None:
|
||||
'''
|
||||
"Trace" the outline of the high-low values of an ohlc sequence
|
||||
as a line such that the maximum deviation (aka disperaion) between
|
||||
bars if preserved.
|
||||
|
||||
This routine is expected to modify input arrays in-place.
|
||||
|
||||
'''
|
||||
last_l = hl['low'][0]
|
||||
last_h = hl['high'][0]
|
||||
|
||||
for i in range(hl.size):
|
||||
row = hl[i]
|
||||
l, h = row['low'], row['high']
|
||||
|
||||
up_diff = h - last_l
|
||||
down_diff = last_h - l
|
||||
|
||||
if up_diff > down_diff:
|
||||
out[2*i + 1] = h
|
||||
out[2*i] = last_l
|
||||
else:
|
||||
out[2*i + 1] = l
|
||||
out[2*i] = last_h
|
||||
|
||||
last_l = l
|
||||
last_h = h
|
||||
|
||||
x[2*i] = int(i) - margin
|
||||
x[2*i + 1] = int(i) + margin
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def ohlc_flatten(
|
||||
ohlc: np.ndarray,
|
||||
use_mxmn: bool = True,
|
||||
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
'''
|
||||
Convert an OHLCV struct-array into a flat ready-for-line-plotting
|
||||
1-d array that is 4 times the size with x-domain values distributed
|
||||
evenly (by 0.5 steps) over each index.
|
||||
|
||||
'''
|
||||
index = ohlc['index']
|
||||
|
||||
if use_mxmn:
|
||||
# traces a line optimally over highs to lows
|
||||
# using numba. NOTE: pretty sure this is faster
|
||||
# and looks about the same as the below output.
|
||||
flat, x = hl2mxmn(ohlc)
|
||||
|
||||
else:
|
||||
flat = rfn.structured_to_unstructured(
|
||||
ohlc[['open', 'high', 'low', 'close']]
|
||||
).flatten()
|
||||
|
||||
x = np.linspace(
|
||||
start=index[0] - 0.5,
|
||||
stop=index[-1] + 0.5,
|
||||
num=len(flat),
|
||||
)
|
||||
return x, flat
|
||||
|
||||
|
||||
def ds_m4(
|
||||
|
|
@ -67,6 +160,16 @@ def ds_m4(
|
|||
This is more or less an OHLC style sampling of a line-style series.
|
||||
|
||||
'''
|
||||
# NOTE: this method is a so called "visualization driven data
|
||||
# aggregation" approach. It gives error-free line chart
|
||||
# downsampling, see
|
||||
# further scientific paper resources:
|
||||
# - http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
# - http://www.vldb.org/2014/program/papers/demo/p997-jugel.pdf
|
||||
|
||||
# Details on implementation of this algo are based in,
|
||||
# https://github.com/pikers/piker/issues/109
|
||||
|
||||
# XXX: from infinite on downsampling viewable graphics:
|
||||
# "one thing i remembered about the binning - if you are
|
||||
# picking a range within your timeseries the start and end bin
|
||||
|
|
@ -88,14 +191,6 @@ def ds_m4(
|
|||
x_end = x[-1] # x end value/highest in domain
|
||||
xrange = (x_end - x_start)
|
||||
|
||||
if xrange < 0:
|
||||
log.error(f'-VE M4 X-RANGE: {x_start} -> {x_end}')
|
||||
# XXX: broken x-range calc-case, likely the x-end points
|
||||
# are wrong and have some default value set (such as
|
||||
# x_end -> <some epoch float> while x_start -> 0.5).
|
||||
# breakpoint()
|
||||
return None
|
||||
|
||||
# XXX: always round up on the input pixels
|
||||
# lnx = len(x)
|
||||
# uppx *= max(4 / (1 + math.log(uppx, 2)), 1)
|
||||
|
|
@ -128,20 +223,14 @@ def ds_m4(
|
|||
assert frames >= (xrange / uppx)
|
||||
|
||||
# call into ``numba``
|
||||
(
|
||||
nb,
|
||||
x_out,
|
||||
y_out,
|
||||
ymn,
|
||||
ymx,
|
||||
) = _m4(
|
||||
nb, i_win, y_out = _m4(
|
||||
x,
|
||||
y,
|
||||
|
||||
frames,
|
||||
|
||||
# TODO: see func below..
|
||||
# x_out,
|
||||
# i_win,
|
||||
# y_out,
|
||||
|
||||
# first index in x data to start at
|
||||
|
|
@ -154,14 +243,14 @@ def ds_m4(
|
|||
# filter out any overshoot in the input allocation arrays by
|
||||
# removing zero-ed tail entries which should start at a certain
|
||||
# index.
|
||||
x_out = x_out[x_out != 0]
|
||||
y_out = y_out[:x_out.size]
|
||||
i_win = i_win[i_win != 0]
|
||||
y_out = y_out[:i_win.size]
|
||||
|
||||
# print(f'M4 output ymn, ymx: {ymn},{ymx}')
|
||||
return nb, x_out, y_out, ymn, ymx
|
||||
return nb, i_win, y_out
|
||||
|
||||
|
||||
@njit(
|
||||
@jit(
|
||||
nopython=True,
|
||||
nogil=True,
|
||||
)
|
||||
def _m4(
|
||||
|
|
@ -171,8 +260,8 @@ def _m4(
|
|||
|
||||
frames: int,
|
||||
|
||||
# TODO: using this approach, having the ``.zeros()`` alloc lines
|
||||
# below in pure python, there were segs faults and alloc crashes..
|
||||
# TODO: using this approach by having the ``.zeros()`` alloc lines
|
||||
# below, in put python was causing segs faults and alloc crashes..
|
||||
# we might need to see how it behaves with shm arrays and consider
|
||||
# allocating them once at startup?
|
||||
|
||||
|
|
@ -185,22 +274,14 @@ def _m4(
|
|||
x_start: int,
|
||||
step: float,
|
||||
|
||||
) -> tuple[
|
||||
int,
|
||||
np.ndarray,
|
||||
np.ndarray,
|
||||
float,
|
||||
float,
|
||||
]:
|
||||
'''
|
||||
Implementation of the m4 algorithm in ``numba``:
|
||||
http://www.vldb.org/pvldb/vol7/p797-jugel.pdf
|
||||
) -> int:
|
||||
# nbins = len(i_win)
|
||||
# count = len(xs)
|
||||
|
||||
'''
|
||||
# these are pre-allocated and mutated by ``numba``
|
||||
# code in-place.
|
||||
y_out = np.zeros((frames, 4), ys.dtype)
|
||||
x_out = np.zeros(frames, xs.dtype)
|
||||
i_win = np.zeros(frames, xs.dtype)
|
||||
|
||||
bincount = 0
|
||||
x_left = x_start
|
||||
|
|
@ -214,34 +295,24 @@ def _m4(
|
|||
|
||||
# set all bins in the left-most entry to the starting left-most x value
|
||||
# (aka a row broadcast).
|
||||
x_out[bincount] = x_left
|
||||
i_win[bincount] = x_left
|
||||
# set all y-values to the first value passed in.
|
||||
y_out[bincount] = ys[0]
|
||||
|
||||
# full input y-data mx and mn
|
||||
mx: float = -np.inf
|
||||
mn: float = np.inf
|
||||
|
||||
# compute OHLC style max / min values per window sized x-frame.
|
||||
for i in range(len(xs)):
|
||||
|
||||
x = xs[i]
|
||||
y = ys[i]
|
||||
|
||||
if x < x_left + step: # the current window "step" is [bin, bin+1)
|
||||
ymn = y_out[bincount, 1] = min(y, y_out[bincount, 1])
|
||||
ymx = y_out[bincount, 2] = max(y, y_out[bincount, 2])
|
||||
y_out[bincount, 1] = min(y, y_out[bincount, 1])
|
||||
y_out[bincount, 2] = max(y, y_out[bincount, 2])
|
||||
y_out[bincount, 3] = y
|
||||
mx = max(mx, ymx)
|
||||
mn = min(mn, ymn)
|
||||
|
||||
else:
|
||||
# Find the next bin
|
||||
while x >= x_left + step:
|
||||
x_left += step
|
||||
|
||||
bincount += 1
|
||||
x_out[bincount] = x_left
|
||||
i_win[bincount] = x_left
|
||||
y_out[bincount] = y
|
||||
|
||||
return bincount, x_out, y_out, mn, mx
|
||||
return bincount, i_win, y_out
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# piker: trading gear for hackers
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
||||
# Copyright (C) Tyler Goodlet (in stewardship for piker0)
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
|
|
@ -18,12 +18,8 @@
|
|||
Mouse interaction graphics
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from functools import partial
|
||||
from typing import (
|
||||
Callable,
|
||||
TYPE_CHECKING,
|
||||
)
|
||||
from typing import Optional, Callable
|
||||
|
||||
import inspect
|
||||
import numpy as np
|
||||
|
|
@ -37,18 +33,9 @@ from ._style import (
|
|||
_font_small,
|
||||
_font,
|
||||
)
|
||||
from ._axes import (
|
||||
YAxisLabel,
|
||||
XAxisLabel,
|
||||
)
|
||||
from ._axes import YAxisLabel, XAxisLabel
|
||||
from ..log import get_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ._chart import (
|
||||
ChartPlotWidget,
|
||||
LinkedSplits,
|
||||
)
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
|
@ -71,9 +58,9 @@ class LineDot(pg.CurvePoint):
|
|||
curve: pg.PlotCurveItem,
|
||||
index: int,
|
||||
|
||||
plot: ChartPlotWidget, # type: ingore # noqa
|
||||
plot: 'ChartPlotWidget', # type: ingore # noqa
|
||||
pos=None,
|
||||
color: str = 'bracket',
|
||||
color: str = 'default_light',
|
||||
|
||||
) -> None:
|
||||
# scale from dpi aware font size
|
||||
|
|
@ -164,12 +151,12 @@ class ContentsLabel(pg.LabelItem):
|
|||
def __init__(
|
||||
self,
|
||||
|
||||
# chart: ChartPlotWidget, # noqa
|
||||
# chart: 'ChartPlotWidget', # noqa
|
||||
view: pg.ViewBox,
|
||||
|
||||
anchor_at: str = ('top', 'right'),
|
||||
justify_text: str = 'left',
|
||||
font_size: int | None = None,
|
||||
font_size: Optional[int] = None,
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -200,11 +187,12 @@ class ContentsLabel(pg.LabelItem):
|
|||
self,
|
||||
|
||||
name: str,
|
||||
ix: int,
|
||||
index: int,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> None:
|
||||
# this being "html" is the dumbest shit :eyeroll:
|
||||
first = array[0]['index']
|
||||
|
||||
self.setText(
|
||||
"<b>i</b>:{index}<br/>"
|
||||
|
|
@ -217,7 +205,7 @@ class ContentsLabel(pg.LabelItem):
|
|||
"<b>C</b>:{}<br/>"
|
||||
"<b>V</b>:{}<br/>"
|
||||
"<b>wap</b>:{}".format(
|
||||
*array[ix][
|
||||
*array[index - first][
|
||||
[
|
||||
'time',
|
||||
'open',
|
||||
|
|
@ -229,7 +217,7 @@ class ContentsLabel(pg.LabelItem):
|
|||
]
|
||||
],
|
||||
name=name,
|
||||
index=ix,
|
||||
index=index,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -237,11 +225,14 @@ class ContentsLabel(pg.LabelItem):
|
|||
self,
|
||||
|
||||
name: str,
|
||||
ix: int,
|
||||
index: int,
|
||||
array: np.ndarray,
|
||||
|
||||
) -> None:
|
||||
data = array[ix][name]
|
||||
|
||||
first = array[0]['index']
|
||||
if index < array[-1]['index'] and index > first:
|
||||
data = array[index - first][name]
|
||||
self.setText(f"{name}: {data:.2f}")
|
||||
|
||||
|
||||
|
|
@ -253,7 +244,7 @@ class ContentsLabels:
|
|||
'''
|
||||
def __init__(
|
||||
self,
|
||||
linkedsplits: LinkedSplits, # type: ignore # noqa
|
||||
linkedsplits: 'LinkedSplits', # type: ignore # noqa
|
||||
|
||||
) -> None:
|
||||
|
||||
|
|
@ -267,20 +258,17 @@ class ContentsLabels:
|
|||
|
||||
def update_labels(
|
||||
self,
|
||||
x_in: int,
|
||||
index: int,
|
||||
|
||||
) -> None:
|
||||
for chart, name, label, update in self._labels:
|
||||
|
||||
viz = chart.get_viz(name)
|
||||
array = viz.shm.array
|
||||
index = array[viz.index_field]
|
||||
start = index[0]
|
||||
stop = index[-1]
|
||||
flow = chart._flows[name]
|
||||
array = flow.shm.array
|
||||
|
||||
if not (
|
||||
x_in >= start
|
||||
and x_in <= stop
|
||||
index >= 0
|
||||
and index < array[-1]['index']
|
||||
):
|
||||
# out of range
|
||||
print('WTF out of range?')
|
||||
|
|
@ -289,10 +277,7 @@ class ContentsLabels:
|
|||
# call provided update func with data point
|
||||
try:
|
||||
label.show()
|
||||
ix = np.searchsorted(index, x_in)
|
||||
if ix > len(array):
|
||||
breakpoint()
|
||||
update(ix, array)
|
||||
update(index, array)
|
||||
|
||||
except IndexError:
|
||||
log.exception(f"Failed to update label: {name}")
|
||||
|
|
@ -304,7 +289,7 @@ class ContentsLabels:
|
|||
def add_label(
|
||||
|
||||
self,
|
||||
chart: ChartPlotWidget, # type: ignore # noqa
|
||||
chart: 'ChartPlotWidget', # type: ignore # noqa
|
||||
name: str,
|
||||
anchor_at: tuple[str, str] = ('top', 'left'),
|
||||
update_func: Callable = ContentsLabel.update_from_value,
|
||||
|
|
@ -331,7 +316,7 @@ class Cursor(pg.GraphicsObject):
|
|||
def __init__(
|
||||
|
||||
self,
|
||||
linkedsplits: LinkedSplits, # noqa
|
||||
linkedsplits: 'LinkedSplits', # noqa
|
||||
digits: int = 0
|
||||
|
||||
) -> None:
|
||||
|
|
@ -340,8 +325,6 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
self.linked = linkedsplits
|
||||
self.graphics: dict[str, pg.GraphicsObject] = {}
|
||||
self.xaxis_label: XAxisLabel | None = None
|
||||
self.always_show_xlabel: bool = True
|
||||
self.plots: list['PlotChartWidget'] = [] # type: ignore # noqa
|
||||
self.active_plot = None
|
||||
self.digits: int = digits
|
||||
|
|
@ -353,7 +336,7 @@ class Cursor(pg.GraphicsObject):
|
|||
# XXX: not sure why these are instance variables?
|
||||
# It's not like we can change them on the fly..?
|
||||
self.pen = pg.mkPen(
|
||||
color=hcolor('bracket'),
|
||||
color=hcolor('default'),
|
||||
style=QtCore.Qt.DashLine,
|
||||
)
|
||||
self.lines_pen = pg.mkPen(
|
||||
|
|
@ -363,14 +346,13 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
# value used for rounding y-axis discreet tick steps
|
||||
# computing once, up front, here cuz why not
|
||||
mkt = self.linked._symbol
|
||||
self._y_tick_mult = 1/float(mkt.price_tick)
|
||||
self._y_incr_mult = 1 / self.linked._symbol.tick_size
|
||||
|
||||
# line width in view coordinates
|
||||
self._lw = self.pixelWidth() * self.lines_pen.width()
|
||||
|
||||
# xhair label's color name
|
||||
self.label_color: str = 'bracket'
|
||||
self.label_color: str = 'default'
|
||||
|
||||
self._y_label_update: bool = True
|
||||
|
||||
|
|
@ -403,7 +385,7 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
def add_plot(
|
||||
self,
|
||||
plot: ChartPlotWidget, # noqa
|
||||
plot: 'ChartPlotWidget', # noqa
|
||||
digits: int = 0,
|
||||
|
||||
) -> None:
|
||||
|
|
@ -423,7 +405,7 @@ class Cursor(pg.GraphicsObject):
|
|||
hl.hide()
|
||||
|
||||
yl = YAxisLabel(
|
||||
pi=plot.plotItem,
|
||||
chart=plot,
|
||||
# parent=plot.getAxis('right'),
|
||||
parent=plot.pi_overlay.get_axis(plot.plotItem, 'right'),
|
||||
digits=digits or self.digits,
|
||||
|
|
@ -487,58 +469,39 @@ class Cursor(pg.GraphicsObject):
|
|||
|
||||
def add_curve_cursor(
|
||||
self,
|
||||
chart: ChartPlotWidget, # noqa
|
||||
plot: 'ChartPlotWidget', # noqa
|
||||
curve: 'PlotCurveItem', # noqa
|
||||
|
||||
) -> LineDot:
|
||||
# if this chart contains curves add line dot "cursors" to denote
|
||||
# if this plot contains curves add line dot "cursors" to denote
|
||||
# the current sample under the mouse
|
||||
main_viz = chart.get_viz(chart.name)
|
||||
|
||||
main_flow = plot._flows[plot.name]
|
||||
# read out last index
|
||||
i = main_viz.shm.array[-1]['index']
|
||||
i = main_flow.shm.array[-1]['index']
|
||||
cursor = LineDot(
|
||||
curve,
|
||||
index=i,
|
||||
plot=chart
|
||||
plot=plot
|
||||
)
|
||||
chart.addItem(cursor)
|
||||
self.graphics[chart].setdefault('cursors', []).append(cursor)
|
||||
plot.addItem(cursor)
|
||||
self.graphics[plot].setdefault('cursors', []).append(cursor)
|
||||
return cursor
|
||||
|
||||
def mouseAction(
|
||||
self,
|
||||
action: str,
|
||||
plot: ChartPlotWidget,
|
||||
|
||||
) -> None: # noqa
|
||||
|
||||
def mouseAction(self, action, plot): # noqa
|
||||
log.debug(f"{(action, plot.name)}")
|
||||
if action == 'Enter':
|
||||
self.active_plot = plot
|
||||
plot.linked.godwidget._active_cursor = self
|
||||
|
||||
# show horiz line and y-label
|
||||
self.graphics[plot]['hl'].show()
|
||||
self.graphics[plot]['yl'].show()
|
||||
|
||||
if (
|
||||
not self.always_show_xlabel
|
||||
and not self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.show()
|
||||
else: # Leave
|
||||
|
||||
# Leave: hide horiz line and y-label
|
||||
else:
|
||||
# hide horiz line and y-label
|
||||
self.graphics[plot]['hl'].hide()
|
||||
self.graphics[plot]['yl'].hide()
|
||||
|
||||
if (
|
||||
not self.always_show_xlabel
|
||||
and self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.hide()
|
||||
|
||||
def mouseMoved(
|
||||
self,
|
||||
coords: tuple[QPointF], # noqa
|
||||
|
|
@ -572,15 +535,9 @@ class Cursor(pg.GraphicsObject):
|
|||
line_offset = self._lw / 2
|
||||
|
||||
# round y value to nearest tick step
|
||||
m = self._y_tick_mult
|
||||
m = self._y_incr_mult
|
||||
iy = round(y * m) / m
|
||||
vl_y = iy - line_offset
|
||||
# print(
|
||||
# f'tick: {self._y_tick}\n'
|
||||
# f'y: {y}\n'
|
||||
# f'iy: {iy}\n'
|
||||
# f'vl_y: {vl_y}\n'
|
||||
# )
|
||||
|
||||
# update y-range items
|
||||
if iy != last_iy:
|
||||
|
|
@ -633,10 +590,6 @@ class Cursor(pg.GraphicsObject):
|
|||
left_axis_width += left.width()
|
||||
|
||||
# map back to abs (label-local) coordinates
|
||||
if (
|
||||
self.always_show_xlabel
|
||||
or self.xaxis_label.isVisible()
|
||||
):
|
||||
self.xaxis_label.update_label(
|
||||
abs_pos=(
|
||||
plot.mapFromView(QPointF(vl_x, iy)) -
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ Fast, smooth, sexy curves.
|
|||
|
||||
"""
|
||||
from contextlib import contextmanager as cm
|
||||
from typing import Callable
|
||||
from typing import Optional, Callable
|
||||
|
||||
import numpy as np
|
||||
import pyqtgraph as pg
|
||||
|
|
@ -28,7 +28,10 @@ from PyQt5.QtWidgets import QGraphicsItem
|
|||
from PyQt5.QtCore import (
|
||||
Qt,
|
||||
QLineF,
|
||||
QSizeF,
|
||||
QRectF,
|
||||
# QRect,
|
||||
QPointF,
|
||||
)
|
||||
from PyQt5.QtGui import (
|
||||
QPainter,
|
||||
|
|
@ -36,8 +39,11 @@ from PyQt5.QtGui import (
|
|||
)
|
||||
from .._profile import pg_profile_enabled, ms_slower_then
|
||||
from ._style import hcolor
|
||||
# from ._compression import (
|
||||
# # ohlc_to_m4_line,
|
||||
# ds_m4,
|
||||
# )
|
||||
from ..log import get_logger
|
||||
from .._profile import Profiler
|
||||
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
|
@ -51,144 +57,11 @@ _line_styles: dict[str, int] = {
|
|||
}
|
||||
|
||||
|
||||
class FlowGraphic(pg.GraphicsObject):
|
||||
'''
|
||||
Base class with minimal interface for `QPainterPath` implemented,
|
||||
real-time updated "data flow" graphics.
|
||||
|
||||
See subtypes below.
|
||||
|
||||
'''
|
||||
# sub-type customization methods
|
||||
declare_paintables: Callable | None = None
|
||||
sub_paint: Callable | None = None
|
||||
|
||||
# XXX-NOTE-XXX: graphics caching B)
|
||||
# see explanation for different caching modes:
|
||||
# https://stackoverflow.com/a/39410081
|
||||
cache_mode: int = QGraphicsItem.DeviceCoordinateCache
|
||||
# XXX: WARNING item caching seems to only be useful
|
||||
# if we don't re-generate the entire QPainterPath every time
|
||||
# don't ever use this - it's a colossal nightmare of artefacts
|
||||
# and is disastrous for performance.
|
||||
# QGraphicsItem.ItemCoordinateCache
|
||||
# TODO: still questions todo with coord-cacheing that we should
|
||||
# probably talk to a core dev about:
|
||||
# - if this makes trasform interactions slower (such as zooming)
|
||||
# and if so maybe if/when we implement a "history" mode for the
|
||||
# view we disable this in that mode?
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
name: str | None = None,
|
||||
|
||||
# line styling
|
||||
color: str = 'bracket',
|
||||
last_step_color: str | None = None,
|
||||
fill_color: str | None = None,
|
||||
style: str = 'solid',
|
||||
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
||||
self._name = name
|
||||
|
||||
# primary graphics item used for history
|
||||
self.path: QPainterPath = QPainterPath()
|
||||
|
||||
# additional path that can be optionally used for appends which
|
||||
# tries to avoid triggering an update/redraw of the presumably
|
||||
# larger historical ``.path`` above. the flag to enable
|
||||
# this behaviour is found in `Renderer.render()`.
|
||||
self.fast_path: QPainterPath | None = None
|
||||
|
||||
# TODO: evaluating the path capacity stuff and see
|
||||
# if it really makes much diff pre-allocating it.
|
||||
# self._last_cap: int = 0
|
||||
# cap = path.capacity()
|
||||
# if cap != self._last_cap:
|
||||
# print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
|
||||
# self._last_cap = cap
|
||||
|
||||
# all history of curve is drawn in single px thickness
|
||||
self._color: str = color
|
||||
pen = pg.mkPen(hcolor(color), width=1)
|
||||
pen.setStyle(_line_styles[style])
|
||||
|
||||
if 'dash' in style:
|
||||
pen.setDashPattern([8, 3])
|
||||
|
||||
self._pen = pen
|
||||
self._brush = pg.functions.mkBrush(
|
||||
hcolor(fill_color or color)
|
||||
)
|
||||
|
||||
# last segment is drawn in 2px thickness for emphasis
|
||||
if last_step_color:
|
||||
self.last_step_pen = pg.mkPen(
|
||||
hcolor(last_step_color),
|
||||
width=2,
|
||||
)
|
||||
else:
|
||||
self.last_step_pen = pg.mkPen(
|
||||
self._pen,
|
||||
width=2,
|
||||
)
|
||||
|
||||
self._last_line: QLineF = QLineF()
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# apply cache mode
|
||||
self.setCacheMode(self.cache_mode)
|
||||
|
||||
def x_uppx(self) -> int:
|
||||
|
||||
px_vecs = self.pixelVectors()[0]
|
||||
if px_vecs:
|
||||
return px_vecs.x()
|
||||
else:
|
||||
return 0
|
||||
|
||||
def x_last(self) -> float | None:
|
||||
'''
|
||||
Return the last most x value of the last line segment or if not
|
||||
drawn yet, ``None``.
|
||||
|
||||
'''
|
||||
if self._last_line:
|
||||
return self._last_line.x1()
|
||||
|
||||
return None
|
||||
|
||||
# XXX: due to a variety of weird jitter bugs and "smearing"
|
||||
# artifacts when click-drag panning and viewing history time series,
|
||||
# we offer this ctx-mngr interface to allow temporarily disabling
|
||||
# Qt's graphics caching mode; this is now currently used from
|
||||
# ``ChartView.start/signal_ic()`` methods which also disable the
|
||||
# rt-display loop when the user is moving around a view.
|
||||
@cm
|
||||
def reset_cache(self) -> None:
|
||||
try:
|
||||
none = QGraphicsItem.NoCache
|
||||
log.debug(
|
||||
f'{self._name} -> CACHE DISABLE: {none}'
|
||||
)
|
||||
self.setCacheMode(none)
|
||||
yield
|
||||
finally:
|
||||
mode = self.cache_mode
|
||||
log.debug(f'{self._name} -> CACHE ENABLE {mode}')
|
||||
self.setCacheMode(mode)
|
||||
|
||||
|
||||
class Curve(FlowGraphic):
|
||||
class Curve(pg.GraphicsObject):
|
||||
'''
|
||||
A faster, simpler, append friendly version of
|
||||
``pyqtgraph.PlotCurveItem`` built for highly customizable real-time
|
||||
updates; a graphics object to render a simple "line" plot.
|
||||
updates.
|
||||
|
||||
This type is a much stripped down version of a ``pyqtgraph`` style
|
||||
"graphics object" in the sense that the internal lower level
|
||||
|
|
@ -201,7 +74,7 @@ class Curve(FlowGraphic):
|
|||
lower level graphics data can be rendered in different threads and
|
||||
then read and drawn in this main thread without having to worry
|
||||
about dealing with Qt's concurrency primitives. See
|
||||
``piker.ui._render.Renderer`` for details and logic related to lower
|
||||
``piker.ui._flows.Renderer`` for details and logic related to lower
|
||||
level path generation and incremental update. The main differences in
|
||||
the path generation code include:
|
||||
|
||||
|
|
@ -213,38 +86,127 @@ class Curve(FlowGraphic):
|
|||
updates don't trigger a full path redraw.
|
||||
|
||||
'''
|
||||
# TODO: can we remove this?
|
||||
# sub_br: Callable | None = None
|
||||
|
||||
# sub-type customization methods
|
||||
sub_br: Optional[Callable] = None
|
||||
sub_paint: Optional[Callable] = None
|
||||
declare_paintables: Optional[Callable] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
|
||||
# color: str = 'default_lightest',
|
||||
# fill_color: str | None = None,
|
||||
# style: str = 'solid',
|
||||
step_mode: bool = False,
|
||||
color: str = 'default_lightest',
|
||||
fill_color: Optional[str] = None,
|
||||
style: str = 'solid',
|
||||
name: Optional[str] = None,
|
||||
use_fpath: bool = True,
|
||||
|
||||
**kwargs
|
||||
|
||||
) -> None:
|
||||
|
||||
self._name = name
|
||||
|
||||
# brutaaalll, see comments within..
|
||||
self.yData = None
|
||||
self.xData = None
|
||||
|
||||
# self._last_cap: int = 0
|
||||
self.path: Optional[QPainterPath] = None
|
||||
|
||||
# additional path used for appends which tries to avoid
|
||||
# triggering an update/redraw of the presumably larger
|
||||
# historical ``.path`` above.
|
||||
self.use_fpath = use_fpath
|
||||
self.fast_path: Optional[QPainterPath] = None
|
||||
|
||||
# TODO: we can probably just dispense with the parent since
|
||||
# we're basically only using the pen setting now...
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self._last_line: QLineF = QLineF()
|
||||
# all history of curve is drawn in single px thickness
|
||||
pen = pg.mkPen(hcolor(color))
|
||||
pen.setStyle(_line_styles[style])
|
||||
|
||||
if 'dash' in style:
|
||||
pen.setDashPattern([8, 3])
|
||||
|
||||
self._pen = pen
|
||||
|
||||
# last segment is drawn in 2px thickness for emphasis
|
||||
# self.last_step_pen = pg.mkPen(hcolor(color), width=2)
|
||||
self.last_step_pen = pg.mkPen(pen, width=2)
|
||||
|
||||
# self._last_line: Optional[QLineF] = None
|
||||
self._last_line = QLineF()
|
||||
self._last_w: float = 1
|
||||
|
||||
# flat-top style histogram-like discrete curve
|
||||
# self._step_mode: bool = step_mode
|
||||
|
||||
# self._fill = True
|
||||
self._brush = pg.functions.mkBrush(hcolor(fill_color or color))
|
||||
|
||||
# NOTE: this setting seems to mostly prevent redraws on mouse
|
||||
# interaction which is a huge boon for avg interaction latency.
|
||||
|
||||
# TODO: one question still remaining is if this makes trasform
|
||||
# interactions slower (such as zooming) and if so maybe if/when
|
||||
# we implement a "history" mode for the view we disable this in
|
||||
# that mode?
|
||||
# don't enable caching by default for the case where the
|
||||
# only thing drawn is the "last" line segment which can
|
||||
# have a weird artifact where it won't be fully drawn to its
|
||||
# endpoint (something we saw on trade rate curves)
|
||||
self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
# XXX: see explanation for different caching modes:
|
||||
# https://stackoverflow.com/a/39410081
|
||||
# seems to only be useful if we don't re-generate the entire
|
||||
# QPainterPath every time
|
||||
# curve.setCacheMode(QtWidgets.QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
# don't ever use this - it's a colossal nightmare of artefacts
|
||||
# and is disastrous for performance.
|
||||
# curve.setCacheMode(QtWidgets.QGraphicsItem.ItemCoordinateCache)
|
||||
|
||||
# allow sub-type customization
|
||||
declare = self.declare_paintables
|
||||
if declare:
|
||||
declare()
|
||||
|
||||
# TODO: probably stick this in a new parent
|
||||
# type which will contain our own version of
|
||||
# what ``PlotCurveItem`` had in terms of base
|
||||
# functionality? A `FlowGraphic` maybe?
|
||||
def x_uppx(self) -> int:
|
||||
|
||||
px_vecs = self.pixelVectors()[0]
|
||||
if px_vecs:
|
||||
xs_in_px = px_vecs.x()
|
||||
return round(xs_in_px)
|
||||
else:
|
||||
return 0
|
||||
|
||||
def px_width(self) -> float:
|
||||
|
||||
vb = self.getViewBox()
|
||||
if not vb:
|
||||
return 0
|
||||
|
||||
vr = self.viewRect()
|
||||
l, r = int(vr.left()), int(vr.right())
|
||||
|
||||
start, stop = self._xrange
|
||||
lbar = max(l, start)
|
||||
rbar = min(r, stop)
|
||||
|
||||
return vb.mapViewToDevice(
|
||||
QLineF(lbar, 0, rbar, 0)
|
||||
).length()
|
||||
|
||||
# XXX: lol brutal, the internals of `CurvePoint` (inherited by
|
||||
# our `LineDot`) required ``.getData()`` to work..
|
||||
def getData(self):
|
||||
|
|
@ -268,8 +230,14 @@ class Curve(FlowGraphic):
|
|||
self.path.clear()
|
||||
|
||||
if self.fast_path:
|
||||
self.fast_path.clear()
|
||||
# self.fast_path = None
|
||||
# self.fast_path.clear()
|
||||
self.fast_path = None
|
||||
|
||||
@cm
|
||||
def reset_cache(self) -> None:
|
||||
self.setCacheMode(QtWidgets.QGraphicsItem.NoCache)
|
||||
yield
|
||||
self.setCacheMode(QGraphicsItem.DeviceCoordinateCache)
|
||||
|
||||
def boundingRect(self):
|
||||
'''
|
||||
|
|
@ -283,65 +251,77 @@ class Curve(FlowGraphic):
|
|||
self.boundingRect = self._path_br
|
||||
return self._path_br()
|
||||
|
||||
# Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect
|
||||
def _path_br(self):
|
||||
'''
|
||||
Post init ``.boundingRect()```.
|
||||
|
||||
'''
|
||||
# profiler = Profiler(
|
||||
# msg=f'Curve.boundingRect(): `{self._name}`',
|
||||
# disabled=not pg_profile_enabled(),
|
||||
# ms_threshold=ms_slower_then,
|
||||
# hb = self.path.boundingRect()
|
||||
hb = self.path.controlPointRect()
|
||||
hb_size = hb.size()
|
||||
|
||||
fp = self.fast_path
|
||||
if fp:
|
||||
fhb = fp.controlPointRect()
|
||||
hb_size = fhb.size() + hb_size
|
||||
|
||||
# print(f'hb_size: {hb_size}')
|
||||
|
||||
# if self._last_step_rect:
|
||||
# hb_size += self._last_step_rect.size()
|
||||
|
||||
# if self._line:
|
||||
# br = self._last_step_rect.bottomRight()
|
||||
|
||||
# tl = QPointF(
|
||||
# # self._vr[0],
|
||||
# # hb.topLeft().y(),
|
||||
# # 0,
|
||||
# # hb_size.height() + 1
|
||||
# )
|
||||
pr = self.path.controlPointRect()
|
||||
hb_tl, hb_br = (
|
||||
pr.topLeft(),
|
||||
pr.bottomRight(),
|
||||
)
|
||||
mn_y = hb_tl.y()
|
||||
mx_y = hb_br.y()
|
||||
most_left = hb_tl.x()
|
||||
most_right = hb_br.x()
|
||||
# profiler('calc path vertices')
|
||||
|
||||
# TODO: if/when we get fast path appends working in the
|
||||
# `Renderer`, then we might need to actually use this..
|
||||
# fp = self.fast_path
|
||||
# if fp:
|
||||
# fhb = fp.controlPointRect()
|
||||
# # hb_size = fhb.size() + hb_size
|
||||
# br = pr.united(fhb)
|
||||
# br = self._last_step_rect.bottomRight()
|
||||
|
||||
# XXX: *was* a way to allow sub-types to extend the
|
||||
# boundingrect calc, but in the one use case for a step curve
|
||||
# doesn't seem like we need it as long as the last line segment
|
||||
# is drawn as it is?
|
||||
|
||||
# sbr = self.sub_br
|
||||
# if sbr:
|
||||
# # w, h = self.sub_br(w, h)
|
||||
# sub_br = sbr()
|
||||
# br = br.united(sub_br)
|
||||
w = hb_size.width()
|
||||
h = hb_size.height()
|
||||
|
||||
sbr = self.sub_br
|
||||
if sbr:
|
||||
w, h = self.sub_br(w, h)
|
||||
else:
|
||||
# assume plain line graphic and use
|
||||
# default unit step in each direction.
|
||||
ll = self._last_line
|
||||
y1, y2 = ll.y1(), ll.y2()
|
||||
x1, x2 = ll.x1(), ll.x2()
|
||||
|
||||
ymn = min(y1, y2, mn_y)
|
||||
ymx = max(y1, y2, mx_y)
|
||||
most_left = min(x1, x2, most_left)
|
||||
most_right = max(x1, x2, most_right)
|
||||
# profiler('calc last line vertices')
|
||||
# only on a plane line do we include
|
||||
# and extra index step's worth of width
|
||||
# since in the step case the end of the curve
|
||||
# actually terminates earlier so we don't need
|
||||
# this for the last step.
|
||||
w += self._last_w
|
||||
# ll = self._last_line
|
||||
h += 1 # ll.y2() - ll.y1()
|
||||
|
||||
return QRectF(
|
||||
most_left,
|
||||
ymn,
|
||||
most_right - most_left + 1,
|
||||
ymx,
|
||||
# br = QPointF(
|
||||
# self._vr[-1],
|
||||
# # tl.x() + w,
|
||||
# tl.y() + h,
|
||||
# )
|
||||
|
||||
br = QRectF(
|
||||
|
||||
# top left
|
||||
# hb.topLeft()
|
||||
# tl,
|
||||
QPointF(hb.topLeft()),
|
||||
|
||||
# br,
|
||||
# total size
|
||||
# QSizeF(hb_size)
|
||||
# hb_size,
|
||||
QSizeF(w, h)
|
||||
)
|
||||
# print(f'bounding rect: {br}')
|
||||
return br
|
||||
|
||||
def paint(
|
||||
self,
|
||||
|
|
@ -351,7 +331,7 @@ class Curve(FlowGraphic):
|
|||
|
||||
) -> None:
|
||||
|
||||
profiler = Profiler(
|
||||
profiler = pg.debug.Profiler(
|
||||
msg=f'Curve.paint(): `{self._name}`',
|
||||
disabled=not pg_profile_enabled(),
|
||||
ms_threshold=ms_slower_then,
|
||||
|
|
@ -359,14 +339,18 @@ class Curve(FlowGraphic):
|
|||
|
||||
sub_paint = self.sub_paint
|
||||
if sub_paint:
|
||||
sub_paint(p)
|
||||
sub_paint(p, profiler)
|
||||
|
||||
p.setPen(self.last_step_pen)
|
||||
p.drawLine(self._last_line)
|
||||
profiler('last datum `.drawLine()`')
|
||||
|
||||
profiler('.drawLine()')
|
||||
p.setPen(self._pen)
|
||||
|
||||
path = self.path
|
||||
# cap = path.capacity()
|
||||
# if cap != self._last_cap:
|
||||
# print(f'NEW CAPACITY: {self._last_cap} -> {cap}')
|
||||
# self._last_cap = cap
|
||||
|
||||
if path:
|
||||
p.drawPath(path)
|
||||
|
|
@ -389,29 +373,22 @@ class Curve(FlowGraphic):
|
|||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
) -> None:
|
||||
# default line draw last call
|
||||
x = src_data[index_field]
|
||||
y = src_data[array_key]
|
||||
|
||||
x_last = x[-1]
|
||||
x_2last = x[-2]
|
||||
# with self.reset_cache():
|
||||
x = render_data['index']
|
||||
y = render_data[array_key]
|
||||
|
||||
# draw the "current" step graphic segment so it
|
||||
# lines up with the "middle" of the current
|
||||
# (OHLC) sample.
|
||||
self._last_line = QLineF(
|
||||
|
||||
# NOTE: currently we draw in x-domain
|
||||
# from last datum to current such that
|
||||
# the end of line touches the "beginning"
|
||||
# of the current datum step span.
|
||||
x_2last, y[-2],
|
||||
x_last, y[-1],
|
||||
x[-2], y[-2],
|
||||
x[-1], y[-1],
|
||||
)
|
||||
|
||||
return x, y
|
||||
|
|
@ -422,31 +399,18 @@ class Curve(FlowGraphic):
|
|||
# element such that the current datum in view can be shown
|
||||
# (via it's max / min) even when highly zoomed out.
|
||||
class FlattenedOHLC(Curve):
|
||||
'''
|
||||
More or less the exact same as a standard line ``Curve`` above
|
||||
but meant to handle a traced-and-downsampled OHLC time series.
|
||||
_
|
||||
_| | _
|
||||
|_ | |_ | |
|
||||
_| => |_| |
|
||||
| |
|
||||
|_ |_
|
||||
|
||||
The main implementation different is that ``.draw_last_datum()``
|
||||
expects an underlying OHLC array for the ``src_data`` input.
|
||||
|
||||
'''
|
||||
def draw_last_datum(
|
||||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
) -> None:
|
||||
lasts = src_data[-2:]
|
||||
x = lasts[index_field]
|
||||
x = lasts['index']
|
||||
y = lasts['close']
|
||||
|
||||
# draw the "current" step graphic segment so it
|
||||
|
|
@ -460,19 +424,7 @@ class FlattenedOHLC(Curve):
|
|||
|
||||
|
||||
class StepCurve(Curve):
|
||||
'''
|
||||
A familiar rectangle-with-y-height-per-datum type curve:
|
||||
|
||||
||
|
||||
|| ||
|
||||
|| || ||||
|
||||
_||_||_||_||||_ where each datum's y-value is drawn as
|
||||
a nearly full rectangle, each "level" spans some x-step size.
|
||||
|
||||
This is most often used for vlm and option OI style curves and/or
|
||||
the very popular "bar chart".
|
||||
|
||||
'''
|
||||
def declare_paintables(
|
||||
self,
|
||||
) -> None:
|
||||
|
|
@ -482,9 +434,9 @@ class StepCurve(Curve):
|
|||
self,
|
||||
path: QPainterPath,
|
||||
src_data: np.ndarray,
|
||||
render_data: np.ndarray,
|
||||
reset: bool,
|
||||
array_key: str,
|
||||
index_field: str,
|
||||
|
||||
w: float = 0.5,
|
||||
|
||||
|
|
@ -493,31 +445,40 @@ class StepCurve(Curve):
|
|||
# TODO: remove this and instead place all step curve
|
||||
# updating into pre-path data render callbacks.
|
||||
# full input data
|
||||
x = src_data[index_field]
|
||||
x = src_data['index']
|
||||
y = src_data[array_key]
|
||||
|
||||
x_last = x[-1]
|
||||
x_2last = x[-2]
|
||||
y_last = y[-1]
|
||||
step_size = x_last - x_2last
|
||||
|
||||
# lol, commenting this makes step curves
|
||||
# all "black" for me :eyeroll:..
|
||||
self._last_line = QLineF(
|
||||
x_2last, 0,
|
||||
x_last, 0,
|
||||
x_last - w, 0,
|
||||
x_last + w, 0,
|
||||
)
|
||||
self._last_step_rect = QRectF(
|
||||
x_last, 0,
|
||||
step_size, y_last,
|
||||
x_last - w, 0,
|
||||
x_last + w, y_last,
|
||||
)
|
||||
return x, y
|
||||
|
||||
def sub_paint(
|
||||
self,
|
||||
p: QPainter,
|
||||
profiler: pg.debug.Profiler,
|
||||
|
||||
) -> None:
|
||||
# p.drawLines(*tuple(filter(bool, self._last_step_lines)))
|
||||
# p.drawRect(self._last_step_rect)
|
||||
p.fillRect(self._last_step_rect, self._brush)
|
||||
profiler('.fillRect()')
|
||||
|
||||
def sub_br(
|
||||
self,
|
||||
path_w: float,
|
||||
path_h: float,
|
||||
|
||||
) -> (float, float):
|
||||
# passthrough
|
||||
return path_w, path_h
|
||||
|
|
|
|||
1502
piker/ui/_dataviz.py
1502
piker/ui/_dataviz.py
File diff suppressed because it is too large
Load Diff
1582
piker/ui/_display.py
1582
piker/ui/_display.py
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue