From 277461161734510ef765752bd7c57c59b8b85bc0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Apr 2019 19:13:24 -0400 Subject: [PATCH 001/206] Blind stab at a basic chart --- piker/ui/qt/stackof_candle.py | 67 +++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 piker/ui/qt/stackof_candle.py diff --git a/piker/ui/qt/stackof_candle.py b/piker/ui/qt/stackof_candle.py new file mode 100644 index 00000000..0bcd37e4 --- /dev/null +++ b/piker/ui/qt/stackof_candle.py @@ -0,0 +1,67 @@ +import sys + +from PySide2.QtCharts import QtCharts +from PySide2.QtWidgets import QApplication, QMainWindow +from PySide2.QtCore import Qt, QPointF +from PySide2 import QtGui +import qdarkstyle + +data = ((1, 7380, 7520, 7380, 7510, 7324), + (2, 7520, 7580, 7410, 7440, 7372), + (3, 7440, 7650, 7310, 7520, 7434), + (4, 7450, 7640, 7450, 7550, 7480), + (5, 7510, 7590, 7460, 7490, 7502), + (6, 7500, 7590, 7480, 7560, 7512), + (7, 7560, 7830, 7540, 7800, 7584)) + + +app = QApplication([]) +# set dark stylesheet +# import pdb; pdb.set_trace() +app.setStyleSheet(qdarkstyle.load_stylesheet_pyside()) + +series = QtCharts.QCandlestickSeries() +series.setDecreasingColor(Qt.darkRed) +series.setIncreasingColor(Qt.darkGreen) + +ma5 = QtCharts.QLineSeries() # 5-days average data line +tm = [] # stores str type data + +# in a loop, series and ma5 append corresponding data +for num, o, h, l, c, m in data: + candle = QtCharts.QCandlestickSet(o, h, l, c) + series.append(candle) + ma5.append(QPointF(num, m)) + tm.append(str(num)) + +pen = candle.pen() +# import pdb; pdb.set_trace() + +chart = QtCharts.QChart() + +# import pdb; pdb.set_trace() +series.setBodyOutlineVisible(False) +series.setCapsVisible(False) +# brush = QtGui.QBrush() +# brush.setColor(Qt.green) +# series.setBrush(brush) +chart.addSeries(series) # candle +chart.addSeries(ma5) # ma5 line + +chart.setAnimationOptions(QtCharts.QChart.SeriesAnimations) +chart.createDefaultAxes() +chart.legend().hide() + +chart.axisX(series).setCategories(tm) +chart.axisX(ma5).setVisible(False) + +view = QtCharts.QChartView(chart) +view.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeDark) +view.setRubberBand(QtCharts.QChartView.HorizontalRubberBand) +# chartview.chart().setTheme(QtCharts.QChart.ChartTheme.ChartThemeBlueCerulean) + +ui = QMainWindow() +# ui.setGeometry(50, 50, 500, 300) +ui.setCentralWidget(view) +ui.show() +sys.exit(app.exec_()) From 503aa8a64a5a207e25fcac9a444d050fabbefe58 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 May 2019 09:24:40 -0400 Subject: [PATCH 002/206] Use darkstyle pkg --- Pipfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Pipfile b/Pipfile index 9c2dc6d0..bbe8a956 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ tractor = {git = "git://github.com/goodboy/tractor.git"} toml = "*" pyqtgraph = "*" pyside2 = "*" +qdarkstyle = "*" [dev-packages] pytest = "*" From bb81d7881c190f3419deafb29c7f7ee6c9884e33 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Jun 2020 12:28:24 -0400 Subject: [PATCH 003/206] Use qt5 and trio guest mode --- Pipfile | 5 +++-- piker/ui/kivy/utils_async.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Pipfile b/Pipfile index bbe8a956..8e055400 100644 --- a/Pipfile +++ b/Pipfile @@ -5,7 +5,8 @@ name = "pypi" [packages] e1839a8 = {path = ".",editable = true} -trio = "*" +# use master branch for "guest mode" +trio = {git = "git://github.com/python-trio/trio.git"} Cython = "*" # use master branch kivy since wheels seem borked (due to cython stuff) kivy = {git = "git://github.com/kivy/kivy.git"} @@ -14,8 +15,8 @@ msgpack = "*" tractor = {git = "git://github.com/goodboy/tractor.git"} toml = "*" pyqtgraph = "*" -pyside2 = "*" qdarkstyle = "*" +pyqt5 = "*" [dev-packages] pytest = "*" diff --git a/piker/ui/kivy/utils_async.py b/piker/ui/kivy/utils_async.py index fd4b82da..ef6a76a0 100644 --- a/piker/ui/kivy/utils_async.py +++ b/piker/ui/kivy/utils_async.py @@ -61,10 +61,10 @@ class AsyncCallbackQueue(object): return self async def __anext__(self): - self.event.clear() + self.event = async_lib.Event() while not self.callback_result and not self.quit: await self.event.wait() - self.event.clear() + self.event = async_lib.Event() if self.callback_result: return self.callback_result.popleft() From 9c84e3c45d3b7114ef68c953d742b444861bb153 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Jun 2020 12:48:35 -0400 Subject: [PATCH 004/206] Add initial Qt-trio integration Use the new "guest mode" available on trio master branch. Add entrypoint for `pyqtgraph` based charting based on the `Quantdom` project. --- piker/ui/qt/__init__.py | 0 piker/ui/qt/_exec.py | 117 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 piker/ui/qt/__init__.py create mode 100644 piker/ui/qt/_exec.py diff --git a/piker/ui/qt/__init__.py b/piker/ui/qt/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/piker/ui/qt/_exec.py b/piker/ui/qt/_exec.py new file mode 100644 index 00000000..90286681 --- /dev/null +++ b/piker/ui/qt/_exec.py @@ -0,0 +1,117 @@ +""" +Trio - Qt integration + +Run ``trio`` in guest mode on top of the Qt event loop. +All global Qt runtime settings are mostly defined here. +""" +import traceback +from datetime import datetime + +import PyQt5 # noqa +from pyqtgraph import QtGui +from PyQt5 import QtCore +from PyQt5.QtCore import pyqtRemoveInputHook +import qdarkstyle +import trio +from outcome import Error + +from _chart import QuotesTabWidget +from quantdom.base import Symbol +from quantdom.loaders import get_quotes + + +# Taken from Quantdom +class MainWindow(QtGui.QMainWindow): + + size = (800, 500) + title = 'piker: chart' + + def __init__(self, parent=None): + super().__init__(parent) + # self.setMinimumSize(*self.size) + self.setWindowTitle(self.title) + + +def run_qtrio( + func, + args, + main_widget, +) -> None: + + # avoids annoying message when entering debugger from qt loop + pyqtRemoveInputHook() + + app = QtGui.QApplication.instance() + if app is None: + app = PyQt5.QtWidgets.QApplication([]) + + # This code is from Nathaniel: + + # This is substantially faster than using a signal... for some + # reason Qt signal dispatch is really slow (and relies on events + # underneath anyway, so this is strictly less work) + REENTER_EVENT = QtCore.QEvent.Type(QtCore.QEvent.registerEventType()) + + class ReenterEvent(QtCore.QEvent): + pass + + class Reenter(QtCore.QObject): + def event(self, event): + event.fn() + return False + + reenter = Reenter() + + def run_sync_soon_threadsafe(fn): + event = ReenterEvent(REENTER_EVENT) + event.fn = fn + app.postEvent(reenter, event) + + def done_callback(outcome): + print(f"Outcome: {outcome}") + if isinstance(outcome, Error): + exc = outcome.error + traceback.print_exception(type(exc), exc, exc.__traceback__) + app.quit() + + # load dark theme + app.setStyleSheet(qdarkstyle.load_stylesheet(qt_api='pyqt5')) + + # make window and exec + window = MainWindow() + instance = main_widget() + + widgets = { + 'window': window, + 'main': instance, + } + + # guest mode + trio.lowlevel.start_guest_run( + func, + widgets, + run_sync_soon_threadsafe=run_sync_soon_threadsafe, + done_callback=done_callback, + ) + + window.main_widget = main_widget + window.setCentralWidget(instance) + window.show() + app.exec_() + + +async def plot_aapl(widgets): + qtw = widgets['main'] + s = Symbol(ticker='AAPL', mode=Symbol.SHARES) + get_quotes( + symbol=s.ticker, + date_from=datetime(1900, 1, 1), + date_to=datetime(2030, 12, 31), + ) + # spawn chart + qtw.update_chart(s) + await trio.sleep_forever() + + +if __name__ == '__main__': + run_qtrio(plot_aapl, (), QuotesTabWidget) From eddd8aacab03d270c4dd54fa9b061de14bd3bae8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Jun 2020 12:50:09 -0400 Subject: [PATCH 005/206] Add charting components from `Quantdom` Hand select necessary components to get real-time charting with `pyqtgraph` from the `Quantdom` projects: https://github.com/constverum/Quantdom We've offered to collaborate with the author but have received no response and the project has not been updated in over a year. Given this, we are moving forward with taking the required components to make further improvements upon especially since the `pyqtgraph` project is now being actively maintained again. If the author comes back we will be more then happy to contribute modified components upstream: https://github.com/constverum/Quantdom/issues/18 Relates to #80 --- piker/ui/qt/quantdom/__init__.py | 10 + piker/ui/qt/quantdom/base.py | 132 +++++ piker/ui/qt/quantdom/charts.py | 799 ++++++++++++++++++++++++++++ piker/ui/qt/quantdom/const.py | 36 ++ piker/ui/qt/quantdom/loaders.py | 172 ++++++ piker/ui/qt/quantdom/performance.py | 350 ++++++++++++ piker/ui/qt/quantdom/portfolio.py | 410 ++++++++++++++ piker/ui/qt/quantdom/utils.py | 82 +++ 8 files changed, 1991 insertions(+) create mode 100644 piker/ui/qt/quantdom/__init__.py create mode 100644 piker/ui/qt/quantdom/base.py create mode 100644 piker/ui/qt/quantdom/charts.py create mode 100644 piker/ui/qt/quantdom/const.py create mode 100644 piker/ui/qt/quantdom/loaders.py create mode 100644 piker/ui/qt/quantdom/performance.py create mode 100644 piker/ui/qt/quantdom/portfolio.py create mode 100644 piker/ui/qt/quantdom/utils.py diff --git a/piker/ui/qt/quantdom/__init__.py b/piker/ui/qt/quantdom/__init__.py new file mode 100644 index 00000000..ae7e2e8c --- /dev/null +++ b/piker/ui/qt/quantdom/__init__.py @@ -0,0 +1,10 @@ +""" +Curated set of components from ``Quantdom`` used as a starting +draft for real-time charting with ``pyqtgraph``. + +Much thanks to the author: +https://github.com/constverum/Quantdom + +Note this code is licensed Apache 2.0: +https://github.com/constverum/Quantdom/blob/master/LICENSE +""" diff --git a/piker/ui/qt/quantdom/base.py b/piker/ui/qt/quantdom/base.py new file mode 100644 index 00000000..a53b18c2 --- /dev/null +++ b/piker/ui/qt/quantdom/base.py @@ -0,0 +1,132 @@ +"""Base classes.""" + +from enum import Enum, auto + +import numpy as np +import pandas as pd + +from .const import ChartType, TimeFrame + +__all__ = ('Indicator', 'Symbol', 'Quotes') + + +class BaseQuotes(np.recarray): + def __new__(cls, shape=None, dtype=None, order='C'): + dt = np.dtype( + [ + ('id', int), + ('time', float), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', int), + ] + ) + shape = shape or (1,) + return np.ndarray.__new__(cls, shape, (np.record, dt), order=order) + + def _nan_to_closest_num(self): + """Return interpolated values instead of NaN.""" + for col in ['open', 'high', 'low', 'close']: + mask = np.isnan(self[col]) + if not mask.size: + continue + self[col][mask] = np.interp( + np.flatnonzero(mask), np.flatnonzero(~mask), self[col][~mask] + ) + + def _set_time_frame(self, default_tf): + tf = { + 1: TimeFrame.M1, + 5: TimeFrame.M5, + 15: TimeFrame.M15, + 30: TimeFrame.M30, + 60: TimeFrame.H1, + 240: TimeFrame.H4, + 1440: TimeFrame.D1, + } + minutes = int(np.diff(self.time[-10:]).min() / 60) + self.timeframe = tf.get(minutes) or tf[default_tf] + + def new(self, data, source=None, default_tf=None): + shape = (len(data),) + self.resize(shape, refcheck=False) + + if isinstance(data, pd.DataFrame): + data.reset_index(inplace=True) + data.insert(0, 'id', data.index) + data.Date = self.convert_dates(data.Date) + data = data.rename( + columns={ + 'Date': 'time', + 'Open': 'open', + 'High': 'high', + 'Low': 'low', + 'Close': 'close', + 'Volume': 'volume', + } + ) + for name in self.dtype.names: + self[name] = data[name] + elif isinstance(data, (np.recarray, BaseQuotes)): + self[:] = data[:] + + self._nan_to_closest_num() + self._set_time_frame(default_tf) + return self + + def convert_dates(self, dates): + return np.array([d.timestamp() for d in dates]) + + +class SymbolType(Enum): + FOREX = auto() + CFD = auto() + FUTURES = auto() + SHARES = auto() + + +class Symbol: + + FOREX = SymbolType.FOREX + CFD = SymbolType.CFD + FUTURES = SymbolType.FUTURES + SHARES = SymbolType.SHARES + + def __init__(self, ticker, mode, tick_size=0, tick_value=None): + self.ticker = ticker + self.mode = mode + if self.mode in [self.FOREX, self.CFD]: + # number of units of the commodity, currency + # or financial asset in one lot + self.contract_size = 100_000 # (100000 == 1 Lot) + elif self.mode == self.FUTURES: + # cost of a single price change point ($10) / + # one minimum price movement + self.tick_value = tick_value + # minimum price change step (0.0001) + self.tick_size = tick_size + if isinstance(tick_size, float): + self.digits = len(str(tick_size).split('.')[1]) + else: + self.digits = 0 + + def __repr__(self): + return 'Symbol (%s | %s)' % (self.ticker, self.mode) + + +class Indicator: + def __init__( + self, label=None, window=None, data=None, tp=None, base=None, **kwargs + ): + self.label = label + self.window = window + self.data = data or [0] + self.type = tp or ChartType.LINE + self.base = base or {'linewidth': 0.5, 'color': 'black'} + self.lineStyle = {'linestyle': '-', 'linewidth': 0.5, 'color': 'blue'} + self.lineStyle.update(kwargs) + + +Quotes = BaseQuotes() diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py new file mode 100644 index 00000000..c1cf3712 --- /dev/null +++ b/piker/ui/qt/quantdom/charts.py @@ -0,0 +1,799 @@ +"""Chart.""" + +import numpy as np +import pyqtgraph as pg +from PyQt5 import QtCore, QtGui + +from .base import Quotes +from .const import ChartType +from .portfolio import Order, Portfolio +from .utils import fromtimestamp, timeit + +__all__ = ('QuotesChart', 'EquityChart') + + +# pg.setConfigOption('background', 'w') +CHART_MARGINS = (0, 0, 20, 5) + + +class SampleLegendItem(pg.graphicsItems.LegendItem.ItemSample): + def paint(self, p, *args): + p.setRenderHint(p.Antialiasing) + if isinstance(self.item, tuple): + positive = self.item[0].opts + negative = self.item[1].opts + p.setPen(pg.mkPen(positive['pen'])) + p.setBrush(pg.mkBrush(positive['brush'])) + p.drawPolygon( + QtGui.QPolygonF( + [ + QtCore.QPointF(0, 0), + QtCore.QPointF(18, 0), + QtCore.QPointF(18, 18), + ] + ) + ) + p.setPen(pg.mkPen(negative['pen'])) + p.setBrush(pg.mkBrush(negative['brush'])) + p.drawPolygon( + QtGui.QPolygonF( + [ + QtCore.QPointF(0, 0), + QtCore.QPointF(0, 18), + QtCore.QPointF(18, 18), + ] + ) + ) + else: + opts = self.item.opts + p.setPen(pg.mkPen(opts['pen'])) + p.drawRect(0, 10, 18, 0.5) + + +class PriceAxis(pg.AxisItem): + def __init__(self): + super().__init__(orientation='right') + self.style.update({'textFillLimits': [(0, 0.8)]}) + + def tickStrings(self, vals, scale, spacing): + digts = max(0, np.ceil(-np.log10(spacing * scale))) + return [ + ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals + ] + + +class DateAxis(pg.AxisItem): + tick_tpl = {'D1': '%d %b\n%Y'} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.quotes_count = len(Quotes) - 1 + + def tickStrings(self, values, scale, spacing): + s_period = 'D1' + strings = [] + for ibar in values: + if ibar > self.quotes_count: + return strings + dt_tick = fromtimestamp(Quotes[int(ibar)].time) + strings.append(dt_tick.strftime(self.tick_tpl[s_period])) + return strings + + +class CenteredTextItem(QtGui.QGraphicsTextItem): + def __init__( + self, + text='', + parent=None, + pos=(0, 0), + pen=None, + brush=None, + valign=None, + opacity=0.1, + ): + super().__init__(text, parent) + + self.pen = pen + self.brush = brush + self.opacity = opacity + self.valign = valign + self.text_flags = QtCore.Qt.AlignCenter + self.setPos(*pos) + self.setFlag(self.ItemIgnoresTransformations) + + def boundingRect(self): # noqa + r = super().boundingRect() + if self.valign == QtCore.Qt.AlignTop: + return QtCore.QRectF(-r.width() / 2, -37, r.width(), r.height()) + elif self.valign == QtCore.Qt.AlignBottom: + return QtCore.QRectF(-r.width() / 2, 15, r.width(), r.height()) + + def paint(self, p, option, widget): + p.setRenderHint(p.Antialiasing, False) + p.setRenderHint(p.TextAntialiasing, True) + p.setPen(self.pen) + if self.brush.style() != QtCore.Qt.NoBrush: + p.setOpacity(self.opacity) + p.fillRect(option.rect, self.brush) + p.setOpacity(1) + p.drawText(option.rect, self.text_flags, self.toPlainText()) + + +class AxisLabel(pg.GraphicsObject): + + bg_color = pg.mkColor('#dbdbdb') + fg_color = pg.mkColor('#000000') + + def __init__(self, parent=None, digits=0, color=None, opacity=1, **kwargs): + super().__init__(parent) + self.parent = parent + self.opacity = opacity + self.label_str = '' + self.digits = digits + self.quotes_count = len(Quotes) - 1 + if isinstance(color, QtGui.QPen): + self.bg_color = color.color() + self.fg_color = pg.mkColor('#ffffff') + elif isinstance(color, list): + self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} + self.fg_color = pg.mkColor('#ffffff') + self.setFlag(self.ItemIgnoresTransformations) + + def tick_to_string(self, tick_pos): + raise NotImplementedError() + + def boundingRect(self): # noqa + raise NotImplementedError() + + def update_label(self, evt_post, point_view): + raise NotImplementedError() + + def update_label_test(self, ypos=0, ydata=0): + self.label_str = self.tick_to_string(ydata) + height = self.boundingRect().height() + offset = 0 # if have margins + new_pos = QtCore.QPointF(0, ypos - height / 2 - offset) + self.setPos(new_pos) + + def paint(self, p, option, widget): + p.setRenderHint(p.TextAntialiasing, True) + p.setPen(self.fg_color) + if self.label_str: + if not isinstance(self.bg_color, dict): + bg_color = self.bg_color + else: + if int(self.label_str.replace(' ', '')) > 0: + bg_color = self.bg_color['>0'] + else: + bg_color = self.bg_color['<0'] + p.setOpacity(self.opacity) + p.fillRect(option.rect, bg_color) + p.setOpacity(1) + p.drawText(option.rect, self.text_flags, self.label_str) + + +class XAxisLabel(AxisLabel): + + text_flags = ( + QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter | QtCore.Qt.AlignTop + ) + + def tick_to_string(self, tick_pos): + # TODO: change to actual period + tpl = self.parent.tick_tpl['D1'] + return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) + + def boundingRect(self): # noqa + return QtCore.QRectF(0, 0, 60, 38) + + def update_label(self, evt_post, point_view): + ibar = point_view.x() + if ibar > self.quotes_count: + return + self.label_str = self.tick_to_string(ibar) + width = self.boundingRect().width() + offset = 0 # if have margins + new_pos = QtCore.QPointF(evt_post.x() - width / 2 - offset, 0) + self.setPos(new_pos) + + +class YAxisLabel(AxisLabel): + + text_flags = ( + QtCore.Qt.TextDontClip | QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter + ) + + def tick_to_string(self, tick_pos): + return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') + + def boundingRect(self): # noqa + return QtCore.QRectF(0, 0, 74, 24) + + def update_label(self, evt_post, point_view): + self.label_str = self.tick_to_string(point_view.y()) + height = self.boundingRect().height() + offset = 0 # if have margins + new_pos = QtCore.QPointF(0, evt_post.y() - height / 2 - offset) + self.setPos(new_pos) + + +class CustomPlotWidget(pg.PlotWidget): + sig_mouse_leave = QtCore.Signal(object) + sig_mouse_enter = QtCore.Signal(object) + + def enterEvent(self, ev): # noqa + self.sig_mouse_enter.emit(self) + + def leaveEvent(self, ev): # noqa + self.sig_mouse_leave.emit(self) + self.scene().leaveEvent(ev) + + +_rate_limit = 30 + + +class CrossHairItem(pg.GraphicsObject): + + def __init__(self, parent, indicators=None, digits=0): + super().__init__() + # self.pen = pg.mkPen('#000000') + self.pen = pg.mkPen('#a9a9a9') + self.parent = parent + self.indicators = {} + self.activeIndicator = None + self.xaxis = self.parent.getAxis('bottom') + self.yaxis = self.parent.getAxis('right') + + self.vline = self.parent.addLine(x=0, pen=self.pen, movable=False) + self.hline = self.parent.addLine(y=0, pen=self.pen, movable=False) + + self.proxy_moved = pg.SignalProxy( + self.parent.scene().sigMouseMoved, + rateLimit=_rate_limit, + slot=self.mouseMoved, + ) + + self.yaxis_label = YAxisLabel( + parent=self.yaxis, digits=digits, opacity=1 + ) + + indicators = indicators or [] + if indicators: + last_ind = indicators[-1] + self.xaxis_label = XAxisLabel( + parent=last_ind.getAxis('bottom'), opacity=1 + ) + self.proxy_enter = pg.SignalProxy( + self.parent.sig_mouse_enter, + rateLimit=_rate_limit, + slot=lambda: self.mouseAction('Enter', False), + ) + self.proxy_leave = pg.SignalProxy( + self.parent.sig_mouse_leave, + rateLimit=_rate_limit, + slot=lambda: self.mouseAction('Leave', False), + ) + else: + self.xaxis_label = XAxisLabel(parent=self.xaxis, opacity=1) + + for i in indicators: + vl = i.addLine(x=0, pen=self.pen, movable=False) + hl = i.addLine(y=0, pen=self.pen, movable=False) + yl = YAxisLabel(parent=i.getAxis('right'), opacity=1) + px_moved = pg.SignalProxy( + i.scene().sigMouseMoved, rateLimit=_rate_limit, slot=self.mouseMoved + ) + px_enter = pg.SignalProxy( + i.sig_mouse_enter, + rateLimit=_rate_limit, + slot=lambda: self.mouseAction('Enter', i), + ) + px_leave = pg.SignalProxy( + i.sig_mouse_leave, + rateLimit=_rate_limit, + slot=lambda: self.mouseAction('Leave', i), + ) + self.indicators[i] = { + 'vl': vl, + 'hl': hl, + 'yl': yl, + 'px': (px_moved, px_enter, px_leave), + } + + def mouseAction(self, action, ind=False): # noqa + if action == 'Enter': + if ind: + self.indicators[ind]['hl'].show() + self.indicators[ind]['yl'].show() + self.activeIndicator = ind + else: + self.yaxis_label.show() + self.hline.show() + else: # Leave + if ind: + self.indicators[ind]['hl'].hide() + self.indicators[ind]['yl'].hide() + self.activeIndicator = None + else: + self.yaxis_label.hide() + self.hline.hide() + + def mouseMoved(self, evt): # noqa + pos = evt[0] + if self.parent.sceneBoundingRect().contains(pos): + # mouse_point = self.vb.mapSceneToView(pos) + mouse_point = self.parent.mapToView(pos) + self.vline.setX(mouse_point.x()) + self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) + for opts in self.indicators.values(): + opts['vl'].setX(mouse_point.x()) + + if self.activeIndicator: + mouse_point_ind = self.activeIndicator.mapToView(pos) + self.indicators[self.activeIndicator]['hl'].setY( + mouse_point_ind.y() + ) + self.indicators[self.activeIndicator]['yl'].update_label( + evt_post=pos, point_view=mouse_point_ind + ) + else: + self.hline.setY(mouse_point.y()) + self.yaxis_label.update_label( + evt_post=pos, point_view=mouse_point + ) + + def paint(self, p, *args): + pass + + def boundingRect(self): + return self.parent.boundingRect() + + +class BarItem(pg.GraphicsObject): + + w = 0.35 + bull_brush = pg.mkPen('#00cc00') + bear_brush = pg.mkPen('#fa0000') + + def __init__(self): + super().__init__() + self.generatePicture() + + def _generate(self, p): + hl = np.array( + [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] + ) + op = np.array( + [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) for q in Quotes] + ) + cl = np.array( + [ + QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) + for q in Quotes + ] + ) + lines = np.concatenate([hl, op, cl]) + long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) + + p.setPen(self.bull_brush) + p.drawLines(*lines[long_bars]) + + p.setPen(self.bear_brush) + p.drawLines(*lines[short_bars]) + + @timeit + def generatePicture(self): + self.picture = QtGui.QPicture() + p = QtGui.QPainter(self.picture) + self._generate(p) + p.end() + + def paint(self, p, *args): + p.drawPicture(0, 0, self.picture) + + def boundingRect(self): + return QtCore.QRectF(self.picture.boundingRect()) + + +class CandlestickItem(BarItem): + + w2 = 0.7 + line_pen = pg.mkPen('#000000') + bull_brush = pg.mkBrush('#00ff00') + bear_brush = pg.mkBrush('#ff0000') + + def _generate(self, p): + rects = np.array( + [ + QtCore.QRectF(q.id - self.w, q.open, self.w2, q.close - q.open) + for q in Quotes + ] + ) + + p.setPen(self.line_pen) + p.drawLines([QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes]) + + p.setBrush(self.bull_brush) + p.drawRects(*rects[Quotes.close > Quotes.open]) + + p.setBrush(self.bear_brush) + p.drawRects(*rects[Quotes.close < Quotes.open]) + + +class QuotesChart(QtGui.QWidget): + + long_pen = pg.mkPen('#006000') + long_brush = pg.mkBrush('#00ff00') + short_pen = pg.mkPen('#600000') + short_brush = pg.mkBrush('#ff0000') + + zoomIsDisabled = QtCore.pyqtSignal(bool) + + def __init__(self): + super().__init__() + self.signals_visible = False + self.style = ChartType.BAR + self.indicators = [] + + self.xaxis = DateAxis(orientation='bottom') + self.xaxis.setStyle( + tickTextOffset=7, textFillLimits=[(0, 0.80)], showValues=False + ) + + self.xaxis_ind = DateAxis(orientation='bottom') + self.xaxis_ind.setStyle(tickTextOffset=7, textFillLimits=[(0, 0.80)]) + + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + + self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) + self.splitter.setHandleWidth(4) + + self.layout.addWidget(self.splitter) + + def _show_text_signals(self, lbar, rbar): + signals = [ + sig + for sig in self.signals_text_items[lbar:rbar] + if isinstance(sig, CenteredTextItem) + ] + if len(signals) <= 50: + for sig in signals: + sig.show() + else: + for sig in signals: + sig.hide() + + def _remove_signals(self): + self.chart.removeItem(self.signals_group_arrow) + self.chart.removeItem(self.signals_group_text) + del self.signals_text_items + del self.signals_group_arrow + del self.signals_group_text + self.signals_visible = False + + def _update_quotes_chart(self): + self.chart.hideAxis('left') + self.chart.showAxis('right') + self.chart.addItem(_get_chart_points(self.style)) + self.chart.setLimits( + xMin=Quotes[0].id, + xMax=Quotes[-1].id, + minXRange=60, + yMin=Quotes.low.min() * 0.98, + yMax=Quotes.high.max() * 1.02, + ) + self.chart.showGrid(x=True, y=True) + self.chart.setCursor(QtCore.Qt.BlankCursor) + self.chart.sigXRangeChanged.connect(self._update_yrange_limits) + + def _update_ind_charts(self): + for ind, d in self.indicators: + curve = pg.PlotDataItem(d, pen='b', antialias=True) + ind.addItem(curve) + ind.hideAxis('left') + ind.showAxis('right') + # ind.setAspectLocked(1) + ind.setXLink(self.chart) + ind.setLimits( + xMin=Quotes[0].id, + xMax=Quotes[-1].id, + minXRange=60, + yMin=Quotes.open.min() * 0.98, + yMax=Quotes.open.max() * 1.02, + ) + ind.showGrid(x=True, y=True) + ind.setCursor(QtCore.Qt.BlankCursor) + + def _update_sizes(self): + min_h_ind = int(self.height() * 0.3 / len(self.indicators)) + sizes = [int(self.height() * 0.7)] + sizes.extend([min_h_ind] * len(self.indicators)) + self.splitter.setSizes(sizes) # , int(self.height()*0.2) + + def _update_yrange_limits(self): + vr = self.chart.viewRect() + lbar, rbar = int(vr.left()), int(vr.right()) + if self.signals_visible: + self._show_text_signals(lbar, rbar) + bars = Quotes[lbar:rbar] + ylow = bars.low.min() * 0.98 + yhigh = bars.high.max() * 1.02 + + std = np.std(bars.close) + self.chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + self.chart.setYRange(ylow, yhigh) + for i, d in self.indicators: + # ydata = i.plotItem.items[0].getData()[1] + ydata = d[lbar:rbar] + ylow = ydata.min() * 0.98 + yhigh = ydata.max() * 1.02 + std = np.std(ydata) + i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + i.setYRange(ylow, yhigh) + + def plot(self, symbol): + self.digits = symbol.digits + self.chart = CustomPlotWidget( + parent=self.splitter, + axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, + enableMenu=False, + ) + # self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + + inds = [Quotes.open] + + for d in inds: + ind = CustomPlotWidget( + parent=self.splitter, + axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, + enableMenu=False, + ) + ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + ind.getPlotItem().setContentsMargins(*CHART_MARGINS) + # self.splitter.addWidget(ind) + self.indicators.append((ind, d)) + + self._update_quotes_chart() + self._update_ind_charts() + self._update_sizes() + + ch = CrossHairItem( + self.chart, [_ind for _ind, d in self.indicators], self.digits + ) + self.chart.addItem(ch) + + def add_signals(self): + self.signals_group_text = QtGui.QGraphicsItemGroup() + self.signals_group_arrow = QtGui.QGraphicsItemGroup() + self.signals_text_items = np.empty(len(Quotes), dtype=object) + + for p in Portfolio.positions: + x, price = p.id_bar_open, p.open_price + if p.type == Order.BUY: + y = Quotes[x].low * 0.99 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.long_pen, + brush=self.long_brush, + angle=90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.long_pen, + brush=self.long_brush, + text=('Buy at {:.%df}' % self.digits).format(price), + valign=QtCore.Qt.AlignBottom, + ) + text_sig.hide() + else: + y = Quotes[x].high * 1.01 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.short_pen, + brush=self.short_brush, + angle=-90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.short_pen, + brush=self.short_brush, + text=('Sell at {:.%df}' % self.digits).format(price), + valign=QtCore.Qt.AlignTop, + ) + text_sig.hide() + + self.signals_text_items[x] = text_sig + + self.chart.addItem(self.signals_group_arrow) + self.chart.addItem(self.signals_group_text) + self.signals_visible = True + + +class EquityChart(QtGui.QWidget): + + eq_pen_pos_color = pg.mkColor('#00cc00') + eq_pen_neg_color = pg.mkColor('#cc0000') + eq_brush_pos_color = pg.mkColor('#40ee40') + eq_brush_neg_color = pg.mkColor('#ee4040') + long_pen_color = pg.mkColor('#008000') + short_pen_color = pg.mkColor('#800000') + buy_and_hold_pen_color = pg.mkColor('#4444ff') + + def __init__(self): + super().__init__() + self.xaxis = DateAxis(orientation='bottom') + self.xaxis.setStyle(tickTextOffset=7, textFillLimits=[(0, 0.80)]) + self.yaxis = PriceAxis() + + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + + self.chart = pg.PlotWidget( + axisItems={'bottom': self.xaxis, 'right': self.yaxis}, + enableMenu=False, + ) + self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart.showGrid(x=True, y=True) + self.chart.hideAxis('left') + self.chart.showAxis('right') + + self.chart.setCursor(QtCore.Qt.BlankCursor) + self.chart.sigXRangeChanged.connect(self._update_yrange_limits) + + self.layout.addWidget(self.chart) + + def _add_legend(self): + legend = pg.LegendItem((140, 100), offset=(10, 10)) + legend.setParentItem(self.chart.getPlotItem()) + + for arr, item in self.curves: + legend.addItem( + SampleLegendItem(item), + item.opts['name'] + if not isinstance(item, tuple) + else item[0].opts['name'], + ) + + def _add_ylabels(self): + self.ylabels = [] + for arr, item in self.curves: + color = ( + item.opts['pen'] + if not isinstance(item, tuple) + else [i.opts['pen'] for i in item] + ) + label = YAxisLabel(parent=self.yaxis, color=color) + self.ylabels.append(label) + + def _update_ylabels(self, vb, rbar): + for i, curve in enumerate(self.curves): + arr, item = curve + ylast = arr[rbar] + ypos = vb.mapFromView(QtCore.QPointF(0, ylast)).y() + axlabel = self.ylabels[i] + axlabel.update_label_test(ypos=ypos, ydata=ylast) + + def _update_yrange_limits(self, vb=None): + if not hasattr(self, 'min_curve'): + return + vr = self.chart.viewRect() + lbar, rbar = int(vr.left()), int(vr.right()) + ylow = self.min_curve[lbar:rbar].min() * 1.1 + yhigh = self.max_curve[lbar:rbar].max() * 1.1 + + std = np.std(self.max_curve[lbar:rbar]) * 4 + self.chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + self.chart.setYRange(ylow, yhigh) + self._update_ylabels(vb, rbar) + + @timeit + def plot(self): + equity_curve = Portfolio.equity_curve + eq_pos = np.zeros_like(equity_curve) + eq_neg = np.zeros_like(equity_curve) + eq_pos[equity_curve >= 0] = equity_curve[equity_curve >= 0] + eq_neg[equity_curve <= 0] = equity_curve[equity_curve <= 0] + + # Equity + self.eq_pos_curve = pg.PlotCurveItem( + eq_pos, + name='Equity', + fillLevel=0, + antialias=True, + pen=self.eq_pen_pos_color, + brush=self.eq_brush_pos_color, + ) + self.eq_neg_curve = pg.PlotCurveItem( + eq_neg, + name='Equity', + fillLevel=0, + antialias=True, + pen=self.eq_pen_neg_color, + brush=self.eq_brush_neg_color, + ) + self.chart.addItem(self.eq_pos_curve) + self.chart.addItem(self.eq_neg_curve) + + # Only Long + self.long_curve = pg.PlotCurveItem( + Portfolio.long_curve, + name='Only Long', + pen=self.long_pen_color, + antialias=True, + ) + self.chart.addItem(self.long_curve) + + # Only Short + self.short_curve = pg.PlotCurveItem( + Portfolio.short_curve, + name='Only Short', + pen=self.short_pen_color, + antialias=True, + ) + self.chart.addItem(self.short_curve) + + # Buy and Hold + self.buy_and_hold_curve = pg.PlotCurveItem( + Portfolio.buy_and_hold_curve, + name='Buy and Hold', + pen=self.buy_and_hold_pen_color, + antialias=True, + ) + self.chart.addItem(self.buy_and_hold_curve) + + self.curves = [ + (Portfolio.equity_curve, (self.eq_pos_curve, self.eq_neg_curve)), + (Portfolio.long_curve, self.long_curve), + (Portfolio.short_curve, self.short_curve), + (Portfolio.buy_and_hold_curve, self.buy_and_hold_curve), + ] + + self._add_legend() + self._add_ylabels() + + ch = CrossHairItem(self.chart) + self.chart.addItem(ch) + + arrs = ( + Portfolio.equity_curve, + Portfolio.buy_and_hold_curve, + Portfolio.long_curve, + Portfolio.short_curve, + ) + np_arrs = np.concatenate(arrs) + _min = abs(np_arrs.min()) * -1.1 + _max = np_arrs.max() * 1.1 + + self.chart.setLimits( + xMin=Quotes[0].id, + xMax=Quotes[-1].id, + yMin=_min, + yMax=_max, + minXRange=60, + ) + + self.min_curve = arrs[0].copy() + self.max_curve = arrs[0].copy() + for arr in arrs[1:]: + self.min_curve = np.minimum(self.min_curve, arr) + self.max_curve = np.maximum(self.max_curve, arr) + + +def _get_chart_points(style): + if style == ChartType.CANDLESTICK: + return CandlestickItem() + elif style == ChartType.BAR: + return BarItem() + return pg.PlotDataItem(Quotes.close, pen='b') diff --git a/piker/ui/qt/quantdom/const.py b/piker/ui/qt/quantdom/const.py new file mode 100644 index 00000000..eee1f55e --- /dev/null +++ b/piker/ui/qt/quantdom/const.py @@ -0,0 +1,36 @@ +"""Constants.""" + +from enum import Enum, auto + +__all__ = ('ChartType', 'TimeFrame') + + +class ChartType(Enum): + BAR = auto() + CANDLESTICK = auto() + LINE = auto() + + +class TimeFrame(Enum): + M1 = auto() + M5 = auto() + M15 = auto() + M30 = auto() + H1 = auto() + H4 = auto() + D1 = auto() + W1 = auto() + MN = auto() + + +ANNUAL_PERIOD = 252 # number of trading days in a year + +# # TODO: 6.5 - US trading hours (trading session); fix it for fx +# ANNUALIZATION_FACTORS = { +# TimeFrame.M1: int(252 * 6.5 * 60), +# TimeFrame.M5: int(252 * 6.5 * 12), +# TimeFrame.M15: int(252 * 6.5 * 4), +# TimeFrame.M30: int(252 * 6.5 * 2), +# TimeFrame.H1: int(252 * 6.5), +# TimeFrame.D1: 252, +# } diff --git a/piker/ui/qt/quantdom/loaders.py b/piker/ui/qt/quantdom/loaders.py new file mode 100644 index 00000000..0bb18d89 --- /dev/null +++ b/piker/ui/qt/quantdom/loaders.py @@ -0,0 +1,172 @@ +"""Parser.""" + +import logging +import os.path +import pickle + +import pandas as pd +import pandas_datareader.data as web +from pandas_datareader._utils import RemoteDataError +from pandas_datareader.data import ( + get_data_quandl, + get_data_yahoo, + get_data_alphavantage, +) +from pandas_datareader.nasdaq_trader import get_nasdaq_symbols +from pandas_datareader.exceptions import ImmediateDeprecationError + +from .base import Quotes +from .utils import get_data_path, timeit + +__all__ = ( + 'YahooQuotesLoader', + 'QuandleQuotesLoader', + 'get_symbols', + 'get_quotes', +) + + +logger = logging.getLogger(__name__) + + +class QuotesLoader: + + source = None + timeframe = '1D' + sort_index = False + default_tf = None + name_format = '%(symbol)s_%(tf)s_%(date_from)s_%(date_to)s.%(ext)s' + + @classmethod + def _get(cls, symbol, date_from, date_to): + quotes = web.DataReader( + symbol, cls.source, start=date_from, end=date_to + ) + if cls.sort_index: + quotes.sort_index(inplace=True) + return quotes + + @classmethod + def _get_file_path(cls, symbol, tf, date_from, date_to): + fname = cls.name_format % { + 'symbol': symbol, + 'tf': tf, + 'date_from': date_from.isoformat(), + 'date_to': date_to.isoformat(), + 'ext': 'qdom', + } + return os.path.join(get_data_path('stock_data'), fname) + + @classmethod + def _save_to_disk(cls, fpath, data): + logger.debug('Saving quotes to a file: %s', fpath) + with open(fpath, 'wb') as f: + pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + + @classmethod + def _load_from_disk(cls, fpath): + logger.debug('Loading quotes from a file: %s', fpath) + with open(fpath, 'rb') as f: + return pickle.load(f) + + @classmethod + @timeit + def get_quotes(cls, symbol, date_from, date_to): + quotes = None + fpath = cls._get_file_path(symbol, cls.timeframe, date_from, date_to) + if os.path.exists(fpath): + quotes = Quotes.new(cls._load_from_disk(fpath)) + else: + quotes_raw = cls._get(symbol, date_from, date_to) + quotes = Quotes.new( + quotes_raw, source=cls.source, default_tf=cls.default_tf + ) + cls._save_to_disk(fpath, quotes) + return quotes + + +class YahooQuotesLoader(QuotesLoader): + + source = 'yahoo' + + @classmethod + def _get(cls, symbol, date_from, date_to): + return get_data_yahoo(symbol, date_from, date_to) + + +class QuandleQuotesLoader(QuotesLoader): + + source = 'quandle' + + @classmethod + def _get(cls, symbol, date_from, date_to): + quotes = get_data_quandl(symbol, date_from, date_to) + quotes.sort_index(inplace=True) + return quotes + + +class AlphaVantageQuotesLoader(QuotesLoader): + + source = 'alphavantage' + api_key = 'demo' + + @classmethod + def _get(cls, symbol, date_from, date_to): + quotes = get_data_alphavantage( + symbol, date_from, date_to, api_key=cls.api_key + ) + return quotes + + +class StooqQuotesLoader(QuotesLoader): + + source = 'stooq' + sort_index = True + default_tf = 1440 + + +class IEXQuotesLoader(QuotesLoader): + + source = 'iex' + + @classmethod + def _get(cls, symbol, date_from, date_to): + quotes = web.DataReader( + symbol, cls.source, start=date_from, end=date_to + ) + quotes['Date'] = pd.to_datetime(quotes.index) + return quotes + + +class RobinhoodQuotesLoader(QuotesLoader): + + source = 'robinhood' + + +def get_symbols(): + fpath = os.path.join(get_data_path('stock_data'), 'symbols.qdom') + if os.path.exists(fpath): + with open(fpath, 'rb') as f: + symbols = pickle.load(f) + else: + symbols = get_nasdaq_symbols() + symbols.reset_index(inplace=True) + with open(fpath, 'wb') as f: + pickle.dump(symbols, f, pickle.HIGHEST_PROTOCOL) + return symbols + + +def get_quotes(*args, **kwargs): + quotes = [] + # don't work: + # GoogleQuotesLoader, QuandleQuotesLoader, + # AlphaVantageQuotesLoader, RobinhoodQuotesLoader + loaders = [YahooQuotesLoader, IEXQuotesLoader, StooqQuotesLoader] + while loaders: + loader = loaders.pop(0) + try: + quotes = loader.get_quotes(*args, **kwargs) + break + except (RemoteDataError, ImmediateDeprecationError) as e: + logger.error('get_quotes => error: %r', e) + return quotes diff --git a/piker/ui/qt/quantdom/performance.py b/piker/ui/qt/quantdom/performance.py new file mode 100644 index 00000000..f95fecca --- /dev/null +++ b/piker/ui/qt/quantdom/performance.py @@ -0,0 +1,350 @@ +"""Performance.""" + +import codecs +import json +from collections import OrderedDict, defaultdict + +import numpy as np + +from .base import Quotes +from .const import ANNUAL_PERIOD +from .utils import fromtimestamp, get_resource_path + +__all__ = ( + 'BriefPerformance', + 'Performance', + 'Stats', + 'REPORT_COLUMNS', + 'REPORT_ROWS', +) + + +REPORT_COLUMNS = ('All', 'Long', 'Short', 'Market') +with codecs.open( + get_resource_path('report_rows.json'), mode='r', encoding='utf-8' +) as f: + REPORT_ROWS = OrderedDict(json.load(f)) + + +class Stats(np.recarray): + def __new__(cls, positions, shape=None, dtype=None, order='C'): + shape = shape or (len(positions['All']),) + dtype = np.dtype( + [ + ('type', object), + ('symbol', object), + ('volume', float), + ('open_time', float), + ('close_time', float), + ('open_price', float), + ('close_price', float), + ('total_profit', float), + ('entry_name', object), + ('exit_name', object), + ('status', object), + ('comment', object), + ('abs', float), + ('perc', float), + ('bars', float), + ('on_bar', float), + ('mae', float), + ('mfe', float), + ] + ) + dt = [(col, dtype) for col in REPORT_COLUMNS] + return np.ndarray.__new__(cls, shape, (np.record, dt), order=order) + + def __init__(self, positions, **kwargs): + for col, _positions in positions.items(): + for i, p in enumerate(_positions): + self._add_position(p, col, i) + + def _add_position(self, p, col, i): + self[col][i].type = p.type + self[col][i].symbol = p.symbol + self[col][i].volume = p.volume + self[col][i].open_time = p.open_time + self[col][i].close_time = p.close_time + self[col][i].open_price = p.open_price + self[col][i].close_price = p.close_price + self[col][i].total_profit = p.total_profit + self[col][i].entry_name = p.entry_name + self[col][i].exit_name = p.exit_name + self[col][i].status = p.status + self[col][i].comment = p.comment + self[col][i].abs = p.profit + self[col][i].perc = p.profit_perc + + quotes_on_trade = Quotes[p.id_bar_open : p.id_bar_close] + + if not quotes_on_trade.size: + # if position was opened and closed on the last bar + quotes_on_trade = Quotes[p.id_bar_open : p.id_bar_close + 1] + + kwargs = { + 'low': quotes_on_trade.low.min(), + 'high': quotes_on_trade.high.max(), + } + self[col][i].mae = p.calc_mae(**kwargs) + self[col][i].mfe = p.calc_mfe(**kwargs) + + bars = p.id_bar_close - p.id_bar_open + self[col][i].bars = bars + self[col][i].on_bar = p.profit_perc / bars + + +class BriefPerformance(np.recarray): + def __new__(cls, shape=None, dtype=None, order='C'): + dt = np.dtype( + [ + ('kwargs', object), + ('net_profit_abs', float), + ('net_profit_perc', float), + ('year_profit', float), + ('win_average_profit_perc', float), + ('loss_average_profit_perc', float), + ('max_drawdown_abs', float), + ('total_trades', int), + ('win_trades_abs', int), + ('win_trades_perc', float), + ('profit_factor', float), + ('recovery_factor', float), + ('payoff_ratio', float), + ] + ) + shape = shape or (1,) + return np.ndarray.__new__(cls, shape, (np.record, dt), order=order) + + def _days_count(self, positions): + if hasattr(self, 'days'): + return self.days + self.days = ( + ( + fromtimestamp(positions[-1].close_time) + - fromtimestamp(positions[0].open_time) + ).days + if positions + else 1 + ) + return self.days + + def add(self, initial_balance, positions, i, kwargs): + position_count = len(positions) + profit = np.recarray( + (position_count,), dtype=[('abs', float), ('perc', float)] + ) + for n, position in enumerate(positions): + profit[n].abs = position.profit + profit[n].perc = position.profit_perc + s = self[i] + s.kwargs = kwargs + s.net_profit_abs = np.sum(profit.abs) + s.net_profit_perc = np.sum(profit.perc) + days = self._days_count(positions) + gain_factor = (s.net_profit_abs + initial_balance) / initial_balance + s.year_profit = (gain_factor ** (365 / days) - 1) * 100 + s.win_average_profit_perc = np.mean(profit.perc[profit.perc > 0]) + s.loss_average_profit_perc = np.mean(profit.perc[profit.perc < 0]) + s.max_drawdown_abs = profit.abs.min() + s.total_trades = position_count + wins = profit.abs[profit.abs > 0] + loss = profit.abs[profit.abs < 0] + s.win_trades_abs = len(wins) + s.win_trades_perc = round(s.win_trades_abs / s.total_trades * 100, 2) + s.profit_factor = abs(np.sum(wins) / np.sum(loss)) + s.recovery_factor = abs(s.net_profit_abs / s.max_drawdown_abs) + s.payoff_ratio = abs(np.mean(wins) / np.mean(loss)) + + +class Performance: + """Performance Metrics.""" + + rows = REPORT_ROWS + columns = REPORT_COLUMNS + + def __init__(self, initial_balance, stats, positions): + self._data = {} + for col in self.columns: + column = type('Column', (object,), dict.fromkeys(self.rows, 0)) + column.initial_balance = initial_balance + self._data[col] = column + self.calculate(column, stats[col], positions[col]) + + def __getitem__(self, col): + return self._data[col] + + def _calc_trade_series(self, col, positions): + win_in_series, loss_in_series = 0, 0 + for i, p in enumerate(positions): + if p.profit >= 0: + win_in_series += 1 + loss_in_series = 0 + if win_in_series > col.win_in_series: + col.win_in_series = win_in_series + else: + win_in_series = 0 + loss_in_series += 1 + if loss_in_series > col.loss_in_series: + col.loss_in_series = loss_in_series + + def calculate(self, col, stats, positions): + self._calc_trade_series(col, positions) + + col.total_trades = len(positions) + + profit_abs = stats[np.flatnonzero(stats.abs)].abs + profit_perc = stats[np.flatnonzero(stats.perc)].perc + bars = stats[np.flatnonzero(stats.bars)].bars + on_bar = stats[np.flatnonzero(stats.on_bar)].on_bar + + gt_zero_abs = stats[stats.abs > 0].abs + gt_zero_perc = stats[stats.perc > 0].perc + win_bars = stats[stats.perc > 0].bars + + lt_zero_abs = stats[stats.abs < 0].abs + lt_zero_perc = stats[stats.perc < 0].perc + los_bars = stats[stats.perc < 0].bars + + col.average_profit_abs = np.mean(profit_abs) if profit_abs.size else 0 + col.average_profit_perc = ( + np.mean(profit_perc) if profit_perc.size else 0 + ) + col.bars_on_trade = np.mean(bars) if bars.size else 0 + col.bar_profit = np.mean(on_bar) if on_bar.size else 0 + + col.win_average_profit_abs = ( + np.mean(gt_zero_abs) if gt_zero_abs.size else 0 + ) + col.win_average_profit_perc = ( + np.mean(gt_zero_perc) if gt_zero_perc.size else 0 + ) + col.win_bars_on_trade = np.mean(win_bars) if win_bars.size else 0 + + col.loss_average_profit_abs = ( + np.mean(lt_zero_abs) if lt_zero_abs.size else 0 + ) + col.loss_average_profit_perc = ( + np.mean(lt_zero_perc) if lt_zero_perc.size else 0 + ) + col.loss_bars_on_trade = np.mean(los_bars) if los_bars.size else 0 + + col.win_trades_abs = len(gt_zero_abs) + col.win_trades_perc = ( + round(col.win_trades_abs / col.total_trades * 100, 2) + if col.total_trades + else 0 + ) + + col.loss_trades_abs = len(lt_zero_abs) + col.loss_trades_perc = ( + round(col.loss_trades_abs / col.total_trades * 100, 2) + if col.total_trades + else 0 + ) + + col.total_profit = np.sum(gt_zero_abs) + col.total_loss = np.sum(lt_zero_abs) + col.net_profit_abs = np.sum(stats.abs) + col.net_profit_perc = np.sum(stats.perc) + col.total_mae = np.sum(stats.mae) + col.total_mfe = np.sum(stats.mfe) + + # https://financial-calculators.com/roi-calculator + + days = ( + ( + fromtimestamp(positions[-1].close_time) + - fromtimestamp(positions[0].open_time) + ).days + if positions + else 1 + ) + gain_factor = ( + col.net_profit_abs + col.initial_balance + ) / col.initial_balance + col.year_profit = (gain_factor ** (365 / days) - 1) * 100 + col.month_profit = (gain_factor ** (365 / days / 12) - 1) * 100 + + col.max_profit_abs = stats.abs.max() + col.max_profit_perc = stats.perc.max() + col.max_profit_abs_day = fromtimestamp( + stats.close_time[stats.abs == col.max_profit_abs][0] + ) + col.max_profit_perc_day = fromtimestamp( + stats.close_time[stats.perc == col.max_profit_perc][0] + ) + + col.max_drawdown_abs = stats.abs.min() + col.max_drawdown_perc = stats.perc.min() + col.max_drawdown_abs_day = fromtimestamp( + stats.close_time[stats.abs == col.max_drawdown_abs][0] + ) + col.max_drawdown_perc_day = fromtimestamp( + stats.close_time[stats.perc == col.max_drawdown_perc][0] + ) + + col.profit_factor = ( + abs(col.total_profit / col.total_loss) if col.total_loss else 0 + ) + col.recovery_factor = ( + abs(col.net_profit_abs / col.max_drawdown_abs) + if col.max_drawdown_abs + else 0 + ) + col.payoff_ratio = ( + abs(col.win_average_profit_abs / col.loss_average_profit_abs) + if col.loss_average_profit_abs + else 0 + ) + col.sharpe_ratio = annualized_sharpe_ratio(stats) + col.sortino_ratio = annualized_sortino_ratio(stats) + + # TODO: + col.alpha_ratio = np.nan + col.beta_ratio = np.nan + + +def day_percentage_returns(stats): + days = defaultdict(float) + trade_count = np.count_nonzero(stats) + + if trade_count == 1: + # market position, so returns should based on quotes + # calculate percentage changes on a list of quotes + changes = np.diff(Quotes.close) / Quotes[:-1].close * 100 + data = np.column_stack((Quotes[1:].time, changes)) # np.c_ + else: + # slice `:trade_count` to exclude zero values in long/short columns + data = stats[['close_time', 'perc']][:trade_count] + + # FIXME: [FutureWarning] https://github.com/numpy/numpy/issues/8383 + for close_time, perc in data: + days[fromtimestamp(close_time).date()] += perc + returns = np.array(list(days.values())) + + # if np.count_nonzero(stats) == 1: + # import pudb; pudb.set_trace() + if len(returns) >= ANNUAL_PERIOD: + return returns + + _returns = np.zeros(ANNUAL_PERIOD) + _returns[: len(returns)] = returns + return _returns + + +def annualized_sharpe_ratio(stats): + # risk_free = 0 + returns = day_percentage_returns(stats) + return np.sqrt(ANNUAL_PERIOD) * np.mean(returns) / np.std(returns) + + +def annualized_sortino_ratio(stats): + # http://www.cmegroup.com/education/files/sortino-a-sharper-ratio.pdf + required_return = 0 + returns = day_percentage_returns(stats) + mask = [returns < required_return] + tdd = np.zeros(len(returns)) + tdd[mask] = returns[mask] # keep only negative values and zeros + # "or 1" to prevent division by zero, if we don't have negative returns + tdd = np.sqrt(np.mean(np.square(tdd))) or 1 + return np.sqrt(ANNUAL_PERIOD) * np.mean(returns) / tdd diff --git a/piker/ui/qt/quantdom/portfolio.py b/piker/ui/qt/quantdom/portfolio.py new file mode 100644 index 00000000..79908d61 --- /dev/null +++ b/piker/ui/qt/quantdom/portfolio.py @@ -0,0 +1,410 @@ +"""Portfolio.""" + +import itertools +from contextlib import contextmanager +from enum import Enum, auto + +import numpy as np + +from .base import Quotes +from .performance import BriefPerformance, Performance, Stats +from .utils import fromtimestamp, timeit + +__all__ = ('Portfolio', 'Position', 'Order') + + +class BasePortfolio: + def __init__(self, balance=100_000, leverage=5): + self._initial_balance = balance + self.balance = balance + self.equity = None + # TODO: + # self.cash + # self.currency + self.leverage = leverage + self.positions = [] + + self.balance_curve = None + self.equity_curve = None + self.long_curve = None + self.short_curve = None + self.mae_curve = None + self.mfe_curve = None + + self.stats = None + self.performance = None + self.brief_performance = None + + def clear(self): + self.positions.clear() + self.balance = self._initial_balance + + @property + def initial_balance(self): + return self._initial_balance + + @initial_balance.setter + def initial_balance(self, value): + self._initial_balance = value + + def add_position(self, position): + position.ticket = len(self.positions) + 1 + self.positions.append(position) + + def position_count(self, tp=None): + if tp == Order.BUY: + return len([p for p in self.positions if p.type == Order.BUY]) + elif tp == Order.SELL: + return len([p for p in self.positions if p.type == Order.SELL]) + return len(self.positions) + + def _close_open_positions(self): + for p in self.positions: + if p.status == Position.OPEN: + p.close( + price=Quotes[-1].open, volume=p.volume, time=Quotes[-1].time + ) + + def _get_market_position(self): + p = self.positions[0] # real postions + p = Position( + symbol=p.symbol, + ptype=Order.BUY, + volume=p.volume, + price=Quotes[0].open, + open_time=Quotes[0].time, + close_price=Quotes[-1].close, + close_time=Quotes[-1].time, + id_bar_close=len(Quotes) - 1, + status=Position.CLOSED, + ) + p.profit = p.calc_profit(close_price=Quotes[-1].close) + p.profit_perc = p.profit / self._initial_balance * 100 + return p + + def _calc_equity_curve(self): + """Equity curve.""" + self.equity_curve = np.zeros_like(Quotes.time) + for i, p in enumerate(self.positions): + balance = np.sum(self.stats['All'][:i].abs) + for ibar in range(p.id_bar_open, p.id_bar_close): + profit = p.calc_profit(close_price=Quotes[ibar].close) + self.equity_curve[ibar] = balance + profit + # taking into account the real balance after the last trade + self.equity_curve[-1] = self.balance_curve[-1] + + def _calc_buy_and_hold_curve(self): + """Buy and Hold.""" + p = self._get_market_position() + self.buy_and_hold_curve = np.array( + [p.calc_profit(close_price=price) for price in Quotes.close] + ) + + def _calc_long_short_curves(self): + """Only Long/Short positions curve.""" + self.long_curve = np.zeros_like(Quotes.time) + self.short_curve = np.zeros_like(Quotes.time) + + for i, p in enumerate(self.positions): + if p.type == Order.BUY: + name = 'Long' + curve = self.long_curve + else: + name = 'Short' + curve = self.short_curve + balance = np.sum(self.stats[name][:i].abs) + # Calculate equity for this position + for ibar in range(p.id_bar_open, p.id_bar_close): + profit = p.calc_profit(close_price=Quotes[ibar].close) + curve[ibar] = balance + profit + + for name, curve in [ + ('Long', self.long_curve), + ('Short', self.short_curve), + ]: + curve[:] = fill_zeros_with_last(curve) + # taking into account the real balance after the last trade + curve[-1] = np.sum(self.stats[name].abs) + + def _calc_curves(self): + self.mae_curve = np.cumsum(self.stats['All'].mae) + self.mfe_curve = np.cumsum(self.stats['All'].mfe) + self.balance_curve = np.cumsum(self.stats['All'].abs) + self._calc_equity_curve() + self._calc_buy_and_hold_curve() + self._calc_long_short_curves() + + @contextmanager + def optimization_mode(self): + """Backup and restore current balance and positions.""" + # mode='general', + self.backup_balance = self.balance + self.backup_positions = self.positions.copy() + self.balance = self._initial_balance + self.positions.clear() + yield + self.balance = self.backup_balance + self.positions = self.backup_positions.copy() + self.backup_positions.clear() + + @timeit + def run_optimization(self, strategy, params): + keys = list(params.keys()) + vals = list(params.values()) + variants = list(itertools.product(*vals)) + self.brief_performance = BriefPerformance(shape=(len(variants),)) + with self.optimization_mode(): + for i, vals in enumerate(variants): + kwargs = {keys[n]: val for n, val in enumerate(vals)} + strategy.start(**kwargs) + self._close_open_positions() + self.brief_performance.add( + self._initial_balance, self.positions, i, kwargs + ) + self.clear() + + @timeit + def summarize(self): + self._close_open_positions() + positions = { + 'All': self.positions, + 'Long': [p for p in self.positions if p.type == Order.BUY], + 'Short': [p for p in self.positions if p.type == Order.SELL], + 'Market': [self._get_market_position()], + } + self.stats = Stats(positions) + self.performance = Performance( + self._initial_balance, self.stats, positions + ) + self._calc_curves() + + +Portfolio = BasePortfolio() + + +class PositionStatus(Enum): + OPEN = auto() + CLOSED = auto() + CANCELED = auto() + + +class Position: + + OPEN = PositionStatus.OPEN + CLOSED = PositionStatus.CLOSED + CANCELED = PositionStatus.CANCELED + + __slots__ = ( + 'type', + 'symbol', + 'ticket', + 'open_price', + 'close_price', + 'open_time', + 'close_time', + 'volume', + 'sl', + 'tp', + 'status', + 'profit', + 'profit_perc', + 'commis', + 'id_bar_open', + 'id_bar_close', + 'entry_name', + 'exit_name', + 'total_profit', + 'comment', + ) + + def __init__( + self, + symbol, + ptype, + price, + volume, + open_time, + sl=None, + tp=None, + status=OPEN, + entry_name='', + exit_name='', + comment='', + **kwargs, + ): + self.type = ptype + self.symbol = symbol + self.ticket = None + self.open_price = price + self.close_price = None + self.open_time = open_time + self.close_time = None + self.volume = volume + self.sl = sl + self.tp = tp + self.status = status + self.profit = None + self.profit_perc = None + self.commis = None + self.id_bar_open = np.where(Quotes.time == self.open_time)[0][0] + self.id_bar_close = None + self.entry_name = entry_name + self.exit_name = exit_name + self.total_profit = 0 + self.comment = comment + # self.bars_on_trade = None + # self.is_profitable = False + + for k, v in kwargs.items(): + setattr(self, k, v) + + def __repr__(self): + _type = 'LONG' if self.type == Order.BUY else 'SHORT' + time = fromtimestamp(self.open_time).strftime('%d.%m.%y %H:%M') + return '%s/%s/[%s - %.4f]' % ( + self.status.name, + _type, + time, + self.open_price, + ) + + def close(self, price, time, volume=None): + # TODO: allow closing only part of the volume + self.close_price = price + self.close_time = time + self.id_bar_close = np.where(Quotes.time == self.close_time)[0][0] + self.profit = self.calc_profit(volume=volume or self.volume) + self.profit_perc = self.profit / Portfolio.balance * 100 + + Portfolio.balance += self.profit + + self.total_profit = Portfolio.balance - Portfolio.initial_balance + self.status = self.CLOSED + + def calc_profit(self, volume=None, close_price=None): + # TODO: rewrite it + close_price = close_price or self.close_price + volume = volume or self.volume + factor = 1 if self.type == Order.BUY else -1 + price_delta = (close_price - self.open_price) * factor + if self.symbol.mode in [self.symbol.FOREX, self.symbol.CFD]: + # Margin: Lots*Contract_Size/Leverage + if ( + self.symbol.mode == self.symbol.FOREX + and self.symbol.ticker[:3] == 'USD' + ): + # Example: 'USD/JPY' + # Прибыль Размер Объем Текущий + # в пунктах пункта позиции курс + # 1 * 0.0001 * 100000 / 1.00770 + # USD/CHF: 1*0.0001*100000/1.00770 = $9.92 + # 0.01 + # USD/JPY: 1*0.01*100000/121.35 = $8.24 + # (1.00770-1.00595)/0.0001 = 17.5 пунктов + # (1.00770-1.00595)/0.0001*0.0001*100000*1/1.00770*1 + _points = price_delta / self.symbol.tick_size + _profit = ( + _points + * self.symbol.tick_size + * self.symbol.contract_size + / close_price + * volume + ) + elif ( + self.symbol.mode == self.symbol.FOREX + and self.symbol.ticker[-3:] == 'USD' + ): + # Example: 'EUR/USD' + # Profit: (close_price-open_price)*Contract_Size*Lots + # EUR/USD BUY: (1.05875-1.05850)*100000*1 = +$25 (без комиссии) + _profit = price_delta * self.symbol.contract_size * volume + else: + # Cross rates. Example: 'GBP/CHF' + # Цена пункта = + # объем поз.*размер п.*тек.курс баз.вал. к USD/тек. кросс-курс + # GBP/CHF: 100000*0.0001*1.48140/1.48985 = $9.94 + # TODO: temporary patch (same as the previous choice) - + # in the future connect to some quotes provider and get rates + _profit = price_delta * self.symbol.contract_size * volume + elif self.symbol.mode == self.symbol.FUTURES: + # Margin: Lots *InitialMargin*Percentage/100 + # Profit: (close_price-open_price)*TickPrice/TickSize*Lots + # CL BUY: (46.35-46.30)*10/0.01*1 = $50 (без учета комиссии!) + # EuroFX(6E) BUY:(1.05875-1.05850)*12.50/0.0001*1 =$31.25 (без ком) + # RTS (RIH5) BUY:(84510-84500)*12.26506/10*1 = @12.26506 (без ком) + # E-miniSP500 BUY:(2065.95-2065.25)*12.50/0.25 = $35 (без ком) + # http://americanclearing.ru/specifications.php + # http://www.moex.com/ru/contract.aspx?code=RTS-3.18 + # http://www.cmegroup.com/trading/equity-index/us-index/e-mini-sandp500_contract_specifications.html + _profit = ( + price_delta + * self.symbol.tick_value + / self.symbol.tick_size + * volume + ) + else: + # shares + _profit = price_delta * volume + + return _profit + + def calc_mae(self, low, high): + """Return [MAE] Maximum Adverse Excursion.""" + if self.type == Order.BUY: + return self.calc_profit(close_price=low) + return self.calc_profit(close_price=high) + + def calc_mfe(self, low, high): + """Return [MFE] Maximum Favorable Excursion.""" + if self.type == Order.BUY: + return self.calc_profit(close_price=high) + return self.calc_profit(close_price=low) + + +class OrderType(Enum): + BUY = auto() + SELL = auto() + BUY_LIMIT = auto() + SELL_LIMIT = auto() + BUY_STOP = auto() + SELL_STOP = auto() + + +class Order: + + BUY = OrderType.BUY + SELL = OrderType.SELL + BUY_LIMIT = OrderType.BUY_LIMIT + SELL_LIMIT = OrderType.SELL_LIMIT + BUY_STOP = OrderType.BUY_STOP + SELL_STOP = OrderType.SELL_STOP + + @staticmethod + def open(symbol, otype, price, volume, time, sl=None, tp=None): + # TODO: add margin calculation + # and if the margin is not enough - do not open the position + position = Position( + symbol=symbol, + ptype=otype, + price=price, + volume=volume, + open_time=time, + sl=sl, + tp=tp, + ) + Portfolio.add_position(position) + return position + + @staticmethod + def close(position, price, time, volume=None): + # FIXME: may be closed not the whole volume, but + # the position status will be changed to CLOSED + position.close(price=price, time=time, volume=volume) + + +def fill_zeros_with_last(arr): + """Fill empty(zero) elements (between positions).""" + index = np.arange(len(arr)) + index[arr == 0] = 0 + index = np.maximum.accumulate(index) + return arr[index] diff --git a/piker/ui/qt/quantdom/utils.py b/piker/ui/qt/quantdom/utils.py new file mode 100644 index 00000000..0324b16e --- /dev/null +++ b/piker/ui/qt/quantdom/utils.py @@ -0,0 +1,82 @@ +"""Utils.""" + +import importlib.util +import inspect +import logging +import os +import os.path +import sys +import time +from datetime import datetime +from functools import wraps + +from PyQt5 import QtCore + +__all__ = ( + 'BASE_DIR', + 'Settings', + 'timeit', + 'fromtimestamp', + 'get_data_path', + 'get_resource_path', + 'strategies_from_file', +) + + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def get_data_path(path=''): + data_path = QtCore.QStandardPaths.writableLocation( + QtCore.QStandardPaths.AppDataLocation + ) + data_path = os.path.join(data_path, path) + os.makedirs(data_path, mode=0o755, exist_ok=True) + return data_path + + +def get_resource_path(relative_path): + # PyInstaller creates a temp folder and stores path in _MEIPASS + base_path = getattr(sys, '_MEIPASS', BASE_DIR) + return os.path.join(base_path, relative_path) + + +config_path = os.path.join(get_data_path(), 'Quantdom', 'config.ini') +Settings = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat) + + +def timeit(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + t = time.time() + res = fn(*args, **kwargs) + logger = logging.getLogger('runtime') + logger.debug( + '%s.%s: %.4f sec' + % (fn.__module__, fn.__qualname__, time.time() - t) + ) + return res + + return wrapper + + +def fromtimestamp(timestamp): + if timestamp == 0: + # on Win zero timestamp cause error + return datetime(1970, 1, 1) + return datetime.fromtimestamp(timestamp) + + +def strategies_from_file(filepath): + from .strategy import AbstractStrategy + + spec = importlib.util.spec_from_file_location('Strategy', filepath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + is_strategy = lambda _class: ( # noqa:E731 + inspect.isclass(_class) + and issubclass(_class, AbstractStrategy) + and _class.__name__ != 'AbstractStrategy' + ) + return [_class for _, _class in inspect.getmembers(module, is_strategy)] From b670af484c37b78788f467d368ee7c7f2be9f63f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Jun 2020 13:48:21 -0400 Subject: [PATCH 006/206] Move UI spawning cmds to new module --- piker/brokers/cli.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index b18219bf..c05be413 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -131,7 +131,6 @@ def bars(config, symbol, count, df_output): click.echo(colorize_json(bars)) - @cli.command() @click.option('--rate', '-r', default=5, help='Logging level') @click.option('--filename', '-f', default='quotestream.jsonstream', @@ -226,43 +225,3 @@ def optsquote(config, symbol, df_output, date): click.echo(df) else: click.echo(colorize_json(quotes)) - - -@cli.command() -@click.argument('tickers', nargs=-1, required=True) -@click.pass_obj -def symbol_info(config, tickers): - """Print symbol quotes to the console - """ - # global opts - brokermod = config['brokermod'] - - quotes = trio.run(partial(core.symbol_info, brokermod, tickers)) - if not quotes: - log.error(f"No quotes could be found for {tickers}?") - return - - if len(quotes) < len(tickers): - syms = tuple(map(itemgetter('symbol'), quotes)) - for ticker in tickers: - if ticker not in syms: - brokermod.log.warn(f"Could not find symbol {ticker}?") - - click.echo(colorize_json(quotes)) - - -@cli.command() -@click.argument('pattern', required=True) -@click.pass_obj -def search(config, pattern): - """Search for symbols from broker backend(s). - """ - # global opts - brokermod = config['brokermod'] - - quotes = trio.run(partial(core.symbol_search, brokermod, pattern)) - if not quotes: - log.error(f"No matches could be found for {pattern}?") - return - - click.echo(colorize_json(quotes)) From 2ad3b6f080b322698b1a79059d62ac7b2ba5b9f8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 10 Jun 2020 13:56:09 -0400 Subject: [PATCH 007/206] Add piker chart command --- piker/ui/cli.py | 32 ++++++++++++++++++++++++++++++++ piker/ui/qt/__init__.py | 3 +++ piker/ui/qt/_exec.py | 22 ---------------------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index a869e307..d5f6a9ed 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -1,9 +1,11 @@ """ Console interface to UI components. """ +from datetime import datetime from functools import partial import os import click +import trio import tractor from ..cli import cli @@ -103,3 +105,33 @@ def optschain(config, symbol, date, tl, rate, test): loglevel=loglevel if tl else None, start_method='forkserver', ) + + +@cli.command() +@click.option('--tl', is_flag=True, help='Enable tractor logging') +@click.option('--date', '-d', help='Contracts expiry date') +@click.option('--test', '-t', help='Test quote stream file') +@click.option('--rate', '-r', default=1, help='Logging level') +@click.argument('symbol', required=True) +@click.pass_obj +def chart(config, symbol, date, tl, rate, test): + """Start an option chain UI + """ + from .qt._exec import run_qtrio + from .qt._chart import QuotesTabWidget + from .qt.quantdom.base import Symbol + from .qt.quantdom.loaders import get_quotes + + async def plot_symbol(widgets): + qtw = widgets['main'] + s = Symbol(ticker=symbol, mode=Symbol.SHARES) + get_quotes( + symbol=s.ticker, + date_from=datetime(1900, 1, 1), + date_to=datetime(2030, 12, 31), + ) + # spawn chart + qtw.update_chart(s) + await trio.sleep_forever() + + run_qtrio(plot_symbol, (), QuotesTabWidget) diff --git a/piker/ui/qt/__init__.py b/piker/ui/qt/__init__.py index e69de29b..8513b317 100644 --- a/piker/ui/qt/__init__.py +++ b/piker/ui/qt/__init__.py @@ -0,0 +1,3 @@ +""" +Super hawt Qt UI components +""" diff --git a/piker/ui/qt/_exec.py b/piker/ui/qt/_exec.py index 90286681..41b216d3 100644 --- a/piker/ui/qt/_exec.py +++ b/piker/ui/qt/_exec.py @@ -5,7 +5,6 @@ Run ``trio`` in guest mode on top of the Qt event loop. All global Qt runtime settings are mostly defined here. """ import traceback -from datetime import datetime import PyQt5 # noqa from pyqtgraph import QtGui @@ -15,10 +14,6 @@ import qdarkstyle import trio from outcome import Error -from _chart import QuotesTabWidget -from quantdom.base import Symbol -from quantdom.loaders import get_quotes - # Taken from Quantdom class MainWindow(QtGui.QMainWindow): @@ -98,20 +93,3 @@ def run_qtrio( window.setCentralWidget(instance) window.show() app.exec_() - - -async def plot_aapl(widgets): - qtw = widgets['main'] - s = Symbol(ticker='AAPL', mode=Symbol.SHARES) - get_quotes( - symbol=s.ticker, - date_from=datetime(1900, 1, 1), - date_to=datetime(2030, 12, 31), - ) - # spawn chart - qtw.update_chart(s) - await trio.sleep_forever() - - -if __name__ == '__main__': - run_qtrio(plot_aapl, (), QuotesTabWidget) From c8afdb0adcb450e5bce3db60d021f4fc4694a64c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 13 Jun 2020 09:49:21 -0400 Subject: [PATCH 008/206] Styling, start re-org, commenting - Move out equity plotting to new module. - Make axis margins and fonts look good on i3. - Adjust axis labels colors to gray. - Start commenting a lot of the code after figuring out what it all does when cross referencing with ``pyqtgraph``. - Add option to move date axis to middle. --- piker/ui/qt/_exec.py | 1 - piker/ui/qt/quantdom/_equity.py | 188 ++++++++++++++ piker/ui/qt/quantdom/base.py | 16 +- piker/ui/qt/quantdom/charts.py | 419 +++++++++++++----------------- piker/ui/qt/quantdom/loaders.py | 14 +- piker/ui/qt/quantdom/portfolio.py | 4 +- 6 files changed, 405 insertions(+), 237 deletions(-) create mode 100644 piker/ui/qt/quantdom/_equity.py diff --git a/piker/ui/qt/_exec.py b/piker/ui/qt/_exec.py index 41b216d3..524a79e5 100644 --- a/piker/ui/qt/_exec.py +++ b/piker/ui/qt/_exec.py @@ -32,7 +32,6 @@ def run_qtrio( args, main_widget, ) -> None: - # avoids annoying message when entering debugger from qt loop pyqtRemoveInputHook() diff --git a/piker/ui/qt/quantdom/_equity.py b/piker/ui/qt/quantdom/_equity.py new file mode 100644 index 00000000..e31c899d --- /dev/null +++ b/piker/ui/qt/quantdom/_equity.py @@ -0,0 +1,188 @@ +""" +Strategy and performance charting +""" +import numpy as np +import pyqtgraph as pg +from PyQt5 import QtCore, QtGui + +from .base import Quotes +from .portfolio import Portfolio +from .utils import timeit +from .charts import ( + PriceAxis, + CHART_MARGINS, + SampleLegendItem, + YAxisLabel, + CrossHairItem, +) + + +class EquityChart(QtGui.QWidget): + + eq_pen_pos_color = pg.mkColor('#00cc00') + eq_pen_neg_color = pg.mkColor('#cc0000') + eq_brush_pos_color = pg.mkColor('#40ee40') + eq_brush_neg_color = pg.mkColor('#ee4040') + long_pen_color = pg.mkColor('#008000') + short_pen_color = pg.mkColor('#800000') + buy_and_hold_pen_color = pg.mkColor('#4444ff') + + def __init__(self): + super().__init__() + self.xaxis = pg.DateAxisItem() + self.xaxis.setStyle(tickTextOffset=7, textFillLimits=[(0, 0.80)]) + self.yaxis = PriceAxis() + + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + + self.chart = pg.PlotWidget( + axisItems={'bottom': self.xaxis, 'right': self.yaxis}, + enableMenu=False, + ) + self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart.showGrid(x=True, y=True) + self.chart.hideAxis('left') + self.chart.showAxis('right') + + self.chart.setCursor(QtCore.Qt.BlankCursor) + self.chart.sigXRangeChanged.connect(self._update_yrange_limits) + + self.layout.addWidget(self.chart) + + def _add_legend(self): + legend = pg.LegendItem((140, 100), offset=(10, 10)) + legend.setParentItem(self.chart.getPlotItem()) + + for arr, item in self.curves: + legend.addItem( + SampleLegendItem(item), + item.opts['name'] + if not isinstance(item, tuple) + else item[0].opts['name'], + ) + + def _add_ylabels(self): + self.ylabels = [] + for arr, item in self.curves: + color = ( + item.opts['pen'] + if not isinstance(item, tuple) + else [i.opts['pen'] for i in item] + ) + label = YAxisLabel(parent=self.yaxis, color=color) + self.ylabels.append(label) + + def _update_ylabels(self, vb, rbar): + for i, curve in enumerate(self.curves): + arr, item = curve + ylast = arr[rbar] + ypos = vb.mapFromView(QtCore.QPointF(0, ylast)).y() + axlabel = self.ylabels[i] + axlabel.update_label_test(ypos=ypos, ydata=ylast) + + def _update_yrange_limits(self, vb=None): + if not hasattr(self, 'min_curve'): + return + vr = self.chart.viewRect() + lbar, rbar = int(vr.left()), int(vr.right()) + ylow = self.min_curve[lbar:rbar].min() * 1.1 + yhigh = self.max_curve[lbar:rbar].max() * 1.1 + + std = np.std(self.max_curve[lbar:rbar]) * 4 + self.chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + self.chart.setYRange(ylow, yhigh) + self._update_ylabels(vb, rbar) + + @timeit + def plot(self): + equity_curve = Portfolio.equity_curve + eq_pos = np.zeros_like(equity_curve) + eq_neg = np.zeros_like(equity_curve) + eq_pos[equity_curve >= 0] = equity_curve[equity_curve >= 0] + eq_neg[equity_curve <= 0] = equity_curve[equity_curve <= 0] + + # Equity + self.eq_pos_curve = pg.PlotCurveItem( + eq_pos, + name='Equity', + fillLevel=0, + antialias=True, + pen=self.eq_pen_pos_color, + brush=self.eq_brush_pos_color, + ) + self.eq_neg_curve = pg.PlotCurveItem( + eq_neg, + name='Equity', + fillLevel=0, + antialias=True, + pen=self.eq_pen_neg_color, + brush=self.eq_brush_neg_color, + ) + self.chart.addItem(self.eq_pos_curve) + self.chart.addItem(self.eq_neg_curve) + + # Only Long + self.long_curve = pg.PlotCurveItem( + Portfolio.long_curve, + name='Only Long', + pen=self.long_pen_color, + antialias=True, + ) + self.chart.addItem(self.long_curve) + + # Only Short + self.short_curve = pg.PlotCurveItem( + Portfolio.short_curve, + name='Only Short', + pen=self.short_pen_color, + antialias=True, + ) + self.chart.addItem(self.short_curve) + + # Buy and Hold + self.buy_and_hold_curve = pg.PlotCurveItem( + Portfolio.buy_and_hold_curve, + name='Buy and Hold', + pen=self.buy_and_hold_pen_color, + antialias=True, + ) + self.chart.addItem(self.buy_and_hold_curve) + + self.curves = [ + (Portfolio.equity_curve, (self.eq_pos_curve, self.eq_neg_curve)), + (Portfolio.long_curve, self.long_curve), + (Portfolio.short_curve, self.short_curve), + (Portfolio.buy_and_hold_curve, self.buy_and_hold_curve), + ] + + self._add_legend() + self._add_ylabels() + + ch = CrossHairItem(self.chart) + self.chart.addItem(ch) + + arrs = ( + Portfolio.equity_curve, + Portfolio.buy_and_hold_curve, + Portfolio.long_curve, + Portfolio.short_curve, + ) + np_arrs = np.concatenate(arrs) + _min = abs(np_arrs.min()) * -1.1 + _max = np_arrs.max() * 1.1 + + self.chart.setLimits( + xMin=Quotes[0].id, + xMax=Quotes[-1].id, + yMin=_min, + yMax=_max, + minXRange=60, + ) + + self.min_curve = arrs[0].copy() + self.max_curve = arrs[0].copy() + for arr in arrs[1:]: + self.min_curve = np.minimum(self.min_curve, arr) + self.max_curve = np.maximum(self.max_curve, arr) diff --git a/piker/ui/qt/quantdom/base.py b/piker/ui/qt/quantdom/base.py index a53b18c2..9f5eb074 100644 --- a/piker/ui/qt/quantdom/base.py +++ b/piker/ui/qt/quantdom/base.py @@ -10,6 +10,16 @@ from .const import ChartType, TimeFrame __all__ = ('Indicator', 'Symbol', 'Quotes') +# I actually can't think of a worse reason to override an array than +# this: +# - a method .new() that mutates the data from an input data frame +# - mutating the time column wholesale based on a setting +# - enforcing certain fields / columns +# - zero overriding of any of the array interface for the purposes of +# a different underlying implementation. + +# Literally all this can be done in a simple function with way less +# confusion for the reader. class BaseQuotes(np.recarray): def __new__(cls, shape=None, dtype=None, order='C'): dt = np.dtype( @@ -49,6 +59,8 @@ class BaseQuotes(np.recarray): minutes = int(np.diff(self.time[-10:]).min() / 60) self.timeframe = tf.get(minutes) or tf[default_tf] + # bruh this isn't creating anything it's copying data in + # from a data frame... def new(self, data, source=None, default_tf=None): shape = (len(data),) self.resize(shape, refcheck=False) @@ -77,7 +89,7 @@ class BaseQuotes(np.recarray): return self def convert_dates(self, dates): - return np.array([d.timestamp() for d in dates]) + return np.array([d.timestamp().time for d in dates]) class SymbolType(Enum): @@ -129,4 +141,6 @@ class Indicator: self.lineStyle.update(kwargs) +# This creates a global array that seems to be shared between all +# charting UI components Quotes = BaseQuotes() diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index c1cf3712..ef158a70 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -1,7 +1,11 @@ -"""Chart.""" +""" +Real-time quotes charting components +""" +from typing import Callable import numpy as np import pyqtgraph as pg +from pyqtgraph import functions as fn from PyQt5 import QtCore, QtGui from .base import Quotes @@ -9,14 +13,22 @@ from .const import ChartType from .portfolio import Order, Portfolio from .utils import fromtimestamp, timeit -__all__ = ('QuotesChart', 'EquityChart') +__all__ = ('QuotesChart') +# white background for tinas like xb # pg.setConfigOption('background', 'w') -CHART_MARGINS = (0, 0, 20, 5) + +# margins +CHART_MARGINS = (0, 0, 10, 3) + +# chart-wide font +_font = QtGui.QFont("Hack", 4) +_i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) class SampleLegendItem(pg.graphicsItems.LegendItem.ItemSample): + def paint(self, p, *args): p.setRenderHint(p.Antialiasing) if isinstance(self.item, tuple): @@ -51,32 +63,58 @@ class SampleLegendItem(pg.graphicsItems.LegendItem.ItemSample): class PriceAxis(pg.AxisItem): + def __init__(self): super().__init__(orientation='right') - self.style.update({'textFillLimits': [(0, 0.8)]}) + # self.setStyle(**{ + # 'textFillLimits': [(0, 0.8)], + # # 'tickTextWidth': 5, + # # 'tickTextHeight': 5, + # # 'autoExpandTextSpace': True, + # # 'maxTickLength': -20, + # }) + # self.setLabel(**{'font-size':'10pt'}) + self.setTickFont(_font) - def tickStrings(self, vals, scale, spacing): - digts = max(0, np.ceil(-np.log10(spacing * scale))) - return [ - ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals - ] + # XXX: drop for now since it just eats up h space + + # def tickStrings(self, vals, scale, spacing): + # digts = max(0, np.ceil(-np.log10(spacing * scale))) + # return [ + # ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals + # ] -class DateAxis(pg.AxisItem): - tick_tpl = {'D1': '%d %b\n%Y'} +class FromTimeFieldDateAxis(pg.AxisItem): + tick_tpl = {'D1': '%Y-%b-%d'} def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.setTickFont(_font) self.quotes_count = len(Quotes) - 1 + # default styling + self.setStyle( + tickTextOffset=7, + textFillLimits=[(0, 0.90)], + # TODO: doesn't seem to work -> bug in pyqtgraph? + # tickTextHeight=2, + ) + def tickStrings(self, values, scale, spacing): + # if len(values) > 1 or not values: + # values = Quotes.time + + # strings = super().tickStrings(values, scale, spacing) s_period = 'D1' strings = [] for ibar in values: if ibar > self.quotes_count: return strings dt_tick = fromtimestamp(Quotes[int(ibar)].time) - strings.append(dt_tick.strftime(self.tick_tpl[s_period])) + strings.append( + dt_tick.strftime(self.tick_tpl[s_period]) + ) return strings @@ -121,7 +159,8 @@ class CenteredTextItem(QtGui.QGraphicsTextItem): class AxisLabel(pg.GraphicsObject): - bg_color = pg.mkColor('#dbdbdb') + # bg_color = pg.mkColor('#a9a9a9') + bg_color = pg.mkColor('#808080') fg_color = pg.mkColor('#000000') def __init__(self, parent=None, digits=0, color=None, opacity=1, **kwargs): @@ -130,13 +169,15 @@ class AxisLabel(pg.GraphicsObject): self.opacity = opacity self.label_str = '' self.digits = digits - self.quotes_count = len(Quotes) - 1 + # self.quotes_count = len(Quotes) - 1 + if isinstance(color, QtGui.QPen): self.bg_color = color.color() self.fg_color = pg.mkColor('#ffffff') elif isinstance(color, list): self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} self.fg_color = pg.mkColor('#ffffff') + self.setFlag(self.ItemIgnoresTransformations) def tick_to_string(self, tick_pos): @@ -169,6 +210,8 @@ class AxisLabel(pg.GraphicsObject): p.setOpacity(self.opacity) p.fillRect(option.rect, bg_color) p.setOpacity(1) + p.setFont(_font) + p.drawText(option.rect, self.text_flags, self.label_str) @@ -184,12 +227,12 @@ class XAxisLabel(AxisLabel): return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 60, 38) + return QtCore.QRectF(0, 0, 145, 50) def update_label(self, evt_post, point_view): ibar = point_view.x() - if ibar > self.quotes_count: - return + # if ibar > self.quotes_count: + # return self.label_str = self.tick_to_string(ibar) width = self.boundingRect().width() offset = 0 # if have margins @@ -207,7 +250,7 @@ class YAxisLabel(AxisLabel): return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 74, 24) + return QtCore.QRectF(0, 0, 80, 40) def update_label(self, evt_post, point_view): self.label_str = self.tick_to_string(point_view.y()) @@ -217,10 +260,28 @@ class YAxisLabel(AxisLabel): self.setPos(new_pos) +# TODO: convert this to a ``ViewBox`` type giving us +# control over mouse scrolling and a context menu class CustomPlotWidget(pg.PlotWidget): sig_mouse_leave = QtCore.Signal(object) sig_mouse_enter = QtCore.Signal(object) + # def wheelEvent(self, ev, axis=None): + # if axis in (0, 1): + # mask = [False, False] + # mask[axis] = self.state['mouseEnabled'][axis] + # else: + # mask = self.state['mouseEnabled'][:] + + # # actual scaling factor + # s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) + # s = [(None if m is False else s) for m in mask] + + # self._resetTarget() + # self.scaleBy(s, center) + # ev.accept() + # self.sigRangeChangedManually.emit(mask) + def enterEvent(self, ev): # noqa self.sig_mouse_enter.emit(self) @@ -229,7 +290,8 @@ class CustomPlotWidget(pg.PlotWidget): self.scene().leaveEvent(ev) -_rate_limit = 30 +_mouse_rate_limit = 60 +_xaxis_at = 'bottom' class CrossHairItem(pg.GraphicsObject): @@ -249,7 +311,7 @@ class CrossHairItem(pg.GraphicsObject): self.proxy_moved = pg.SignalProxy( self.parent.scene().sigMouseMoved, - rateLimit=_rate_limit, + rateLimit=_mouse_rate_limit, slot=self.mouseMoved, ) @@ -258,39 +320,52 @@ class CrossHairItem(pg.GraphicsObject): ) indicators = indicators or [] + if indicators: + # when there are indicators present in sub-plot rows + # take the last one (nearest to the bottom) and place the + # crosshair label on it's x-axis. last_ind = indicators[-1] - self.xaxis_label = XAxisLabel( - parent=last_ind.getAxis('bottom'), opacity=1 - ) + self.proxy_enter = pg.SignalProxy( self.parent.sig_mouse_enter, - rateLimit=_rate_limit, + rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Enter', False), ) self.proxy_leave = pg.SignalProxy( self.parent.sig_mouse_leave, - rateLimit=_rate_limit, + rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Leave', False), ) + + if _xaxis_at == 'bottom': + # place below is last indicator subplot + self.xaxis_label = XAxisLabel( + parent=last_ind.getAxis('bottom'), opacity=1 + ) else: + # keep x-axis right below main chart self.xaxis_label = XAxisLabel(parent=self.xaxis, opacity=1) for i in indicators: + # add vertial and horizonal lines and a y-axis label vl = i.addLine(x=0, pen=self.pen, movable=False) hl = i.addLine(y=0, pen=self.pen, movable=False) yl = YAxisLabel(parent=i.getAxis('right'), opacity=1) + px_moved = pg.SignalProxy( - i.scene().sigMouseMoved, rateLimit=_rate_limit, slot=self.mouseMoved + i.scene().sigMouseMoved, + rateLimit=_mouse_rate_limit, + slot=self.mouseMoved ) px_enter = pg.SignalProxy( i.sig_mouse_enter, - rateLimit=_rate_limit, + rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Enter', i), ) px_leave = pg.SignalProxy( i.sig_mouse_leave, - rateLimit=_rate_limit, + rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Leave', i), ) self.indicators[i] = { @@ -302,6 +377,7 @@ class CrossHairItem(pg.GraphicsObject): def mouseAction(self, action, ind=False): # noqa if action == 'Enter': + # show horiz line and y-label if ind: self.indicators[ind]['hl'].show() self.indicators[ind]['yl'].show() @@ -310,6 +386,7 @@ class CrossHairItem(pg.GraphicsObject): self.yaxis_label.show() self.hline.show() else: # Leave + # hide horiz line and y-label if ind: self.indicators[ind]['hl'].hide() self.indicators[ind]['yl'].hide() @@ -319,16 +396,29 @@ class CrossHairItem(pg.GraphicsObject): self.hline.hide() def mouseMoved(self, evt): # noqa + """Update horizonal and vertical lines when mouse moves inside + either the main chart or any indicator subplot. + """ + pos = evt[0] + + # if the mouse is within the parent ``CustomPlotWidget`` if self.parent.sceneBoundingRect().contains(pos): # mouse_point = self.vb.mapSceneToView(pos) mouse_point = self.parent.mapToView(pos) + + # move the vertial line to the current x coordinate self.vline.setX(mouse_point.x()) + + # update the label on the bottom of the crosshair self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) + + # update the vertical line in any indicators subplots for opts in self.indicators.values(): opts['vl'].setX(mouse_point.x()) if self.activeIndicator: + # vertial position of the mouse is inside an indicator mouse_point_ind = self.activeIndicator.mapToView(pos) self.indicators[self.activeIndicator]['hl'].setY( mouse_point_ind.y() @@ -337,6 +427,7 @@ class CrossHairItem(pg.GraphicsObject): evt_post=pos, point_view=mouse_point_ind ) else: + # vertial position of the mouse is inside and main chart self.hline.setY(mouse_point.y()) self.yaxis_label.update_label( evt_post=pos, point_view=mouse_point @@ -351,28 +442,31 @@ class CrossHairItem(pg.GraphicsObject): class BarItem(pg.GraphicsObject): - w = 0.35 - bull_brush = pg.mkPen('#00cc00') - bear_brush = pg.mkPen('#fa0000') + w = 0.5 + + bull_brush = bear_brush = pg.mkPen('#808080') + # bull_brush = pg.mkPen('#00cc00') + # bear_brush = pg.mkPen('#fa0000') def __init__(self): super().__init__() self.generatePicture() def _generate(self, p): - hl = np.array( + high_to_low = np.array( [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] ) - op = np.array( - [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) for q in Quotes] + open_stick = np.array( + [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) + for q in Quotes] ) - cl = np.array( + close_stick = np.array( [ QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) for q in Quotes ] ) - lines = np.concatenate([hl, op, cl]) + lines = np.concatenate([high_to_low, open_stick, close_stick]) long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) @@ -382,6 +476,7 @@ class BarItem(pg.GraphicsObject): p.setPen(self.bear_brush) p.drawLines(*lines[short_bars]) + # TODO: this is the routine to be retriggered for redraw @timeit def generatePicture(self): self.picture = QtGui.QPicture() @@ -412,7 +507,10 @@ class CandlestickItem(BarItem): ) p.setPen(self.line_pen) - p.drawLines([QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes]) + p.drawLines( + [QtCore.QLineF(q.id, q.low, q.id, q.high) + for q in Quotes] + ) p.setBrush(self.bull_brush) p.drawRects(*rects[Quotes.close > Quotes.open]) @@ -421,6 +519,34 @@ class CandlestickItem(BarItem): p.drawRects(*rects[Quotes.close < Quotes.open]) +def _configure_quotes_chart( + chart: CustomPlotWidget, + style: ChartType, + update_yrange_limits: Callable, +) -> None: + """Update and format a chart with quotes data. + """ + + chart.hideAxis('left') + chart.showAxis('right') + chart.addItem(_get_chart_points(style)) + chart.setLimits( + xMin=Quotes[0].id, + xMax=Quotes[-1].id, + minXRange=60, + yMin=Quotes.low.min() * 0.98, + yMax=Quotes.high.max() * 1.02, + ) + chart.showGrid(x=True, y=True) + chart.setCursor(QtCore.Qt.BlankCursor) + + # assign callback for rescaling y-axis automatically + # based on y-range contents + # TODO: this can likely be ported to built-in: .enableAutoRange() + # but needs testing + chart.sigXRangeChanged.connect(update_yrange_limits) + + class QuotesChart(QtGui.QWidget): long_pen = pg.mkPen('#006000') @@ -436,19 +562,21 @@ class QuotesChart(QtGui.QWidget): self.style = ChartType.BAR self.indicators = [] - self.xaxis = DateAxis(orientation='bottom') - self.xaxis.setStyle( - tickTextOffset=7, textFillLimits=[(0, 0.80)], showValues=False - ) + self.xaxis = FromTimeFieldDateAxis(orientation='bottom') + # self.xaxis = pg.DateAxisItem() - self.xaxis_ind = DateAxis(orientation='bottom') - self.xaxis_ind.setStyle(tickTextOffset=7, textFillLimits=[(0, 0.80)]) + self.xaxis_ind = FromTimeFieldDateAxis(orientation='bottom') + + if _xaxis_at == 'bottom': + self.xaxis.setStyle(showValues=False) + else: + self.xaxis_ind.setStyle(showValues=False) self.layout = QtGui.QVBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) - self.splitter.setHandleWidth(4) + self.splitter.setHandleWidth(5) self.layout.addWidget(self.splitter) @@ -473,21 +601,6 @@ class QuotesChart(QtGui.QWidget): del self.signals_group_text self.signals_visible = False - def _update_quotes_chart(self): - self.chart.hideAxis('left') - self.chart.showAxis('right') - self.chart.addItem(_get_chart_points(self.style)) - self.chart.setLimits( - xMin=Quotes[0].id, - xMax=Quotes[-1].id, - minXRange=60, - yMin=Quotes.low.min() * 0.98, - yMax=Quotes.high.max() * 1.02, - ) - self.chart.showGrid(x=True, y=True) - self.chart.setCursor(QtCore.Qt.BlankCursor) - self.chart.sigXRangeChanged.connect(self._update_yrange_limits) - def _update_ind_charts(self): for ind, d in self.indicators: curve = pg.PlotDataItem(d, pen='b', antialias=True) @@ -513,10 +626,14 @@ class QuotesChart(QtGui.QWidget): self.splitter.setSizes(sizes) # , int(self.height()*0.2) def _update_yrange_limits(self): + """Callback for each y-range update. + """ vr = self.chart.viewRect() lbar, rbar = int(vr.left()), int(vr.right()) + if self.signals_visible: self._show_text_signals(lbar, rbar) + bars = Quotes[lbar:rbar] ylow = bars.low.min() * 0.98 yhigh = bars.high.max() * 1.02 @@ -540,15 +657,17 @@ class QuotesChart(QtGui.QWidget): axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, enableMenu=False, ) - # self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + # TODO: this is where we would load an indicator chain inds = [Quotes.open] for d in inds: ind = CustomPlotWidget( parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, + # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, enableMenu=False, ) ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) @@ -556,7 +675,11 @@ class QuotesChart(QtGui.QWidget): # self.splitter.addWidget(ind) self.indicators.append((ind, d)) - self._update_quotes_chart() + _configure_quotes_chart( + self.chart, + self.style, + self._update_yrange_limits + ) self._update_ind_charts() self._update_sizes() @@ -620,177 +743,9 @@ class QuotesChart(QtGui.QWidget): self.signals_visible = True -class EquityChart(QtGui.QWidget): - - eq_pen_pos_color = pg.mkColor('#00cc00') - eq_pen_neg_color = pg.mkColor('#cc0000') - eq_brush_pos_color = pg.mkColor('#40ee40') - eq_brush_neg_color = pg.mkColor('#ee4040') - long_pen_color = pg.mkColor('#008000') - short_pen_color = pg.mkColor('#800000') - buy_and_hold_pen_color = pg.mkColor('#4444ff') - - def __init__(self): - super().__init__() - self.xaxis = DateAxis(orientation='bottom') - self.xaxis.setStyle(tickTextOffset=7, textFillLimits=[(0, 0.80)]) - self.yaxis = PriceAxis() - - self.layout = QtGui.QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) - - self.chart = pg.PlotWidget( - axisItems={'bottom': self.xaxis, 'right': self.yaxis}, - enableMenu=False, - ) - self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) - self.chart.showGrid(x=True, y=True) - self.chart.hideAxis('left') - self.chart.showAxis('right') - - self.chart.setCursor(QtCore.Qt.BlankCursor) - self.chart.sigXRangeChanged.connect(self._update_yrange_limits) - - self.layout.addWidget(self.chart) - - def _add_legend(self): - legend = pg.LegendItem((140, 100), offset=(10, 10)) - legend.setParentItem(self.chart.getPlotItem()) - - for arr, item in self.curves: - legend.addItem( - SampleLegendItem(item), - item.opts['name'] - if not isinstance(item, tuple) - else item[0].opts['name'], - ) - - def _add_ylabels(self): - self.ylabels = [] - for arr, item in self.curves: - color = ( - item.opts['pen'] - if not isinstance(item, tuple) - else [i.opts['pen'] for i in item] - ) - label = YAxisLabel(parent=self.yaxis, color=color) - self.ylabels.append(label) - - def _update_ylabels(self, vb, rbar): - for i, curve in enumerate(self.curves): - arr, item = curve - ylast = arr[rbar] - ypos = vb.mapFromView(QtCore.QPointF(0, ylast)).y() - axlabel = self.ylabels[i] - axlabel.update_label_test(ypos=ypos, ydata=ylast) - - def _update_yrange_limits(self, vb=None): - if not hasattr(self, 'min_curve'): - return - vr = self.chart.viewRect() - lbar, rbar = int(vr.left()), int(vr.right()) - ylow = self.min_curve[lbar:rbar].min() * 1.1 - yhigh = self.max_curve[lbar:rbar].max() * 1.1 - - std = np.std(self.max_curve[lbar:rbar]) * 4 - self.chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - self.chart.setYRange(ylow, yhigh) - self._update_ylabels(vb, rbar) - - @timeit - def plot(self): - equity_curve = Portfolio.equity_curve - eq_pos = np.zeros_like(equity_curve) - eq_neg = np.zeros_like(equity_curve) - eq_pos[equity_curve >= 0] = equity_curve[equity_curve >= 0] - eq_neg[equity_curve <= 0] = equity_curve[equity_curve <= 0] - - # Equity - self.eq_pos_curve = pg.PlotCurveItem( - eq_pos, - name='Equity', - fillLevel=0, - antialias=True, - pen=self.eq_pen_pos_color, - brush=self.eq_brush_pos_color, - ) - self.eq_neg_curve = pg.PlotCurveItem( - eq_neg, - name='Equity', - fillLevel=0, - antialias=True, - pen=self.eq_pen_neg_color, - brush=self.eq_brush_neg_color, - ) - self.chart.addItem(self.eq_pos_curve) - self.chart.addItem(self.eq_neg_curve) - - # Only Long - self.long_curve = pg.PlotCurveItem( - Portfolio.long_curve, - name='Only Long', - pen=self.long_pen_color, - antialias=True, - ) - self.chart.addItem(self.long_curve) - - # Only Short - self.short_curve = pg.PlotCurveItem( - Portfolio.short_curve, - name='Only Short', - pen=self.short_pen_color, - antialias=True, - ) - self.chart.addItem(self.short_curve) - - # Buy and Hold - self.buy_and_hold_curve = pg.PlotCurveItem( - Portfolio.buy_and_hold_curve, - name='Buy and Hold', - pen=self.buy_and_hold_pen_color, - antialias=True, - ) - self.chart.addItem(self.buy_and_hold_curve) - - self.curves = [ - (Portfolio.equity_curve, (self.eq_pos_curve, self.eq_neg_curve)), - (Portfolio.long_curve, self.long_curve), - (Portfolio.short_curve, self.short_curve), - (Portfolio.buy_and_hold_curve, self.buy_and_hold_curve), - ] - - self._add_legend() - self._add_ylabels() - - ch = CrossHairItem(self.chart) - self.chart.addItem(ch) - - arrs = ( - Portfolio.equity_curve, - Portfolio.buy_and_hold_curve, - Portfolio.long_curve, - Portfolio.short_curve, - ) - np_arrs = np.concatenate(arrs) - _min = abs(np_arrs.min()) * -1.1 - _max = np_arrs.max() * 1.1 - - self.chart.setLimits( - xMin=Quotes[0].id, - xMax=Quotes[-1].id, - yMin=_min, - yMax=_max, - minXRange=60, - ) - - self.min_curve = arrs[0].copy() - self.max_curve = arrs[0].copy() - for arr in arrs[1:]: - self.min_curve = np.minimum(self.min_curve, arr) - self.max_curve = np.maximum(self.max_curve, arr) - - +# this function is borderline rediculous. +# The creation of these chart types mutates all the input data +# inside each type's constructor (mind blown) def _get_chart_points(style): if style == ChartType.CANDLESTICK: return CandlestickItem() diff --git a/piker/ui/qt/quantdom/loaders.py b/piker/ui/qt/quantdom/loaders.py index 0bb18d89..c3bc6069 100644 --- a/piker/ui/qt/quantdom/loaders.py +++ b/piker/ui/qt/quantdom/loaders.py @@ -3,7 +3,9 @@ import logging import os.path import pickle +import datetime +import numpy as np import pandas as pd import pandas_datareader.data as web from pandas_datareader._utils import RemoteDataError @@ -15,7 +17,7 @@ from pandas_datareader.data import ( from pandas_datareader.nasdaq_trader import get_nasdaq_symbols from pandas_datareader.exceptions import ImmediateDeprecationError -from .base import Quotes +from .base import Quotes, Symbol from .utils import get_data_path, timeit __all__ = ( @@ -71,7 +73,15 @@ class QuotesLoader: @classmethod @timeit - def get_quotes(cls, symbol, date_from, date_to): + def get_quotes( + cls, + symbol: Symbol, + date_from: datetime.datetime, + date_to: datetime.datetime, + ) -> Quotes: + """Retrieve quotes data from a provider and return a ``numpy.ndarray`` subtype. + """ + quotes = None fpath = cls._get_file_path(symbol, cls.timeframe, date_from, date_to) if os.path.exists(fpath): diff --git a/piker/ui/qt/quantdom/portfolio.py b/piker/ui/qt/quantdom/portfolio.py index 79908d61..40f9b44b 100644 --- a/piker/ui/qt/quantdom/portfolio.py +++ b/piker/ui/qt/quantdom/portfolio.py @@ -62,7 +62,9 @@ class BasePortfolio: for p in self.positions: if p.status == Position.OPEN: p.close( - price=Quotes[-1].open, volume=p.volume, time=Quotes[-1].time + price=Quotes[-1].open, + volume=p.volume, + time=Quotes[-1].time ) def _get_market_position(self): From 730241bb8a496ed4ece5b85b353a5b5294cd1eb4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 13 Jun 2020 19:25:44 -0400 Subject: [PATCH 009/206] Add scrolling from right and cross-hair Modify the default ``ViewBox`` scroll to zoom behaviour such that whatever right-most point is visible is used as the "center" for zooming. Add a "traditional" cross-hair cursor. --- piker/ui/qt/_exec.py | 7 +-- piker/ui/qt/quantdom/charts.py | 100 +++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 34 deletions(-) diff --git a/piker/ui/qt/_exec.py b/piker/ui/qt/_exec.py index 524a79e5..39aea685 100644 --- a/piker/ui/qt/_exec.py +++ b/piker/ui/qt/_exec.py @@ -39,11 +39,10 @@ def run_qtrio( if app is None: app = PyQt5.QtWidgets.QApplication([]) - # This code is from Nathaniel: - - # This is substantially faster than using a signal... for some + # This code is from Nathaniel, and I quote: + # "This is substantially faster than using a signal... for some # reason Qt signal dispatch is really slow (and relies on events - # underneath anyway, so this is strictly less work) + # underneath anyway, so this is strictly less work)." REENTER_EVENT = QtCore.QEvent.Type(QtCore.QEvent.registerEventType()) class ReenterEvent(QtCore.QEvent): diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index ef158a70..25a23a23 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -260,32 +260,54 @@ class YAxisLabel(AxisLabel): self.setPos(new_pos) +class ScrollFromRightView(pg.ViewBox): + + def wheelEvent(self, ev, axis=None): + if axis in (0, 1): + mask = [False, False] + mask[axis] = self.state['mouseEnabled'][axis] + else: + mask = self.state['mouseEnabled'][:] + + # actual scaling factor + s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) + s = [(None if m is False else s) for m in mask] + + # XXX: scroll "around" the right most element in the view + # center = pg.Point( + # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) + # ) + furthest_right_coord = self.boundingRect().topRight() + center = pg.Point( + fn.invertQTransform(self.childGroup.transform()).map(furthest_right_coord) + ) + + self._resetTarget() + self.scaleBy(s, center) + ev.accept() + self.sigRangeChangedManually.emit(mask) + + # TODO: convert this to a ``ViewBox`` type giving us # control over mouse scrolling and a context menu +# This is a sub-class of ``GracphicView`` which can +# take a ``background`` color setting. class CustomPlotWidget(pg.PlotWidget): + """``GraphicsView`` subtype containing a single ``PlotItem``. + + (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we + eventually want multiple plots managed together). + """ + sig_mouse_leave = QtCore.Signal(object) sig_mouse_enter = QtCore.Signal(object) - # def wheelEvent(self, ev, axis=None): - # if axis in (0, 1): - # mask = [False, False] - # mask[axis] = self.state['mouseEnabled'][axis] - # else: - # mask = self.state['mouseEnabled'][:] - - # # actual scaling factor - # s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) - # s = [(None if m is False else s) for m in mask] - - # self._resetTarget() - # self.scaleBy(s, center) - # ev.accept() - # self.sigRangeChangedManually.emit(mask) - def enterEvent(self, ev): # noqa + pg.PlotWidget.enterEvent(self, ev) self.sig_mouse_enter.emit(self) def leaveEvent(self, ev): # noqa + pg.PlotWidget.leaveEvent(self, ev) self.sig_mouse_leave.emit(self) self.scene().leaveEvent(ev) @@ -441,6 +463,8 @@ class CrossHairItem(pg.GraphicsObject): class BarItem(pg.GraphicsObject): + # XXX: From the customGraphicsItem.py example: + # The only required methods are paint() and boundingRect() w = 0.5 @@ -452,7 +476,19 @@ class BarItem(pg.GraphicsObject): super().__init__() self.generatePicture() + # TODO: this is the routine to be retriggered for redraw + @timeit + def generatePicture(self): + # pre-computing a QPicture object allows paint() to run much + # more quickly, rather than re-drawing the shapes every time. + self.picture = QtGui.QPicture() + p = QtGui.QPainter(self.picture) + self._generate(p) + p.end() + def _generate(self, p): + # XXX: overloaded method to allow drawing other candle types + high_to_low = np.array( [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] ) @@ -476,18 +512,14 @@ class BarItem(pg.GraphicsObject): p.setPen(self.bear_brush) p.drawLines(*lines[short_bars]) - # TODO: this is the routine to be retriggered for redraw - @timeit - def generatePicture(self): - self.picture = QtGui.QPicture() - p = QtGui.QPainter(self.picture) - self._generate(p) - p.end() - def paint(self, p, *args): p.drawPicture(0, 0, self.picture) def boundingRect(self): + # boundingRect _must_ indicate the entire area that will be + # drawn on or else we will get artifacts and possibly crashing. + # (in this case, QPicture does all the work of computing the + # bouning rect for us) return QtCore.QRectF(self.picture.boundingRect()) @@ -529,7 +561,13 @@ def _configure_quotes_chart( chart.hideAxis('left') chart.showAxis('right') + + # adds all bar/candle graphics objects for each + # data point in the np array buffer to + # be drawn on next render cycle chart.addItem(_get_chart_points(style)) + + # set panning limits chart.setLimits( xMin=Quotes[0].id, xMax=Quotes[-1].id, @@ -537,8 +575,8 @@ def _configure_quotes_chart( yMin=Quotes.low.min() * 0.98, yMax=Quotes.high.max() * 1.02, ) - chart.showGrid(x=True, y=True) - chart.setCursor(QtCore.Qt.BlankCursor) + chart.showGrid(x=True, y=True, alpha=0.4) + chart.setCursor(QtCore.Qt.CrossCursor) # assign callback for rescaling y-axis automatically # based on y-range contents @@ -607,6 +645,7 @@ class QuotesChart(QtGui.QWidget): ind.addItem(curve) ind.hideAxis('left') ind.showAxis('right') + # XXX: never do this lol # ind.setAspectLocked(1) ind.setXLink(self.chart) ind.setLimits( @@ -616,8 +655,8 @@ class QuotesChart(QtGui.QWidget): yMin=Quotes.open.min() * 0.98, yMax=Quotes.open.max() * 1.02, ) - ind.showGrid(x=True, y=True) - ind.setCursor(QtCore.Qt.BlankCursor) + ind.showGrid(x=True, y=True, alpha=0.4) + ind.setCursor(QtCore.Qt.CrossCursor) def _update_sizes(self): min_h_ind = int(self.height() * 0.3 / len(self.indicators)) @@ -655,7 +694,8 @@ class QuotesChart(QtGui.QWidget): self.chart = CustomPlotWidget( parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, - enableMenu=False, + viewBox=ScrollFromRightView, + # enableMenu=False, ) self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) @@ -668,7 +708,7 @@ class QuotesChart(QtGui.QWidget): parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, - enableMenu=False, + # enableMenu=False, ) ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) ind.getPlotItem().setContentsMargins(*CHART_MARGINS) From a7fe18cba9185deb174a37004b10950f92cdd485 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Jun 2020 11:48:48 -0400 Subject: [PATCH 010/206] Factor common chart configuration --- piker/ui/qt/quantdom/charts.py | 103 ++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index 25a23a23..9a0486e6 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -1,7 +1,7 @@ """ Real-time quotes charting components """ -from typing import Callable +from typing import Callable, List, Tuple import numpy as np import pyqtgraph as pg @@ -279,7 +279,9 @@ class ScrollFromRightView(pg.ViewBox): # ) furthest_right_coord = self.boundingRect().topRight() center = pg.Point( - fn.invertQTransform(self.childGroup.transform()).map(furthest_right_coord) + fn.invertQTransform( + self.childGroup.transform() + ).map(furthest_right_coord) ) self._resetTarget() @@ -288,13 +290,15 @@ class ScrollFromRightView(pg.ViewBox): self.sigRangeChangedManually.emit(mask) -# TODO: convert this to a ``ViewBox`` type giving us -# control over mouse scrolling and a context menu -# This is a sub-class of ``GracphicView`` which can +# TODO: This is a sub-class of ``GracphicView`` which can # take a ``background`` color setting. -class CustomPlotWidget(pg.PlotWidget): +class ChartPlotWidget(pg.PlotWidget): """``GraphicsView`` subtype containing a single ``PlotItem``. + Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing + a single ``PlotItem``) to intercept and and re-emit mouse enter/exit + events. + (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we eventually want multiple plots managed together). """ @@ -360,6 +364,7 @@ class CrossHairItem(pg.GraphicsObject): slot=lambda: self.mouseAction('Leave', False), ) + # determine where to place x-axis label if _xaxis_at == 'bottom': # place below is last indicator subplot self.xaxis_label = XAxisLabel( @@ -407,7 +412,8 @@ class CrossHairItem(pg.GraphicsObject): else: self.yaxis_label.show() self.hline.show() - else: # Leave + # Leave + else: # hide horiz line and y-label if ind: self.indicators[ind]['hl'].hide() @@ -424,7 +430,7 @@ class CrossHairItem(pg.GraphicsObject): pos = evt[0] - # if the mouse is within the parent ``CustomPlotWidget`` + # if the mouse is within the parent ``ChartPlotWidget`` if self.parent.sceneBoundingRect().contains(pos): # mouse_point = self.vb.mapSceneToView(pos) mouse_point = self.parent.mapToView(pos) @@ -551,22 +557,16 @@ class CandlestickItem(BarItem): p.drawRects(*rects[Quotes.close < Quotes.open]) -def _configure_quotes_chart( - chart: CustomPlotWidget, - style: ChartType, - update_yrange_limits: Callable, +def _configure_chart( + chart: ChartPlotWidget, ) -> None: - """Update and format a chart with quotes data. + """Configure a chart with common settings. """ + # show only right side axes chart.hideAxis('left') chart.showAxis('right') - # adds all bar/candle graphics objects for each - # data point in the np array buffer to - # be drawn on next render cycle - chart.addItem(_get_chart_points(style)) - # set panning limits chart.setLimits( xMin=Quotes[0].id, @@ -575,16 +575,53 @@ def _configure_quotes_chart( yMin=Quotes.low.min() * 0.98, yMax=Quotes.high.max() * 1.02, ) + # show background grid chart.showGrid(x=True, y=True, alpha=0.4) + + # use cross-hair for cursor chart.setCursor(QtCore.Qt.CrossCursor) + +def _configure_quotes_chart( + chart: ChartPlotWidget, + style: ChartType, + update_yrange_limits: Callable, +) -> None: + """Update and format a chart with quotes data. + """ + _configure_chart(chart) + + # adds all bar/candle graphics objects for each + # data point in the np array buffer to + # be drawn on next render cycle + chart.addItem(_get_chart_points(style)) + # assign callback for rescaling y-axis automatically # based on y-range contents + # TODO: this can likely be ported to built-in: .enableAutoRange() # but needs testing chart.sigXRangeChanged.connect(update_yrange_limits) +def _configure_ind_charts( + indicators: List[Tuple[ChartPlotWidget, np.ndarray]], + xlink_to_chart: ChartPlotWidget, +) -> None: + for ind_chart, d in indicators: + # default config + _configure_chart(ind_chart) + + curve = pg.PlotDataItem(d, pen='b', antialias=True) + ind_chart.addItem(curve) + + # XXX: never do this lol + # ind.setAspectLocked(1) + + # link chart x-axis to main quotes chart + ind_chart.setXLink(xlink_to_chart) + + class QuotesChart(QtGui.QWidget): long_pen = pg.mkPen('#006000') @@ -639,25 +676,6 @@ class QuotesChart(QtGui.QWidget): del self.signals_group_text self.signals_visible = False - def _update_ind_charts(self): - for ind, d in self.indicators: - curve = pg.PlotDataItem(d, pen='b', antialias=True) - ind.addItem(curve) - ind.hideAxis('left') - ind.showAxis('right') - # XXX: never do this lol - # ind.setAspectLocked(1) - ind.setXLink(self.chart) - ind.setLimits( - xMin=Quotes[0].id, - xMax=Quotes[-1].id, - minXRange=60, - yMin=Quotes.open.min() * 0.98, - yMax=Quotes.open.max() * 1.02, - ) - ind.showGrid(x=True, y=True, alpha=0.4) - ind.setCursor(QtCore.Qt.CrossCursor) - def _update_sizes(self): min_h_ind = int(self.height() * 0.3 / len(self.indicators)) sizes = [int(self.height() * 0.7)] @@ -691,7 +709,7 @@ class QuotesChart(QtGui.QWidget): def plot(self, symbol): self.digits = symbol.digits - self.chart = CustomPlotWidget( + self.chart = ChartPlotWidget( parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, viewBox=ScrollFromRightView, @@ -704,7 +722,7 @@ class QuotesChart(QtGui.QWidget): inds = [Quotes.open] for d in inds: - ind = CustomPlotWidget( + ind = ChartPlotWidget( parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, @@ -720,7 +738,10 @@ class QuotesChart(QtGui.QWidget): self.style, self._update_yrange_limits ) - self._update_ind_charts() + _configure_ind_charts( + self.indicators, + xlink_to_chart=self.chart + ) self._update_sizes() ch = CrossHairItem( @@ -783,7 +804,7 @@ class QuotesChart(QtGui.QWidget): self.signals_visible = True -# this function is borderline rediculous. +# this function is borderline ridiculous. # The creation of these chart types mutates all the input data # inside each type's constructor (mind blown) def _get_chart_points(style): From fbce0334adadf90e76cf5fb47917808b0911c4e3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Jun 2020 15:09:32 -0400 Subject: [PATCH 011/206] Lol I guess we probably need this --- piker/ui/qt/_chart.py | 55 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 piker/ui/qt/_chart.py diff --git a/piker/ui/qt/_chart.py b/piker/ui/qt/_chart.py new file mode 100644 index 00000000..8deda737 --- /dev/null +++ b/piker/ui/qt/_chart.py @@ -0,0 +1,55 @@ +""" +High level Qt chart wrapping widgets. +""" +from PyQt5 import QtGui + +from .quantdom.charts import SplitterChart + + +class QuotesTabWidget(QtGui.QWidget): + def __init__(self, parent=None): + super().__init__(parent) + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + self.toolbar_layout = QtGui.QHBoxLayout() + self.toolbar_layout.setContentsMargins(10, 10, 15, 0) + self.chart_layout = QtGui.QHBoxLayout() + + # self.init_timeframes_ui() + # self.init_strategy_ui() + + self.layout.addLayout(self.toolbar_layout) + self.layout.addLayout(self.chart_layout) + + def init_timeframes_ui(self): + self.tf_layout = QtGui.QHBoxLayout() + self.tf_layout.setSpacing(0) + self.tf_layout.setContentsMargins(0, 12, 0, 0) + time_frames = ('1M', '5M', '15M', '30M', '1H', '1D', '1W', 'MN') + btn_prefix = 'TF' + for tf in time_frames: + btn_name = ''.join([btn_prefix, tf]) + btn = QtGui.QPushButton(tf) + # TODO: + btn.setEnabled(False) + setattr(self, btn_name, btn) + self.tf_layout.addWidget(btn) + self.toolbar_layout.addLayout(self.tf_layout) + + # XXX: strat loader/saver that we don't need yet. + # def init_strategy_ui(self): + # self.strategy_box = StrategyBoxWidget(self) + # self.toolbar_layout.addWidget(self.strategy_box) + + # TODO: this needs to be changed to ``load_symbol()`` + # which will not only load historical data but also a real-time + # stream and schedule the redraw events on new quotes + def update_chart(self, symbol): + if not self.chart_layout.isEmpty(): + self.chart_layout.removeWidget(self.chart) + self.chart = SplitterChart() + self.chart.plot(symbol) + self.chart_layout.addWidget(self.chart) + + def add_signals(self): + self.chart.add_signals() From d8ca799504f230bfa0597ad35b3a90a61d3d3cac Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 14 Jun 2020 15:10:08 -0400 Subject: [PATCH 012/206] Start grouping interactions into a ``ViewBox`` Move chart resize code into our ``ViewBox`` subtype (a ``ChartView``) in an effort to start organizing interaction behaviour closer to the appropriate underlying objects. Add some docs for all this and do some renaming. --- piker/ui/qt/_exec.py | 1 + piker/ui/qt/quantdom/charts.py | 149 +++++++++++++++++++++------------ 2 files changed, 97 insertions(+), 53 deletions(-) diff --git a/piker/ui/qt/_exec.py b/piker/ui/qt/_exec.py index 39aea685..c2eda278 100644 --- a/piker/ui/qt/_exec.py +++ b/piker/ui/qt/_exec.py @@ -89,5 +89,6 @@ def run_qtrio( window.main_widget = main_widget window.setCentralWidget(instance) + # actually render to screen window.show() app.exec_() diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index 9a0486e6..3b81e6ba 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -1,7 +1,7 @@ """ Real-time quotes charting components """ -from typing import Callable, List, Tuple +from typing import List, Tuple import numpy as np import pyqtgraph as pg @@ -13,10 +13,10 @@ from .const import ChartType from .portfolio import Order, Portfolio from .utils import fromtimestamp, timeit -__all__ = ('QuotesChart') +__all__ = ('SplitterChart') -# white background for tinas like xb +# white background (for tinas like our pal xb) # pg.setConfigOption('background', 'w') # margins @@ -260,9 +260,79 @@ class YAxisLabel(AxisLabel): self.setPos(new_pos) -class ScrollFromRightView(pg.ViewBox): +class ChartView(pg.ViewBox): + """Price chart view box with interaction behaviors you'd expect from + an interactive platform: + + - zoom on mouse scroll that auto fits y-axis + - no vertical scrolling + - zoom to a "fixed point" on the y-axis + """ + def __init__( + self, + parent=None, + **kwargs, + # invertY=False, + ): + super().__init__(parent=parent, **kwargs) + self.chart = parent + + # disable vertical scrolling + self.setMouseEnabled(x=True, y=False) + + # assign callback for rescaling y-axis automatically + # based on y-range contents + self.chart.sigXRangeChanged.connect(self._update_yrange_limits) + + def _update_yrange_limits(self): + """Callback for each y-range update. + + This adds auto-scaling like zoom on the scroll wheel such + that data always fits nicely inside the current view of the + data set. + """ + # TODO: this can likely be ported in part to the built-ins: + # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) + # self.setMouseEnabled(x=True, y=False) + # self.setXRange(Quotes[0].id, Quotes[-1].id) + # self.setAutoVisible(x=False, y=True) + # self.enableAutoRange(x=False, y=True) + + chart = self.chart + chart_parent = chart.parent + + vr = self.chart.viewRect() + lbar, rbar = int(vr.left()), int(vr.right()) + + if chart_parent.signals_visible: + chart_parent._show_text_signals(lbar, rbar) + + bars = Quotes[lbar:rbar] + ylow = bars.low.min() * 0.98 + yhigh = bars.high.max() * 1.02 + + std = np.std(bars.close) + chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + chart.setYRange(ylow, yhigh) + + for i, d in chart_parent.indicators: + # ydata = i.plotItem.items[0].getData()[1] + ydata = d[lbar:rbar] + ylow = ydata.min() * 0.98 + yhigh = ydata.max() * 1.02 + std = np.std(ydata) + i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + i.setYRange(ylow, yhigh) def wheelEvent(self, ev, axis=None): + """Override "center-point" location for scrolling. + + This is an override of the ``ViewBox`` method simply changing + the center of the zoom to be the y-axis. + + TODO: PR a method into ``pyqtgraph`` to make this configurable + """ + if axis in (0, 1): mask = [False, False] mask[axis] = self.state['mouseEnabled'][axis] @@ -307,11 +377,11 @@ class ChartPlotWidget(pg.PlotWidget): sig_mouse_enter = QtCore.Signal(object) def enterEvent(self, ev): # noqa - pg.PlotWidget.enterEvent(self, ev) + # pg.PlotWidget.enterEvent(self, ev) self.sig_mouse_enter.emit(self) def leaveEvent(self, ev): # noqa - pg.PlotWidget.leaveEvent(self, ev) + # pg.PlotWidget.leaveEvent(self, ev) self.sig_mouse_leave.emit(self) self.scene().leaveEvent(ev) @@ -562,16 +632,19 @@ def _configure_chart( ) -> None: """Configure a chart with common settings. """ - # show only right side axes chart.hideAxis('left') chart.showAxis('right') + # highest = Quotes.high.max() * 1.02 + # lowest = Quotes.low.min() * 0.98 + # set panning limits chart.setLimits( xMin=Quotes[0].id, xMax=Quotes[-1].id, - minXRange=60, + minXRange=40, + # maxYRange=highest-lowest, yMin=Quotes.low.min() * 0.98, yMax=Quotes.high.max() * 1.02, ) @@ -584,8 +657,7 @@ def _configure_chart( def _configure_quotes_chart( chart: ChartPlotWidget, - style: ChartType, - update_yrange_limits: Callable, + style: ChartType = ChartType.BAR, ) -> None: """Update and format a chart with quotes data. """ @@ -596,19 +668,15 @@ def _configure_quotes_chart( # be drawn on next render cycle chart.addItem(_get_chart_points(style)) - # assign callback for rescaling y-axis automatically - # based on y-range contents - - # TODO: this can likely be ported to built-in: .enableAutoRange() - # but needs testing - chart.sigXRangeChanged.connect(update_yrange_limits) - def _configure_ind_charts( indicators: List[Tuple[ChartPlotWidget, np.ndarray]], xlink_to_chart: ChartPlotWidget, ) -> None: for ind_chart, d in indicators: + # link chart x-axis to main quotes chart + ind_chart.setXLink(xlink_to_chart) + # default config _configure_chart(ind_chart) @@ -618,11 +686,8 @@ def _configure_ind_charts( # XXX: never do this lol # ind.setAspectLocked(1) - # link chart x-axis to main quotes chart - ind_chart.setXLink(xlink_to_chart) - -class QuotesChart(QtGui.QWidget): +class SplitterChart(QtGui.QWidget): long_pen = pg.mkPen('#006000') long_brush = pg.mkBrush('#00ff00') @@ -634,7 +699,6 @@ class QuotesChart(QtGui.QWidget): def __init__(self): super().__init__() self.signals_visible = False - self.style = ChartType.BAR self.indicators = [] self.xaxis = FromTimeFieldDateAxis(orientation='bottom') @@ -677,44 +741,23 @@ class QuotesChart(QtGui.QWidget): self.signals_visible = False def _update_sizes(self): - min_h_ind = int(self.height() * 0.3 / len(self.indicators)) - sizes = [int(self.height() * 0.7)] + min_h_ind = int(self.height() * 0.2 / len(self.indicators)) + sizes = [int(self.height() * 0.8)] sizes.extend([min_h_ind] * len(self.indicators)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) - def _update_yrange_limits(self): - """Callback for each y-range update. - """ - vr = self.chart.viewRect() - lbar, rbar = int(vr.left()), int(vr.right()) - - if self.signals_visible: - self._show_text_signals(lbar, rbar) - - bars = Quotes[lbar:rbar] - ylow = bars.low.min() * 0.98 - yhigh = bars.high.max() * 1.02 - - std = np.std(bars.close) - self.chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - self.chart.setYRange(ylow, yhigh) - for i, d in self.indicators: - # ydata = i.plotItem.items[0].getData()[1] - ydata = d[lbar:rbar] - ylow = ydata.min() * 0.98 - yhigh = ydata.max() * 1.02 - std = np.std(ydata) - i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - i.setYRange(ylow, yhigh) - def plot(self, symbol): self.digits = symbol.digits self.chart = ChartPlotWidget( parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, - viewBox=ScrollFromRightView, + viewBox=ChartView, # enableMenu=False, ) + # TODO: ``pyqtgraph`` doesn't pass through a parent to the + # ``PlotItem`` by default; maybe we should PR this in? + self.chart.plotItem.parent = self + self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) @@ -726,8 +769,10 @@ class QuotesChart(QtGui.QWidget): parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, - # enableMenu=False, + viewBox=ChartView, ) + ind.plotItem.parent = self + ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) ind.getPlotItem().setContentsMargins(*CHART_MARGINS) # self.splitter.addWidget(ind) @@ -735,8 +780,6 @@ class QuotesChart(QtGui.QWidget): _configure_quotes_chart( self.chart, - self.style, - self._update_yrange_limits ) _configure_ind_charts( self.indicators, From ac389c30d9cbd1d2ec38afcf42fb00901a5915fc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Jun 2020 09:56:13 -0400 Subject: [PATCH 013/206] Move drawing and resize behavior into chart widget --- piker/ui/qt/quantdom/charts.py | 275 ++++++++++++++++++--------------- 1 file changed, 150 insertions(+), 125 deletions(-) diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index 3b81e6ba..6a87ce2d 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -66,14 +66,14 @@ class PriceAxis(pg.AxisItem): def __init__(self): super().__init__(orientation='right') - # self.setStyle(**{ - # 'textFillLimits': [(0, 0.8)], - # # 'tickTextWidth': 5, - # # 'tickTextHeight': 5, - # # 'autoExpandTextSpace': True, - # # 'maxTickLength': -20, - # }) - # self.setLabel(**{'font-size':'10pt'}) + self.setStyle(**{ + 'textFillLimits': [(0, 0.8)], + # 'tickTextWidth': 5, + # 'tickTextHeight': 5, + # 'autoExpandTextSpace': True, + # 'maxTickLength': -20, + }) + self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font) # XXX: drop for now since it just eats up h space @@ -224,6 +224,8 @@ class XAxisLabel(AxisLabel): def tick_to_string(self, tick_pos): # TODO: change to actual period tpl = self.parent.tick_tpl['D1'] + if tick_pos > len(Quotes): + return 'Unknown Time' return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) def boundingRect(self): # noqa @@ -280,50 +282,6 @@ class ChartView(pg.ViewBox): # disable vertical scrolling self.setMouseEnabled(x=True, y=False) - # assign callback for rescaling y-axis automatically - # based on y-range contents - self.chart.sigXRangeChanged.connect(self._update_yrange_limits) - - def _update_yrange_limits(self): - """Callback for each y-range update. - - This adds auto-scaling like zoom on the scroll wheel such - that data always fits nicely inside the current view of the - data set. - """ - # TODO: this can likely be ported in part to the built-ins: - # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) - # self.setMouseEnabled(x=True, y=False) - # self.setXRange(Quotes[0].id, Quotes[-1].id) - # self.setAutoVisible(x=False, y=True) - # self.enableAutoRange(x=False, y=True) - - chart = self.chart - chart_parent = chart.parent - - vr = self.chart.viewRect() - lbar, rbar = int(vr.left()), int(vr.right()) - - if chart_parent.signals_visible: - chart_parent._show_text_signals(lbar, rbar) - - bars = Quotes[lbar:rbar] - ylow = bars.low.min() * 0.98 - yhigh = bars.high.max() * 1.02 - - std = np.std(bars.close) - chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - chart.setYRange(ylow, yhigh) - - for i, d in chart_parent.indicators: - # ydata = i.plotItem.items[0].getData()[1] - ydata = d[lbar:rbar] - ylow = ydata.min() * 0.98 - yhigh = ydata.max() * 1.02 - std = np.std(ydata) - i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - i.setYRange(ylow, yhigh) - def wheelEvent(self, ev, axis=None): """Override "center-point" location for scrolling. @@ -343,10 +301,11 @@ class ChartView(pg.ViewBox): s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) s = [(None if m is False else s) for m in mask] - # XXX: scroll "around" the right most element in the view # center = pg.Point( # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) # ) + + # XXX: scroll "around" the right most element in the view furthest_right_coord = self.boundingRect().topRight() center = pg.Point( fn.invertQTransform( @@ -372,10 +331,130 @@ class ChartPlotWidget(pg.PlotWidget): (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we eventually want multiple plots managed together). """ - sig_mouse_leave = QtCore.Signal(object) sig_mouse_enter = QtCore.Signal(object) + def __init__( + self, + split_charts, + **kwargs, + # parent=None, + # background='default', + # plotItem=None, + # **kargs + ): + """Configure chart display settings. + """ + + super().__init__(**kwargs) + # label = pg.LabelItem(justify='left') + # self.addItem(label) + # label.setText("Yo yoyo") + # label.setText("x=") + self.parent = split_charts + + # show only right side axes + self.hideAxis('left') + self.showAxis('right') + + # show background grid + self.showGrid(x=True, y=True, alpha=0.4) + + # use cross-hair for cursor + self.setCursor(QtCore.Qt.CrossCursor) + + # set panning limits + min_points_to_show = 20 + min_bars_in_view = 10 + max_lookahead = min_points_to_show - min_bars_in_view + last = Quotes[-1].id + self.setLimits( + xMin=Quotes[0].id, + xMax=last + max_lookahead, + minXRange=min_points_to_show, + # maxYRange=highest-lowest, + yMin=Quotes.low.min() * 0.98, + yMax=Quotes.high.max() * 1.02, + ) + + # show last 50 points on startup + self.plotItem.vb.setXRange(last - 50, last + max_lookahead) + + # assign callback for rescaling y-axis automatically + # based on y-range contents + self.sigXRangeChanged.connect(self._update_yrange_limits) + self._update_yrange_limits() + + def bars_range(self): + """Return a range tuple for the bars present in view. + """ + + vr = self.viewRect() + lbar, rbar = int(vr.left()), int(min(vr.right(), len(Quotes) - 1)) + return lbar, rbar + + def draw_ohlc( + self, + style: ChartType = ChartType.BAR, + ) -> None: + """Draw OHLC datums to chart. + """ + + # adds all bar/candle graphics objects for each + # data point in the np array buffer to + # be drawn on next render cycle + self.addItem(_get_chart_points(style)) + + def draw_curve( + self, + data: np.ndarray, + ) -> None: + # draw the indicator as a plain curve + curve = pg.PlotDataItem(data, antialias=True) + ind_chart.addItem(curve) + + def _update_yrange_limits(self): + """Callback for each y-range update. + + This adds auto-scaling like zoom on the scroll wheel such + that data always fits nicely inside the current view of the + data set. + """ + # TODO: this can likely be ported in part to the built-ins: + # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) + # self.setMouseEnabled(x=True, y=False) + # self.setXRange(Quotes[0].id, Quotes[-1].id) + # self.setAutoVisible(x=False, y=True) + # self.enableAutoRange(x=False, y=True) + + chart = self + chart_parent = self.parent + + lbar, rbar = self.bars_range() + # vr = chart.viewRect() + # lbar, rbar = int(vr.left()), int(vr.right()) + + if chart_parent.signals_visible: + chart_parent._show_text_signals(lbar, rbar) + + bars = Quotes[lbar:rbar] + ylow = bars.low.min() * 0.98 + yhigh = bars.high.max() * 1.02 + + std = np.std(bars.close) + chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + chart.setYRange(ylow, yhigh) + + for i, d in chart_parent.indicators: + # ydata = i.plotItem.items[0].getData()[1] + ydata = d[lbar:rbar] + ylow = ydata.min() * 0.98 + yhigh = ydata.max() * 1.02 + std = np.std(ydata) + i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + i.setYRange(ylow, yhigh) + + def enterEvent(self, ev): # noqa # pg.PlotWidget.enterEvent(self, ev) self.sig_mouse_enter.emit(self) @@ -525,7 +604,7 @@ class CrossHairItem(pg.GraphicsObject): evt_post=pos, point_view=mouse_point_ind ) else: - # vertial position of the mouse is inside and main chart + # vertial position of the mouse is inside the main chart self.hline.setY(mouse_point.y()) self.yaxis_label.update_label( evt_post=pos, point_view=mouse_point @@ -627,66 +706,6 @@ class CandlestickItem(BarItem): p.drawRects(*rects[Quotes.close < Quotes.open]) -def _configure_chart( - chart: ChartPlotWidget, -) -> None: - """Configure a chart with common settings. - """ - # show only right side axes - chart.hideAxis('left') - chart.showAxis('right') - - # highest = Quotes.high.max() * 1.02 - # lowest = Quotes.low.min() * 0.98 - - # set panning limits - chart.setLimits( - xMin=Quotes[0].id, - xMax=Quotes[-1].id, - minXRange=40, - # maxYRange=highest-lowest, - yMin=Quotes.low.min() * 0.98, - yMax=Quotes.high.max() * 1.02, - ) - # show background grid - chart.showGrid(x=True, y=True, alpha=0.4) - - # use cross-hair for cursor - chart.setCursor(QtCore.Qt.CrossCursor) - - -def _configure_quotes_chart( - chart: ChartPlotWidget, - style: ChartType = ChartType.BAR, -) -> None: - """Update and format a chart with quotes data. - """ - _configure_chart(chart) - - # adds all bar/candle graphics objects for each - # data point in the np array buffer to - # be drawn on next render cycle - chart.addItem(_get_chart_points(style)) - - -def _configure_ind_charts( - indicators: List[Tuple[ChartPlotWidget, np.ndarray]], - xlink_to_chart: ChartPlotWidget, -) -> None: - for ind_chart, d in indicators: - # link chart x-axis to main quotes chart - ind_chart.setXLink(xlink_to_chart) - - # default config - _configure_chart(ind_chart) - - curve = pg.PlotDataItem(d, pen='b', antialias=True) - ind_chart.addItem(curve) - - # XXX: never do this lol - # ind.setAspectLocked(1) - - class SplitterChart(QtGui.QWidget): long_pen = pg.mkPen('#006000') @@ -711,12 +730,12 @@ class SplitterChart(QtGui.QWidget): else: self.xaxis_ind.setStyle(showValues=False) - self.layout = QtGui.QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) - self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) self.splitter.setHandleWidth(5) + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + self.layout.addWidget(self.splitter) def _show_text_signals(self, lbar, rbar): @@ -749,6 +768,7 @@ class SplitterChart(QtGui.QWidget): def plot(self, symbol): self.digits = symbol.digits self.chart = ChartPlotWidget( + split_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, viewBox=ChartView, @@ -766,6 +786,7 @@ class SplitterChart(QtGui.QWidget): for d in inds: ind = ChartPlotWidget( + split_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, @@ -778,13 +799,17 @@ class SplitterChart(QtGui.QWidget): # self.splitter.addWidget(ind) self.indicators.append((ind, d)) - _configure_quotes_chart( - self.chart, - ) - _configure_ind_charts( - self.indicators, - xlink_to_chart=self.chart - ) + self.chart.draw_ohlc() + + for ind_chart, d in self.indicators: + + # link chart x-axis to main quotes chart + ind_chart.setXLink(self.chart) + + # XXX: never do this lol + # ind.setAspectLocked(1) + ind_chart.draw_curve(d) + self._update_sizes() ch = CrossHairItem( From 6fa173a1c179d7dbe3fe4246e0a9d3ae9514924a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Jun 2020 10:48:00 -0400 Subject: [PATCH 014/206] Factor components into more suitably named modules --- piker/ui/qt/_axes.py | 171 +++++++ piker/ui/qt/_chart.py | 406 ++++++++++++++++- piker/ui/qt/_graphics.py | 252 +++++++++++ piker/ui/qt/_style.py | 18 + piker/ui/qt/quantdom/charts.py | 804 --------------------------------- 5 files changed, 844 insertions(+), 807 deletions(-) create mode 100644 piker/ui/qt/_axes.py create mode 100644 piker/ui/qt/_graphics.py create mode 100644 piker/ui/qt/_style.py diff --git a/piker/ui/qt/_axes.py b/piker/ui/qt/_axes.py new file mode 100644 index 00000000..96724a26 --- /dev/null +++ b/piker/ui/qt/_axes.py @@ -0,0 +1,171 @@ +""" +Chart axes graphics and behavior. +""" +import pyqtgraph as pg +from PyQt5 import QtCore, QtGui + + +from .quantdom.base import Quotes +from .quantdom.utils import fromtimestamp +from ._style import _font + + +class PriceAxis(pg.AxisItem): + + def __init__(self): + super().__init__(orientation='right') + self.setStyle(**{ + 'textFillLimits': [(0, 0.8)], + # 'tickTextWidth': 5, + # 'tickTextHeight': 5, + # 'autoExpandTextSpace': True, + # 'maxTickLength': -20, + }) + self.setLabel(**{'font-size': '10pt'}) + self.setTickFont(_font) + + # XXX: drop for now since it just eats up h space + + # def tickStrings(self, vals, scale, spacing): + # digts = max(0, np.ceil(-np.log10(spacing * scale))) + # return [ + # ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals + # ] + + +class FromTimeFieldDateAxis(pg.AxisItem): + tick_tpl = {'D1': '%Y-%b-%d'} + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.setTickFont(_font) + self.quotes_count = len(Quotes) - 1 + + # default styling + self.setStyle( + tickTextOffset=7, + textFillLimits=[(0, 0.90)], + # TODO: doesn't seem to work -> bug in pyqtgraph? + # tickTextHeight=2, + ) + + def tickStrings(self, values, scale, spacing): + # if len(values) > 1 or not values: + # values = Quotes.time + + # strings = super().tickStrings(values, scale, spacing) + s_period = 'D1' + strings = [] + for ibar in values: + if ibar > self.quotes_count: + return strings + dt_tick = fromtimestamp(Quotes[int(ibar)].time) + strings.append( + dt_tick.strftime(self.tick_tpl[s_period]) + ) + return strings + + +class AxisLabel(pg.GraphicsObject): + + # bg_color = pg.mkColor('#a9a9a9') + bg_color = pg.mkColor('#808080') + fg_color = pg.mkColor('#000000') + + def __init__(self, parent=None, digits=0, color=None, opacity=1, **kwargs): + super().__init__(parent) + self.parent = parent + self.opacity = opacity + self.label_str = '' + self.digits = digits + # self.quotes_count = len(Quotes) - 1 + + if isinstance(color, QtGui.QPen): + self.bg_color = color.color() + self.fg_color = pg.mkColor('#ffffff') + elif isinstance(color, list): + self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} + self.fg_color = pg.mkColor('#ffffff') + + self.setFlag(self.ItemIgnoresTransformations) + + def tick_to_string(self, tick_pos): + raise NotImplementedError() + + def boundingRect(self): # noqa + raise NotImplementedError() + + def update_label(self, evt_post, point_view): + raise NotImplementedError() + + def update_label_test(self, ypos=0, ydata=0): + self.label_str = self.tick_to_string(ydata) + height = self.boundingRect().height() + offset = 0 # if have margins + new_pos = QtCore.QPointF(0, ypos - height / 2 - offset) + self.setPos(new_pos) + + def paint(self, p, option, widget): + p.setRenderHint(p.TextAntialiasing, True) + p.setPen(self.fg_color) + if self.label_str: + if not isinstance(self.bg_color, dict): + bg_color = self.bg_color + else: + if int(self.label_str.replace(' ', '')) > 0: + bg_color = self.bg_color['>0'] + else: + bg_color = self.bg_color['<0'] + p.setOpacity(self.opacity) + p.fillRect(option.rect, bg_color) + p.setOpacity(1) + p.setFont(_font) + + p.drawText(option.rect, self.text_flags, self.label_str) + + +class XAxisLabel(AxisLabel): + + text_flags = ( + QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter | QtCore.Qt.AlignTop + ) + + def tick_to_string(self, tick_pos): + # TODO: change to actual period + tpl = self.parent.tick_tpl['D1'] + if tick_pos > len(Quotes): + return 'Unknown Time' + return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) + + def boundingRect(self): # noqa + return QtCore.QRectF(0, 0, 145, 50) + + def update_label(self, evt_post, point_view): + ibar = point_view.x() + # if ibar > self.quotes_count: + # return + self.label_str = self.tick_to_string(ibar) + width = self.boundingRect().width() + offset = 0 # if have margins + new_pos = QtCore.QPointF(evt_post.x() - width / 2 - offset, 0) + self.setPos(new_pos) + + +class YAxisLabel(AxisLabel): + + text_flags = ( + QtCore.Qt.TextDontClip | QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter + ) + + def tick_to_string(self, tick_pos): + return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') + + def boundingRect(self): # noqa + return QtCore.QRectF(0, 0, 80, 40) + + def update_label(self, evt_post, point_view): + self.label_str = self.tick_to_string(point_view.y()) + height = self.boundingRect().height() + offset = 0 # if have margins + new_pos = QtCore.QPointF(0, evt_post.y() - height / 2 - offset) + self.setPos(new_pos) diff --git a/piker/ui/qt/_chart.py b/piker/ui/qt/_chart.py index 8deda737..e77f1191 100644 --- a/piker/ui/qt/_chart.py +++ b/piker/ui/qt/_chart.py @@ -1,9 +1,29 @@ """ -High level Qt chart wrapping widgets. +High level Qt chart widgets. """ -from PyQt5 import QtGui +import numpy as np +import pyqtgraph as pg +from pyqtgraph import functions as fn +from PyQt5 import QtCore, QtGui -from .quantdom.charts import SplitterChart +from ._axes import ( + FromTimeFieldDateAxis, + PriceAxis, +) +from ._graphics import CrossHairItem, CandlestickItem, BarItem +from ._style import _xaxis_at + +from .quantdom.charts import CenteredTextItem +from .quantdom.base import Quotes +from .quantdom.const import ChartType +from .quantdom.portfolio import Order, Portfolio + + +# white background (for tinas like our pal xb) +# pg.setConfigOption('background', 'w') + +# margins +CHART_MARGINS = (0, 0, 10, 3) class QuotesTabWidget(QtGui.QWidget): @@ -53,3 +73,383 @@ class QuotesTabWidget(QtGui.QWidget): def add_signals(self): self.chart.add_signals() + + +class SplitterChart(QtGui.QWidget): + + long_pen = pg.mkPen('#006000') + long_brush = pg.mkBrush('#00ff00') + short_pen = pg.mkPen('#600000') + short_brush = pg.mkBrush('#ff0000') + + zoomIsDisabled = QtCore.pyqtSignal(bool) + + def __init__(self): + super().__init__() + self.signals_visible = False + self.indicators = [] + + self.xaxis = FromTimeFieldDateAxis(orientation='bottom') + # self.xaxis = pg.DateAxisItem() + + self.xaxis_ind = FromTimeFieldDateAxis(orientation='bottom') + + if _xaxis_at == 'bottom': + self.xaxis.setStyle(showValues=False) + else: + self.xaxis_ind.setStyle(showValues=False) + + self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) + self.splitter.setHandleWidth(5) + + self.layout = QtGui.QVBoxLayout(self) + self.layout.setContentsMargins(0, 0, 0, 0) + + self.layout.addWidget(self.splitter) + + def _show_text_signals(self, lbar, rbar): + signals = [ + sig + for sig in self.signals_text_items[lbar:rbar] + if isinstance(sig, CenteredTextItem) + ] + if len(signals) <= 50: + for sig in signals: + sig.show() + else: + for sig in signals: + sig.hide() + + def _remove_signals(self): + self.chart.removeItem(self.signals_group_arrow) + self.chart.removeItem(self.signals_group_text) + del self.signals_text_items + del self.signals_group_arrow + del self.signals_group_text + self.signals_visible = False + + def _update_sizes(self): + min_h_ind = int(self.height() * 0.2 / len(self.indicators)) + sizes = [int(self.height() * 0.8)] + sizes.extend([min_h_ind] * len(self.indicators)) + self.splitter.setSizes(sizes) # , int(self.height()*0.2) + + def plot(self, symbol): + self.digits = symbol.digits + self.chart = ChartPlotWidget( + split_charts=self, + parent=self.splitter, + axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, + viewBox=ChartView, + # enableMenu=False, + ) + # TODO: ``pyqtgraph`` doesn't pass through a parent to the + # ``PlotItem`` by default; maybe we should PR this in? + self.chart.plotItem.parent = self + + self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + + # TODO: this is where we would load an indicator chain + inds = [Quotes.open] + + for d in inds: + ind = ChartPlotWidget( + split_charts=self, + parent=self.splitter, + axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, + # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, + viewBox=ChartView, + ) + ind.plotItem.parent = self + + ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + ind.getPlotItem().setContentsMargins(*CHART_MARGINS) + # self.splitter.addWidget(ind) + self.indicators.append((ind, d)) + + self.chart.draw_ohlc() + + for ind_chart, d in self.indicators: + + # link chart x-axis to main quotes chart + ind_chart.setXLink(self.chart) + + # XXX: never do this lol + # ind.setAspectLocked(1) + ind_chart.draw_curve(d) + + self._update_sizes() + + ch = CrossHairItem( + self.chart, [_ind for _ind, d in self.indicators], self.digits + ) + self.chart.addItem(ch) + + def add_signals(self): + self.signals_group_text = QtGui.QGraphicsItemGroup() + self.signals_group_arrow = QtGui.QGraphicsItemGroup() + self.signals_text_items = np.empty(len(Quotes), dtype=object) + + for p in Portfolio.positions: + x, price = p.id_bar_open, p.open_price + if p.type == Order.BUY: + y = Quotes[x].low * 0.99 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.long_pen, + brush=self.long_brush, + angle=90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.long_pen, + brush=self.long_brush, + text=('Buy at {:.%df}' % self.digits).format(price), + valign=QtCore.Qt.AlignBottom, + ) + text_sig.hide() + else: + y = Quotes[x].high * 1.01 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.short_pen, + brush=self.short_brush, + angle=-90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.short_pen, + brush=self.short_brush, + text=('Sell at {:.%df}' % self.digits).format(price), + valign=QtCore.Qt.AlignTop, + ) + text_sig.hide() + + self.signals_text_items[x] = text_sig + + self.chart.addItem(self.signals_group_arrow) + self.chart.addItem(self.signals_group_text) + self.signals_visible = True + + +# TODO: This is a sub-class of ``GracphicView`` which can +# take a ``background`` color setting. +class ChartPlotWidget(pg.PlotWidget): + """``GraphicsView`` subtype containing a single ``PlotItem``. + + Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing + a single ``PlotItem``) to intercept and and re-emit mouse enter/exit + events. + + (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we + eventually want multiple plots managed together). + """ + sig_mouse_leave = QtCore.Signal(object) + sig_mouse_enter = QtCore.Signal(object) + + def __init__( + self, + split_charts, + **kwargs, + # parent=None, + # background='default', + # plotItem=None, + # **kargs + ): + """Configure chart display settings. + """ + + super().__init__(**kwargs) + # label = pg.LabelItem(justify='left') + # self.addItem(label) + # label.setText("Yo yoyo") + # label.setText("x=") + self.parent = split_charts + + # show only right side axes + self.hideAxis('left') + self.showAxis('right') + + # show background grid + self.showGrid(x=True, y=True, alpha=0.4) + + # use cross-hair for cursor + self.setCursor(QtCore.Qt.CrossCursor) + + # set panning limits + min_points_to_show = 20 + min_bars_in_view = 10 + max_lookahead = min_points_to_show - min_bars_in_view + last = Quotes[-1].id + self.setLimits( + xMin=Quotes[0].id, + xMax=last + max_lookahead, + minXRange=min_points_to_show, + # maxYRange=highest-lowest, + yMin=Quotes.low.min() * 0.98, + yMax=Quotes.high.max() * 1.02, + ) + + # show last 50 points on startup + self.plotItem.vb.setXRange(last - 50, last + max_lookahead) + + # assign callback for rescaling y-axis automatically + # based on y-range contents + self.sigXRangeChanged.connect(self._update_yrange_limits) + self._update_yrange_limits() + + def bars_range(self): + """Return a range tuple for the bars present in view. + """ + + vr = self.viewRect() + lbar, rbar = int(vr.left()), int(min(vr.right(), len(Quotes) - 1)) + return lbar, rbar + + def draw_ohlc( + self, + style: ChartType = ChartType.BAR, + ) -> None: + """Draw OHLC datums to chart. + """ + + # adds all bar/candle graphics objects for each + # data point in the np array buffer to + # be drawn on next render cycle + self.addItem(_get_chart_points(style)) + + def draw_curve( + self, + data: np.ndarray, + ) -> None: + # draw the indicator as a plain curve + curve = pg.PlotDataItem(data, antialias=True) + self.addItem(curve) + + def _update_yrange_limits(self): + """Callback for each y-range update. + + This adds auto-scaling like zoom on the scroll wheel such + that data always fits nicely inside the current view of the + data set. + """ + # TODO: this can likely be ported in part to the built-ins: + # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) + # self.setMouseEnabled(x=True, y=False) + # self.setXRange(Quotes[0].id, Quotes[-1].id) + # self.setAutoVisible(x=False, y=True) + # self.enableAutoRange(x=False, y=True) + + chart = self + chart_parent = self.parent + + lbar, rbar = self.bars_range() + # vr = chart.viewRect() + # lbar, rbar = int(vr.left()), int(vr.right()) + + if chart_parent.signals_visible: + chart_parent._show_text_signals(lbar, rbar) + + bars = Quotes[lbar:rbar] + ylow = bars.low.min() * 0.98 + yhigh = bars.high.max() * 1.02 + + std = np.std(bars.close) + chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + chart.setYRange(ylow, yhigh) + + for i, d in chart_parent.indicators: + # ydata = i.plotItem.items[0].getData()[1] + ydata = d[lbar:rbar] + ylow = ydata.min() * 0.98 + yhigh = ydata.max() * 1.02 + std = np.std(ydata) + i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + i.setYRange(ylow, yhigh) + + + def enterEvent(self, ev): # noqa + # pg.PlotWidget.enterEvent(self, ev) + self.sig_mouse_enter.emit(self) + + def leaveEvent(self, ev): # noqa + # pg.PlotWidget.leaveEvent(self, ev) + self.sig_mouse_leave.emit(self) + self.scene().leaveEvent(ev) + + +class ChartView(pg.ViewBox): + """Price chart view box with interaction behaviors you'd expect from + an interactive platform: + + - zoom on mouse scroll that auto fits y-axis + - no vertical scrolling + - zoom to a "fixed point" on the y-axis + """ + def __init__( + self, + parent=None, + **kwargs, + # invertY=False, + ): + super().__init__(parent=parent, **kwargs) + self.chart = parent + + # disable vertical scrolling + self.setMouseEnabled(x=True, y=False) + + def wheelEvent(self, ev, axis=None): + """Override "center-point" location for scrolling. + + This is an override of the ``ViewBox`` method simply changing + the center of the zoom to be the y-axis. + + TODO: PR a method into ``pyqtgraph`` to make this configurable + """ + + if axis in (0, 1): + mask = [False, False] + mask[axis] = self.state['mouseEnabled'][axis] + else: + mask = self.state['mouseEnabled'][:] + + # actual scaling factor + s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) + s = [(None if m is False else s) for m in mask] + + # center = pg.Point( + # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) + # ) + + # XXX: scroll "around" the right most element in the view + furthest_right_coord = self.boundingRect().topRight() + center = pg.Point( + fn.invertQTransform( + self.childGroup.transform() + ).map(furthest_right_coord) + ) + + self._resetTarget() + self.scaleBy(s, center) + ev.accept() + self.sigRangeChangedManually.emit(mask) + + +# this function is borderline ridiculous. +# The creation of these chart types mutates all the input data +# inside each type's constructor (mind blown) +def _get_chart_points(style): + if style == ChartType.CANDLESTICK: + return CandlestickItem() + elif style == ChartType.BAR: + return BarItem() + return pg.PlotDataItem(Quotes.close, pen='b') diff --git a/piker/ui/qt/_graphics.py b/piker/ui/qt/_graphics.py new file mode 100644 index 00000000..bff8c043 --- /dev/null +++ b/piker/ui/qt/_graphics.py @@ -0,0 +1,252 @@ +""" +Chart graphics for displaying a slew of different data types. +""" +import numpy as np +import pyqtgraph as pg +from PyQt5 import QtCore, QtGui + +from .quantdom.utils import timeit +from .quantdom.base import Quotes + +from ._style import _xaxis_at +from ._axes import YAxisLabel, XAxisLabel + + +_mouse_rate_limit = 60 + + +class CrossHairItem(pg.GraphicsObject): + + def __init__(self, parent, indicators=None, digits=0): + super().__init__() + # self.pen = pg.mkPen('#000000') + self.pen = pg.mkPen('#a9a9a9') + self.parent = parent + self.indicators = {} + self.activeIndicator = None + self.xaxis = self.parent.getAxis('bottom') + self.yaxis = self.parent.getAxis('right') + + self.vline = self.parent.addLine(x=0, pen=self.pen, movable=False) + self.hline = self.parent.addLine(y=0, pen=self.pen, movable=False) + + self.proxy_moved = pg.SignalProxy( + self.parent.scene().sigMouseMoved, + rateLimit=_mouse_rate_limit, + slot=self.mouseMoved, + ) + + self.yaxis_label = YAxisLabel( + parent=self.yaxis, digits=digits, opacity=1 + ) + + indicators = indicators or [] + + if indicators: + # when there are indicators present in sub-plot rows + # take the last one (nearest to the bottom) and place the + # crosshair label on it's x-axis. + last_ind = indicators[-1] + + self.proxy_enter = pg.SignalProxy( + self.parent.sig_mouse_enter, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Enter', False), + ) + self.proxy_leave = pg.SignalProxy( + self.parent.sig_mouse_leave, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Leave', False), + ) + + # determine where to place x-axis label + if _xaxis_at == 'bottom': + # place below is last indicator subplot + self.xaxis_label = XAxisLabel( + parent=last_ind.getAxis('bottom'), opacity=1 + ) + else: + # keep x-axis right below main chart + self.xaxis_label = XAxisLabel(parent=self.xaxis, opacity=1) + + for i in indicators: + # add vertial and horizonal lines and a y-axis label + vl = i.addLine(x=0, pen=self.pen, movable=False) + hl = i.addLine(y=0, pen=self.pen, movable=False) + yl = YAxisLabel(parent=i.getAxis('right'), opacity=1) + + px_moved = pg.SignalProxy( + i.scene().sigMouseMoved, + rateLimit=_mouse_rate_limit, + slot=self.mouseMoved + ) + px_enter = pg.SignalProxy( + i.sig_mouse_enter, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Enter', i), + ) + px_leave = pg.SignalProxy( + i.sig_mouse_leave, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Leave', i), + ) + self.indicators[i] = { + 'vl': vl, + 'hl': hl, + 'yl': yl, + 'px': (px_moved, px_enter, px_leave), + } + + def mouseAction(self, action, ind=False): # noqa + if action == 'Enter': + # show horiz line and y-label + if ind: + self.indicators[ind]['hl'].show() + self.indicators[ind]['yl'].show() + self.activeIndicator = ind + else: + self.yaxis_label.show() + self.hline.show() + # Leave + else: + # hide horiz line and y-label + if ind: + self.indicators[ind]['hl'].hide() + self.indicators[ind]['yl'].hide() + self.activeIndicator = None + else: + self.yaxis_label.hide() + self.hline.hide() + + def mouseMoved(self, evt): # noqa + """Update horizonal and vertical lines when mouse moves inside + either the main chart or any indicator subplot. + """ + + pos = evt[0] + + # if the mouse is within the parent ``ChartPlotWidget`` + if self.parent.sceneBoundingRect().contains(pos): + # mouse_point = self.vb.mapSceneToView(pos) + mouse_point = self.parent.mapToView(pos) + + # move the vertial line to the current x coordinate + self.vline.setX(mouse_point.x()) + + # update the label on the bottom of the crosshair + self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) + + # update the vertical line in any indicators subplots + for opts in self.indicators.values(): + opts['vl'].setX(mouse_point.x()) + + if self.activeIndicator: + # vertial position of the mouse is inside an indicator + mouse_point_ind = self.activeIndicator.mapToView(pos) + self.indicators[self.activeIndicator]['hl'].setY( + mouse_point_ind.y() + ) + self.indicators[self.activeIndicator]['yl'].update_label( + evt_post=pos, point_view=mouse_point_ind + ) + else: + # vertial position of the mouse is inside the main chart + self.hline.setY(mouse_point.y()) + self.yaxis_label.update_label( + evt_post=pos, point_view=mouse_point + ) + + def paint(self, p, *args): + pass + + def boundingRect(self): + return self.parent.boundingRect() + + +class BarItem(pg.GraphicsObject): + # XXX: From the customGraphicsItem.py example: + # The only required methods are paint() and boundingRect() + + w = 0.5 + + bull_brush = bear_brush = pg.mkPen('#808080') + # bull_brush = pg.mkPen('#00cc00') + # bear_brush = pg.mkPen('#fa0000') + + def __init__(self): + super().__init__() + self.generatePicture() + + # TODO: this is the routine to be retriggered for redraw + @timeit + def generatePicture(self): + # pre-computing a QPicture object allows paint() to run much + # more quickly, rather than re-drawing the shapes every time. + self.picture = QtGui.QPicture() + p = QtGui.QPainter(self.picture) + self._generate(p) + p.end() + + def _generate(self, p): + # XXX: overloaded method to allow drawing other candle types + + high_to_low = np.array( + [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] + ) + open_stick = np.array( + [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) + for q in Quotes] + ) + close_stick = np.array( + [ + QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) + for q in Quotes + ] + ) + lines = np.concatenate([high_to_low, open_stick, close_stick]) + long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) + + p.setPen(self.bull_brush) + p.drawLines(*lines[long_bars]) + + p.setPen(self.bear_brush) + p.drawLines(*lines[short_bars]) + + def paint(self, p, *args): + p.drawPicture(0, 0, self.picture) + + def boundingRect(self): + # boundingRect _must_ indicate the entire area that will be + # drawn on or else we will get artifacts and possibly crashing. + # (in this case, QPicture does all the work of computing the + # bouning rect for us) + return QtCore.QRectF(self.picture.boundingRect()) + + +class CandlestickItem(BarItem): + + w2 = 0.7 + line_pen = pg.mkPen('#000000') + bull_brush = pg.mkBrush('#00ff00') + bear_brush = pg.mkBrush('#ff0000') + + def _generate(self, p): + rects = np.array( + [ + QtCore.QRectF(q.id - self.w, q.open, self.w2, q.close - q.open) + for q in Quotes + ] + ) + + p.setPen(self.line_pen) + p.drawLines( + [QtCore.QLineF(q.id, q.low, q.id, q.high) + for q in Quotes] + ) + + p.setBrush(self.bull_brush) + p.drawRects(*rects[Quotes.close > Quotes.open]) + + p.setBrush(self.bear_brush) + p.drawRects(*rects[Quotes.close < Quotes.open]) diff --git a/piker/ui/qt/_style.py b/piker/ui/qt/_style.py new file mode 100644 index 00000000..8630d67d --- /dev/null +++ b/piker/ui/qt/_style.py @@ -0,0 +1,18 @@ +""" +Qt styling. +""" +from PyQt5 import QtGui + + +# TODO: add "tina mode" to make everything look "conventional" +# white background (for tinas like our pal xb) +# pg.setConfigOption('background', 'w') + + +# chart-wide font +_font = QtGui.QFont("Hack", 4) +_i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) + + +# splitter widget config +_xaxis_at = 'bottom' diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/qt/quantdom/charts.py index 6a87ce2d..e2da8586 100644 --- a/piker/ui/qt/quantdom/charts.py +++ b/piker/ui/qt/quantdom/charts.py @@ -1,31 +1,9 @@ """ Real-time quotes charting components """ -from typing import List, Tuple - -import numpy as np import pyqtgraph as pg -from pyqtgraph import functions as fn from PyQt5 import QtCore, QtGui -from .base import Quotes -from .const import ChartType -from .portfolio import Order, Portfolio -from .utils import fromtimestamp, timeit - -__all__ = ('SplitterChart') - - -# white background (for tinas like our pal xb) -# pg.setConfigOption('background', 'w') - -# margins -CHART_MARGINS = (0, 0, 10, 3) - -# chart-wide font -_font = QtGui.QFont("Hack", 4) -_i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) - class SampleLegendItem(pg.graphicsItems.LegendItem.ItemSample): @@ -62,62 +40,6 @@ class SampleLegendItem(pg.graphicsItems.LegendItem.ItemSample): p.drawRect(0, 10, 18, 0.5) -class PriceAxis(pg.AxisItem): - - def __init__(self): - super().__init__(orientation='right') - self.setStyle(**{ - 'textFillLimits': [(0, 0.8)], - # 'tickTextWidth': 5, - # 'tickTextHeight': 5, - # 'autoExpandTextSpace': True, - # 'maxTickLength': -20, - }) - self.setLabel(**{'font-size': '10pt'}) - self.setTickFont(_font) - - # XXX: drop for now since it just eats up h space - - # def tickStrings(self, vals, scale, spacing): - # digts = max(0, np.ceil(-np.log10(spacing * scale))) - # return [ - # ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals - # ] - - -class FromTimeFieldDateAxis(pg.AxisItem): - tick_tpl = {'D1': '%Y-%b-%d'} - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.setTickFont(_font) - self.quotes_count = len(Quotes) - 1 - - # default styling - self.setStyle( - tickTextOffset=7, - textFillLimits=[(0, 0.90)], - # TODO: doesn't seem to work -> bug in pyqtgraph? - # tickTextHeight=2, - ) - - def tickStrings(self, values, scale, spacing): - # if len(values) > 1 or not values: - # values = Quotes.time - - # strings = super().tickStrings(values, scale, spacing) - s_period = 'D1' - strings = [] - for ibar in values: - if ibar > self.quotes_count: - return strings - dt_tick = fromtimestamp(Quotes[int(ibar)].time) - strings.append( - dt_tick.strftime(self.tick_tpl[s_period]) - ) - return strings - - class CenteredTextItem(QtGui.QGraphicsTextItem): def __init__( self, @@ -155,729 +77,3 @@ class CenteredTextItem(QtGui.QGraphicsTextItem): p.fillRect(option.rect, self.brush) p.setOpacity(1) p.drawText(option.rect, self.text_flags, self.toPlainText()) - - -class AxisLabel(pg.GraphicsObject): - - # bg_color = pg.mkColor('#a9a9a9') - bg_color = pg.mkColor('#808080') - fg_color = pg.mkColor('#000000') - - def __init__(self, parent=None, digits=0, color=None, opacity=1, **kwargs): - super().__init__(parent) - self.parent = parent - self.opacity = opacity - self.label_str = '' - self.digits = digits - # self.quotes_count = len(Quotes) - 1 - - if isinstance(color, QtGui.QPen): - self.bg_color = color.color() - self.fg_color = pg.mkColor('#ffffff') - elif isinstance(color, list): - self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} - self.fg_color = pg.mkColor('#ffffff') - - self.setFlag(self.ItemIgnoresTransformations) - - def tick_to_string(self, tick_pos): - raise NotImplementedError() - - def boundingRect(self): # noqa - raise NotImplementedError() - - def update_label(self, evt_post, point_view): - raise NotImplementedError() - - def update_label_test(self, ypos=0, ydata=0): - self.label_str = self.tick_to_string(ydata) - height = self.boundingRect().height() - offset = 0 # if have margins - new_pos = QtCore.QPointF(0, ypos - height / 2 - offset) - self.setPos(new_pos) - - def paint(self, p, option, widget): - p.setRenderHint(p.TextAntialiasing, True) - p.setPen(self.fg_color) - if self.label_str: - if not isinstance(self.bg_color, dict): - bg_color = self.bg_color - else: - if int(self.label_str.replace(' ', '')) > 0: - bg_color = self.bg_color['>0'] - else: - bg_color = self.bg_color['<0'] - p.setOpacity(self.opacity) - p.fillRect(option.rect, bg_color) - p.setOpacity(1) - p.setFont(_font) - - p.drawText(option.rect, self.text_flags, self.label_str) - - -class XAxisLabel(AxisLabel): - - text_flags = ( - QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter | QtCore.Qt.AlignTop - ) - - def tick_to_string(self, tick_pos): - # TODO: change to actual period - tpl = self.parent.tick_tpl['D1'] - if tick_pos > len(Quotes): - return 'Unknown Time' - return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) - - def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 145, 50) - - def update_label(self, evt_post, point_view): - ibar = point_view.x() - # if ibar > self.quotes_count: - # return - self.label_str = self.tick_to_string(ibar) - width = self.boundingRect().width() - offset = 0 # if have margins - new_pos = QtCore.QPointF(evt_post.x() - width / 2 - offset, 0) - self.setPos(new_pos) - - -class YAxisLabel(AxisLabel): - - text_flags = ( - QtCore.Qt.TextDontClip | QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter - ) - - def tick_to_string(self, tick_pos): - return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') - - def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 80, 40) - - def update_label(self, evt_post, point_view): - self.label_str = self.tick_to_string(point_view.y()) - height = self.boundingRect().height() - offset = 0 # if have margins - new_pos = QtCore.QPointF(0, evt_post.y() - height / 2 - offset) - self.setPos(new_pos) - - -class ChartView(pg.ViewBox): - """Price chart view box with interaction behaviors you'd expect from - an interactive platform: - - - zoom on mouse scroll that auto fits y-axis - - no vertical scrolling - - zoom to a "fixed point" on the y-axis - """ - def __init__( - self, - parent=None, - **kwargs, - # invertY=False, - ): - super().__init__(parent=parent, **kwargs) - self.chart = parent - - # disable vertical scrolling - self.setMouseEnabled(x=True, y=False) - - def wheelEvent(self, ev, axis=None): - """Override "center-point" location for scrolling. - - This is an override of the ``ViewBox`` method simply changing - the center of the zoom to be the y-axis. - - TODO: PR a method into ``pyqtgraph`` to make this configurable - """ - - if axis in (0, 1): - mask = [False, False] - mask[axis] = self.state['mouseEnabled'][axis] - else: - mask = self.state['mouseEnabled'][:] - - # actual scaling factor - s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) - s = [(None if m is False else s) for m in mask] - - # center = pg.Point( - # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) - # ) - - # XXX: scroll "around" the right most element in the view - furthest_right_coord = self.boundingRect().topRight() - center = pg.Point( - fn.invertQTransform( - self.childGroup.transform() - ).map(furthest_right_coord) - ) - - self._resetTarget() - self.scaleBy(s, center) - ev.accept() - self.sigRangeChangedManually.emit(mask) - - -# TODO: This is a sub-class of ``GracphicView`` which can -# take a ``background`` color setting. -class ChartPlotWidget(pg.PlotWidget): - """``GraphicsView`` subtype containing a single ``PlotItem``. - - Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing - a single ``PlotItem``) to intercept and and re-emit mouse enter/exit - events. - - (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we - eventually want multiple plots managed together). - """ - sig_mouse_leave = QtCore.Signal(object) - sig_mouse_enter = QtCore.Signal(object) - - def __init__( - self, - split_charts, - **kwargs, - # parent=None, - # background='default', - # plotItem=None, - # **kargs - ): - """Configure chart display settings. - """ - - super().__init__(**kwargs) - # label = pg.LabelItem(justify='left') - # self.addItem(label) - # label.setText("Yo yoyo") - # label.setText("x=") - self.parent = split_charts - - # show only right side axes - self.hideAxis('left') - self.showAxis('right') - - # show background grid - self.showGrid(x=True, y=True, alpha=0.4) - - # use cross-hair for cursor - self.setCursor(QtCore.Qt.CrossCursor) - - # set panning limits - min_points_to_show = 20 - min_bars_in_view = 10 - max_lookahead = min_points_to_show - min_bars_in_view - last = Quotes[-1].id - self.setLimits( - xMin=Quotes[0].id, - xMax=last + max_lookahead, - minXRange=min_points_to_show, - # maxYRange=highest-lowest, - yMin=Quotes.low.min() * 0.98, - yMax=Quotes.high.max() * 1.02, - ) - - # show last 50 points on startup - self.plotItem.vb.setXRange(last - 50, last + max_lookahead) - - # assign callback for rescaling y-axis automatically - # based on y-range contents - self.sigXRangeChanged.connect(self._update_yrange_limits) - self._update_yrange_limits() - - def bars_range(self): - """Return a range tuple for the bars present in view. - """ - - vr = self.viewRect() - lbar, rbar = int(vr.left()), int(min(vr.right(), len(Quotes) - 1)) - return lbar, rbar - - def draw_ohlc( - self, - style: ChartType = ChartType.BAR, - ) -> None: - """Draw OHLC datums to chart. - """ - - # adds all bar/candle graphics objects for each - # data point in the np array buffer to - # be drawn on next render cycle - self.addItem(_get_chart_points(style)) - - def draw_curve( - self, - data: np.ndarray, - ) -> None: - # draw the indicator as a plain curve - curve = pg.PlotDataItem(data, antialias=True) - ind_chart.addItem(curve) - - def _update_yrange_limits(self): - """Callback for each y-range update. - - This adds auto-scaling like zoom on the scroll wheel such - that data always fits nicely inside the current view of the - data set. - """ - # TODO: this can likely be ported in part to the built-ins: - # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) - # self.setMouseEnabled(x=True, y=False) - # self.setXRange(Quotes[0].id, Quotes[-1].id) - # self.setAutoVisible(x=False, y=True) - # self.enableAutoRange(x=False, y=True) - - chart = self - chart_parent = self.parent - - lbar, rbar = self.bars_range() - # vr = chart.viewRect() - # lbar, rbar = int(vr.left()), int(vr.right()) - - if chart_parent.signals_visible: - chart_parent._show_text_signals(lbar, rbar) - - bars = Quotes[lbar:rbar] - ylow = bars.low.min() * 0.98 - yhigh = bars.high.max() * 1.02 - - std = np.std(bars.close) - chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - chart.setYRange(ylow, yhigh) - - for i, d in chart_parent.indicators: - # ydata = i.plotItem.items[0].getData()[1] - ydata = d[lbar:rbar] - ylow = ydata.min() * 0.98 - yhigh = ydata.max() * 1.02 - std = np.std(ydata) - i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - i.setYRange(ylow, yhigh) - - - def enterEvent(self, ev): # noqa - # pg.PlotWidget.enterEvent(self, ev) - self.sig_mouse_enter.emit(self) - - def leaveEvent(self, ev): # noqa - # pg.PlotWidget.leaveEvent(self, ev) - self.sig_mouse_leave.emit(self) - self.scene().leaveEvent(ev) - - -_mouse_rate_limit = 60 -_xaxis_at = 'bottom' - - -class CrossHairItem(pg.GraphicsObject): - - def __init__(self, parent, indicators=None, digits=0): - super().__init__() - # self.pen = pg.mkPen('#000000') - self.pen = pg.mkPen('#a9a9a9') - self.parent = parent - self.indicators = {} - self.activeIndicator = None - self.xaxis = self.parent.getAxis('bottom') - self.yaxis = self.parent.getAxis('right') - - self.vline = self.parent.addLine(x=0, pen=self.pen, movable=False) - self.hline = self.parent.addLine(y=0, pen=self.pen, movable=False) - - self.proxy_moved = pg.SignalProxy( - self.parent.scene().sigMouseMoved, - rateLimit=_mouse_rate_limit, - slot=self.mouseMoved, - ) - - self.yaxis_label = YAxisLabel( - parent=self.yaxis, digits=digits, opacity=1 - ) - - indicators = indicators or [] - - if indicators: - # when there are indicators present in sub-plot rows - # take the last one (nearest to the bottom) and place the - # crosshair label on it's x-axis. - last_ind = indicators[-1] - - self.proxy_enter = pg.SignalProxy( - self.parent.sig_mouse_enter, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Enter', False), - ) - self.proxy_leave = pg.SignalProxy( - self.parent.sig_mouse_leave, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Leave', False), - ) - - # determine where to place x-axis label - if _xaxis_at == 'bottom': - # place below is last indicator subplot - self.xaxis_label = XAxisLabel( - parent=last_ind.getAxis('bottom'), opacity=1 - ) - else: - # keep x-axis right below main chart - self.xaxis_label = XAxisLabel(parent=self.xaxis, opacity=1) - - for i in indicators: - # add vertial and horizonal lines and a y-axis label - vl = i.addLine(x=0, pen=self.pen, movable=False) - hl = i.addLine(y=0, pen=self.pen, movable=False) - yl = YAxisLabel(parent=i.getAxis('right'), opacity=1) - - px_moved = pg.SignalProxy( - i.scene().sigMouseMoved, - rateLimit=_mouse_rate_limit, - slot=self.mouseMoved - ) - px_enter = pg.SignalProxy( - i.sig_mouse_enter, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Enter', i), - ) - px_leave = pg.SignalProxy( - i.sig_mouse_leave, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Leave', i), - ) - self.indicators[i] = { - 'vl': vl, - 'hl': hl, - 'yl': yl, - 'px': (px_moved, px_enter, px_leave), - } - - def mouseAction(self, action, ind=False): # noqa - if action == 'Enter': - # show horiz line and y-label - if ind: - self.indicators[ind]['hl'].show() - self.indicators[ind]['yl'].show() - self.activeIndicator = ind - else: - self.yaxis_label.show() - self.hline.show() - # Leave - else: - # hide horiz line and y-label - if ind: - self.indicators[ind]['hl'].hide() - self.indicators[ind]['yl'].hide() - self.activeIndicator = None - else: - self.yaxis_label.hide() - self.hline.hide() - - def mouseMoved(self, evt): # noqa - """Update horizonal and vertical lines when mouse moves inside - either the main chart or any indicator subplot. - """ - - pos = evt[0] - - # if the mouse is within the parent ``ChartPlotWidget`` - if self.parent.sceneBoundingRect().contains(pos): - # mouse_point = self.vb.mapSceneToView(pos) - mouse_point = self.parent.mapToView(pos) - - # move the vertial line to the current x coordinate - self.vline.setX(mouse_point.x()) - - # update the label on the bottom of the crosshair - self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) - - # update the vertical line in any indicators subplots - for opts in self.indicators.values(): - opts['vl'].setX(mouse_point.x()) - - if self.activeIndicator: - # vertial position of the mouse is inside an indicator - mouse_point_ind = self.activeIndicator.mapToView(pos) - self.indicators[self.activeIndicator]['hl'].setY( - mouse_point_ind.y() - ) - self.indicators[self.activeIndicator]['yl'].update_label( - evt_post=pos, point_view=mouse_point_ind - ) - else: - # vertial position of the mouse is inside the main chart - self.hline.setY(mouse_point.y()) - self.yaxis_label.update_label( - evt_post=pos, point_view=mouse_point - ) - - def paint(self, p, *args): - pass - - def boundingRect(self): - return self.parent.boundingRect() - - -class BarItem(pg.GraphicsObject): - # XXX: From the customGraphicsItem.py example: - # The only required methods are paint() and boundingRect() - - w = 0.5 - - bull_brush = bear_brush = pg.mkPen('#808080') - # bull_brush = pg.mkPen('#00cc00') - # bear_brush = pg.mkPen('#fa0000') - - def __init__(self): - super().__init__() - self.generatePicture() - - # TODO: this is the routine to be retriggered for redraw - @timeit - def generatePicture(self): - # pre-computing a QPicture object allows paint() to run much - # more quickly, rather than re-drawing the shapes every time. - self.picture = QtGui.QPicture() - p = QtGui.QPainter(self.picture) - self._generate(p) - p.end() - - def _generate(self, p): - # XXX: overloaded method to allow drawing other candle types - - high_to_low = np.array( - [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] - ) - open_stick = np.array( - [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) - for q in Quotes] - ) - close_stick = np.array( - [ - QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) - for q in Quotes - ] - ) - lines = np.concatenate([high_to_low, open_stick, close_stick]) - long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) - short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) - - p.setPen(self.bull_brush) - p.drawLines(*lines[long_bars]) - - p.setPen(self.bear_brush) - p.drawLines(*lines[short_bars]) - - def paint(self, p, *args): - p.drawPicture(0, 0, self.picture) - - def boundingRect(self): - # boundingRect _must_ indicate the entire area that will be - # drawn on or else we will get artifacts and possibly crashing. - # (in this case, QPicture does all the work of computing the - # bouning rect for us) - return QtCore.QRectF(self.picture.boundingRect()) - - -class CandlestickItem(BarItem): - - w2 = 0.7 - line_pen = pg.mkPen('#000000') - bull_brush = pg.mkBrush('#00ff00') - bear_brush = pg.mkBrush('#ff0000') - - def _generate(self, p): - rects = np.array( - [ - QtCore.QRectF(q.id - self.w, q.open, self.w2, q.close - q.open) - for q in Quotes - ] - ) - - p.setPen(self.line_pen) - p.drawLines( - [QtCore.QLineF(q.id, q.low, q.id, q.high) - for q in Quotes] - ) - - p.setBrush(self.bull_brush) - p.drawRects(*rects[Quotes.close > Quotes.open]) - - p.setBrush(self.bear_brush) - p.drawRects(*rects[Quotes.close < Quotes.open]) - - -class SplitterChart(QtGui.QWidget): - - long_pen = pg.mkPen('#006000') - long_brush = pg.mkBrush('#00ff00') - short_pen = pg.mkPen('#600000') - short_brush = pg.mkBrush('#ff0000') - - zoomIsDisabled = QtCore.pyqtSignal(bool) - - def __init__(self): - super().__init__() - self.signals_visible = False - self.indicators = [] - - self.xaxis = FromTimeFieldDateAxis(orientation='bottom') - # self.xaxis = pg.DateAxisItem() - - self.xaxis_ind = FromTimeFieldDateAxis(orientation='bottom') - - if _xaxis_at == 'bottom': - self.xaxis.setStyle(showValues=False) - else: - self.xaxis_ind.setStyle(showValues=False) - - self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) - self.splitter.setHandleWidth(5) - - self.layout = QtGui.QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) - - self.layout.addWidget(self.splitter) - - def _show_text_signals(self, lbar, rbar): - signals = [ - sig - for sig in self.signals_text_items[lbar:rbar] - if isinstance(sig, CenteredTextItem) - ] - if len(signals) <= 50: - for sig in signals: - sig.show() - else: - for sig in signals: - sig.hide() - - def _remove_signals(self): - self.chart.removeItem(self.signals_group_arrow) - self.chart.removeItem(self.signals_group_text) - del self.signals_text_items - del self.signals_group_arrow - del self.signals_group_text - self.signals_visible = False - - def _update_sizes(self): - min_h_ind = int(self.height() * 0.2 / len(self.indicators)) - sizes = [int(self.height() * 0.8)] - sizes.extend([min_h_ind] * len(self.indicators)) - self.splitter.setSizes(sizes) # , int(self.height()*0.2) - - def plot(self, symbol): - self.digits = symbol.digits - self.chart = ChartPlotWidget( - split_charts=self, - parent=self.splitter, - axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, - viewBox=ChartView, - # enableMenu=False, - ) - # TODO: ``pyqtgraph`` doesn't pass through a parent to the - # ``PlotItem`` by default; maybe we should PR this in? - self.chart.plotItem.parent = self - - self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) - self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - - # TODO: this is where we would load an indicator chain - inds = [Quotes.open] - - for d in inds: - ind = ChartPlotWidget( - split_charts=self, - parent=self.splitter, - axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, - # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, - viewBox=ChartView, - ) - ind.plotItem.parent = self - - ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - ind.getPlotItem().setContentsMargins(*CHART_MARGINS) - # self.splitter.addWidget(ind) - self.indicators.append((ind, d)) - - self.chart.draw_ohlc() - - for ind_chart, d in self.indicators: - - # link chart x-axis to main quotes chart - ind_chart.setXLink(self.chart) - - # XXX: never do this lol - # ind.setAspectLocked(1) - ind_chart.draw_curve(d) - - self._update_sizes() - - ch = CrossHairItem( - self.chart, [_ind for _ind, d in self.indicators], self.digits - ) - self.chart.addItem(ch) - - def add_signals(self): - self.signals_group_text = QtGui.QGraphicsItemGroup() - self.signals_group_arrow = QtGui.QGraphicsItemGroup() - self.signals_text_items = np.empty(len(Quotes), dtype=object) - - for p in Portfolio.positions: - x, price = p.id_bar_open, p.open_price - if p.type == Order.BUY: - y = Quotes[x].low * 0.99 - pg.ArrowItem( - parent=self.signals_group_arrow, - pos=(x, y), - pen=self.long_pen, - brush=self.long_brush, - angle=90, - headLen=12, - tipAngle=50, - ) - text_sig = CenteredTextItem( - parent=self.signals_group_text, - pos=(x, y), - pen=self.long_pen, - brush=self.long_brush, - text=('Buy at {:.%df}' % self.digits).format(price), - valign=QtCore.Qt.AlignBottom, - ) - text_sig.hide() - else: - y = Quotes[x].high * 1.01 - pg.ArrowItem( - parent=self.signals_group_arrow, - pos=(x, y), - pen=self.short_pen, - brush=self.short_brush, - angle=-90, - headLen=12, - tipAngle=50, - ) - text_sig = CenteredTextItem( - parent=self.signals_group_text, - pos=(x, y), - pen=self.short_pen, - brush=self.short_brush, - text=('Sell at {:.%df}' % self.digits).format(price), - valign=QtCore.Qt.AlignTop, - ) - text_sig.hide() - - self.signals_text_items[x] = text_sig - - self.chart.addItem(self.signals_group_arrow) - self.chart.addItem(self.signals_group_text) - self.signals_visible = True - - -# this function is borderline ridiculous. -# The creation of these chart types mutates all the input data -# inside each type's constructor (mind blown) -def _get_chart_points(style): - if style == ChartType.CANDLESTICK: - return CandlestickItem() - elif style == ChartType.BAR: - return BarItem() - return pg.PlotDataItem(Quotes.close, pen='b') From 507368a13accc88793055cf141174e8d0fa54041 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Jun 2020 11:40:41 -0400 Subject: [PATCH 015/206] Don't scroll right after max zoom --- piker/ui/qt/_chart.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/piker/ui/qt/_chart.py b/piker/ui/qt/_chart.py index e77f1191..b01a4110 100644 --- a/piker/ui/qt/_chart.py +++ b/piker/ui/qt/_chart.py @@ -19,9 +19,6 @@ from .quantdom.const import ChartType from .quantdom.portfolio import Order, Portfolio -# white background (for tinas like our pal xb) -# pg.setConfigOption('background', 'w') - # margins CHART_MARGINS = (0, 0, 10, 3) @@ -135,7 +132,10 @@ class SplitterChart(QtGui.QWidget): self.splitter.setSizes(sizes) # , int(self.height()*0.2) def plot(self, symbol): + """Start up and show price chart and all registered indicators. + """ self.digits = symbol.digits + self.chart = ChartPlotWidget( split_charts=self, parent=self.splitter, @@ -145,7 +145,7 @@ class SplitterChart(QtGui.QWidget): ) # TODO: ``pyqtgraph`` doesn't pass through a parent to the # ``PlotItem`` by default; maybe we should PR this in? - self.chart.plotItem.parent = self + self.chart.plotItem.vb.splitter_widget = self self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) @@ -161,7 +161,7 @@ class SplitterChart(QtGui.QWidget): # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, viewBox=ChartView, ) - ind.plotItem.parent = self + self.chart.plotItem.vb.splitter_widget = self ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) ind.getPlotItem().setContentsMargins(*CHART_MARGINS) @@ -241,6 +241,10 @@ class SplitterChart(QtGui.QWidget): self.signals_visible = True +_min_points_to_show = 20 +_min_bars_in_view = 10 + + # TODO: This is a sub-class of ``GracphicView`` which can # take a ``background`` color setting. class ChartPlotWidget(pg.PlotWidget): @@ -286,14 +290,12 @@ class ChartPlotWidget(pg.PlotWidget): self.setCursor(QtCore.Qt.CrossCursor) # set panning limits - min_points_to_show = 20 - min_bars_in_view = 10 - max_lookahead = min_points_to_show - min_bars_in_view + max_lookahead = _min_points_to_show - _min_bars_in_view last = Quotes[-1].id self.setLimits( xMin=Quotes[0].id, xMax=last + max_lookahead, - minXRange=min_points_to_show, + minXRange=_min_points_to_show, # maxYRange=highest-lowest, yMin=Quotes.low.min() * 0.98, yMax=Quotes.high.max() * 1.02, @@ -402,8 +404,6 @@ class ChartView(pg.ViewBox): # invertY=False, ): super().__init__(parent=parent, **kwargs) - self.chart = parent - # disable vertical scrolling self.setMouseEnabled(x=True, y=False) @@ -422,6 +422,11 @@ class ChartView(pg.ViewBox): else: mask = self.state['mouseEnabled'][:] + lbar, rbar = self.splitter_widget.chart.bars_range() + if ev.delta() >= 0 and rbar - lbar <= _min_points_to_show: + # don't zoom more then the min points setting + return + # actual scaling factor s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) s = [(None if m is False else s) for m in mask] From 613564b0f55db1b5e4e9ae77eeb015a5ade724bc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 15 Jun 2020 23:11:18 -0400 Subject: [PATCH 016/206] Add ui package mod --- piker/ui/__init__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 piker/ui/__init__.py diff --git a/piker/ui/__init__.py b/piker/ui/__init__.py new file mode 100644 index 00000000..ace44590 --- /dev/null +++ b/piker/ui/__init__.py @@ -0,0 +1,3 @@ +""" +Stuff for your eyes. +""" From f77a39ceb79f2b836f621586a9dd40a28143ec3e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Jun 2020 11:55:37 -0400 Subject: [PATCH 017/206] Add symbol-info command --- piker/brokers/cli.py | 23 +++++++++++++++++++++++ piker/brokers/core.py | 12 ------------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index c05be413..9f1ee4fa 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -225,3 +225,26 @@ def optsquote(config, symbol, df_output, date): click.echo(df) else: click.echo(colorize_json(quotes)) + + +@cli.command() +@click.argument('tickers', nargs=-1, required=True) +@click.pass_obj +def symbol_info(config, tickers): + """Print symbol quotes to the console + """ + # global opts + brokermod = config['brokermod'] + + quotes = trio.run(partial(core.symbol_info, brokermod, tickers)) + if not quotes: + log.error(f"No quotes could be found for {tickers}?") + return + + if len(quotes) < len(tickers): + syms = tuple(map(itemgetter('symbol'), quotes)) + for ticker in tickers: + if ticker not in syms: + brokermod.log.warn(f"Could not find symbol {ticker}?") + + click.echo(colorize_json(quotes)) diff --git a/piker/brokers/core.py b/piker/brokers/core.py index 2e672c61..e65fcb40 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -108,15 +108,3 @@ async def symbol_info( """ async with brokermod.get_client() as client: return await client.symbol_info(symbol, **kwargs) - - -async def symbol_search( - brokermod: ModuleType, - symbol: str, - **kwargs, -) -> Dict[str, Dict[str, Dict[str, Any]]]: - """Return symbol info from broker. - """ - async with brokermod.get_client() as client: - # TODO: support multiple asset type concurrent searches. - return await client.search_stocks(symbol, **kwargs) From 45906c2729089bdcba2a8937018f9653a46e2dcf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Jun 2020 13:32:03 -0400 Subject: [PATCH 018/206] Render plots from provided input sequence(s) Previously graphics were loaded and rendered implicitly during the import and creation of certain objects. Remove all this and instead expect client code to pass in the OHLC sequence to plot. Speed up the bars graphics rendering by simplifying to a single iteration of the input array; gives about a 2x speedup. --- piker/ui/cli.py | 18 ++++--- piker/ui/qt/_axes.py | 9 +++- piker/ui/qt/_chart.py | 110 ++++++++++++++++++++++----------------- piker/ui/qt/_graphics.py | 105 +++++++++++++++++++++++++------------ piker/ui/qt/_source.py | 97 ++++++++++++++++++++++++++++++++++ piker/ui/qt/_style.py | 18 +++++-- 6 files changed, 263 insertions(+), 94 deletions(-) create mode 100644 piker/ui/qt/_source.py diff --git a/piker/ui/cli.py b/piker/ui/cli.py index d5f6a9ed..54a77eee 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -118,20 +118,24 @@ def chart(config, symbol, date, tl, rate, test): """Start an option chain UI """ from .qt._exec import run_qtrio - from .qt._chart import QuotesTabWidget - from .qt.quantdom.base import Symbol + from .qt._chart import Chart + + # uses pandas_datareader from .qt.quantdom.loaders import get_quotes async def plot_symbol(widgets): + """Main Qt-trio routine invoked by the Qt loop with + the widgets ``dict``. + """ + qtw = widgets['main'] - s = Symbol(ticker=symbol, mode=Symbol.SHARES) - get_quotes( - symbol=s.ticker, + quotes = get_quotes( + symbol=symbol, date_from=datetime(1900, 1, 1), date_to=datetime(2030, 12, 31), ) # spawn chart - qtw.update_chart(s) + qtw.load_symbol(symbol, quotes) await trio.sleep_forever() - run_qtrio(plot_symbol, (), QuotesTabWidget) + run_qtrio(plot_symbol, (), Chart) diff --git a/piker/ui/qt/_axes.py b/piker/ui/qt/_axes.py index 96724a26..90b5c5f2 100644 --- a/piker/ui/qt/_axes.py +++ b/piker/ui/qt/_axes.py @@ -72,7 +72,14 @@ class AxisLabel(pg.GraphicsObject): bg_color = pg.mkColor('#808080') fg_color = pg.mkColor('#000000') - def __init__(self, parent=None, digits=0, color=None, opacity=1, **kwargs): + def __init__( + self, + parent=None, + digits=0, + color=None, + opacity=1, + **kwargs + ): super().__init__(parent) self.parent = parent self.opacity = opacity diff --git a/piker/ui/qt/_chart.py b/piker/ui/qt/_chart.py index b01a4110..cdf6b988 100644 --- a/piker/ui/qt/_chart.py +++ b/piker/ui/qt/_chart.py @@ -10,12 +10,13 @@ from ._axes import ( FromTimeFieldDateAxis, PriceAxis, ) -from ._graphics import CrossHairItem, CandlestickItem, BarItem +from ._graphics import CrossHairItem, ChartType from ._style import _xaxis_at +from ._source import Symbol from .quantdom.charts import CenteredTextItem from .quantdom.base import Quotes -from .quantdom.const import ChartType +# from .quantdom.const import ChartType from .quantdom.portfolio import Order, Portfolio @@ -23,20 +24,21 @@ from .quantdom.portfolio import Order, Portfolio CHART_MARGINS = (0, 0, 10, 3) -class QuotesTabWidget(QtGui.QWidget): +class Chart(QtGui.QWidget): def __init__(self, parent=None): super().__init__(parent) - self.layout = QtGui.QVBoxLayout(self) - self.layout.setContentsMargins(0, 0, 0, 0) + self.v_layout = QtGui.QVBoxLayout(self) + self.v_layout.setContentsMargins(0, 0, 0, 0) self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(10, 10, 15, 0) - self.chart_layout = QtGui.QHBoxLayout() + self.h_layout = QtGui.QHBoxLayout() # self.init_timeframes_ui() # self.init_strategy_ui() - self.layout.addLayout(self.toolbar_layout) - self.layout.addLayout(self.chart_layout) + self.v_layout.addLayout(self.toolbar_layout) + self.v_layout.addLayout(self.h_layout) + self._plot_cache = {} def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() @@ -58,21 +60,32 @@ class QuotesTabWidget(QtGui.QWidget): # self.strategy_box = StrategyBoxWidget(self) # self.toolbar_layout.addWidget(self.strategy_box) - # TODO: this needs to be changed to ``load_symbol()`` - # which will not only load historical data but also a real-time - # stream and schedule the redraw events on new quotes - def update_chart(self, symbol): - if not self.chart_layout.isEmpty(): - self.chart_layout.removeWidget(self.chart) - self.chart = SplitterChart() - self.chart.plot(symbol) - self.chart_layout.addWidget(self.chart) + def load_symbol( + self, + symbol: str, + data: np.ndarray, + ) -> None: + """Load a new contract into the charting app. + """ + # XXX: let's see if this causes mem problems + self.chart = self._plot_cache.setdefault(symbol, SplitterPlots()) + s = Symbol(key=symbol) - def add_signals(self): - self.chart.add_signals() + # remove any existing plots + if not self.h_layout.isEmpty(): + self.h_layout.removeWidget(self.chart) + + self.chart.plot(s, data) + self.h_layout.addWidget(self.chart) + + # TODO: add signalling painter system + # def add_signals(self): + # self.chart.add_signals() -class SplitterChart(QtGui.QWidget): +class SplitterPlots(QtGui.QWidget): + """Widget that holds a price chart plus indicators separated by splitters. + """ long_pen = pg.mkPen('#006000') long_brush = pg.mkBrush('#00ff00') @@ -131,16 +144,21 @@ class SplitterChart(QtGui.QWidget): sizes.extend([min_h_ind] * len(self.indicators)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) - def plot(self, symbol): + def plot( + self, + symbol: Symbol, + data: np.ndarray, + ): """Start up and show price chart and all registered indicators. """ - self.digits = symbol.digits + self.digits = symbol.digits() + cv = ChartView() self.chart = ChartPlotWidget( split_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, - viewBox=ChartView, + viewBox=cv, # enableMenu=False, ) # TODO: ``pyqtgraph`` doesn't pass through a parent to the @@ -150,27 +168,28 @@ class SplitterChart(QtGui.QWidget): self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + self.chart.draw_ohlc(data) + # TODO: this is where we would load an indicator chain inds = [Quotes.open] for d in inds: - ind = ChartPlotWidget( + cv = ChartView() + ind_chart = ChartPlotWidget( split_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, - viewBox=ChartView, + viewBox=cv, ) self.chart.plotItem.vb.splitter_widget = self - ind.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - ind.getPlotItem().setContentsMargins(*CHART_MARGINS) - # self.splitter.addWidget(ind) - self.indicators.append((ind, d)) - - self.chart.draw_ohlc() - - for ind_chart, d in self.indicators: + ind_chart.setFrameStyle( + QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain + ) + ind_chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + # self.splitter.addWidget(ind_chart) + self.indicators.append((ind_chart, d)) # link chart x-axis to main quotes chart ind_chart.setXLink(self.chart) @@ -245,8 +264,6 @@ _min_points_to_show = 20 _min_bars_in_view = 10 -# TODO: This is a sub-class of ``GracphicView`` which can -# take a ``background`` color setting. class ChartPlotWidget(pg.PlotWidget): """``GraphicsView`` subtype containing a single ``PlotItem``. @@ -260,6 +277,9 @@ class ChartPlotWidget(pg.PlotWidget): sig_mouse_leave = QtCore.Signal(object) sig_mouse_enter = QtCore.Signal(object) + # TODO: can take a ``background`` color setting - maybe there's + # a better one? + def __init__( self, split_charts, @@ -319,15 +339,18 @@ class ChartPlotWidget(pg.PlotWidget): def draw_ohlc( self, + data: np.ndarray, style: ChartType = ChartType.BAR, ) -> None: """Draw OHLC datums to chart. """ - + # remember it's an enum type.. + graphics = style.value() # adds all bar/candle graphics objects for each # data point in the np array buffer to # be drawn on next render cycle - self.addItem(_get_chart_points(style)) + graphics.draw_from_data(data) + self.addItem(graphics) def draw_curve( self, @@ -422,9 +445,9 @@ class ChartView(pg.ViewBox): else: mask = self.state['mouseEnabled'][:] + # don't zoom more then the min points setting lbar, rbar = self.splitter_widget.chart.bars_range() if ev.delta() >= 0 and rbar - lbar <= _min_points_to_show: - # don't zoom more then the min points setting return # actual scaling factor @@ -447,14 +470,3 @@ class ChartView(pg.ViewBox): self.scaleBy(s, center) ev.accept() self.sigRangeChangedManually.emit(mask) - - -# this function is borderline ridiculous. -# The creation of these chart types mutates all the input data -# inside each type's constructor (mind blown) -def _get_chart_points(style): - if style == ChartType.CANDLESTICK: - return CandlestickItem() - elif style == ChartType.BAR: - return BarItem() - return pg.PlotDataItem(Quotes.close, pen='b') diff --git a/piker/ui/qt/_graphics.py b/piker/ui/qt/_graphics.py index bff8c043..9b90e8cf 100644 --- a/piker/ui/qt/_graphics.py +++ b/piker/ui/qt/_graphics.py @@ -1,14 +1,18 @@ """ Chart graphics for displaying a slew of different data types. """ +from enum import Enum +from contextlib import contextmanager + import numpy as np import pyqtgraph as pg from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import QLineF from .quantdom.utils import timeit from .quantdom.base import Quotes -from ._style import _xaxis_at +from ._style import _xaxis_at, _tina_mode from ._axes import YAxisLabel, XAxisLabel @@ -163,11 +167,10 @@ class CrossHairItem(pg.GraphicsObject): return self.parent.boundingRect() -class BarItem(pg.GraphicsObject): - # XXX: From the customGraphicsItem.py example: - # The only required methods are paint() and boundingRect() - - w = 0.5 +class BarItems(pg.GraphicsObject): + """Price range bars graphics rendered from a OHLC sequence. + """ + w: float = 0.5 bull_brush = bear_brush = pg.mkPen('#808080') # bull_brush = pg.mkPen('#00cc00') @@ -175,44 +178,72 @@ class BarItem(pg.GraphicsObject): def __init__(self): super().__init__() - self.generatePicture() + self.picture = QtGui.QPicture() + self.lines = None + # self.generatePicture() # TODO: this is the routine to be retriggered for redraw - @timeit - def generatePicture(self): + @contextmanager + def painter(self): # pre-computing a QPicture object allows paint() to run much # more quickly, rather than re-drawing the shapes every time. - self.picture = QtGui.QPicture() p = QtGui.QPainter(self.picture) - self._generate(p) + yield p p.end() - def _generate(self, p): + @timeit + def draw_from_data(self, data): # XXX: overloaded method to allow drawing other candle types - high_to_low = np.array( - [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] - ) - open_stick = np.array( - [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) - for q in Quotes] - ) - close_stick = np.array( - [ - QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) - for q in Quotes - ] - ) - lines = np.concatenate([high_to_low, open_stick, close_stick]) - long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) - short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) + high_to_low = np.empty_like(data, dtype=object) + open_sticks = np.empty_like(data, dtype=object) + close_sticks = np.empty_like(data, dtype=object) + with self.painter() as p: + import time + start = time.time() + for i, q in enumerate(data): + high_to_low[i] = QLineF(q['id'], q['low'], q['id'], q['high']) + open_sticks[i] = QLineF( + q['id'] - self.w, q['open'], q['id'], q['open']) + close_sticks[i] = QtCore.QLineF( + q['id'] + self.w, q['close'], q['id'], q['close']) - p.setPen(self.bull_brush) - p.drawLines(*lines[long_bars]) + # high_to_low = np.array( + # [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] + # ) + # open_sticks = np.array( + # [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) + # for q in Quotes] + # ) + # close_sticks = np.array( + # [ + # QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) + # for q in Quotes + # ] + # ) + print(f"took {time.time() - start}") + self.lines = lines = np.concatenate([high_to_low, open_sticks, close_sticks]) - p.setPen(self.bear_brush) - p.drawLines(*lines[short_bars]) + if _tina_mode: + long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) + ups = lines[long_bars] + downs = lines[short_bars] + # draw "up" bars + p.setPen(self.bull_brush) + p.drawLines(*ups) + + # draw "down" bars + p.setPen(self.bear_brush) + p.drawLines(*downs) + + else: # piker mode + p.setPen(self.bull_brush) + p.drawLines(*lines) + + # XXX: From the customGraphicsItem.py example: + # The only required methods are paint() and boundingRect() def paint(self, p, *args): p.drawPicture(0, 0, self.picture) @@ -224,7 +255,7 @@ class BarItem(pg.GraphicsObject): return QtCore.QRectF(self.picture.boundingRect()) -class CandlestickItem(BarItem): +class CandlestickItems(BarItems): w2 = 0.7 line_pen = pg.mkPen('#000000') @@ -250,3 +281,11 @@ class CandlestickItem(BarItem): p.setBrush(self.bear_brush) p.drawRects(*rects[Quotes.close < Quotes.open]) + + +class ChartType(Enum): + """Bar type to graphics class map. + """ + BAR = BarItems + CANDLESTICK = CandlestickItems + LINE = pg.PlotDataItem diff --git a/piker/ui/qt/_source.py b/piker/ui/qt/_source.py new file mode 100644 index 00000000..3d6fbae7 --- /dev/null +++ b/piker/ui/qt/_source.py @@ -0,0 +1,97 @@ +""" +Numpy data source machinery. +""" +import math +from dataclasses import dataclass + +import numpy as np +import pandas as pd + + +OHLC_dtype = np.dtype( + [ + ('id', int), + ('time', float), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', int), + ] +) + +# tf = { +# 1: TimeFrame.M1, +# 5: TimeFrame.M5, +# 15: TimeFrame.M15, +# 30: TimeFrame.M30, +# 60: TimeFrame.H1, +# 240: TimeFrame.H4, +# 1440: TimeFrame.D1, +# } + + +@dataclass +class Symbol: + """I guess this is some kinda container thing for dealing with + all the different meta-data formats from brokers? + """ + key: str = '' + min_tick: float = 0.01 + contract: str = '' + + def digits(self) -> int: + """Return the trailing number of digits specified by the + min tick size for the instrument. + """ + return int(math.log(self.min_tick, 0.1)) + + +def from_df( + df: pd.DataFrame, + source=None, + default_tf=None +): + """Cast OHLC ``pandas.DataFrame`` to ``numpy.structarray``. + """ + # shape = (len(df),) + # array.resize(shape, refcheck=False) + array = np.array([], dtype=OHLC_dtype) + + df.reset_index(inplace=True) + df.insert(0, 'id', df.index) + array['time'] = np.array([d.timestamp().time for d in df.Date]) + + # try to rename from some camel case + df = df.rename( + columns={ + # 'Date': 'time', + 'Open': 'open', + 'High': 'high', + 'Low': 'low', + 'Close': 'close', + 'Volume': 'volume', + } + ) + for name in array.dtype.names: + array[name] = df[name] + + array[:] = df[:] + + _nan_to_closest_num(array) + # self._set_time_frame(default_tf) + + return array + + +def _nan_to_closest_num(array: np.ndarray): + """Return interpolated values instead of NaN. + """ + + for col in ['open', 'high', 'low', 'close']: + mask = np.isnan(array[col]) + if not mask.size: + continue + array[col][mask] = np.interp( + np.flatnonzero(mask), np.flatnonzero(~mask), array[col][~mask] + ) diff --git a/piker/ui/qt/_style.py b/piker/ui/qt/_style.py index 8630d67d..b35e2e47 100644 --- a/piker/ui/qt/_style.py +++ b/piker/ui/qt/_style.py @@ -4,10 +4,6 @@ Qt styling. from PyQt5 import QtGui -# TODO: add "tina mode" to make everything look "conventional" -# white background (for tinas like our pal xb) -# pg.setConfigOption('background', 'w') - # chart-wide font _font = QtGui.QFont("Hack", 4) @@ -16,3 +12,17 @@ _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config _xaxis_at = 'bottom' + + +_tina_mode = False + + +def enable_tina_mode() -> None: + """Enable "tina mode" to make everything look "conventional" + like your pet hedgehog always wanted. + """ + + _tina_mode = True + + # white background (for tinas like our pal xb) + pg.setConfigOption('background', 'w') From b82587f665263559243d231d7f9b621f7b2d1b2a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 16 Jun 2020 14:24:24 -0400 Subject: [PATCH 019/206] Use a single array for all lines Speed up the lines array creation using proper slice assignment. This gives another 10% speedup to the historical price rendering. Drop ``_tina_mode`` support for now since we're not testing it. --- piker/ui/qt/_graphics.py | 67 +++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 38 deletions(-) diff --git a/piker/ui/qt/_graphics.py b/piker/ui/qt/_graphics.py index 9b90e8cf..093d66b4 100644 --- a/piker/ui/qt/_graphics.py +++ b/piker/ui/qt/_graphics.py @@ -193,50 +193,41 @@ class BarItems(pg.GraphicsObject): @timeit def draw_from_data(self, data): - # XXX: overloaded method to allow drawing other candle types - - high_to_low = np.empty_like(data, dtype=object) - open_sticks = np.empty_like(data, dtype=object) - close_sticks = np.empty_like(data, dtype=object) + self.lines = lines = np.empty_like(data, shape=(data.shape[0]*3,), dtype=object) + # open_sticks = np.empty_like(data, dtype=object) + # close_sticks = np.empty_like(data, dtype=object) with self.painter() as p: - import time - start = time.time() + # import time + # start = time.time() for i, q in enumerate(data): - high_to_low[i] = QLineF(q['id'], q['low'], q['id'], q['high']) - open_sticks[i] = QLineF( - q['id'] - self.w, q['open'], q['id'], q['open']) - close_sticks[i] = QtCore.QLineF( - q['id'] + self.w, q['close'], q['id'], q['close']) - - # high_to_low = np.array( - # [QtCore.QLineF(q.id, q.low, q.id, q.high) for q in Quotes] - # ) - # open_sticks = np.array( - # [QtCore.QLineF(q.id - self.w, q.open, q.id, q.open) - # for q in Quotes] - # ) - # close_sticks = np.array( - # [ - # QtCore.QLineF(q.id + self.w, q.close, q.id, q.close) - # for q in Quotes - # ] - # ) - print(f"took {time.time() - start}") - self.lines = lines = np.concatenate([high_to_low, open_sticks, close_sticks]) + # indexing here is as per the below comments + lines[3*i:3*i+3] = ( + # high_to_low + QLineF(q['id'], q['low'], q['id'], q['high']), + # open_sticks + QLineF(q['id'] - self.w, q['open'], q['id'], q['open']), + # close_sticks + QtCore.QLineF(q['id'] + self.w, q['close'], q['id'], q['close']) + ) + # print(f"took {time.time() - start}") + # self.lines = lines = np.concatenate([high_to_low, open_sticks, close_sticks]) if _tina_mode: - long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) - short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) - ups = lines[long_bars] - downs = lines[short_bars] + pass + # use traditional up/down green/red coloring + # long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + # short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) - # draw "up" bars - p.setPen(self.bull_brush) - p.drawLines(*ups) + # ups = lines[long_bars] + # downs = lines[short_bars] - # draw "down" bars - p.setPen(self.bear_brush) - p.drawLines(*downs) + # # draw "up" bars + # p.setPen(self.bull_brush) + # p.drawLines(*ups) + + # # draw "down" bars + # p.setPen(self.bear_brush) + # p.drawLines(*downs) else: # piker mode p.setPen(self.bull_brush) From 51f302191ae3594879ce0e5b4f83dd6daeb65ded Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 09:29:18 -0400 Subject: [PATCH 020/206] Add update method for last bars graphic --- piker/ui/qt/_graphics.py | 100 ++++++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 28 deletions(-) diff --git a/piker/ui/qt/_graphics.py b/piker/ui/qt/_graphics.py index 093d66b4..78f14d8e 100644 --- a/piker/ui/qt/_graphics.py +++ b/piker/ui/qt/_graphics.py @@ -1,6 +1,7 @@ """ Chart graphics for displaying a slew of different data types. """ +from typing import Dict, Any from enum import Enum from contextlib import contextmanager @@ -12,11 +13,12 @@ from PyQt5.QtCore import QLineF from .quantdom.utils import timeit from .quantdom.base import Quotes -from ._style import _xaxis_at, _tina_mode +from ._style import _xaxis_at # , _tina_mode from ._axes import YAxisLabel, XAxisLabel +# TODO: checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 60 +_mouse_rate_limit = 50 class CrossHairItem(pg.GraphicsObject): @@ -170,9 +172,13 @@ class CrossHairItem(pg.GraphicsObject): class BarItems(pg.GraphicsObject): """Price range bars graphics rendered from a OHLC sequence. """ + sigPlotChanged = QtCore.Signal(object) + w: float = 0.5 bull_brush = bear_brush = pg.mkPen('#808080') + + # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') # bear_brush = pg.mkPen('#fa0000') @@ -180,7 +186,7 @@ class BarItems(pg.GraphicsObject): super().__init__() self.picture = QtGui.QPicture() self.lines = None - # self.generatePicture() + self._last_quote = {} # TODO: this is the routine to be retriggered for redraw @contextmanager @@ -192,13 +198,18 @@ class BarItems(pg.GraphicsObject): p.end() @timeit - def draw_from_data(self, data): - self.lines = lines = np.empty_like(data, shape=(data.shape[0]*3,), dtype=object) - # open_sticks = np.empty_like(data, dtype=object) - # close_sticks = np.empty_like(data, dtype=object) + def draw_from_data( + self, + data: np.recarray, + ): + """Draw OHLC datum graphics from a ``np.recarray``. + """ + # XXX: not sure this actually needs to be an array other + # then for the old tina mode calcs for up/down bars below? + self.lines = lines = np.empty_like( + data, shape=(data.shape[0]*3,), dtype=object) + with self.painter() as p: - # import time - # start = time.time() for i, q in enumerate(data): # indexing here is as per the below comments lines[3*i:3*i+3] = ( @@ -207,31 +218,64 @@ class BarItems(pg.GraphicsObject): # open_sticks QLineF(q['id'] - self.w, q['open'], q['id'], q['open']), # close_sticks - QtCore.QLineF(q['id'] + self.w, q['close'], q['id'], q['close']) + QtCore.QLineF( + q['id'] + self.w, q['close'], q['id'], q['close']) ) - # print(f"took {time.time() - start}") - # self.lines = lines = np.concatenate([high_to_low, open_sticks, close_sticks]) + else: + self._last_quote = q + # if not _tina_mode: # piker mode + p.setPen(self.bull_brush) + p.drawLines(*lines) + # else _tina_mode: + # self.lines = lines = np.concatenate( + # [high_to_low, open_sticks, close_sticks]) + # use traditional up/down green/red coloring + # long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + # short_bars = np.resize( + # Quotes.close < Quotes.open, len(lines)) - if _tina_mode: - pass - # use traditional up/down green/red coloring - # long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) - # short_bars = np.resize(Quotes.close < Quotes.open, len(lines)) + # ups = lines[long_bars] + # downs = lines[short_bars] - # ups = lines[long_bars] - # downs = lines[short_bars] + # # draw "up" bars + # p.setPen(self.bull_brush) + # p.drawLines(*ups) - # # draw "up" bars - # p.setPen(self.bull_brush) - # p.drawLines(*ups) + # # draw "down" bars + # p.setPen(self.bear_brush) + # p.drawLines(*downs) - # # draw "down" bars - # p.setPen(self.bear_brush) - # p.drawLines(*downs) + def update_last_bar( + self, + quote: Dict[str, Any], + ) -> None: + """Update the last datum's bar graphic from a quote ``dict``. + """ + last = quote['last'] + body, larm, rarm = self.lines[-3:] # close line is absolute last - else: # piker mode - p.setPen(self.bull_brush) - p.drawLines(*lines) + # XXX: is there a faster way to modify this? + + # update right arm + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # update body + high = body.y2() + low = body.y1() + if last < low: + low = last + + if last > high: + high = last + + body.setLine(body.x1(), low, body.x2(), high) + + with self.painter() as p: + p.setPen(self.bull_brush) + p.drawLines(*self.lines) + + # trigger re-draw + self.update() # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() From 36ac26cdcf0976484b9532b756263aa46cd82b56 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 11:44:54 -0400 Subject: [PATCH 021/206] Add zeroed ohlc array constructor --- piker/ui/qt/_source.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/piker/ui/qt/_source.py b/piker/ui/qt/_source.py index 3d6fbae7..a23c9f41 100644 --- a/piker/ui/qt/_source.py +++ b/piker/ui/qt/_source.py @@ -31,6 +31,15 @@ OHLC_dtype = np.dtype( # } +def ohlc_zeros(length: int) -> np.ndarray: + """Construct an OHLC field formatted structarray. + + For "why a structarray" see here: https://stackoverflow.com/a/52443038 + Bottom line, they're faster then ``np.recarray``. + """ + return np.zeros(length, dtype=OHLC_dtype) + + @dataclass class Symbol: """I guess this is some kinda container thing for dealing with From 9d6dffe5ecdd5cc79fb3a72e03887c77e7a99ee9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 11:45:43 -0400 Subject: [PATCH 022/206] Cleanup yrange auto-update callback This was a mess before with a weird loop using the parent split charts to update all "indicators". Instead just have each plot do its own yrange updates since the signals are being handled just fine per plot. Handle both the OHLC and plane line chart cases with a hacky `try:, except IndexError:` for now. Oh, and move the main entry point for the chart app to the relevant module. I added some WIP bar update code for the moment. --- piker/ui/cli.py | 25 +----- piker/ui/qt/_chart.py | 173 +++++++++++++++++++++++++++++++----------- 2 files changed, 129 insertions(+), 69 deletions(-) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 54a77eee..59d7bfb5 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -1,11 +1,9 @@ """ Console interface to UI components. """ -from datetime import datetime from functools import partial import os import click -import trio import tractor from ..cli import cli @@ -117,25 +115,6 @@ def optschain(config, symbol, date, tl, rate, test): def chart(config, symbol, date, tl, rate, test): """Start an option chain UI """ - from .qt._exec import run_qtrio - from .qt._chart import Chart + from .qt._chart import main - # uses pandas_datareader - from .qt.quantdom.loaders import get_quotes - - async def plot_symbol(widgets): - """Main Qt-trio routine invoked by the Qt loop with - the widgets ``dict``. - """ - - qtw = widgets['main'] - quotes = get_quotes( - symbol=symbol, - date_from=datetime(1900, 1, 1), - date_to=datetime(2030, 12, 31), - ) - # spawn chart - qtw.load_symbol(symbol, quotes) - await trio.sleep_forever() - - run_qtrio(plot_symbol, (), Chart) + main(symbol) diff --git a/piker/ui/qt/_chart.py b/piker/ui/qt/_chart.py index cdf6b988..28425605 100644 --- a/piker/ui/qt/_chart.py +++ b/piker/ui/qt/_chart.py @@ -1,6 +1,7 @@ """ High level Qt chart widgets. """ +import trio import numpy as np import pyqtgraph as pg from pyqtgraph import functions as fn @@ -12,7 +13,7 @@ from ._axes import ( ) from ._graphics import CrossHairItem, ChartType from ._style import _xaxis_at -from ._source import Symbol +from ._source import Symbol, ohlc_zeros from .quantdom.charts import CenteredTextItem from .quantdom.base import Quotes @@ -77,6 +78,7 @@ class Chart(QtGui.QWidget): self.chart.plot(s, data) self.h_layout.addWidget(self.chart) + return self.chart # TODO: add signalling painter system # def add_signals(self): @@ -163,6 +165,7 @@ class SplitterPlots(QtGui.QWidget): ) # TODO: ``pyqtgraph`` doesn't pass through a parent to the # ``PlotItem`` by default; maybe we should PR this in? + cv.splitter_widget = self self.chart.plotItem.vb.splitter_widget = self self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) @@ -182,6 +185,7 @@ class SplitterPlots(QtGui.QWidget): # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, viewBox=cv, ) + cv.splitter_widget = self self.chart.plotItem.vb.splitter_widget = self ind_chart.setFrameStyle( @@ -260,7 +264,7 @@ class SplitterPlots(QtGui.QWidget): self.signals_visible = True -_min_points_to_show = 20 +_min_points_to_show = 15 _min_bars_in_view = 10 @@ -291,7 +295,6 @@ class ChartPlotWidget(pg.PlotWidget): ): """Configure chart display settings. """ - super().__init__(**kwargs) # label = pg.LabelItem(justify='left') # self.addItem(label) @@ -299,6 +302,12 @@ class ChartPlotWidget(pg.PlotWidget): # label.setText("x=") self.parent = split_charts + # placeholder for source of data + self._array = ohlc_zeros(1) + + # to be filled in when data is loaded + self._graphics = {} + # show only right side axes self.hideAxis('left') self.showAxis('right') @@ -306,51 +315,75 @@ class ChartPlotWidget(pg.PlotWidget): # show background grid self.showGrid(x=True, y=True, alpha=0.4) + self.plotItem.vb.setXRange(0, 0) + # use cross-hair for cursor self.setCursor(QtCore.Qt.CrossCursor) - # set panning limits - max_lookahead = _min_points_to_show - _min_bars_in_view - last = Quotes[-1].id - self.setLimits( - xMin=Quotes[0].id, - xMax=last + max_lookahead, - minXRange=_min_points_to_show, - # maxYRange=highest-lowest, - yMin=Quotes.low.min() * 0.98, - yMax=Quotes.high.max() * 1.02, - ) - - # show last 50 points on startup - self.plotItem.vb.setXRange(last - 50, last + max_lookahead) - # assign callback for rescaling y-axis automatically # based on y-range contents self.sigXRangeChanged.connect(self._update_yrange_limits) + + def set_view_limits(self, xfirst, xlast, ymin, ymax): + # max_lookahead = _min_points_to_show - _min_bars_in_view + + # set panning limits + # last = data[-1]['id'] + self.setLimits( + # xMin=data[0]['id'], + xMin=xfirst, + # xMax=last + _min_points_to_show - 3, + xMax=xlast + _min_points_to_show - 3, + minXRange=_min_points_to_show, + # maxYRange=highest-lowest, + # yMin=data['low'].min() * 0.98, + # yMax=data['high'].max() * 1.02, + yMin=ymin * 0.98, + yMax=ymax * 1.02, + ) + + # show last 50 points on startup + # self.plotItem.vb.setXRange(last - 50, last + 50) + self.plotItem.vb.setXRange(xlast - 50, xlast + 50) + + # fit y self._update_yrange_limits() def bars_range(self): """Return a range tuple for the bars present in view. """ - vr = self.viewRect() - lbar, rbar = int(vr.left()), int(min(vr.right(), len(Quotes) - 1)) + lbar = int(vr.left()) + rbar = int(min(vr.right(), len(self._array) - 1)) return lbar, rbar def draw_ohlc( self, data: np.ndarray, + # XXX: pretty sure this is dumb and we don't need an Enum style: ChartType = ChartType.BAR, ) -> None: """Draw OHLC datums to chart. """ # remember it's an enum type.. graphics = style.value() - # adds all bar/candle graphics objects for each - # data point in the np array buffer to - # be drawn on next render cycle + + # adds all bar/candle graphics objects for each data point in + # the np array buffer to be drawn on next render cycle graphics.draw_from_data(data) + self._graphics['ohlc'] = graphics self.addItem(graphics) + self._array = data + + # update view limits + self.set_view_limits( + data[0]['id'], + data[-1]['id'], + data['low'].min(), + data['high'].max() + ) + + return graphics def draw_curve( self, @@ -360,6 +393,12 @@ class ChartPlotWidget(pg.PlotWidget): curve = pg.PlotDataItem(data, antialias=True) self.addItem(curve) + # update view limits + self.set_view_limits(0, len(data)-1, data.min(), data.max()) + self._array = data + + return curve + def _update_yrange_limits(self): """Callback for each y-range update. @@ -374,34 +413,40 @@ class ChartPlotWidget(pg.PlotWidget): # self.setAutoVisible(x=False, y=True) # self.enableAutoRange(x=False, y=True) - chart = self - chart_parent = self.parent - lbar, rbar = self.bars_range() - # vr = chart.viewRect() - # lbar, rbar = int(vr.left()), int(vr.right()) - if chart_parent.signals_visible: - chart_parent._show_text_signals(lbar, rbar) + # if chart_parent.signals_visible: + # chart_parent._show_text_signals(lbar, rbar) - bars = Quotes[lbar:rbar] - ylow = bars.low.min() * 0.98 - yhigh = bars.high.max() * 1.02 + bars = self._array[lbar:rbar] + if not len(bars): + # likely no data loaded yet + return - std = np.std(bars.close) - chart.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) + # TODO: should probably just have some kinda attr mark + # that determines this behavior based on array type + try: + ylow = bars['low'].min() + yhigh = bars['high'].max() + std = np.std(bars['close']) + except IndexError: + # must be non-ohlc array? + ylow = bars.min() + yhigh = bars.max() + std = np.std(bars) + + # view margins + ylow *= 0.98 + yhigh *= 1.02 + + chart = self + chart.setLimits( + yMin=ylow, + yMax=yhigh, + minYRange=std + ) chart.setYRange(ylow, yhigh) - for i, d in chart_parent.indicators: - # ydata = i.plotItem.items[0].getData()[1] - ydata = d[lbar:rbar] - ylow = ydata.min() * 0.98 - yhigh = ydata.max() * 1.02 - std = np.std(ydata) - i.setLimits(yMin=ylow, yMax=yhigh, minYRange=std) - i.setYRange(ylow, yhigh) - - def enterEvent(self, ev): # noqa # pg.PlotWidget.enterEvent(self, ev) self.sig_mouse_enter.emit(self) @@ -429,6 +474,7 @@ class ChartView(pg.ViewBox): super().__init__(parent=parent, **kwargs) # disable vertical scrolling self.setMouseEnabled(x=True, y=False) + self.splitter_widget = None def wheelEvent(self, ev, axis=None): """Override "center-point" location for scrolling. @@ -447,11 +493,12 @@ class ChartView(pg.ViewBox): # don't zoom more then the min points setting lbar, rbar = self.splitter_widget.chart.bars_range() + # breakpoint() if ev.delta() >= 0 and rbar - lbar <= _min_points_to_show: return # actual scaling factor - s = 1.02 ** (ev.delta() * self.state['wheelScaleFactor']) + s = 1.02 ** (ev.delta() * -1/10) # self.state['wheelScaleFactor']) s = [(None if m is False else s) for m in mask] # center = pg.Point( @@ -470,3 +517,37 @@ class ChartView(pg.ViewBox): self.scaleBy(s, center) ev.accept() self.sigRangeChangedManually.emit(mask) + + +def main(symbol): + """Entry point to spawn a chart app. + """ + from datetime import datetime + + from ._exec import run_qtrio + # uses pandas_datareader + from .quantdom.loaders import get_quotes + + async def _main(widgets): + """Main Qt-trio routine invoked by the Qt loop with + the widgets ``dict``. + """ + + chart_app = widgets['main'] + quotes = get_quotes( + symbol=symbol, + date_from=datetime(1900, 1, 1), + date_to=datetime(2030, 12, 31), + ) + # spawn chart + splitter_chart = chart_app.load_symbol(symbol, quotes) + import itertools + nums = itertools.cycle([315., 320., 325.]) + while True: + await trio.sleep(0.05) + splitter_chart.chart._graphics['ohlc'].update_last_bar( + {'last': next(nums)}) + # splitter_chart.chart.plotItem.sigPlotChanged.emit(self) + # breakpoint() + + run_qtrio(_main, (), Chart) From 82a5daf91bef8ddc0f9eaef5d56f35267b9abc35 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 14:51:29 -0400 Subject: [PATCH 023/206] Move all kivy ui components to subpackage --- piker/brokers/questrade.py | 8 +++++++- piker/ui/{ => kivy}/monitor.py | 4 ++-- piker/ui/kivy/mouse_over.py | 4 ++-- piker/ui/{ => kivy}/option_chain.py | 7 +++---- piker/ui/{ => kivy}/pager.py | 4 ++-- piker/ui/{ => kivy}/tabular.py | 4 ++-- 6 files changed, 18 insertions(+), 13 deletions(-) rename piker/ui/{ => kivy}/monitor.py (99%) rename piker/ui/{ => kivy}/option_chain.py (99%) rename piker/ui/{ => kivy}/pager.py (99%) rename piker/ui/{ => kivy}/tabular.py (99%) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 58b31fcf..f3708d3f 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -989,13 +989,19 @@ def format_option_quote( # change = percent_change(previous, last) computed = { # why QT do you have to be an asshole shipping null values!!! - '$ vol': round((quote['VWAP'] or 0) * (quote['volume'] or 0), 3), + # '$ vol': round((quote['VWAP'] or 0) * (quote['volume'] or 0), 3), # '%': round(change, 3), # 'close': previous, } new = {} displayable = {} + vwap = quote.get('VWAP') + volume = quote.get('volume') + if volume is not None: # could be 0 + # why questrade do you have to be an asshole shipping null values!!! + computed['$ vol'] = round((vwap or 0) * (volume or 0), 3) + # structuring and normalization for key, new_key in keymap.items(): display_value = value = computed.get(key) or quote.get(key) diff --git a/piker/ui/monitor.py b/piker/ui/kivy/monitor.py similarity index 99% rename from piker/ui/monitor.py rename to piker/ui/kivy/monitor.py index cd73c3fa..932b581e 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/kivy/monitor.py @@ -15,11 +15,11 @@ from kivy.lang import Builder from kivy.app import async_runTouchApp from kivy.core.window import Window -from ..brokers.data import DataFeed +from ...brokers.data import DataFeed from .tabular import ( Row, TickerTable, _kv, _black_rgba, colorcode, ) -from ..log import get_logger +from ...log import get_logger from .pager import PagerView diff --git a/piker/ui/kivy/mouse_over.py b/piker/ui/kivy/mouse_over.py index c9ad149b..766371cd 100644 --- a/piker/ui/kivy/mouse_over.py +++ b/piker/ui/kivy/mouse_over.py @@ -11,7 +11,6 @@ from kivy.properties import BooleanProperty, ObjectProperty from kivy.core.window import Window from kivy.clock import Clock - from ...log import get_logger @@ -100,7 +99,8 @@ class MouseOverBehavior(object): # throttle at 10ms latency @triggered(timeout=0.01, interval=False) def _on_mouse_pos(cls, *args): - log.debug(f"{cls} time since last call: {time.time() - cls._last_time}") + log.debug( + f"{cls} time since last call: {time.time() - cls._last_time}") cls._last_time = time.time() # XXX: how to still do this at the class level? # don't proceed if I'm not displayed <=> If have no parent diff --git a/piker/ui/option_chain.py b/piker/ui/kivy/option_chain.py similarity index 99% rename from piker/ui/option_chain.py rename to piker/ui/kivy/option_chain.py index e2421103..cfa0e665 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/kivy/option_chain.py @@ -15,11 +15,10 @@ from kivy.app import async_runTouchApp from kivy.core.window import Window from kivy.uix.label import Label -from ..log import get_logger, get_console_log -from ..brokers.data import DataFeed -from ..brokers import get_brokermod +from ...log import get_logger, get_console_log +from ...brokers.data import DataFeed +from ...brokers import get_brokermod from .pager import PagerView - from .tabular import Row, HeaderCell, Cell, TickerTable from .monitor import update_quotes diff --git a/piker/ui/pager.py b/piker/ui/kivy/pager.py similarity index 99% rename from piker/ui/pager.py rename to piker/ui/kivy/pager.py index ca17935b..20a8baef 100644 --- a/piker/ui/pager.py +++ b/piker/ui/kivy/pager.py @@ -9,8 +9,8 @@ from kivy.uix.widget import Widget from kivy.uix.textinput import TextInput from kivy.uix.scrollview import ScrollView -from ..log import get_logger -from .kivy.utils_async import async_bind +from ...log import get_logger +from .utils_async import async_bind log = get_logger('keyboard') diff --git a/piker/ui/tabular.py b/piker/ui/kivy/tabular.py similarity index 99% rename from piker/ui/tabular.py rename to piker/ui/kivy/tabular.py index 27c1e091..5a5a6b3a 100644 --- a/piker/ui/tabular.py +++ b/piker/ui/kivy/tabular.py @@ -12,8 +12,8 @@ from kivy.uix.button import Button from kivy import utils from kivy.properties import BooleanProperty -from ..log import get_logger -from .kivy.mouse_over import new_mouse_over_group +from ...log import get_logger +from .mouse_over import new_mouse_over_group HoverBehavior = new_mouse_over_group() From 0b5af4b5905d56176071d94d9d062f9bdccce650 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 19:20:54 -0400 Subject: [PATCH 024/206] Move all Qt components into top level ui module --- piker/ui/{qt => }/_axes.py | 0 piker/ui/{qt => }/_chart.py | 0 piker/ui/{qt => }/_exec.py | 0 piker/ui/{qt => }/_graphics.py | 0 piker/ui/{qt => }/_source.py | 0 piker/ui/{qt => }/_style.py | 0 piker/ui/cli.py | 2 +- piker/ui/{qt => }/quantdom/__init__.py | 0 piker/ui/{qt => }/quantdom/_equity.py | 0 piker/ui/{qt => }/quantdom/base.py | 1 + piker/ui/{qt => }/quantdom/charts.py | 0 piker/ui/{qt => }/quantdom/const.py | 0 piker/ui/{qt => }/quantdom/loaders.py | 23 ++++++++++++++--------- piker/ui/{qt => }/quantdom/performance.py | 0 piker/ui/{qt => }/quantdom/portfolio.py | 0 piker/ui/{qt => }/quantdom/utils.py | 0 16 files changed, 16 insertions(+), 10 deletions(-) rename piker/ui/{qt => }/_axes.py (100%) rename piker/ui/{qt => }/_chart.py (100%) rename piker/ui/{qt => }/_exec.py (100%) rename piker/ui/{qt => }/_graphics.py (100%) rename piker/ui/{qt => }/_source.py (100%) rename piker/ui/{qt => }/_style.py (100%) rename piker/ui/{qt => }/quantdom/__init__.py (100%) rename piker/ui/{qt => }/quantdom/_equity.py (100%) rename piker/ui/{qt => }/quantdom/base.py (99%) rename piker/ui/{qt => }/quantdom/charts.py (100%) rename piker/ui/{qt => }/quantdom/const.py (100%) rename piker/ui/{qt => }/quantdom/loaders.py (90%) rename piker/ui/{qt => }/quantdom/performance.py (100%) rename piker/ui/{qt => }/quantdom/portfolio.py (100%) rename piker/ui/{qt => }/quantdom/utils.py (100%) diff --git a/piker/ui/qt/_axes.py b/piker/ui/_axes.py similarity index 100% rename from piker/ui/qt/_axes.py rename to piker/ui/_axes.py diff --git a/piker/ui/qt/_chart.py b/piker/ui/_chart.py similarity index 100% rename from piker/ui/qt/_chart.py rename to piker/ui/_chart.py diff --git a/piker/ui/qt/_exec.py b/piker/ui/_exec.py similarity index 100% rename from piker/ui/qt/_exec.py rename to piker/ui/_exec.py diff --git a/piker/ui/qt/_graphics.py b/piker/ui/_graphics.py similarity index 100% rename from piker/ui/qt/_graphics.py rename to piker/ui/_graphics.py diff --git a/piker/ui/qt/_source.py b/piker/ui/_source.py similarity index 100% rename from piker/ui/qt/_source.py rename to piker/ui/_source.py diff --git a/piker/ui/qt/_style.py b/piker/ui/_style.py similarity index 100% rename from piker/ui/qt/_style.py rename to piker/ui/_style.py diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 59d7bfb5..78c749df 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -115,6 +115,6 @@ def optschain(config, symbol, date, tl, rate, test): def chart(config, symbol, date, tl, rate, test): """Start an option chain UI """ - from .qt._chart import main + from ._chart import main main(symbol) diff --git a/piker/ui/qt/quantdom/__init__.py b/piker/ui/quantdom/__init__.py similarity index 100% rename from piker/ui/qt/quantdom/__init__.py rename to piker/ui/quantdom/__init__.py diff --git a/piker/ui/qt/quantdom/_equity.py b/piker/ui/quantdom/_equity.py similarity index 100% rename from piker/ui/qt/quantdom/_equity.py rename to piker/ui/quantdom/_equity.py diff --git a/piker/ui/qt/quantdom/base.py b/piker/ui/quantdom/base.py similarity index 99% rename from piker/ui/qt/quantdom/base.py rename to piker/ui/quantdom/base.py index 9f5eb074..fed59508 100644 --- a/piker/ui/qt/quantdom/base.py +++ b/piker/ui/quantdom/base.py @@ -89,6 +89,7 @@ class BaseQuotes(np.recarray): return self def convert_dates(self, dates): + breakpoint() return np.array([d.timestamp().time for d in dates]) diff --git a/piker/ui/qt/quantdom/charts.py b/piker/ui/quantdom/charts.py similarity index 100% rename from piker/ui/qt/quantdom/charts.py rename to piker/ui/quantdom/charts.py diff --git a/piker/ui/qt/quantdom/const.py b/piker/ui/quantdom/const.py similarity index 100% rename from piker/ui/qt/quantdom/const.py rename to piker/ui/quantdom/const.py diff --git a/piker/ui/qt/quantdom/loaders.py b/piker/ui/quantdom/loaders.py similarity index 90% rename from piker/ui/qt/quantdom/loaders.py rename to piker/ui/quantdom/loaders.py index c3bc6069..8de6c274 100644 --- a/piker/ui/qt/quantdom/loaders.py +++ b/piker/ui/quantdom/loaders.py @@ -62,14 +62,18 @@ class QuotesLoader: @classmethod def _save_to_disk(cls, fpath, data): logger.debug('Saving quotes to a file: %s', fpath) + breakpoint() with open(fpath, 'wb') as f: + pass pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + d = pickle.load(f) @classmethod def _load_from_disk(cls, fpath): logger.debug('Loading quotes from a file: %s', fpath) with open(fpath, 'rb') as f: - return pickle.load(f) + breakpoint() + data = pickle.load(f) @classmethod @timeit @@ -84,14 +88,15 @@ class QuotesLoader: quotes = None fpath = cls._get_file_path(symbol, cls.timeframe, date_from, date_to) - if os.path.exists(fpath): - quotes = Quotes.new(cls._load_from_disk(fpath)) - else: - quotes_raw = cls._get(symbol, date_from, date_to) - quotes = Quotes.new( - quotes_raw, source=cls.source, default_tf=cls.default_tf - ) - cls._save_to_disk(fpath, quotes) + # if os.path.exists(fpath): + # quotes = Quotes.new(cls._load_from_disk(fpath)) + # else: + quotes_raw = cls._get(symbol, date_from, date_to) + breakpoint() + quotes = Quotes.new( + quotes_raw, source=cls.source, default_tf=cls.default_tf + ) + cls._save_to_disk(fpath, quotes) return quotes diff --git a/piker/ui/qt/quantdom/performance.py b/piker/ui/quantdom/performance.py similarity index 100% rename from piker/ui/qt/quantdom/performance.py rename to piker/ui/quantdom/performance.py diff --git a/piker/ui/qt/quantdom/portfolio.py b/piker/ui/quantdom/portfolio.py similarity index 100% rename from piker/ui/qt/quantdom/portfolio.py rename to piker/ui/quantdom/portfolio.py diff --git a/piker/ui/qt/quantdom/utils.py b/piker/ui/quantdom/utils.py similarity index 100% rename from piker/ui/qt/quantdom/utils.py rename to piker/ui/quantdom/utils.py From 14bff66ec5841fb9c67b7b5bc2da1f71e9fafd82 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 19:22:37 -0400 Subject: [PATCH 025/206] Add a sane pandas.DataFrame to recarray converter --- piker/ui/_source.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/piker/ui/_source.py b/piker/ui/_source.py index a23c9f41..f0380b72 100644 --- a/piker/ui/_source.py +++ b/piker/ui/_source.py @@ -63,30 +63,24 @@ def from_df( ): """Cast OHLC ``pandas.DataFrame`` to ``numpy.structarray``. """ - # shape = (len(df),) - # array.resize(shape, refcheck=False) - array = np.array([], dtype=OHLC_dtype) - df.reset_index(inplace=True) - df.insert(0, 'id', df.index) - array['time'] = np.array([d.timestamp().time for d in df.Date]) + df['Date'] = [d.timestamp() for d in df.Date] # try to rename from some camel case - df = df.rename( - columns={ - # 'Date': 'time', - 'Open': 'open', - 'High': 'high', - 'Low': 'low', - 'Close': 'close', - 'Volume': 'volume', - } - ) - for name in array.dtype.names: - array[name] = df[name] - - array[:] = df[:] - + columns={ + 'Date': 'time', + 'Open': 'open', + 'High': 'high', + 'Low': 'low', + 'Close': 'close', + 'Volume': 'volume', + } + for name in df.columns: + if name not in columns: + del df[name] + df = df.rename(columns=columns) + # df.insert(0, 'id', df.index) + array = df.to_records() _nan_to_closest_num(array) # self._set_time_frame(default_tf) From cc4b51cb17497b508835257685ab978c9ae84312 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 20:45:47 -0400 Subject: [PATCH 026/206] Rip out all usage of `quantdom.bases.Quotes` smh. --- piker/ui/_axes.py | 19 ++++++++++++------- piker/ui/_chart.py | 41 +++++++++++++++++++++++++++++------------ piker/ui/_graphics.py | 8 ++++---- 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 90b5c5f2..51a9e497 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -5,7 +5,7 @@ import pyqtgraph as pg from PyQt5 import QtCore, QtGui -from .quantdom.base import Quotes +# from .quantdom.base import Quotes from .quantdom.utils import fromtimestamp from ._style import _font @@ -36,10 +36,11 @@ class PriceAxis(pg.AxisItem): class FromTimeFieldDateAxis(pg.AxisItem): tick_tpl = {'D1': '%Y-%b-%d'} - def __init__(self, *args, **kwargs): + def __init__(self, splitter, *args, **kwargs): super().__init__(*args, **kwargs) + self.splitter = splitter self.setTickFont(_font) - self.quotes_count = len(Quotes) - 1 + # self.quotes_count = len(self.splitter.chart._array) - 1 # default styling self.setStyle( @@ -56,10 +57,13 @@ class FromTimeFieldDateAxis(pg.AxisItem): # strings = super().tickStrings(values, scale, spacing) s_period = 'D1' strings = [] + quotes_count = len(self.splitter.chart._array) - 1 + for ibar in values: - if ibar > self.quotes_count: + if ibar > quotes_count: return strings - dt_tick = fromtimestamp(Quotes[int(ibar)].time) + bars = self.splitter.chart._array + dt_tick = fromtimestamp(bars[int(ibar)].time) strings.append( dt_tick.strftime(self.tick_tpl[s_period]) ) @@ -140,9 +144,10 @@ class XAxisLabel(AxisLabel): def tick_to_string(self, tick_pos): # TODO: change to actual period tpl = self.parent.tick_tpl['D1'] - if tick_pos > len(Quotes): + bars = self.parent.splitter.chart._array + if tick_pos > len(bars): return 'Unknown Time' - return fromtimestamp(Quotes[round(tick_pos)].time).strftime(tpl) + return fromtimestamp(bars[round(tick_pos)].time).strftime(tpl) def boundingRect(self): # noqa return QtCore.QRectF(0, 0, 145, 50) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 28425605..e10b2698 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -16,7 +16,7 @@ from ._style import _xaxis_at from ._source import Symbol, ohlc_zeros from .quantdom.charts import CenteredTextItem -from .quantdom.base import Quotes +# from .quantdom.base import Quotes # from .quantdom.const import ChartType from .quantdom.portfolio import Order, Portfolio @@ -101,10 +101,11 @@ class SplitterPlots(QtGui.QWidget): self.signals_visible = False self.indicators = [] - self.xaxis = FromTimeFieldDateAxis(orientation='bottom') + self.xaxis = FromTimeFieldDateAxis(orientation='bottom', splitter=self) # self.xaxis = pg.DateAxisItem() - self.xaxis_ind = FromTimeFieldDateAxis(orientation='bottom') + self.xaxis_ind = FromTimeFieldDateAxis( + orientation='bottom', splitter=self) if _xaxis_at == 'bottom': self.xaxis.setStyle(showValues=False) @@ -174,7 +175,9 @@ class SplitterPlots(QtGui.QWidget): self.chart.draw_ohlc(data) # TODO: this is where we would load an indicator chain - inds = [Quotes.open] + # XXX: note, if this isn't index aligned with + # the source data the chart will go haywire. + inds = [data.open] for d in inds: cv = ChartView() @@ -377,8 +380,8 @@ class ChartPlotWidget(pg.PlotWidget): # update view limits self.set_view_limits( - data[0]['id'], - data[-1]['id'], + data[0]['index'], + data[-1]['index'], data['low'].min(), data['high'].max() ) @@ -525,6 +528,7 @@ def main(symbol): from datetime import datetime from ._exec import run_qtrio + from ._source import from_df # uses pandas_datareader from .quantdom.loaders import get_quotes @@ -539,15 +543,28 @@ def main(symbol): date_from=datetime(1900, 1, 1), date_to=datetime(2030, 12, 31), ) + quotes = from_df(quotes) + # spawn chart splitter_chart = chart_app.load_symbol(symbol, quotes) import itertools - nums = itertools.cycle([315., 320., 325.]) + nums = itertools.cycle([315., 320., 325., 310., 3]) + + def gen_nums(): + for i in itertools.count(): + yield quotes[-1].close + i + yield quotes[-1].close - i + + chart = splitter_chart.chart + + nums = gen_nums() while True: - await trio.sleep(0.05) - splitter_chart.chart._graphics['ohlc'].update_last_bar( - {'last': next(nums)}) - # splitter_chart.chart.plotItem.sigPlotChanged.emit(self) - # breakpoint() + await trio.sleep(0.1) + new = next(nums) + quotes[-1].close = new + chart._graphics['ohlc'].update_last_bar({'last': new}) + + # LOL this clearly isn't catching edge cases + chart._update_yrange_limits() run_qtrio(_main, (), Chart) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 78f14d8e..4b57c75b 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -11,7 +11,7 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF from .quantdom.utils import timeit -from .quantdom.base import Quotes +# from .quantdom.base import Quotes from ._style import _xaxis_at # , _tina_mode from ._axes import YAxisLabel, XAxisLabel @@ -214,12 +214,12 @@ class BarItems(pg.GraphicsObject): # indexing here is as per the below comments lines[3*i:3*i+3] = ( # high_to_low - QLineF(q['id'], q['low'], q['id'], q['high']), + QLineF(q['index'], q['low'], q['index'], q['high']), # open_sticks - QLineF(q['id'] - self.w, q['open'], q['id'], q['open']), + QLineF(q['index'] - self.w, q['open'], q['index'], q['open']), # close_sticks QtCore.QLineF( - q['id'] + self.w, q['close'], q['id'], q['close']) + q['index'] + self.w, q['close'], q['index'], q['close']) ) else: self._last_quote = q From d993147f78972a395baab3e9df1d8b849532c848 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 22:56:27 -0400 Subject: [PATCH 027/206] Factor signalling api into new module --- piker/ui/_chart.py | 97 ++++++---------------------------------- piker/ui/_signalling.py | 98 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 84 deletions(-) create mode 100644 piker/ui/_signalling.py diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index e10b2698..d5a58809 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -15,17 +15,16 @@ from ._graphics import CrossHairItem, ChartType from ._style import _xaxis_at from ._source import Symbol, ohlc_zeros -from .quantdom.charts import CenteredTextItem -# from .quantdom.base import Quotes -# from .quantdom.const import ChartType -from .quantdom.portfolio import Order, Portfolio - # margins CHART_MARGINS = (0, 0, 10, 3) -class Chart(QtGui.QWidget): +class ChartSpace(QtGui.QWidget): + """High level widget which contains layouts for organizing + lower level charts as well as other widgets used to control + or modify them. + """ def __init__(self, parent=None): super().__init__(parent) self.v_layout = QtGui.QVBoxLayout(self) @@ -69,7 +68,7 @@ class Chart(QtGui.QWidget): """Load a new contract into the charting app. """ # XXX: let's see if this causes mem problems - self.chart = self._plot_cache.setdefault(symbol, SplitterPlots()) + self.chart = self._plot_cache.setdefault(symbol, LinkedSplitCharts()) s = Symbol(key=symbol) # remove any existing plots @@ -85,7 +84,7 @@ class Chart(QtGui.QWidget): # self.chart.add_signals() -class SplitterPlots(QtGui.QWidget): +class LinkedSplitCharts(QtGui.QWidget): """Widget that holds a price chart plus indicators separated by splitters. """ @@ -120,27 +119,6 @@ class SplitterPlots(QtGui.QWidget): self.layout.addWidget(self.splitter) - def _show_text_signals(self, lbar, rbar): - signals = [ - sig - for sig in self.signals_text_items[lbar:rbar] - if isinstance(sig, CenteredTextItem) - ] - if len(signals) <= 50: - for sig in signals: - sig.show() - else: - for sig in signals: - sig.hide() - - def _remove_signals(self): - self.chart.removeItem(self.signals_group_arrow) - self.chart.removeItem(self.signals_group_text) - del self.signals_text_items - del self.signals_group_arrow - del self.signals_group_text - self.signals_visible = False - def _update_sizes(self): min_h_ind = int(self.height() * 0.2 / len(self.indicators)) sizes = [int(self.height() * 0.8)] @@ -212,60 +190,6 @@ class SplitterPlots(QtGui.QWidget): ) self.chart.addItem(ch) - def add_signals(self): - self.signals_group_text = QtGui.QGraphicsItemGroup() - self.signals_group_arrow = QtGui.QGraphicsItemGroup() - self.signals_text_items = np.empty(len(Quotes), dtype=object) - - for p in Portfolio.positions: - x, price = p.id_bar_open, p.open_price - if p.type == Order.BUY: - y = Quotes[x].low * 0.99 - pg.ArrowItem( - parent=self.signals_group_arrow, - pos=(x, y), - pen=self.long_pen, - brush=self.long_brush, - angle=90, - headLen=12, - tipAngle=50, - ) - text_sig = CenteredTextItem( - parent=self.signals_group_text, - pos=(x, y), - pen=self.long_pen, - brush=self.long_brush, - text=('Buy at {:.%df}' % self.digits).format(price), - valign=QtCore.Qt.AlignBottom, - ) - text_sig.hide() - else: - y = Quotes[x].high * 1.01 - pg.ArrowItem( - parent=self.signals_group_arrow, - pos=(x, y), - pen=self.short_pen, - brush=self.short_brush, - angle=-90, - headLen=12, - tipAngle=50, - ) - text_sig = CenteredTextItem( - parent=self.signals_group_text, - pos=(x, y), - pen=self.short_pen, - brush=self.short_brush, - text=('Sell at {:.%df}' % self.digits).format(price), - valign=QtCore.Qt.AlignTop, - ) - text_sig.hide() - - self.signals_text_items[x] = text_sig - - self.chart.addItem(self.signals_group_arrow) - self.chart.addItem(self.signals_group_text) - self.signals_visible = True - _min_points_to_show = 15 _min_bars_in_view = 10 @@ -299,6 +223,9 @@ class ChartPlotWidget(pg.PlotWidget): """Configure chart display settings. """ super().__init__(**kwargs) + # XXX: label setting doesn't seem to work? + # likely custom graphics need special handling + # label = pg.LabelItem(justify='left') # self.addItem(label) # label.setText("Yo yoyo") @@ -567,4 +494,6 @@ def main(symbol): # LOL this clearly isn't catching edge cases chart._update_yrange_limits() - run_qtrio(_main, (), Chart) + await trio.sleep_forever() + + run_qtrio(_main, (), ChartSpace) diff --git a/piker/ui/_signalling.py b/piker/ui/_signalling.py new file mode 100644 index 00000000..66a5bf1c --- /dev/null +++ b/piker/ui/_signalling.py @@ -0,0 +1,98 @@ +""" +Signalling graphics and APIs. + +WARNING: this code likely doesn't work at all (yet) + since it was copied from another class that shouldn't + have had it. +""" +import numpy as np +import pyqtgraph as pg +from PyQt5 import QtCore, QtGui + +from .quantdom.charts import CenteredTextItem +from .quantdom.base import Quotes +from .quantdom.portfolio import Order, Portfolio + + +class SignallingApi(object): + def __init__(self, plotgroup): + self.plotgroup = plotgroup + self.chart = plotgroup.chart + + def _show_text_signals(self, lbar, rbar): + signals = [ + sig + for sig in self.signals_text_items[lbar:rbar] + if isinstance(sig, CenteredTextItem) + ] + if len(signals) <= 50: + for sig in signals: + sig.show() + else: + for sig in signals: + sig.hide() + + def _remove_signals(self): + self.chart.removeItem(self.signals_group_arrow) + self.chart.removeItem(self.signals_group_text) + del self.signals_text_items + del self.signals_group_arrow + del self.signals_group_text + self.signals_visible = False + + def add_signals(self): + self.signals_group_text = QtGui.QGraphicsItemGroup() + self.signals_group_arrow = QtGui.QGraphicsItemGroup() + self.signals_text_items = np.empty(len(Quotes), dtype=object) + + for p in Portfolio.positions: + x, price = p.id_bar_open, p.open_price + if p.type == Order.BUY: + y = Quotes[x].low * 0.99 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.plotgroup.long_pen, + brush=self.plotgroup.long_brush, + angle=90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.plotgroup.long_pen, + brush=self.plotgroup.long_brush, + text=( + 'Buy at {:.%df}' % self.plotgroup.digits).format( + price), + valign=QtCore.Qt.AlignBottom, + ) + text_sig.hide() + else: + y = Quotes[x].high * 1.01 + pg.ArrowItem( + parent=self.signals_group_arrow, + pos=(x, y), + pen=self.plotgroup.short_pen, + brush=self.plotgroup.short_brush, + angle=-90, + headLen=12, + tipAngle=50, + ) + text_sig = CenteredTextItem( + parent=self.signals_group_text, + pos=(x, y), + pen=self.plotgroup.short_pen, + brush=self.plotgroup.short_brush, + text=('Sell at {:.%df}' % self.plotgroup.digits).format( + price), + valign=QtCore.Qt.AlignTop, + ) + text_sig.hide() + + self.signals_text_items[x] = text_sig + + self.chart.addItem(self.signals_group_arrow) + self.chart.addItem(self.signals_group_text) + self.signals_visible = True From 6ba0692851ed793e475e5b456bbec5131a8ba0de Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 22:58:54 -0400 Subject: [PATCH 028/206] Revert weird bad .time access --- piker/ui/_source.py | 6 ++---- piker/ui/quantdom/base.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/piker/ui/_source.py b/piker/ui/_source.py index f0380b72..70c586d4 100644 --- a/piker/ui/_source.py +++ b/piker/ui/_source.py @@ -60,8 +60,8 @@ def from_df( df: pd.DataFrame, source=None, default_tf=None -): - """Cast OHLC ``pandas.DataFrame`` to ``numpy.structarray``. +) -> np.recarray: + """Convert OHLC formatted ``pandas.DataFrame`` to ``numpy.recarray``. """ df.reset_index(inplace=True) df['Date'] = [d.timestamp() for d in df.Date] @@ -79,10 +79,8 @@ def from_df( if name not in columns: del df[name] df = df.rename(columns=columns) - # df.insert(0, 'id', df.index) array = df.to_records() _nan_to_closest_num(array) - # self._set_time_frame(default_tf) return array diff --git a/piker/ui/quantdom/base.py b/piker/ui/quantdom/base.py index fed59508..3c853f00 100644 --- a/piker/ui/quantdom/base.py +++ b/piker/ui/quantdom/base.py @@ -90,7 +90,7 @@ class BaseQuotes(np.recarray): def convert_dates(self, dates): breakpoint() - return np.array([d.timestamp().time for d in dates]) + return np.array([d.timestamp() for d in dates]) class SymbolType(Enum): From 048a13dd0ecd91089d70134c903810be599c50da Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 17 Jun 2020 22:59:24 -0400 Subject: [PATCH 029/206] Drop disk caching of quotes --- piker/ui/quantdom/loaders.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/piker/ui/quantdom/loaders.py b/piker/ui/quantdom/loaders.py index 8de6c274..52203976 100644 --- a/piker/ui/quantdom/loaders.py +++ b/piker/ui/quantdom/loaders.py @@ -62,9 +62,7 @@ class QuotesLoader: @classmethod def _save_to_disk(cls, fpath, data): logger.debug('Saving quotes to a file: %s', fpath) - breakpoint() with open(fpath, 'wb') as f: - pass pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) d = pickle.load(f) @@ -92,12 +90,11 @@ class QuotesLoader: # quotes = Quotes.new(cls._load_from_disk(fpath)) # else: quotes_raw = cls._get(symbol, date_from, date_to) - breakpoint() - quotes = Quotes.new( - quotes_raw, source=cls.source, default_tf=cls.default_tf - ) - cls._save_to_disk(fpath, quotes) - return quotes + # quotes = Quotes.new( + # quotes_raw, source=cls.source, default_tf=cls.default_tf + # ) + # cls._save_to_disk(fpath, quotes) + return quotes_raw class YahooQuotesLoader(QuotesLoader): From 5e8e48c7b79e7c96665d16ce552f94387fbbc07f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 19 Jun 2020 07:53:06 -0400 Subject: [PATCH 030/206] Support updating bars graphics from array This makes a OHLC graphics "sequence" update very similar (actually API compatible) with `pg.PlotCurveItem.setData()`. The difference here is that only latest OHLC datum is used to update the charts last bar. --- piker/ui/_graphics.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 4b57c75b..31c71139 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -185,10 +185,10 @@ class BarItems(pg.GraphicsObject): def __init__(self): super().__init__() self.picture = QtGui.QPicture() - self.lines = None - self._last_quote = {} + # lines container + self.lines: np.ndarray = None - # TODO: this is the routine to be retriggered for redraw + # TODO: can we be faster just dropping this? @contextmanager def painter(self): # pre-computing a QPicture object allows paint() to run much @@ -221,8 +221,6 @@ class BarItems(pg.GraphicsObject): QtCore.QLineF( q['index'] + self.w, q['close'], q['index'], q['close']) ) - else: - self._last_quote = q # if not _tina_mode: # piker mode p.setPen(self.bull_brush) p.drawLines(*lines) @@ -245,17 +243,24 @@ class BarItems(pg.GraphicsObject): # p.setPen(self.bear_brush) # p.drawLines(*downs) - def update_last_bar( + def update_from_array( self, - quote: Dict[str, Any], + array: np.ndarray, ) -> None: - """Update the last datum's bar graphic from a quote ``dict``. + """Update the last datum's bar graphic from input data array. + + This routine should be interface compatible with + ``pg.PlotCurveItem.setData()``. Normally this method in + ``pyqtgraph`` seems to update all the data passed to the + graphics object, and then update/rerender, but here we're + assuming the prior graphics havent changed (OHLC history rarely + does) so this "should" be simpler and faster. """ - last = quote['last'] - body, larm, rarm = self.lines[-3:] # close line is absolute last + # do we really need to verify the entire past data set? + last = array['close'][-1] + body, larm, rarm = self.lines[-3:] # XXX: is there a faster way to modify this? - # update right arm rarm.setLine(rarm.x1(), last, rarm.x2(), last) @@ -270,23 +275,27 @@ class BarItems(pg.GraphicsObject): body.setLine(body.x1(), low, body.x2(), high) + # draw the pic with self.painter() as p: p.setPen(self.bull_brush) p.drawLines(*self.lines) - # trigger re-draw + # trigger re-render self.update() + # be compat with ``pg.PlotCurveItem`` + setData = update_from_array + # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() - def paint(self, p, *args): + def paint(self, p, opt, widget): p.drawPicture(0, 0, self.picture) def boundingRect(self): # boundingRect _must_ indicate the entire area that will be # drawn on or else we will get artifacts and possibly crashing. # (in this case, QPicture does all the work of computing the - # bouning rect for us) + # bounding rect for us) return QtCore.QRectF(self.picture.boundingRect()) From 2f1fdaf9e5e41df89b6b5abf4a3089b6dc6aae33 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 19 Jun 2020 08:01:10 -0400 Subject: [PATCH 031/206] Rework charting internals for real-time plotting `pg.PlotCurveItem.setData()` is normally used for real-time updates to curves and takes in a whole new array of data to graphics. It makes sense to stick with this interface especially if the current datum graphic will originally be drawn from tick quotes and later filled in when bars data is available (eg. IB has this option in TWS charts for volume). Additionally, having a data feed api where the push process/task can write to shared memory and the UI task(s) can read from that space is ideal. It allows for indicator and algo calculations to be run in parallel (via actors) with initial price draw instructions such that plotting of downstream metrics can be "pipelined" into the chart UI's render loop. This essentially makes the chart UI async programmable from multiple remote processes (or at least that's the goal). Some details: - Only store a single ref to the source array data on the `LinkedSplitCharts`. There should only be one reference since the main relation is **that** x-time aligned sequence. - Add `LinkedSplitCharts.update_from_quote()` which takes in a quote dict and updates the OHLC array from it's contents. - Add `ChartPlotWidget.update_from_array()` method to trigger graphics updates per chart with consideration for overlay curves. --- piker/ui/_axes.py | 15 ++- piker/ui/_chart.py | 276 +++++++++++++++++++++++++++++---------------- 2 files changed, 186 insertions(+), 105 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 51a9e497..03e68feb 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -33,14 +33,13 @@ class PriceAxis(pg.AxisItem): # ] -class FromTimeFieldDateAxis(pg.AxisItem): +class DynamicDateAxis(pg.AxisItem): tick_tpl = {'D1': '%Y-%b-%d'} - def __init__(self, splitter, *args, **kwargs): + def __init__(self, linked_charts, *args, **kwargs): super().__init__(*args, **kwargs) - self.splitter = splitter + self.linked_charts = linked_charts self.setTickFont(_font) - # self.quotes_count = len(self.splitter.chart._array) - 1 # default styling self.setStyle( @@ -57,12 +56,12 @@ class FromTimeFieldDateAxis(pg.AxisItem): # strings = super().tickStrings(values, scale, spacing) s_period = 'D1' strings = [] - quotes_count = len(self.splitter.chart._array) - 1 + bars = self.linked_charts._array + quotes_count = len(bars) - 1 for ibar in values: if ibar > quotes_count: return strings - bars = self.splitter.chart._array dt_tick = fromtimestamp(bars[int(ibar)].time) strings.append( dt_tick.strftime(self.tick_tpl[s_period]) @@ -144,7 +143,7 @@ class XAxisLabel(AxisLabel): def tick_to_string(self, tick_pos): # TODO: change to actual period tpl = self.parent.tick_tpl['D1'] - bars = self.parent.splitter.chart._array + bars = self.parent.linked_charts._array if tick_pos > len(bars): return 'Unknown Time' return fromtimestamp(bars[round(tick_pos)].time).strftime(tpl) @@ -173,7 +172,7 @@ class YAxisLabel(AxisLabel): return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 80, 40) + return QtCore.QRectF(0, 0, 100, 40) def update_label(self, evt_post, point_view): self.label_str = self.tick_to_string(point_view.y()) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index d5a58809..b2402417 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,6 +1,8 @@ """ High level Qt chart widgets. """ +from typing import List, Optional + import trio import numpy as np import pyqtgraph as pg @@ -8,12 +10,12 @@ from pyqtgraph import functions as fn from PyQt5 import QtCore, QtGui from ._axes import ( - FromTimeFieldDateAxis, + DynamicDateAxis, PriceAxis, ) from ._graphics import CrossHairItem, ChartType from ._style import _xaxis_at -from ._source import Symbol, ohlc_zeros +from ._source import Symbol # margins @@ -85,9 +87,13 @@ class ChartSpace(QtGui.QWidget): class LinkedSplitCharts(QtGui.QWidget): - """Widget that holds a price chart plus indicators separated by splitters. - """ + """Widget that holds a central chart plus derived + subcharts computed from the original data set apart + by splitters for resizing. + A single internal references to the data is maintained + for each chart and can be updated externally. + """ long_pen = pg.mkPen('#006000') long_brush = pg.mkBrush('#00ff00') short_pen = pg.mkPen('#600000') @@ -98,13 +104,21 @@ class LinkedSplitCharts(QtGui.QWidget): def __init__(self): super().__init__() self.signals_visible = False + + # main data source + self._array = None + + self._ch = None # crosshair graphics + self._index = 0 + + self.chart = None # main (ohlc) chart self.indicators = [] - self.xaxis = FromTimeFieldDateAxis(orientation='bottom', splitter=self) - # self.xaxis = pg.DateAxisItem() + self.xaxis = DynamicDateAxis( + orientation='bottom', linked_charts=self) - self.xaxis_ind = FromTimeFieldDateAxis( - orientation='bottom', splitter=self) + self.xaxis_ind = DynamicDateAxis( + orientation='bottom', linked_charts=self) if _xaxis_at == 'bottom': self.xaxis.setStyle(showValues=False) @@ -119,24 +133,36 @@ class LinkedSplitCharts(QtGui.QWidget): self.layout.addWidget(self.splitter) - def _update_sizes(self): - min_h_ind = int(self.height() * 0.2 / len(self.indicators)) - sizes = [int(self.height() * 0.8)] + def set_split_sizes( + self, + prop: float = 0.2 + ) -> None: + """Set the proportion of space allocated for linked subcharts. + """ + major = 1 - prop + # 20% allocated to consumer subcharts + min_h_ind = int(self.height() * prop / len(self.indicators)) + sizes = [int(self.height() * major)] sizes.extend([min_h_ind] * len(self.indicators)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) def plot( self, symbol: Symbol, - data: np.ndarray, + array: np.ndarray, + ohlc: bool = True, ): - """Start up and show price chart and all registered indicators. + """Start up and show main (price) chart and all linked subcharts. """ self.digits = symbol.digits() + # XXX: this may eventually be a view onto shared mem + # or some higher level type / API + self._array = array + cv = ChartView() self.chart = ChartPlotWidget( - split_charts=self, + linked_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, viewBox=cv, @@ -144,52 +170,96 @@ class LinkedSplitCharts(QtGui.QWidget): ) # TODO: ``pyqtgraph`` doesn't pass through a parent to the # ``PlotItem`` by default; maybe we should PR this in? - cv.splitter_widget = self - self.chart.plotItem.vb.splitter_widget = self + cv.linked_charts = self + self.chart.plotItem.vb.linked_charts = self self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - self.chart.draw_ohlc(data) + if ohlc: + self.chart.draw_ohlc(array) + else: + raise NotImplementedError( + "Only OHLC linked charts are supported currently" + ) # TODO: this is where we would load an indicator chain # XXX: note, if this isn't index aligned with # the source data the chart will go haywire. - inds = [data.open] + inds = [('open', lambda a: a.close)] - for d in inds: + for name, func in inds: cv = ChartView() ind_chart = ChartPlotWidget( - split_charts=self, + linked_charts=self, parent=self.splitter, axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, viewBox=cv, ) - cv.splitter_widget = self - self.chart.plotItem.vb.splitter_widget = self + # this name will be used to register the primary + # graphics curve managed by the subchart + ind_chart.name = name + cv.linked_charts = self + self.chart.plotItem.vb.linked_charts = self ind_chart.setFrameStyle( QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain ) ind_chart.getPlotItem().setContentsMargins(*CHART_MARGINS) # self.splitter.addWidget(ind_chart) - self.indicators.append((ind_chart, d)) + + # compute historical subchart values from input array + data = func(array) + self.indicators.append((ind_chart, func)) # link chart x-axis to main quotes chart ind_chart.setXLink(self.chart) - # XXX: never do this lol - # ind.setAspectLocked(1) - ind_chart.draw_curve(d) + # draw curve graphics + ind_chart.draw_curve(data, name) - self._update_sizes() + self.set_split_sizes() - ch = CrossHairItem( - self.chart, [_ind for _ind, d in self.indicators], self.digits + ch = self._ch = CrossHairItem( + self.chart, + [_ind for _ind, d in self.indicators], + self.digits ) self.chart.addItem(ch) + def update_from_quote( + self, + quote: dict + ) -> List[pg.GraphicsObject]: + """Update all linked chart graphics with a new quote + datum. + + Return the modified graphics objects in a list. + """ + # TODO: eventually we'll want to update bid/ask labels and other + # data as subscribed by underlying UI consumers. + last = quote['last'] + current = self._array[-1] + + # update ohlc (I guess we're enforcing this for now?) + current['close'] = last + current['high'] = max(current['high'], last) + current['low'] = min(current['low'], last) + + # update the ohlc sequence graphics chart + chart = self.chart + # we send a reference to the whole updated array + chart.update_from_array(self._array) + + # TODO: the "data" here should really be a function + # and it should be managed and computed outside of this UI + for chart, func in self.indicators: + # process array in entirely every update + # TODO: change this for streaming + data = func(self._array) + chart.update_from_array(data, chart.name) + _min_points_to_show = 15 _min_bars_in_view = 10 @@ -198,12 +268,14 @@ _min_bars_in_view = 10 class ChartPlotWidget(pg.PlotWidget): """``GraphicsView`` subtype containing a single ``PlotItem``. - Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing - a single ``PlotItem``) to intercept and and re-emit mouse enter/exit - events. + - The added methods allow for plotting OHLC sequences from + ``np.recarray``s with appropriate field names. + - Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing + a single ``PlotItem``) to intercept and and re-emit mouse enter/exit + events. (Could be replaced with a ``pg.GraphicsLayoutWidget`` if we - eventually want multiple plots managed together). + eventually want multiple plots managed together?) """ sig_mouse_leave = QtCore.Signal(object) sig_mouse_enter = QtCore.Signal(object) @@ -213,29 +285,29 @@ class ChartPlotWidget(pg.PlotWidget): def __init__( self, - split_charts, + linked_charts, **kwargs, # parent=None, # background='default', # plotItem=None, - # **kargs ): """Configure chart display settings. """ super().__init__(**kwargs) + self.parent = linked_charts + # this is the index of that last input array entry and is + # updated and used to figure out how many bars are in view + self._xlast = 0 + # XXX: label setting doesn't seem to work? # likely custom graphics need special handling - # label = pg.LabelItem(justify='left') # self.addItem(label) # label.setText("Yo yoyo") # label.setText("x=") - self.parent = split_charts - # placeholder for source of data - self._array = ohlc_zeros(1) - - # to be filled in when data is loaded + # to be filled in when graphics are rendered + # by name self._graphics = {} # show only right side axes @@ -251,40 +323,32 @@ class ChartPlotWidget(pg.PlotWidget): self.setCursor(QtCore.Qt.CrossCursor) # assign callback for rescaling y-axis automatically - # based on y-range contents - self.sigXRangeChanged.connect(self._update_yrange_limits) + # based on ohlc contents + self.sigXRangeChanged.connect(self._set_yrange) - def set_view_limits(self, xfirst, xlast, ymin, ymax): + def _set_xlimits( + self, + xfirst: int, + xlast: int + ) -> None: + """Set view limits (what's shown in the main chart "pane") + based on max / min x / y coords. + """ # max_lookahead = _min_points_to_show - _min_bars_in_view # set panning limits - # last = data[-1]['id'] self.setLimits( - # xMin=data[0]['id'], xMin=xfirst, - # xMax=last + _min_points_to_show - 3, xMax=xlast + _min_points_to_show - 3, minXRange=_min_points_to_show, - # maxYRange=highest-lowest, - # yMin=data['low'].min() * 0.98, - # yMax=data['high'].max() * 1.02, - yMin=ymin * 0.98, - yMax=ymax * 1.02, ) - # show last 50 points on startup - # self.plotItem.vb.setXRange(last - 50, last + 50) - self.plotItem.vb.setXRange(xlast - 50, xlast + 50) - - # fit y - self._update_yrange_limits() - def bars_range(self): """Return a range tuple for the bars present in view. """ vr = self.viewRect() lbar = int(vr.left()) - rbar = int(min(vr.right(), len(self._array) - 1)) + rbar = int(vr.right()) return lbar, rbar def draw_ohlc( @@ -301,35 +365,55 @@ class ChartPlotWidget(pg.PlotWidget): # adds all bar/candle graphics objects for each data point in # the np array buffer to be drawn on next render cycle graphics.draw_from_data(data) - self._graphics['ohlc'] = graphics + self._graphics['main'] = graphics self.addItem(graphics) - self._array = data - # update view limits - self.set_view_limits( - data[0]['index'], - data[-1]['index'], - data['low'].min(), - data['high'].max() - ) + # set xrange limits + self._xlast = xlast = data[-1]['index'] + self._set_xlimits(data[0]['index'], xlast) + + # show last 50 points on startup + self.plotItem.vb.setXRange(xlast - 50, xlast + 50) return graphics def draw_curve( self, data: np.ndarray, + name: Optional[str] = None, ) -> None: # draw the indicator as a plain curve curve = pg.PlotDataItem(data, antialias=True) self.addItem(curve) - # update view limits - self.set_view_limits(0, len(data)-1, data.min(), data.max()) - self._array = data + # register overlay curve with name + if not self._graphics and name is None: + name = 'main' + self._graphics[name] = curve + + # set a "startup view" + xlast = len(data)-1 + self._set_xlimits(0, xlast) + + # show last 50 points on startup + self.plotItem.vb.setXRange(xlast - 50, xlast + 50) return curve - def _update_yrange_limits(self): + def update_from_array( + self, + array: np.ndarray, + name: str = 'main', + ) -> None: + self._xlast = len(array) - 1 + graphics = self._graphics[name] + graphics.setData(array) + # update view + self._set_yrange() + + def _set_yrange( + self, + ) -> None: """Callback for each y-range update. This adds auto-scaling like zoom on the scroll wheel such @@ -343,12 +427,11 @@ class ChartPlotWidget(pg.PlotWidget): # self.setAutoVisible(x=False, y=True) # self.enableAutoRange(x=False, y=True) + # figure out x-range bars on screen lbar, rbar = self.bars_range() - # if chart_parent.signals_visible: - # chart_parent._show_text_signals(lbar, rbar) - - bars = self._array[lbar:rbar] + # TODO: this should be some kind of numpy view api + bars = self.parent._array[lbar:rbar] if not len(bars): # likely no data loaded yet return @@ -389,22 +472,21 @@ class ChartPlotWidget(pg.PlotWidget): class ChartView(pg.ViewBox): """Price chart view box with interaction behaviors you'd expect from - an interactive platform: + any interactive platform: - - zoom on mouse scroll that auto fits y-axis - - no vertical scrolling - - zoom to a "fixed point" on the y-axis + - zoom on mouse scroll that auto fits y-axis + - no vertical scrolling + - zoom to a "fixed point" on the y-axis """ def __init__( self, parent=None, **kwargs, - # invertY=False, ): super().__init__(parent=parent, **kwargs) # disable vertical scrolling self.setMouseEnabled(x=True, y=False) - self.splitter_widget = None + self.linked_charts = None def wheelEvent(self, ev, axis=None): """Override "center-point" location for scrolling. @@ -422,13 +504,12 @@ class ChartView(pg.ViewBox): mask = self.state['mouseEnabled'][:] # don't zoom more then the min points setting - lbar, rbar = self.splitter_widget.chart.bars_range() - # breakpoint() + lbar, rbar = self.linked_charts.chart.bars_range() if ev.delta() >= 0 and rbar - lbar <= _min_points_to_show: return # actual scaling factor - s = 1.02 ** (ev.delta() * -1/10) # self.state['wheelScaleFactor']) + s = 1.015 ** (ev.delta() * -1/20) # self.state['wheelScaleFactor']) s = [(None if m is False else s) for m in mask] # center = pg.Point( @@ -473,26 +554,27 @@ def main(symbol): quotes = from_df(quotes) # spawn chart - splitter_chart = chart_app.load_symbol(symbol, quotes) + linked_charts = chart_app.load_symbol(symbol, quotes) + + # make some fake update data import itertools nums = itertools.cycle([315., 320., 325., 310., 3]) def gen_nums(): - for i in itertools.count(): - yield quotes[-1].close + i - yield quotes[-1].close - i - - chart = splitter_chart.chart + while True: + yield quotes[-1].close + 1 nums = gen_nums() + + await trio.sleep(10) while True: - await trio.sleep(0.1) new = next(nums) quotes[-1].close = new - chart._graphics['ohlc'].update_last_bar({'last': new}) - - # LOL this clearly isn't catching edge cases - chart._update_yrange_limits() + # this updates the linked_charts internal array + # and then passes that array to all subcharts to + # render downstream graphics + linked_charts.update_from_quote({'last': new}) + await trio.sleep(.1) await trio.sleep_forever() From 99c18abfea100afbccd71100a7c19bbe0147d66a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 24 Jun 2020 14:13:00 -0400 Subject: [PATCH 032/206] Add symbol search to broker api --- piker/brokers/core.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/piker/brokers/core.py b/piker/brokers/core.py index e65fcb40..2e672c61 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -108,3 +108,15 @@ async def symbol_info( """ async with brokermod.get_client() as client: return await client.symbol_info(symbol, **kwargs) + + +async def symbol_search( + brokermod: ModuleType, + symbol: str, + **kwargs, +) -> Dict[str, Dict[str, Dict[str, Any]]]: + """Return symbol info from broker. + """ + async with brokermod.get_client() as client: + # TODO: support multiple asset type concurrent searches. + return await client.search_stocks(symbol, **kwargs) From 7a660b335d417d76c9a5d6ce679ea88c7e6c843f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 2 Jul 2020 16:02:58 -0400 Subject: [PATCH 033/206] Make search work with ib backend --- piker/brokers/cli.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index 9f1ee4fa..b1047b18 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -248,3 +248,24 @@ def symbol_info(config, tickers): brokermod.log.warn(f"Could not find symbol {ticker}?") click.echo(colorize_json(quotes)) + + +@cli.command() +@click.argument('pattern', required=True) +@click.pass_obj +def search(config, pattern): + """Search for symbols from broker backend(s). + """ + # global opts + brokermod = config['brokermod'] + + quotes = tractor.run( + partial(core.symbol_search, brokermod, pattern), + start_method='forkserver', + loglevel='info', + ) + if not quotes: + log.error(f"No matches could be found for {pattern}?") + return + + click.echo(colorize_json(quotes)) From 16e2e27cb8f94ca8bfa06bb6f132a3d3490c50d0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jul 2020 18:08:03 -0400 Subject: [PATCH 034/206] Handle high = low bars For whatever reason if the `QLineF` high/low values are the same a weird little rectangle is drawn (my guess is a `float` precision error of some sort). Instead, if they're the same just use one of the values. Also, store local vars to avoid so many lookups. --- piker/ui/_graphics.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 31c71139..d179cc22 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -18,7 +18,7 @@ from ._axes import YAxisLabel, XAxisLabel # TODO: checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 50 +_mouse_rate_limit = 40 class CrossHairItem(pg.GraphicsObject): @@ -175,8 +175,7 @@ class BarItems(pg.GraphicsObject): sigPlotChanged = QtCore.Signal(object) w: float = 0.5 - - bull_brush = bear_brush = pg.mkPen('#808080') + bull_pen = pg.mkPen('#808080') # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') @@ -211,19 +210,26 @@ class BarItems(pg.GraphicsObject): with self.painter() as p: for i, q in enumerate(data): + low = q['low'] + high = q['high'] + index = q['index'] + + # high - low line + if low != high: + hl = QLineF(index, low, index, high) + else: + # if we don't do it renders a weird rectangle? + hl = QLineF(low, low, low, low) + # open line + o = QLineF(index - self.w, q['open'], index, q['open']) + # close line + c = QLineF(index + self.w, q['close'], index, q['close']) + # indexing here is as per the below comments - lines[3*i:3*i+3] = ( - # high_to_low - QLineF(q['index'], q['low'], q['index'], q['high']), - # open_sticks - QLineF(q['index'] - self.w, q['open'], q['index'], q['open']), - # close_sticks - QtCore.QLineF( - q['index'] + self.w, q['close'], q['index'], q['close']) - ) - # if not _tina_mode: # piker mode - p.setPen(self.bull_brush) + lines[3*i:3*i+3] = (hl, o, c) + p.setPen(self.bull_pen) p.drawLines(*lines) + # if not _tina_mode: # piker mode # else _tina_mode: # self.lines = lines = np.concatenate( # [high_to_low, open_sticks, close_sticks]) From ecfa6d33aa24808421dc2ed6f35d3d25767a1338 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jul 2020 18:32:40 -0400 Subject: [PATCH 035/206] Use msgpack-numpy --- piker/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/piker/__init__.py b/piker/__init__.py index 9ef65671..92553306 100644 --- a/piker/__init__.py +++ b/piker/__init__.py @@ -17,3 +17,8 @@ """ piker: trading gear for hackers. """ +import msgpack # noqa +import msgpack_numpy + +# patch msgpack for numpy arrays +msgpack_numpy.patch() From 4c753b5ee669709f9313a1e872a88d5518102154 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Jul 2020 19:09:57 -0400 Subject: [PATCH 036/206] Add ib --- piker/brokers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index 53597752..587a2b61 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -11,6 +11,7 @@ asks.init('trio') __brokers__ = [ 'questrade', 'robinhood', + 'ib', ] From 013c0fef156e0a0c9be7407b791489e27dccbda8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 4 Jul 2020 17:48:31 -0400 Subject: [PATCH 037/206] Fix a bunch of scrolling / panning logic Don't allow zooming to less then a min number of data points. Allow panning "outside" the data set (i.e. moving one of the sequence "ends" to the middle of the view. Start adding logging. --- piker/ui/_chart.py | 157 +++++++++++++++++++++++++++++++-------------- 1 file changed, 109 insertions(+), 48 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b2402417..73bd4422 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,7 +1,7 @@ """ High level Qt chart widgets. """ -from typing import List, Optional +from typing import List, Optional, Tuple import trio import numpy as np @@ -16,8 +16,12 @@ from ._axes import ( from ._graphics import CrossHairItem, ChartType from ._style import _xaxis_at from ._source import Symbol +from .. import brokers +from .. log import get_logger +log = get_logger(__name__) + # margins CHART_MARGINS = (0, 0, 10, 3) @@ -106,7 +110,7 @@ class LinkedSplitCharts(QtGui.QWidget): self.signals_visible = False # main data source - self._array = None + self._array: np.ndarray = None self._ch = None # crosshair graphics self._index = 0 @@ -135,7 +139,7 @@ class LinkedSplitCharts(QtGui.QWidget): def set_split_sizes( self, - prop: float = 0.2 + prop: float = 0.25 ) -> None: """Set the proportion of space allocated for linked subcharts. """ @@ -186,7 +190,7 @@ class LinkedSplitCharts(QtGui.QWidget): # TODO: this is where we would load an indicator chain # XXX: note, if this isn't index aligned with # the source data the chart will go haywire. - inds = [('open', lambda a: a.close)] + inds = [('open', lambda a: a['close'])] for name, func in inds: cv = ChartView() @@ -261,15 +265,14 @@ class LinkedSplitCharts(QtGui.QWidget): chart.update_from_array(data, chart.name) -_min_points_to_show = 15 -_min_bars_in_view = 10 +_min_points_to_show = 20 class ChartPlotWidget(pg.PlotWidget): """``GraphicsView`` subtype containing a single ``PlotItem``. - The added methods allow for plotting OHLC sequences from - ``np.recarray``s with appropriate field names. + ``np.ndarray``s with appropriate field names. - Overrides a ``pyqtgraph.PlotWidget`` (a ``GraphicsView`` containing a single ``PlotItem``) to intercept and and re-emit mouse enter/exit events. @@ -334,22 +337,24 @@ class ChartPlotWidget(pg.PlotWidget): """Set view limits (what's shown in the main chart "pane") based on max / min x / y coords. """ - # max_lookahead = _min_points_to_show - _min_bars_in_view - # set panning limits self.setLimits( xMin=xfirst, - xMax=xlast + _min_points_to_show - 3, + xMax=xlast, minXRange=_min_points_to_show, ) - def bars_range(self): + def view_range(self) -> Tuple[int, int]: + vr = self.viewRect() + return int(vr.left()), int(vr.right()) + + def bars_range(self) -> Tuple[int, int, int, int]: """Return a range tuple for the bars present in view. """ - vr = self.viewRect() - lbar = int(vr.left()) - rbar = int(vr.right()) - return lbar, rbar + l, r = self.view_range() + lbar = max(l, 0) + rbar = min(r, len(self.parent._array) - 1) + return l, lbar, rbar, r def draw_ohlc( self, @@ -370,7 +375,7 @@ class ChartPlotWidget(pg.PlotWidget): # set xrange limits self._xlast = xlast = data[-1]['index'] - self._set_xlimits(data[0]['index'], xlast) + # self._set_xlimits(data[0]['index'] - 100, xlast) # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -393,7 +398,7 @@ class ChartPlotWidget(pg.PlotWidget): # set a "startup view" xlast = len(data)-1 - self._set_xlimits(0, xlast) + # self._set_xlimits(0, xlast) # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -427,14 +432,33 @@ class ChartPlotWidget(pg.PlotWidget): # self.setAutoVisible(x=False, y=True) # self.enableAutoRange(x=False, y=True) - # figure out x-range bars on screen - lbar, rbar = self.bars_range() + l, lbar, rbar, r = self.bars_range() + + # figure out x-range in view such that user can scroll "off" the data + # set up to the point where ``_min_points_to_show`` are left. + # if l < lbar or r > rbar: + bars_len = rbar - lbar + view_len = r - l + # TODO: logic to check if end of bars in view + extra = view_len - _min_points_to_show + begin = 0 - extra + end = len(self.parent._array) - 1 + extra + + log.trace( + f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" + f"view_len: {view_len}, bars_len: {bars_len}\n" + f"begin: {begin}, end: {end}, extra: {extra}" + ) + self._set_xlimits(begin, end) # TODO: this should be some kind of numpy view api bars = self.parent._array[lbar:rbar] if not len(bars): # likely no data loaded yet + print(f"WTF bars_range = {lbar}:{rbar}") return + elif lbar < 0: + breakpoint() # TODO: should probably just have some kinda attr mark # that determines this behavior based on array type @@ -449,8 +473,9 @@ class ChartPlotWidget(pg.PlotWidget): std = np.std(bars) # view margins - ylow *= 0.98 - yhigh *= 1.02 + diff = yhigh - ylow + ylow = ylow - (diff * 0.08) + yhigh = yhigh + (diff * 0.08) chart = self chart.setLimits( @@ -504,8 +529,14 @@ class ChartView(pg.ViewBox): mask = self.state['mouseEnabled'][:] # don't zoom more then the min points setting - lbar, rbar = self.linked_charts.chart.bars_range() - if ev.delta() >= 0 and rbar - lbar <= _min_points_to_show: + l, lbar, rbar, r = self.linked_charts.chart.bars_range() + vl = r - l + + if ev.delta() > 0 and vl <= _min_points_to_show: + log.trace("Max zoom bruh...") + return + if ev.delta() < 0 and vl >= len(self.linked_charts._array) - 1: + log.trace("Min zoom bruh...") return # actual scaling factor @@ -533,12 +564,9 @@ class ChartView(pg.ViewBox): def main(symbol): """Entry point to spawn a chart app. """ - from datetime import datetime from ._exec import run_qtrio - from ._source import from_df # uses pandas_datareader - from .quantdom.loaders import get_quotes async def _main(widgets): """Main Qt-trio routine invoked by the Qt loop with @@ -546,35 +574,68 @@ def main(symbol): """ chart_app = widgets['main'] - quotes = get_quotes( - symbol=symbol, - date_from=datetime(1900, 1, 1), - date_to=datetime(2030, 12, 31), - ) - quotes = from_df(quotes) + + # from .quantdom.loaders import get_quotes + # from datetime import datetime + # from ._source import from_df + # quotes = get_quotes( + # symbol=symbol, + # date_from=datetime(1900, 1, 1), + # date_to=datetime(2030, 12, 31), + # ) + # quotes = from_df(quotes) + + # data-feed spawning + brokermod = brokers.get_brokermod('ib') + async with brokermod.get_client() as client: + # figure out the exact symbol + bars = await client.bars(symbol='ES') + + # wow, just wow.. non-contiguous eh? + bars = np.array(bars) + + # feed = DataFeed(portal, brokermod) + # quote_gen, quotes = await feed.open_stream( + # symbols, + # 'stock', + # rate=rate, + # test=test, + # ) + + # first_quotes, _ = feed.format_quotes(quotes) + + # if first_quotes[0].get('last') is None: + # log.error("Broker API is down temporarily") + # return # spawn chart - linked_charts = chart_app.load_symbol(symbol, quotes) + linked_charts = chart_app.load_symbol(symbol, bars) + await trio.sleep_forever() # make some fake update data - import itertools - nums = itertools.cycle([315., 320., 325., 310., 3]) + # import itertools + # nums = itertools.cycle([315., 320., 325., 310., 3]) - def gen_nums(): - while True: - yield quotes[-1].close + 1 + # def gen_nums(): + # while True: + # yield quotes[-1].close + 2 + # yield quotes[-1].close - 2 - nums = gen_nums() + # nums = gen_nums() - await trio.sleep(10) - while True: - new = next(nums) - quotes[-1].close = new - # this updates the linked_charts internal array - # and then passes that array to all subcharts to - # render downstream graphics - linked_charts.update_from_quote({'last': new}) - await trio.sleep(.1) + # # await trio.sleep(10) + # import time + # while True: + # new = next(nums) + # quotes[-1].close = new + # # this updates the linked_charts internal array + # # and then passes that array to all subcharts to + # # render downstream graphics + # start = time.time() + # linked_charts.update_from_quote({'last': new}) + # print(f"Render latency {time.time() - start}") + # # 20 Hz seems to be good enough + # await trio.sleep(0.05) await trio.sleep_forever() From 6b1bdbe3eaed0a2b6d78979ca9afb77d63587004 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 7 Jul 2020 10:39:22 -0400 Subject: [PATCH 038/206] Docs the ui pkg mod --- piker/ui/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/ui/__init__.py b/piker/ui/__init__.py index ace44590..a7fb1052 100644 --- a/piker/ui/__init__.py +++ b/piker/ui/__init__.py @@ -1,3 +1,6 @@ """ -Stuff for your eyes. +Stuff for your eyes, aka super hawt Qt UI components. + +Currently we only support PyQt5 due to this issue in Pyside2: +https://bugreports.qt.io/projects/PYSIDE/issues/PYSIDE-1313 """ From 9c1d64413e4d2156d13ac6c0a2ef47c21fed9cf5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 7 Jul 2020 10:44:55 -0400 Subject: [PATCH 039/206] Handle flat bar updates Flat bars have a rendering issue we work around by hacking values in `QLineF` but we have to revert those on any last bar that is being updated in real-time. Comment out candle implementations for now; we can get back to it if/when the tinas unite. Oh, and make bars have a little space between them. --- piker/ui/_graphics.py | 82 +++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 31 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index d179cc22..0c7a019f 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -1,7 +1,7 @@ """ Chart graphics for displaying a slew of different data types. """ -from typing import Dict, Any +# from typing import Dict, Any from enum import Enum from contextlib import contextmanager @@ -18,7 +18,7 @@ from ._axes import YAxisLabel, XAxisLabel # TODO: checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 40 +_mouse_rate_limit = 50 class CrossHairItem(pg.GraphicsObject): @@ -174,7 +174,8 @@ class BarItems(pg.GraphicsObject): """ sigPlotChanged = QtCore.Signal(object) - w: float = 0.5 + # 0.5 is no overlap between arms, 1.0 is full overlap + w: float = 0.4 bull_pen = pg.mkPen('#808080') # XXX: tina mode, see below @@ -218,8 +219,10 @@ class BarItems(pg.GraphicsObject): if low != high: hl = QLineF(index, low, index, high) else: - # if we don't do it renders a weird rectangle? + # XXX: if we don't do it renders a weird rectangle? + # see below too for handling this later... hl = QLineF(low, low, low, low) + hl._flat = True # open line o = QLineF(index - self.w, q['open'], index, q['open']) # close line @@ -227,6 +230,7 @@ class BarItems(pg.GraphicsObject): # indexing here is as per the below comments lines[3*i:3*i+3] = (hl, o, c) + p.setPen(self.bull_pen) p.drawLines(*lines) # if not _tina_mode: # piker mode @@ -263,7 +267,10 @@ class BarItems(pg.GraphicsObject): does) so this "should" be simpler and faster. """ # do we really need to verify the entire past data set? - last = array['close'][-1] + # last = array['close'][-1] + # index, time, open, high, low, close, volume + index, time, _, _, _, close, _ = array[-1] + last = close body, larm, rarm = self.lines[-3:] # XXX: is there a faster way to modify this? @@ -279,15 +286,22 @@ class BarItems(pg.GraphicsObject): if last > high: high = last - body.setLine(body.x1(), low, body.x2(), high) + if getattr(body, '_flat', None) and low != high: + # if the bar was flat it likely does not have + # the index set correctly due to a rendering bug + # see above + body.setLine(index, low, index, high) + body._flat = False + else: + body.setLine(body.x1(), low, body.x2(), high) # draw the pic with self.painter() as p: - p.setPen(self.bull_brush) + p.setPen(self.bull_pen) p.drawLines(*self.lines) - # trigger re-render - self.update() + # trigger re-render + self.update() # be compat with ``pg.PlotCurveItem`` setData = update_from_array @@ -305,37 +319,43 @@ class BarItems(pg.GraphicsObject): return QtCore.QRectF(self.picture.boundingRect()) -class CandlestickItems(BarItems): +# XXX: when we get back to enabling tina mode +# class CandlestickItems(BarItems): - w2 = 0.7 - line_pen = pg.mkPen('#000000') - bull_brush = pg.mkBrush('#00ff00') - bear_brush = pg.mkBrush('#ff0000') +# w2 = 0.7 +# line_pen = pg.mkPen('#000000') +# bull_brush = pg.mkBrush('#00ff00') +# bear_brush = pg.mkBrush('#ff0000') - def _generate(self, p): - rects = np.array( - [ - QtCore.QRectF(q.id - self.w, q.open, self.w2, q.close - q.open) - for q in Quotes - ] - ) +# def _generate(self, p): +# rects = np.array( +# [ +# QtCore.QRectF( +# q.id - self.w, +# q.open, +# self.w2, +# q.close - q.open +# ) +# for q in Quotes +# ] +# ) - p.setPen(self.line_pen) - p.drawLines( - [QtCore.QLineF(q.id, q.low, q.id, q.high) - for q in Quotes] - ) +# p.setPen(self.line_pen) +# p.drawLines( +# [QtCore.QLineF(q.id, q.low, q.id, q.high) +# for q in Quotes] +# ) - p.setBrush(self.bull_brush) - p.drawRects(*rects[Quotes.close > Quotes.open]) +# p.setBrush(self.bull_brush) +# p.drawRects(*rects[Quotes.close > Quotes.open]) - p.setBrush(self.bear_brush) - p.drawRects(*rects[Quotes.close < Quotes.open]) +# p.setBrush(self.bear_brush) +# p.drawRects(*rects[Quotes.close < Quotes.open]) class ChartType(Enum): """Bar type to graphics class map. """ BAR = BarItems - CANDLESTICK = CandlestickItems + # CANDLESTICK = CandlestickItems LINE = pg.PlotDataItem From 3c55f7c6e21c4454a52b6ceb95d1ec8de6b1e7e2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 14:56:45 -0400 Subject: [PATCH 040/206] Use structure array indexing syntax --- piker/ui/_axes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 03e68feb..b64b72ec 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -62,7 +62,7 @@ class DynamicDateAxis(pg.AxisItem): for ibar in values: if ibar > quotes_count: return strings - dt_tick = fromtimestamp(bars[int(ibar)].time) + dt_tick = fromtimestamp(bars[int(ibar)]['time']) strings.append( dt_tick.strftime(self.tick_tpl[s_period]) ) @@ -146,7 +146,7 @@ class XAxisLabel(AxisLabel): bars = self.parent.linked_charts._array if tick_pos > len(bars): return 'Unknown Time' - return fromtimestamp(bars[round(tick_pos)].time).strftime(tpl) + return fromtimestamp(bars[round(tick_pos)]['time']).strftime(tpl) def boundingRect(self): # noqa return QtCore.QRectF(0, 0, 145, 50) From 2dec32e41f600393a259eab52b9c6e0729b33041 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:06:39 -0400 Subject: [PATCH 041/206] Move bar generation into func; support bar appends There's really nothing coupling it to the graphics class (which frankly also seems like it doesn't need to be a class.. Qt). Add support to `.update_from_array()` for diffing with the input array and creating additional bar-lines where necessary. Note, there are still issues with the "correctness" here in terms of bucketing open/close values in the time frame / bar range. Also, this jamming of each bar's 3 lines into a homogeneous array seems like it could be better done with struct arrays and avoid all this "index + 3" stuff. --- piker/ui/_graphics.py | 191 ++++++++++++++++++++++++++---------------- 1 file changed, 119 insertions(+), 72 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 0c7a019f..71b23711 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -169,6 +169,57 @@ class CrossHairItem(pg.GraphicsObject): return self.parent.boundingRect() +def bars_from_ohlc( + data: np.ndarray, + start: int = 0, + w: float = 0.4, +) -> np.ndarray: + lines = np.empty_like(data, shape=(data.shape[0]*3,), dtype=object) + + for i, q in enumerate(data[start:], start=start): + low = q['low'] + high = q['high'] + index = q['index'] + + # high - low line + if low != high: + hl = QLineF(index, low, index, high) + else: + # XXX: if we don't do it renders a weird rectangle? + # see below too for handling this later... + hl = QLineF(low, low, low, low) + hl._flat = True + # open line + o = QLineF(index - w, q['open'], index, q['open']) + # close line + c = QLineF(index + w, q['close'], index, q['close']) + + # indexing here is as per the below comments + lines[3*i:3*i+3] = (hl, o, c) + + # if not _tina_mode: # piker mode + # else _tina_mode: + # self.lines = lines = np.concatenate( + # [high_to_low, open_sticks, close_sticks]) + # use traditional up/down green/red coloring + # long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) + # short_bars = np.resize( + # Quotes.close < Quotes.open, len(lines)) + + # ups = lines[long_bars] + # downs = lines[short_bars] + + # # draw "up" bars + # p.setPen(self.bull_brush) + # p.drawLines(*ups) + + # # draw "down" bars + # p.setPen(self.bear_brush) + # p.drawLines(*downs) + + return lines + + class BarItems(pg.GraphicsObject): """Price range bars graphics rendered from a OHLC sequence. """ @@ -185,8 +236,22 @@ class BarItems(pg.GraphicsObject): def __init__(self): super().__init__() self.picture = QtGui.QPicture() + + # XXX: not sure this actually needs to be an array other + # then for the old tina mode calcs for up/down bars below? # lines container - self.lines: np.ndarray = None + self.lines: np.ndarray = np.empty_like( + [], shape=(int(50e3),), dtype=object) + self._index = 0 + + def _set_index(self, val): + # breakpoint() + self._index = val + + def _get_index(self): + return self._index + + index = property(_get_index, _set_index) # TODO: can we be faster just dropping this? @contextmanager @@ -201,57 +266,21 @@ class BarItems(pg.GraphicsObject): def draw_from_data( self, data: np.recarray, + start: int = 0, ): """Draw OHLC datum graphics from a ``np.recarray``. """ - # XXX: not sure this actually needs to be an array other - # then for the old tina mode calcs for up/down bars below? - self.lines = lines = np.empty_like( - data, shape=(data.shape[0]*3,), dtype=object) + lines = bars_from_ohlc(data, start=start, w=self.w) + + # save graphics for later reference and keep track + # of current internal "last index" + index = len(lines) + self.lines[:index] = lines + self.index = index with self.painter() as p: - for i, q in enumerate(data): - low = q['low'] - high = q['high'] - index = q['index'] - - # high - low line - if low != high: - hl = QLineF(index, low, index, high) - else: - # XXX: if we don't do it renders a weird rectangle? - # see below too for handling this later... - hl = QLineF(low, low, low, low) - hl._flat = True - # open line - o = QLineF(index - self.w, q['open'], index, q['open']) - # close line - c = QLineF(index + self.w, q['close'], index, q['close']) - - # indexing here is as per the below comments - lines[3*i:3*i+3] = (hl, o, c) - p.setPen(self.bull_pen) - p.drawLines(*lines) - # if not _tina_mode: # piker mode - # else _tina_mode: - # self.lines = lines = np.concatenate( - # [high_to_low, open_sticks, close_sticks]) - # use traditional up/down green/red coloring - # long_bars = np.resize(Quotes.close > Quotes.open, len(lines)) - # short_bars = np.resize( - # Quotes.close < Quotes.open, len(lines)) - - # ups = lines[long_bars] - # downs = lines[short_bars] - - # # draw "up" bars - # p.setPen(self.bull_brush) - # p.drawLines(*ups) - - # # draw "down" bars - # p.setPen(self.bear_brush) - # p.drawLines(*downs) + p.drawLines(*self.lines[:index]) def update_from_array( self, @@ -266,39 +295,57 @@ class BarItems(pg.GraphicsObject): assuming the prior graphics havent changed (OHLC history rarely does) so this "should" be simpler and faster. """ - # do we really need to verify the entire past data set? - # last = array['close'][-1] - # index, time, open, high, low, close, volume - index, time, _, _, _, close, _ = array[-1] - last = close - body, larm, rarm = self.lines[-3:] + index = self.index + length = len(array) + idata = int(index/3) - 1 + extra = length - idata + if extra > 0: + # generate new graphics to match provided array + new = array[-extra:] + lines = bars_from_ohlc(new, w=self.w) - # XXX: is there a faster way to modify this? - # update right arm - rarm.setLine(rarm.x1(), last, rarm.x2(), last) + added = len(new) * 3 + assert len(lines) == added - # update body - high = body.y2() - low = body.y1() - if last < low: - low = last + self.lines[index:index + len(lines)] = lines - if last > high: - high = last + self.index += len(lines) - if getattr(body, '_flat', None) and low != high: - # if the bar was flat it likely does not have - # the index set correctly due to a rendering bug - # see above - body.setLine(index, low, index, high) - body._flat = False - else: - body.setLine(body.x1(), low, body.x2(), high) + else: # current bar update + # do we really need to verify the entire past data set? + # index, time, open, high, low, close, volume + i, time, _, _, _, close, _ = array[-1] + last = close + body, larm, rarm = self.lines[index-3:index] + if not rarm: + breakpoint() + + # XXX: is there a faster way to modify this? + # update right arm + rarm.setLine(rarm.x1(), last, rarm.x2(), last) + + # update body + high = body.y2() + low = body.y1() + if last < low: + low = last + + if last > high: + high = last + + if getattr(body, '_flat', None) and low != high: + # if the bar was flat it likely does not have + # the index set correctly due to a rendering bug + # see above + body.setLine(i, low, i, high) + body._flat = False + else: + body.setLine(body.x1(), low, body.x2(), high) # draw the pic with self.painter() as p: p.setPen(self.bull_pen) - p.drawLines(*self.lines) + p.drawLines(*self.lines[:index]) # trigger re-render self.update() @@ -319,7 +366,7 @@ class BarItems(pg.GraphicsObject): return QtCore.QRectF(self.picture.boundingRect()) -# XXX: when we get back to enabling tina mode +# XXX: when we get back to enabling tina mode for xb # class CandlestickItems(BarItems): # w2 = 0.7 From 5f89a2bf081183cb1f7434e96ee849233c9692a7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:40:35 -0400 Subject: [PATCH 042/206] Make run_qtrio invoke tractor at top level --- piker/ui/_exec.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index c2eda278..03d8087b 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -5,6 +5,7 @@ Run ``trio`` in guest mode on top of the Qt event loop. All global Qt runtime settings are mostly defined here. """ import traceback +from functools import partial import PyQt5 # noqa from pyqtgraph import QtGui @@ -12,6 +13,7 @@ from PyQt5 import QtCore from PyQt5.QtCore import pyqtRemoveInputHook import qdarkstyle import trio +import tractor from outcome import Error @@ -31,6 +33,7 @@ def run_qtrio( func, args, main_widget, + loglevel = None, ) -> None: # avoids annoying message when entering debugger from qt loop pyqtRemoveInputHook() @@ -79,10 +82,29 @@ def run_qtrio( 'main': instance, } + # force mp since we may spawn an ib asyncio based backend + tractor._spawn.try_set_start_method('forkserver') + # setup tractor entry point args + args = ( + # async_fn + func, + # args + (widgets,), + # kwargs + {'loglevel': 'info'}, + # arbiter_addr + ( + tractor._default_arbiter_host, + tractor._default_arbiter_port, + ), + # name + 'qtrio', + ) + # guest mode trio.lowlevel.start_guest_run( - func, - widgets, + tractor._main, + *args, run_sync_soon_threadsafe=run_sync_soon_threadsafe, done_callback=done_callback, ) From 4c5bc19ec7f7acfc3576a7212201365e590cdd5e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:41:14 -0400 Subject: [PATCH 043/206] Always convert to posix time --- piker/ui/_source.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/piker/ui/_source.py b/piker/ui/_source.py index 70c586d4..6f0debf7 100644 --- a/piker/ui/_source.py +++ b/piker/ui/_source.py @@ -64,21 +64,35 @@ def from_df( """Convert OHLC formatted ``pandas.DataFrame`` to ``numpy.recarray``. """ df.reset_index(inplace=True) - df['Date'] = [d.timestamp() for d in df.Date] + + # hackery to convert field names + date = 'Date' + if 'date' in df.columns: + date = 'date' + + # convert to POSIX time + df[date] = [d.timestamp() for d in df[date]] # try to rename from some camel case - columns={ + columns = { 'Date': 'time', + 'date': 'time', 'Open': 'open', 'High': 'high', 'Low': 'low', 'Close': 'close', 'Volume': 'volume', } - for name in df.columns: - if name not in columns: - del df[name] + df = df.rename(columns=columns) + + for name in df.columns: + if name not in OHLC_dtype.names: + del df[name] + + # TODO: it turns out column access on recarrays is actually slower: + # https://jakevdp.github.io/PythonDataScienceHandbook/02.09-structured-data-numpy.html#RecordArrays:-Structured-Arrays-with-a-Twist + # it might make sense to make these structured arrays? array = df.to_records() _nan_to_closest_num(array) @@ -88,7 +102,6 @@ def from_df( def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. """ - for col in ['open', 'high', 'low', 'close']: mask = np.isnan(array[col]) if not mask.size: From b9224cd396a088ed85b889abec391cb290756114 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:42:05 -0400 Subject: [PATCH 044/206] Add WIP real-time 5s bar charting --- piker/ui/_chart.py | 161 ++++++++++++++++++++++++++++++++------------- 1 file changed, 115 insertions(+), 46 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 73bd4422..1a9a3a0c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -244,28 +244,46 @@ class LinkedSplitCharts(QtGui.QWidget): # TODO: eventually we'll want to update bid/ask labels and other # data as subscribed by underlying UI consumers. last = quote['last'] - current = self._array[-1] + index, time, open, high, low, close, volume = self._array[-1] # update ohlc (I guess we're enforcing this for now?) - current['close'] = last - current['high'] = max(current['high'], last) - current['low'] = min(current['low'], last) + # self._array[-1]['close'] = last + # self._array[-1]['high'] = max(h, last) + # self._array[-1]['low'] = min(l, last) + # overwrite from quote + self._array[-1] = ( + index, + time, + open, + max(high, last), + min(low, last), + last, + volume, + ) + self.update_from_array(self._array) + + def update_from_array( + self, + array: np.ndarray, + **kwargs, + ) -> None: # update the ohlc sequence graphics chart chart = self.chart + # we send a reference to the whole updated array - chart.update_from_array(self._array) + chart.update_from_array(array, **kwargs) # TODO: the "data" here should really be a function # and it should be managed and computed outside of this UI for chart, func in self.indicators: # process array in entirely every update # TODO: change this for streaming - data = func(self._array) - chart.update_from_array(data, chart.name) + data = func(array) + chart.update_from_array(data, name=chart.name, **kwargs) -_min_points_to_show = 20 +_min_points_to_show = 3 class ChartPlotWidget(pg.PlotWidget): @@ -300,7 +318,7 @@ class ChartPlotWidget(pg.PlotWidget): self.parent = linked_charts # this is the index of that last input array entry and is # updated and used to figure out how many bars are in view - self._xlast = 0 + # self._xlast = 0 # XXX: label setting doesn't seem to work? # likely custom graphics need special handling @@ -353,7 +371,7 @@ class ChartPlotWidget(pg.PlotWidget): """ l, r = self.view_range() lbar = max(l, 0) - rbar = min(r, len(self.parent._array) - 1) + rbar = min(r, len(self.parent._array)) return l, lbar, rbar, r def draw_ohlc( @@ -374,8 +392,7 @@ class ChartPlotWidget(pg.PlotWidget): self.addItem(graphics) # set xrange limits - self._xlast = xlast = data[-1]['index'] - # self._set_xlimits(data[0]['index'] - 100, xlast) + xlast = data[-1]['index'] # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -394,44 +411,46 @@ class ChartPlotWidget(pg.PlotWidget): # register overlay curve with name if not self._graphics and name is None: name = 'main' + self._graphics[name] = curve # set a "startup view" - xlast = len(data)-1 + xlast = len(data) - 1 # self._set_xlimits(0, xlast) # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) + # TODO: we should instead implement a diff based + # "only update with new items" on the pg.PlotDataItem + curve.update_from_array = curve.setData + return curve def update_from_array( self, array: np.ndarray, name: str = 'main', - ) -> None: - self._xlast = len(array) - 1 + **kwargs, + ) -> pg.GraphicsObject: + # self._xlast = len(array) - 1 graphics = self._graphics[name] - graphics.setData(array) + graphics.update_from_array(array, **kwargs) + # update view self._set_yrange() + return graphics + def _set_yrange( self, ) -> None: - """Callback for each y-range update. + """Set the viewable y-range based on embedded data. This adds auto-scaling like zoom on the scroll wheel such that data always fits nicely inside the current view of the data set. """ - # TODO: this can likely be ported in part to the built-ins: - # self.setYRange(Quotes.low.min() * .98, Quotes.high.max() * 1.02) - # self.setMouseEnabled(x=True, y=False) - # self.setXRange(Quotes[0].id, Quotes[-1].id) - # self.setAutoVisible(x=False, y=True) - # self.enableAutoRange(x=False, y=True) - l, lbar, rbar, r = self.bars_range() # figure out x-range in view such that user can scroll "off" the data @@ -474,8 +493,8 @@ class ChartPlotWidget(pg.PlotWidget): # view margins diff = yhigh - ylow - ylow = ylow - (diff * 0.08) - yhigh = yhigh + (diff * 0.08) + ylow = ylow - (diff * 0.1) + yhigh = yhigh + (diff * 0.1) chart = self chart.setLimits( @@ -535,12 +554,12 @@ class ChartView(pg.ViewBox): if ev.delta() > 0 and vl <= _min_points_to_show: log.trace("Max zoom bruh...") return - if ev.delta() < 0 and vl >= len(self.linked_charts._array) - 1: + if ev.delta() < 0 and vl >= len(self.linked_charts._array): log.trace("Min zoom bruh...") return # actual scaling factor - s = 1.015 ** (ev.delta() * -1/20) # self.state['wheelScaleFactor']) + s = 1.015 ** (ev.delta() * -1 / 20) # self.state['wheelScaleFactor']) s = [(None if m is False else s) for m in mask] # center = pg.Point( @@ -550,8 +569,8 @@ class ChartView(pg.ViewBox): # XXX: scroll "around" the right most element in the view furthest_right_coord = self.boundingRect().topRight() center = pg.Point( - fn.invertQTransform( - self.childGroup.transform() + fn.invertQTransform( + self.childGroup.transform() ).map(furthest_right_coord) ) @@ -572,9 +591,74 @@ def main(symbol): """Main Qt-trio routine invoked by the Qt loop with the widgets ``dict``. """ - chart_app = widgets['main'] + # data-feed setup + sym = symbol or 'ES.GLOBEX' + brokermod = brokers.get_brokermod('ib') + async with brokermod.get_client() as client: + # figure out the exact symbol + bars = await client.bars(symbol=sym) + + # ``from_buffer` return read-only + bars = np.array(bars) + linked_charts = chart_app.load_symbol('ES', bars) + + async def add_new_bars(delay_s=5.): + import time + + ohlc = linked_charts._array + + last_5s = ohlc[-1]['time'] + delay = max((last_5s + 4.99) - time.time(), 0) + await trio.sleep(delay) + + while True: + print('new bar') + + # TODO: bunch of stuff: + # - I'm starting to think all this logic should be + # done in one place and "graphics update routines" + # should not be doing any length checking and array diffing. + # - don't keep appending, but instead increase the + # underlying array's size less frequently: + # - handle odd lot orders + # - update last open price correctly instead + # of copying it from last bar's close + # - 5 sec bar lookback-autocorrection like tws does + index, t, open, high, low, close, volume = ohlc[-1] + new = np.append( + ohlc, + np.array( + [(index + 1, t + 5, close, close, close, close, 0)], + dtype=ohlc.dtype + ), + ) + ohlc = linked_charts._array = new + linked_charts.update_from_array(new) + + # sleep until next 5s from last bar + last_5s = ohlc[-1]['time'] + delay = max((last_5s + 4.99) - time.time(), 0) + await trio.sleep(4.9999) + + async with trio.open_nursery() as n: + n.start_soon(add_new_bars) + + async with brokermod.maybe_spawn_brokerd() as portal: + stream = await portal.run( + 'piker.brokers.ib', + 'trio_stream_ticker', + sym=sym, + ) + # TODO: timeframe logic + async for tick in stream: + # breakpoint() + if tick['tickType'] in (48, 77): + linked_charts.update_from_quote( + {'last': tick['price']} + ) + # from .quantdom.loaders import get_quotes # from datetime import datetime # from ._source import from_df @@ -585,15 +669,6 @@ def main(symbol): # ) # quotes = from_df(quotes) - # data-feed spawning - brokermod = brokers.get_brokermod('ib') - async with brokermod.get_client() as client: - # figure out the exact symbol - bars = await client.bars(symbol='ES') - - # wow, just wow.. non-contiguous eh? - bars = np.array(bars) - # feed = DataFeed(portal, brokermod) # quote_gen, quotes = await feed.open_stream( # symbols, @@ -608,10 +683,6 @@ def main(symbol): # log.error("Broker API is down temporarily") # return - # spawn chart - linked_charts = chart_app.load_symbol(symbol, bars) - await trio.sleep_forever() - # make some fake update data # import itertools # nums = itertools.cycle([315., 320., 325., 310., 3]) @@ -637,6 +708,4 @@ def main(symbol): # # 20 Hz seems to be good enough # await trio.sleep(0.05) - await trio.sleep_forever() - run_qtrio(_main, (), ChartSpace) From 2dd596ec6cef205a4ada086ffa8f570b659c8087 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:42:32 -0400 Subject: [PATCH 045/206] Deps bump --- setup.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 81e3d56a..66208cea 100755 --- a/setup.py +++ b/setup.py @@ -47,11 +47,14 @@ setup( install_requires=[ 'click', 'colorlog', - 'trio', 'attrs', - 'async_generator', 'pygments', + # async + 'trio', + # 'tractor', # from github currently + 'async_generator', + # brokers 'asks', 'ib_insync', @@ -61,6 +64,11 @@ setup( 'cython', 'numpy', 'pandas', + 'msgpack-numpy', + + # UI + 'PyQt5', + 'pyqtgraph', # tsdbs 'pymarketstore', @@ -71,13 +79,11 @@ setup( keywords=["async", "trading", "finance", "quant", "charting"], classifiers=[ 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'License :: OSI Approved :: ', 'Operating System :: POSIX :: Linux', "Programming Language :: Python :: Implementation :: CPython", - # "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", 'Intended Audience :: Financial and Insurance Industry', 'Intended Audience :: Science/Research', 'Intended Audience :: Developers', From 4ceffdd83fcc035bd2f02a150f636c026bca9201 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 8 Jul 2020 15:42:51 -0400 Subject: [PATCH 046/206] Drop kivy stuff from docs --- README.rst | 61 ++++++++++++++++++++---------------------------------- 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/README.rst b/README.rst index b5b71d2f..f4851c2d 100644 --- a/README.rst +++ b/README.rst @@ -10,11 +10,13 @@ trading and financial analysis targetted at hardcore Linux users. It tries to use as much bleeding edge tech as possible including (but not limited to): - Python 3.7+ for glue_ and business logic -- trio_ for async -- tractor_ as the underlying actor model +- trio_ for structured concurrency +- tractor_ for distributed, multi-core, real-time streaming - marketstore_ for historical and real-time tick data persistence and sharing - techtonicdb_ for L2 book storage - Qt_ for pristine high performance UIs +- pyqtgraph_ for real-time charting +- ``numpy`` for `fast numerics`_ .. |travis| image:: https://img.shields.io/travis/pikers/piker/master.svg :target: https://travis-ci.org/pikers/piker @@ -24,6 +26,7 @@ It tries to use as much bleeding edge tech as possible including (but not limite .. _techtonicdb: https://github.com/0b01/tectonicdb .. _Qt: https://www.qt.io/ .. _glue: https://numpy.org/doc/stable/user/c-info.python-as-glue.html#using-python-as-glue +.. _fast numerics: https://zerowithdot.com/python-numpy-and-pandas-performance/ Focus and Features: @@ -65,27 +68,7 @@ Install be cloned from this repo and hacked on directly. A couple bleeding edge components are being used atm pertaining to -async ports of libraries for use with `trio`_. - -Before installing make sure you have `pipenv`_ and have installed -``python3.7`` as well as `kivy source build`_ dependencies -since currently there's reliance on an async development branch. - -`kivy` dependencies -=================== -On Archlinux you need the following dependencies:: - - pacman -S python-docutils gstreamer sdl2_ttf sdl2_mixer sdl2_image xclip - -To manually install the async branch of ``kivy`` from github do (though -this should be done as part of the ``pipenv install`` below):: - - pipenv install -e 'git+git://github.com/matham/kivy.git@async-loop#egg=kivy' - - -.. _kivy source build: - https://kivy.org/docs/installation/installation-linux.html#installation-in-a-virtual-environment - +new components within `trio`_. For a development install:: @@ -97,34 +80,36 @@ For a development install:: Broker Support ************** -For live data feeds the only fully functional broker at the moment is Questrade_. -Eventual support is in the works for `IB`, `TD Ameritrade` and `IEX`. +For live data feeds the set of supported brokers is: +- Questrade_ which comes with effectively free L1 +- IB_ via ib_insync +- Webull_ via the reverse engineered public API +- Kraken_ for crypto over their public websocket API + If you want your broker supported and they have an API let us know. .. _Questrade: https://www.questrade.com/api/documentation +.. _IB: https://interactivebrokers.github.io/tws-api/index.html +.. _Webull: https://www.kraken.com/features/api#public-market-data +.. _Kraken: https://www.kraken.com/features/api#public-market-data -Play with some UIs -****************** +Check out some charts +********************* +Bet you weren't expecting this from the foss:: -To start the real-time index monitor with the `questrade` backend:: - - piker -l info monitor indexes + piker chart spy.arca -If you want to see super granular price changes, increase the -broker quote query ``rate`` with ``-r``:: +It is also possible to run a specific broker's data feed as a top +level micro-service daemon:: - piker monitor indexes -r 10 + pikerd -l info -b ib -It is also possible to run the broker data feed micro service as a daemon:: - - pikerd -l info - Then start the client app as normal:: - piker monitor indexes + piker chart -b ib ES.GLOBEX .. _pipenv: https://docs.pipenv.org/ From f9084b8650a978da1831a5d7f165b3130d839c56 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 9 Jul 2020 08:37:30 -0400 Subject: [PATCH 047/206] Store lines graphics in struct array to simplify indexing --- piker/ui/_graphics.py | 153 ++++++++++++++++++++++-------------------- 1 file changed, 79 insertions(+), 74 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 71b23711..976b35a6 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -1,9 +1,9 @@ """ Chart graphics for displaying a slew of different data types. """ -# from typing import Dict, Any +from typing import List from enum import Enum -from contextlib import contextmanager +from itertools import chain import numpy as np import pyqtgraph as pg @@ -169,17 +169,34 @@ class CrossHairItem(pg.GraphicsObject): return self.parent.boundingRect() +def _mk_lines_array(data: List, size: int) -> np.ndarray: + """Create an ndarray to hold lines graphics objects. + """ + # TODO: might want to just make this a 2d array to be faster at + # flattening using .ravel()? + return np.empty_like( + data, + shape=(int(size),), + dtype=[ + ('index', int), + ('body', object), + ('rarm', object), + ('larm', object) + ], + ) + + def bars_from_ohlc( data: np.ndarray, start: int = 0, - w: float = 0.4, + w: float = 0.43, ) -> np.ndarray: - lines = np.empty_like(data, shape=(data.shape[0]*3,), dtype=object) + """Generate an array of lines objects from input ohlc data. + """ + lines = _mk_lines_array(data, data.shape[0]) for i, q in enumerate(data[start:], start=start): - low = q['low'] - high = q['high'] - index = q['index'] + low, high, index = q['low'], q['high'], q['index'] # high - low line if low != high: @@ -195,7 +212,8 @@ def bars_from_ohlc( c = QLineF(index + w, q['close'], index, q['close']) # indexing here is as per the below comments - lines[3*i:3*i+3] = (hl, o, c) + # lines[3*i:3*i+3] = (hl, o, c) + lines[i] = (index, hl, o, c) # if not _tina_mode: # piker mode # else _tina_mode: @@ -240,27 +258,10 @@ class BarItems(pg.GraphicsObject): # XXX: not sure this actually needs to be an array other # then for the old tina mode calcs for up/down bars below? # lines container - self.lines: np.ndarray = np.empty_like( - [], shape=(int(50e3),), dtype=object) - self._index = 0 + self.lines = _mk_lines_array([], 50e3) - def _set_index(self, val): - # breakpoint() - self._index = val - - def _get_index(self): - return self._index - - index = property(_get_index, _set_index) - - # TODO: can we be faster just dropping this? - @contextmanager - def painter(self): - # pre-computing a QPicture object allows paint() to run much - # more quickly, rather than re-drawing the shapes every time. - p = QtGui.QPainter(self.picture) - yield p - p.end() + # track the current length of drawable lines within the larger array + self.index: int = 0 @timeit def draw_from_data( @@ -270,17 +271,28 @@ class BarItems(pg.GraphicsObject): ): """Draw OHLC datum graphics from a ``np.recarray``. """ - lines = bars_from_ohlc(data, start=start, w=self.w) + lines = bars_from_ohlc(data, start=start) # save graphics for later reference and keep track # of current internal "last index" index = len(lines) self.lines[:index] = lines self.index = index + self.draw_lines() - with self.painter() as p: - p.setPen(self.bull_pen) - p.drawLines(*self.lines[:index]) + def draw_lines(self): + """Draw the current line set using the painter. + """ + to_draw = self.lines[ + ['body', 'larm', 'rarm']][:self.index] + + # pre-computing a QPicture object allows paint() to run much + # more quickly, rather than re-drawing the shapes every time. + p = QtGui.QPainter(self.picture) + p.setPen(self.bull_pen) + # TODO: might be better to use 2d array + p.drawLines(*chain.from_iterable(to_draw)) + p.end() def update_from_array( self, @@ -297,58 +309,51 @@ class BarItems(pg.GraphicsObject): """ index = self.index length = len(array) - idata = int(index/3) - 1 - extra = length - idata + extra = length - index if extra > 0: # generate new graphics to match provided array - new = array[-extra:] - lines = bars_from_ohlc(new, w=self.w) + new = array[index:index + extra] + lines = bars_from_ohlc(new) + bars_added = len(lines) + self.lines[index:index + bars_added] = lines + self.index += bars_added - added = len(new) * 3 - assert len(lines) == added + # else: # current bar update + # do we really need to verify the entire past data set? + # index, time, open, high, low, close, volume + i, time, _, _, _, close, _ = array[-1] + last = close + i, body, larm, rarm = self.lines[index-1] + if not rarm: + breakpoint() - self.lines[index:index + len(lines)] = lines + # XXX: is there a faster way to modify this? + # update right arm + rarm.setLine(rarm.x1(), last, rarm.x2(), last) - self.index += len(lines) + # update body + high = body.y2() + low = body.y1() + if last < low: + low = last - else: # current bar update - # do we really need to verify the entire past data set? - # index, time, open, high, low, close, volume - i, time, _, _, _, close, _ = array[-1] - last = close - body, larm, rarm = self.lines[index-3:index] - if not rarm: - breakpoint() + if last > high: + high = last - # XXX: is there a faster way to modify this? - # update right arm - rarm.setLine(rarm.x1(), last, rarm.x2(), last) - - # update body - high = body.y2() - low = body.y1() - if last < low: - low = last - - if last > high: - high = last - - if getattr(body, '_flat', None) and low != high: - # if the bar was flat it likely does not have - # the index set correctly due to a rendering bug - # see above - body.setLine(i, low, i, high) - body._flat = False - else: - body.setLine(body.x1(), low, body.x2(), high) + if getattr(body, '_flat', None) and low != high: + # if the bar was flat it likely does not have + # the index set correctly due to a rendering bug + # see above + body.setLine(i, low, i, high) + body._flat = False + else: + body.setLine(body.x1(), low, body.x2(), high) # draw the pic - with self.painter() as p: - p.setPen(self.bull_pen) - p.drawLines(*self.lines[:index]) + self.draw_lines() - # trigger re-render - self.update() + # trigger re-render + self.update() # be compat with ``pg.PlotCurveItem`` setData = update_from_array From 680267563709bd7ea79295abc0bdd4b7112ab47b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 08:20:29 -0400 Subject: [PATCH 048/206] Add kraken to backend list --- piker/brokers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index 587a2b61..852b3db2 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -12,6 +12,7 @@ __brokers__ = [ 'questrade', 'robinhood', 'ib', + 'kraken', ] From 1a1e7681262159e35ab39084605bf15f03f4c305 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 08:28:13 -0400 Subject: [PATCH 049/206] Port to new data apis --- piker/brokers/core.py | 4 ++-- piker/cli/__init__.py | 1 + piker/data/marketstore.py | 4 +++- piker/ui/cli.py | 6 +++++- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/piker/brokers/core.py b/piker/brokers/core.py index 2e672c61..67255a41 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -112,11 +112,11 @@ async def symbol_info( async def symbol_search( brokermod: ModuleType, - symbol: str, + pattern: str, **kwargs, ) -> Dict[str, Dict[str, Dict[str, Any]]]: """Return symbol info from broker. """ async with brokermod.get_client() as client: # TODO: support multiple asset type concurrent searches. - return await client.search_stocks(symbol, **kwargs) + return await client.search_stocks(pattern=pattern, **kwargs) diff --git a/piker/cli/__init__.py b/piker/cli/__init__.py index ea72b6b6..097f9ad5 100644 --- a/piker/cli/__init__.py +++ b/piker/cli/__init__.py @@ -9,6 +9,7 @@ import tractor from ..log import get_console_log, get_logger from ..brokers import get_brokermod, config + log = get_logger('cli') DEFAULT_BROKER = 'questrade' diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index 84e62ecb..0c68adf7 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -16,6 +16,7 @@ import msgpack import numpy as np import pandas as pd import pymarketstore as pymkts +import tractor from trio_websocket import open_websocket_url from ..log import get_logger, get_console_log @@ -320,7 +321,8 @@ async def stream_quotes( # update cache _cache[symbol].update(quote) else: - quotes = {symbol: [{key.lower(): val for key, val in quote.items()}]} + quotes = { + symbol: [{key.lower(): val for key, val in quote.items()}]} if quotes: yield quotes diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 78c749df..2be6d436 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -117,4 +117,8 @@ def chart(config, symbol, date, tl, rate, test): """ from ._chart import main - main(symbol) + # global opts + loglevel = config['loglevel'] + brokername = config['broker'] + + main(sym=symbol, brokername=brokername, loglevel=loglevel) From 88fb7a89514ee3ee100fdb09530bd0805116a5ef Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 08:28:50 -0400 Subject: [PATCH 050/206] Handle overloaded arg --- piker/brokers/questrade.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index f3708d3f..00e49587 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -565,6 +565,7 @@ class Client: time_frame: str = '1m', count: float = 20e3, is_paid_feed: bool = False, + as_np: bool = False, ) -> List[Dict[str, Any]]: """Retreive OHLCV bars for a symbol over a range to the present. From e5bca1e089d3817544f7020b17cdad73b4fcfd8e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 08:41:29 -0400 Subject: [PATCH 051/206] Fix import error --- piker/ui/_style.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index b35e2e47..a5e3b64e 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -1,6 +1,7 @@ """ Qt styling. """ +import pyqtgraph as pg from PyQt5 import QtGui From 3aebeb580170e2c906036d0e316017eb854c543d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 08:42:01 -0400 Subject: [PATCH 052/206] Standardize ohlc dtype --- piker/ui/_source.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/piker/ui/_source.py b/piker/ui/_source.py index 6f0debf7..0224538b 100644 --- a/piker/ui/_source.py +++ b/piker/ui/_source.py @@ -8,9 +8,9 @@ import numpy as np import pandas as pd -OHLC_dtype = np.dtype( +ohlc_dtype = np.dtype( [ - ('id', int), + ('index', int), ('time', float), ('open', float), ('high', float), @@ -20,15 +20,16 @@ OHLC_dtype = np.dtype( ] ) -# tf = { -# 1: TimeFrame.M1, -# 5: TimeFrame.M5, -# 15: TimeFrame.M15, -# 30: TimeFrame.M30, -# 60: TimeFrame.H1, -# 240: TimeFrame.H4, -# 1440: TimeFrame.D1, -# } +# map time frame "keys" to minutes values +tf_in_1m = { + '1m': 1, + '5m': 5, + '15m': 15, + '30m': 30, + '1h': 60, + '4h': 240, + '1d': 1440, +} def ohlc_zeros(length: int) -> np.ndarray: @@ -37,7 +38,7 @@ def ohlc_zeros(length: int) -> np.ndarray: For "why a structarray" see here: https://stackoverflow.com/a/52443038 Bottom line, they're faster then ``np.recarray``. """ - return np.zeros(length, dtype=OHLC_dtype) + return np.zeros(length, dtype=ohlc_dtype) @dataclass @@ -87,7 +88,7 @@ def from_df( df = df.rename(columns=columns) for name in df.columns: - if name not in OHLC_dtype.names: + if name not in ohlc_dtype.names[1:]: del df[name] # TODO: it turns out column access on recarrays is actually slower: From 788771bd751ff3df9aacf0cf29491f3054599588 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 09:55:09 -0400 Subject: [PATCH 053/206] Change name to qtractor --- piker/ui/_exec.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 03d8087b..06944d0e 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -5,7 +5,6 @@ Run ``trio`` in guest mode on top of the Qt event loop. All global Qt runtime settings are mostly defined here. """ import traceback -from functools import partial import PyQt5 # noqa from pyqtgraph import QtGui @@ -21,7 +20,7 @@ from outcome import Error class MainWindow(QtGui.QMainWindow): size = (800, 500) - title = 'piker: chart' + title = 'piker chart (bby)' def __init__(self, parent=None): super().__init__(parent) @@ -29,11 +28,12 @@ class MainWindow(QtGui.QMainWindow): self.setWindowTitle(self.title) -def run_qtrio( +def run_qtractor( func, args, - main_widget, - loglevel = None, + main_widget: QtGui.QWidget, + window_type: QtGui.QMainWindow = MainWindow, + loglevel: str = None, ) -> None: # avoids annoying message when entering debugger from qt loop pyqtRemoveInputHook() @@ -42,6 +42,12 @@ def run_qtrio( if app is None: app = PyQt5.QtWidgets.QApplication([]) + # TODO: we might not need this if it's desired + # to cancel the tractor machinery on Qt loop + # close, however the details of doing that correctly + # currently seem tricky.. + app.setQuitOnLastWindowClosed(False) + # This code is from Nathaniel, and I quote: # "This is substantially faster than using a signal... for some # reason Qt signal dispatch is really slow (and relies on events @@ -74,8 +80,9 @@ def run_qtrio( app.setStyleSheet(qdarkstyle.load_stylesheet(qt_api='pyqt5')) # make window and exec - window = MainWindow() + window = window_type() instance = main_widget() + instance.window = window widgets = { 'window': window, @@ -98,7 +105,7 @@ def run_qtrio( tractor._default_arbiter_port, ), # name - 'qtrio', + 'qtractor', ) # guest mode From c56aee634717527ffe1cc90370e4fca8d47546be Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 15 Jul 2020 10:59:29 -0400 Subject: [PATCH 054/206] Use array of names for lookup --- piker/ui/_graphics.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 976b35a6..dc7694a4 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -16,7 +16,9 @@ from .quantdom.utils import timeit from ._style import _xaxis_at # , _tina_mode from ._axes import YAxisLabel, XAxisLabel -# TODO: checkout pyqtgraph.PlotCurveItem.setCompositionMode + +# TODO: +# - checkout pyqtgraph.PlotCurveItem.setCompositionMode _mouse_rate_limit = 50 @@ -174,7 +176,7 @@ def _mk_lines_array(data: List, size: int) -> np.ndarray: """ # TODO: might want to just make this a 2d array to be faster at # flattening using .ravel()? - return np.empty_like( + return np.zeros_like( data, shape=(int(size),), dtype=[ @@ -196,7 +198,7 @@ def bars_from_ohlc( lines = _mk_lines_array(data, data.shape[0]) for i, q in enumerate(data[start:], start=start): - low, high, index = q['low'], q['high'], q['index'] + open, high, low, close, index = q[['open', 'high', 'low', 'close', 'index']] # high - low line if low != high: @@ -206,10 +208,11 @@ def bars_from_ohlc( # see below too for handling this later... hl = QLineF(low, low, low, low) hl._flat = True + # open line - o = QLineF(index - w, q['open'], index, q['open']) + o = QLineF(index - w, open, index, open) # close line - c = QLineF(index + w, q['close'], index, q['close']) + c = QLineF(index + w, close, index, close) # indexing here is as per the below comments # lines[3*i:3*i+3] = (hl, o, c) @@ -290,7 +293,11 @@ class BarItems(pg.GraphicsObject): # more quickly, rather than re-drawing the shapes every time. p = QtGui.QPainter(self.picture) p.setPen(self.bull_pen) - # TODO: might be better to use 2d array + + # TODO: might be better to use 2d array? + # try our fsp.rec2array() and a np.ravel() for speedup + # otherwise we might just have to go 2d ndarray of objects. + # see conlusion on speed here: # https://stackoverflow.com/a/60089929 p.drawLines(*chain.from_iterable(to_draw)) p.end() @@ -321,7 +328,7 @@ class BarItems(pg.GraphicsObject): # else: # current bar update # do we really need to verify the entire past data set? # index, time, open, high, low, close, volume - i, time, _, _, _, close, _ = array[-1] + i, time, open, _, _, close, _ = array[-1] last = close i, body, larm, rarm = self.lines[index-1] if not rarm: From b4f1ec79609ce7c088f74329f46c0089b990f8df Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 16 Jul 2020 21:54:24 -0400 Subject: [PATCH 055/206] Massively simplify the cross-hair monstrosity Stop with all this "main chart" special treatment. Manage all lines in the same way across all referenced plots. Add `CrossHair.add_plot()` for adding new plots dynamically. Just, smh. --- piker/ui/_graphics.py | 189 +++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 112 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index dc7694a4..21694d0f 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -23,152 +23,116 @@ from ._axes import YAxisLabel, XAxisLabel _mouse_rate_limit = 50 -class CrossHairItem(pg.GraphicsObject): +class CrossHair(pg.GraphicsObject): - def __init__(self, parent, indicators=None, digits=0): + def __init__(self, parent, digits: int = 0): super().__init__() # self.pen = pg.mkPen('#000000') self.pen = pg.mkPen('#a9a9a9') self.parent = parent - self.indicators = {} - self.activeIndicator = None - self.xaxis = self.parent.getAxis('bottom') - self.yaxis = self.parent.getAxis('right') + self.graphics = {} + self.plots = [] + self.active_plot = None + self.digits = digits - self.vline = self.parent.addLine(x=0, pen=self.pen, movable=False) - self.hline = self.parent.addLine(y=0, pen=self.pen, movable=False) + def add_plot( + self, + plot: 'ChartPlotWidget', # noqa + digits: int = 0, + ) -> None: + # add ``pg.graphicsItems.InfiniteLine``s + # vertical and horizonal lines and a y-axis label + vl = plot.addLine(x=0, pen=self.pen, movable=False) + hl = plot.addLine(y=0, pen=self.pen, movable=False) + yl = YAxisLabel( + parent=plot.getAxis('right'), + digits=digits or self.digits, + opacity=1 + ) - self.proxy_moved = pg.SignalProxy( - self.parent.scene().sigMouseMoved, + # TODO: checkout what ``.sigDelayed`` can be used for + # (emitted once a sufficient delay occurs in mouse movement) + px_moved = pg.SignalProxy( + plot.scene().sigMouseMoved, rateLimit=_mouse_rate_limit, - slot=self.mouseMoved, + slot=self.mouseMoved ) - - self.yaxis_label = YAxisLabel( - parent=self.yaxis, digits=digits, opacity=1 + px_enter = pg.SignalProxy( + plot.sig_mouse_enter, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Enter', plot), ) - - indicators = indicators or [] - - if indicators: - # when there are indicators present in sub-plot rows - # take the last one (nearest to the bottom) and place the - # crosshair label on it's x-axis. - last_ind = indicators[-1] - - self.proxy_enter = pg.SignalProxy( - self.parent.sig_mouse_enter, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Enter', False), - ) - self.proxy_leave = pg.SignalProxy( - self.parent.sig_mouse_leave, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Leave', False), - ) + px_leave = pg.SignalProxy( + plot.sig_mouse_leave, + rateLimit=_mouse_rate_limit, + slot=lambda: self.mouseAction('Leave', plot), + ) + self.graphics[plot] = { + 'vl': vl, + 'hl': hl, + 'yl': yl, + 'px': (px_moved, px_enter, px_leave), + } + self.plots.append(plot) # determine where to place x-axis label if _xaxis_at == 'bottom': - # place below is last indicator subplot + # place below the last plot self.xaxis_label = XAxisLabel( - parent=last_ind.getAxis('bottom'), opacity=1 + parent=self.plots[-1].getAxis('bottom'), + opacity=1 ) else: # keep x-axis right below main chart - self.xaxis_label = XAxisLabel(parent=self.xaxis, opacity=1) + first = self.plots[0] + xaxis = first.getAxis('bottom') + self.xaxis_label = XAxisLabel(parent=xaxis, opacity=1) - for i in indicators: - # add vertial and horizonal lines and a y-axis label - vl = i.addLine(x=0, pen=self.pen, movable=False) - hl = i.addLine(y=0, pen=self.pen, movable=False) - yl = YAxisLabel(parent=i.getAxis('right'), opacity=1) - - px_moved = pg.SignalProxy( - i.scene().sigMouseMoved, - rateLimit=_mouse_rate_limit, - slot=self.mouseMoved - ) - px_enter = pg.SignalProxy( - i.sig_mouse_enter, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Enter', i), - ) - px_leave = pg.SignalProxy( - i.sig_mouse_leave, - rateLimit=_mouse_rate_limit, - slot=lambda: self.mouseAction('Leave', i), - ) - self.indicators[i] = { - 'vl': vl, - 'hl': hl, - 'yl': yl, - 'px': (px_moved, px_enter, px_leave), - } - - def mouseAction(self, action, ind=False): # noqa + def mouseAction(self, action, plot): # noqa + # TODO: why do we no handle all plots the same? + # -> main plot has special path? would simplify code. if action == 'Enter': # show horiz line and y-label - if ind: - self.indicators[ind]['hl'].show() - self.indicators[ind]['yl'].show() - self.activeIndicator = ind - else: - self.yaxis_label.show() - self.hline.show() - # Leave - else: + self.graphics[plot]['hl'].show() + self.graphics[plot]['yl'].show() + self.active_plot = plot + else: # Leave # hide horiz line and y-label - if ind: - self.indicators[ind]['hl'].hide() - self.indicators[ind]['yl'].hide() - self.activeIndicator = None - else: - self.yaxis_label.hide() - self.hline.hide() + self.graphics[plot]['hl'].hide() + self.graphics[plot]['yl'].hide() + self.active_plot = None def mouseMoved(self, evt): # noqa """Update horizonal and vertical lines when mouse moves inside either the main chart or any indicator subplot. """ - pos = evt[0] - # if the mouse is within the parent ``ChartPlotWidget`` - if self.parent.sceneBoundingRect().contains(pos): - # mouse_point = self.vb.mapSceneToView(pos) - mouse_point = self.parent.mapToView(pos) + # find position in main chart + mouse_point = self.plots[0].mapToView(pos) - # move the vertial line to the current x coordinate - self.vline.setX(mouse_point.x()) + # move the vertical line to the current x coordinate in all charts + for opts in self.graphics.values(): + opts['vl'].setX(mouse_point.x()) - # update the label on the bottom of the crosshair - self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) + # update the label on the bottom of the crosshair + self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) - # update the vertical line in any indicators subplots - for opts in self.indicators.values(): - opts['vl'].setX(mouse_point.x()) + # vertical position of the mouse is inside an indicator + mouse_point_ind = self.active_plot.mapToView(pos) - if self.activeIndicator: - # vertial position of the mouse is inside an indicator - mouse_point_ind = self.activeIndicator.mapToView(pos) - self.indicators[self.activeIndicator]['hl'].setY( - mouse_point_ind.y() - ) - self.indicators[self.activeIndicator]['yl'].update_label( - evt_post=pos, point_view=mouse_point_ind - ) - else: - # vertial position of the mouse is inside the main chart - self.hline.setY(mouse_point.y()) - self.yaxis_label.update_label( - evt_post=pos, point_view=mouse_point - ) + self.graphics[self.active_plot]['hl'].setY( + mouse_point_ind.y() + ) + self.graphics[self.active_plot]['yl'].update_label( + evt_post=pos, point_view=mouse_point_ind + ) - def paint(self, p, *args): - pass + # def paint(self, p, *args): + # pass def boundingRect(self): - return self.parent.boundingRect() + return self.plots[0].boundingRect() def _mk_lines_array(data: List, size: int) -> np.ndarray: @@ -198,7 +162,8 @@ def bars_from_ohlc( lines = _mk_lines_array(data, data.shape[0]) for i, q in enumerate(data[start:], start=start): - open, high, low, close, index = q[['open', 'high', 'low', 'close', 'index']] + open, high, low, close, index = q[ + ['open', 'high', 'low', 'close', 'index']] # high - low line if low != high: From bbe02570b37f1a2a60a5aa21764ca85067d07968 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 17 Jul 2020 09:06:20 -0400 Subject: [PATCH 056/206] Allow for dynamically added plots Add `ChartPlotWidget.add_plot()` to add sub charts for indicators which can be updated independently. Clean up rt bar update code and drop some legacy ohlc loading cruft. --- piker/ui/_chart.py | 370 +++++++++++++++++++++++---------------------- piker/ui/cli.py | 4 +- 2 files changed, 188 insertions(+), 186 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 1a9a3a0c..68439ecb 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -2,22 +2,25 @@ High level Qt chart widgets. """ from typing import List, Optional, Tuple +import time -import trio +from PyQt5 import QtCore, QtGui +from pyqtgraph import functions as fn import numpy as np import pyqtgraph as pg -from pyqtgraph import functions as fn -from PyQt5 import QtCore, QtGui +import tractor +import trio from ._axes import ( DynamicDateAxis, PriceAxis, ) -from ._graphics import CrossHairItem, ChartType +from ._graphics import CrossHair, ChartType from ._style import _xaxis_at from ._source import Symbol from .. import brokers -from .. log import get_logger +from .. import data +from ..log import get_logger log = get_logger(__name__) @@ -74,6 +77,7 @@ class ChartSpace(QtGui.QWidget): """Load a new contract into the charting app. """ # XXX: let's see if this causes mem problems + self.window.setWindowTitle(f'piker chart {symbol}') self.chart = self._plot_cache.setdefault(symbol, LinkedSplitCharts()) s = Symbol(key=symbol) @@ -107,22 +111,20 @@ class LinkedSplitCharts(QtGui.QWidget): def __init__(self): super().__init__() - self.signals_visible = False - - # main data source - self._array: np.ndarray = None - - self._ch = None # crosshair graphics - self._index = 0 - - self.chart = None # main (ohlc) chart - self.indicators = [] + self.signals_visible: bool = False + self._array: np.ndarray = None # main data source + self._ch: CrossHair = None # crosshair graphics + self.chart: ChartPlotWidget = None # main (ohlc) chart + self.subplots: List[ChartPlotWidget] = [] self.xaxis = DynamicDateAxis( - orientation='bottom', linked_charts=self) - + orientation='bottom', + linked_charts=self + ) self.xaxis_ind = DynamicDateAxis( - orientation='bottom', linked_charts=self) + orientation='bottom', + linked_charts=self + ) if _xaxis_at == 'bottom': self.xaxis.setStyle(showValues=False) @@ -134,20 +136,18 @@ class LinkedSplitCharts(QtGui.QWidget): self.layout = QtGui.QVBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) - self.layout.addWidget(self.splitter) def set_split_sizes( self, - prop: float = 0.25 + prop: float = 0.25 # proportion allocated to consumer subcharts ) -> None: """Set the proportion of space allocated for linked subcharts. """ major = 1 - prop - # 20% allocated to consumer subcharts - min_h_ind = int(self.height() * prop / len(self.indicators)) + min_h_ind = int(self.height() * prop / len(self.subplots)) sizes = [int(self.height() * major)] - sizes.extend([min_h_ind] * len(self.indicators)) + sizes.extend([min_h_ind] * len(self.subplots)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) def plot( @@ -155,82 +155,94 @@ class LinkedSplitCharts(QtGui.QWidget): symbol: Symbol, array: np.ndarray, ohlc: bool = True, - ): + ) -> None: """Start up and show main (price) chart and all linked subcharts. """ self.digits = symbol.digits() - # XXX: this may eventually be a view onto shared mem + # XXX: this will eventually be a view onto shared mem # or some higher level type / API self._array = array - cv = ChartView() - self.chart = ChartPlotWidget( - linked_charts=self, - parent=self.splitter, - axisItems={'bottom': self.xaxis, 'right': PriceAxis()}, - viewBox=cv, - # enableMenu=False, + # add crosshairs + self._ch = CrossHair( + parent=self, #.chart, + # subplots=[plot for plot, d in self.subplots], + digits=self.digits ) - # TODO: ``pyqtgraph`` doesn't pass through a parent to the - # ``PlotItem`` by default; maybe we should PR this in? - cv.linked_charts = self - self.chart.plotItem.vb.linked_charts = self - - self.chart.getPlotItem().setContentsMargins(*CHART_MARGINS) + self.chart = self.add_plot( + name='main', + array=array, #['close'], + xaxis=self.xaxis, + ohlc=True, + ) + self.chart.addItem(self._ch) self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - if ohlc: - self.chart.draw_ohlc(array) - else: - raise NotImplementedError( - "Only OHLC linked charts are supported currently" - ) - # TODO: this is where we would load an indicator chain # XXX: note, if this isn't index aligned with # the source data the chart will go haywire. inds = [('open', lambda a: a['close'])] for name, func in inds: - cv = ChartView() - ind_chart = ChartPlotWidget( - linked_charts=self, - parent=self.splitter, - axisItems={'bottom': self.xaxis_ind, 'right': PriceAxis()}, - # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, - viewBox=cv, - ) - # this name will be used to register the primary - # graphics curve managed by the subchart - ind_chart.name = name - cv.linked_charts = self - self.chart.plotItem.vb.linked_charts = self - - ind_chart.setFrameStyle( - QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain - ) - ind_chart.getPlotItem().setContentsMargins(*CHART_MARGINS) - # self.splitter.addWidget(ind_chart) # compute historical subchart values from input array data = func(array) - self.indicators.append((ind_chart, func)) - # link chart x-axis to main quotes chart - ind_chart.setXLink(self.chart) + # create sub-plot + ind_chart = self.add_plot(name=name, array=data) - # draw curve graphics - ind_chart.draw_curve(data, name) + self.subplots.append((ind_chart, func)) + # scale split regions self.set_split_sizes() - ch = self._ch = CrossHairItem( - self.chart, - [_ind for _ind, d in self.indicators], - self.digits + def add_plot( + self, + name: str, + array: np.ndarray, + xaxis: DynamicDateAxis = None, + ohlc: bool = False, + ) -> 'ChartPlotWidget': + """Add (sub)plots to chart widget by name. + + If ``name`` == ``"main"`` the chart will be the the primary view. + """ + cv = ChartView() + # use "indicator axis" by default + xaxis = self.xaxis_ind if xaxis is None else xaxis + cpw = ChartPlotWidget( + linked_charts=self, + parent=self.splitter, + axisItems={'bottom': xaxis, 'right': PriceAxis()}, + # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, + viewBox=cv, ) - self.chart.addItem(ch) + # this name will be used to register the primary + # graphics curve managed by the subchart + cpw.name = name + cv.linked_charts = self + cpw.plotItem.vb.linked_charts = self + + cpw.setFrameStyle( + QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain + ) + cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) + # self.splitter.addWidget(cpw) + + # link chart x-axis to main quotes chart + cpw.setXLink(self.chart) + + # draw curve graphics + if ohlc: + cpw.draw_ohlc(array) + else: + cpw.draw_curve(array, name) + + # add to cross-hair's known plots + self._ch.add_plot(cpw) + + return cpw def update_from_quote( self, @@ -243,14 +255,10 @@ class LinkedSplitCharts(QtGui.QWidget): """ # TODO: eventually we'll want to update bid/ask labels and other # data as subscribed by underlying UI consumers. - last = quote['last'] + last = quote.get('last') or quote['close'] index, time, open, high, low, close, volume = self._array[-1] # update ohlc (I guess we're enforcing this for now?) - # self._array[-1]['close'] = last - # self._array[-1]['high'] = max(h, last) - # self._array[-1]['low'] = min(l, last) - # overwrite from quote self._array[-1] = ( index, @@ -268,19 +276,25 @@ class LinkedSplitCharts(QtGui.QWidget): array: np.ndarray, **kwargs, ) -> None: - # update the ohlc sequence graphics chart - chart = self.chart + """Update all linked chart graphics with a new input array. + Return the modified graphics objects in a list. + """ + # update the ohlc sequence graphics chart # we send a reference to the whole updated array - chart.update_from_array(array, **kwargs) + self.chart.update_from_array(array, **kwargs) # TODO: the "data" here should really be a function # and it should be managed and computed outside of this UI - for chart, func in self.indicators: + graphics = [] + for chart, func in self.subplots: # process array in entirely every update # TODO: change this for streaming data = func(array) - chart.update_from_array(data, name=chart.name, **kwargs) + graphic = chart.update_from_array(data, name=chart.name, **kwargs) + graphics.append(graphic) + + return graphics _min_points_to_show = 3 @@ -316,9 +330,6 @@ class ChartPlotWidget(pg.PlotWidget): """ super().__init__(**kwargs) self.parent = linked_charts - # this is the index of that last input array entry and is - # updated and used to figure out how many bars are in view - # self._xlast = 0 # XXX: label setting doesn't seem to work? # likely custom graphics need special handling @@ -327,8 +338,7 @@ class ChartPlotWidget(pg.PlotWidget): # label.setText("Yo yoyo") # label.setText("x=") - # to be filled in when graphics are rendered - # by name + # to be filled in when graphics are rendered by name self._graphics = {} # show only right side axes @@ -353,9 +363,8 @@ class ChartPlotWidget(pg.PlotWidget): xlast: int ) -> None: """Set view limits (what's shown in the main chart "pane") - based on max / min x / y coords. + based on max/min x/y coords. """ - # set panning limits self.setLimits( xMin=xfirst, xMax=xlast, @@ -393,7 +402,6 @@ class ChartPlotWidget(pg.PlotWidget): # set xrange limits xlast = data[-1]['index'] - # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -416,7 +424,6 @@ class ChartPlotWidget(pg.PlotWidget): # set a "startup view" xlast = len(data) - 1 - # self._set_xlimits(0, xlast) # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -433,7 +440,6 @@ class ChartPlotWidget(pg.PlotWidget): name: str = 'main', **kwargs, ) -> pg.GraphicsObject: - # self._xlast = len(array) - 1 graphics = self._graphics[name] graphics.update_from_array(array, **kwargs) @@ -491,7 +497,7 @@ class ChartPlotWidget(pg.PlotWidget): yhigh = bars.max() std = np.std(bars) - # view margins + # view margins: stay within 10% of the "true range" diff = yhigh - ylow ylow = ylow - (diff * 0.1) yhigh = yhigh + (diff * 0.1) @@ -580,12 +586,15 @@ class ChartView(pg.ViewBox): self.sigRangeChangedManually.emit(mask) -def main(symbol): +def _main( + sym: str, + brokername: str, + **qtractor_kwargs, +) -> None: """Entry point to spawn a chart app. """ - - from ._exec import run_qtrio - # uses pandas_datareader + from ._exec import run_qtractor + from ._source import ohlc_dtype async def _main(widgets): """Main Qt-trio routine invoked by the Qt loop with @@ -593,119 +602,112 @@ def main(symbol): """ chart_app = widgets['main'] - # data-feed setup - sym = symbol or 'ES.GLOBEX' - brokermod = brokers.get_brokermod('ib') + # historical data fetch + brokermod = brokers.get_brokermod(brokername) async with brokermod.get_client() as client: # figure out the exact symbol bars = await client.bars(symbol=sym) - # ``from_buffer` return read-only - bars = np.array(bars) - linked_charts = chart_app.load_symbol('ES', bars) + # remember, msgpack-numpy's ``from_buffer` returns read-only array + bars = np.array(bars[list(ohlc_dtype.names)]) + linked_charts = chart_app.load_symbol(sym, bars) - async def add_new_bars(delay_s=5.): - import time + # determine ohlc delay between bars + times = bars['time'] + delay = times[-1] - times[-2] + + async def add_new_bars(delay_s): + """Task which inserts new bars into the ohlc every ``delay_s`` seconds. + """ + # adjust delay to compensate for trio processing time + ad = delay_s - 0.002 ohlc = linked_charts._array - last_5s = ohlc[-1]['time'] - delay = max((last_5s + 4.99) - time.time(), 0) - await trio.sleep(delay) + async def sleep(): + """Sleep until next time frames worth has passed from last bar. + """ + last_ts = ohlc[-1]['time'] + delay = max((last_ts + ad) - time.time(), 0) + await trio.sleep(delay) + + # sleep for duration of current bar + await sleep() while True: - print('new bar') - # TODO: bunch of stuff: # - I'm starting to think all this logic should be # done in one place and "graphics update routines" # should not be doing any length checking and array diffing. # - don't keep appending, but instead increase the - # underlying array's size less frequently: + # underlying array's size less frequently # - handle odd lot orders # - update last open price correctly instead # of copying it from last bar's close - # - 5 sec bar lookback-autocorrection like tws does - index, t, open, high, low, close, volume = ohlc[-1] + # - 5 sec bar lookback-autocorrection like tws does? + (index, t, close) = ohlc[-1][['index', 'time', 'close']] new = np.append( ohlc, np.array( - [(index + 1, t + 5, close, close, close, close, 0)], + [(index + 1, t + delay, close, close, + close, close, 0)], dtype=ohlc.dtype ), ) ohlc = linked_charts._array = new - linked_charts.update_from_array(new) + last_quote = ohlc[-1] - # sleep until next 5s from last bar - last_5s = ohlc[-1]['time'] - delay = max((last_5s + 4.99) - time.time(), 0) - await trio.sleep(4.9999) + # we **don't** update the bar right now + # since the next quote that arrives should + await sleep() + + # if the last bar has not changed print a flat line and + # move to the next + if last_quote == ohlc[-1]: + log.debug("Printing flat line for {sym}") + linked_charts.update_from_array(ohlc) + + async def stream_to_chart(func): + + async with tractor.open_nursery() as n: + portal = await n.run_in_actor( + f'fsp_{func.__name__}', + func, + brokername=brokermod.name, + sym=sym, + loglevel='info', + ) + stream = await portal.result() + + # retreive named layout and style instructions + layout = await stream.__anext__() + + async for quote in stream: + ticks = quote.get('ticks') + if ticks: + for tick in ticks: + print(tick) async with trio.open_nursery() as n: - n.start_soon(add_new_bars) + from piker import fsp - async with brokermod.maybe_spawn_brokerd() as portal: - stream = await portal.run( - 'piker.brokers.ib', - 'trio_stream_ticker', - sym=sym, - ) - # TODO: timeframe logic - async for tick in stream: - # breakpoint() - if tick['tickType'] in (48, 77): - linked_charts.update_from_quote( - {'last': tick['price']} - ) + async with data.open_feed(brokername, [sym]) as stream: + # start graphics tasks + n.start_soon(add_new_bars, delay) + n.start_soon(stream_to_chart, fsp.broker_latency) - # from .quantdom.loaders import get_quotes - # from datetime import datetime - # from ._source import from_df - # quotes = get_quotes( - # symbol=symbol, - # date_from=datetime(1900, 1, 1), - # date_to=datetime(2030, 12, 31), - # ) - # quotes = from_df(quotes) + async for quote in stream: + # XXX: why are we getting both of these again? + ticks = quote.get('ticks') + if ticks: + for tick in ticks: + if tick['tickType'] in (48, 77): + linked_charts.update_from_quote( + {'last': tick['price']} + ) + # else: + # linked_charts.update_from_quote( + # {'last': quote['close']} + # ) - # feed = DataFeed(portal, brokermod) - # quote_gen, quotes = await feed.open_stream( - # symbols, - # 'stock', - # rate=rate, - # test=test, - # ) - - # first_quotes, _ = feed.format_quotes(quotes) - - # if first_quotes[0].get('last') is None: - # log.error("Broker API is down temporarily") - # return - - # make some fake update data - # import itertools - # nums = itertools.cycle([315., 320., 325., 310., 3]) - - # def gen_nums(): - # while True: - # yield quotes[-1].close + 2 - # yield quotes[-1].close - 2 - - # nums = gen_nums() - - # # await trio.sleep(10) - # import time - # while True: - # new = next(nums) - # quotes[-1].close = new - # # this updates the linked_charts internal array - # # and then passes that array to all subcharts to - # # render downstream graphics - # start = time.time() - # linked_charts.update_from_quote({'last': new}) - # print(f"Render latency {time.time() - start}") - # # 20 Hz seems to be good enough - # await trio.sleep(0.05) - - run_qtrio(_main, (), ChartSpace) + run_qtractor(_main, (), ChartSpace, **qtractor_kwargs) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 2be6d436..cc81af85 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -115,10 +115,10 @@ def optschain(config, symbol, date, tl, rate, test): def chart(config, symbol, date, tl, rate, test): """Start an option chain UI """ - from ._chart import main + from ._chart import _main # global opts loglevel = config['loglevel'] brokername = config['broker'] - main(sym=symbol, brokername=brokername, loglevel=loglevel) + _main(sym=symbol, brokername=brokername, loglevel=loglevel) From ad519c10a9752af3c371078ad53b528774f330e9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 17 Jul 2020 10:32:13 -0400 Subject: [PATCH 057/206] Always just look up the current plot on mouse handling --- piker/ui/_graphics.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 21694d0f..e3b8fdc3 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -11,8 +11,6 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF from .quantdom.utils import timeit -# from .quantdom.base import Quotes - from ._style import _xaxis_at # , _tina_mode from ._axes import YAxisLabel, XAxisLabel @@ -27,8 +25,7 @@ class CrossHair(pg.GraphicsObject): def __init__(self, parent, digits: int = 0): super().__init__() - # self.pen = pg.mkPen('#000000') - self.pen = pg.mkPen('#a9a9a9') + self.pen = pg.mkPen('#a9a9a9') # gray? self.parent = parent self.graphics = {} self.plots = [] @@ -89,8 +86,6 @@ class CrossHair(pg.GraphicsObject): self.xaxis_label = XAxisLabel(parent=xaxis, opacity=1) def mouseAction(self, action, plot): # noqa - # TODO: why do we no handle all plots the same? - # -> main plot has special path? would simplify code. if action == 'Enter': # show horiz line and y-label self.graphics[plot]['hl'].show() @@ -108,9 +103,15 @@ class CrossHair(pg.GraphicsObject): """ pos = evt[0] - # find position in main chart - mouse_point = self.plots[0].mapToView(pos) + # find position inside active plot + mouse_point = self.active_plot.mapToView(pos) + self.graphics[self.active_plot]['hl'].setY( + mouse_point.y() + ) + self.graphics[self.active_plot]['yl'].update_label( + evt_post=pos, point_view=mouse_point + ) # move the vertical line to the current x coordinate in all charts for opts in self.graphics.values(): opts['vl'].setX(mouse_point.x()) @@ -118,22 +119,12 @@ class CrossHair(pg.GraphicsObject): # update the label on the bottom of the crosshair self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) - # vertical position of the mouse is inside an indicator - mouse_point_ind = self.active_plot.mapToView(pos) - - self.graphics[self.active_plot]['hl'].setY( - mouse_point_ind.y() - ) - self.graphics[self.active_plot]['yl'].update_label( - evt_post=pos, point_view=mouse_point_ind - ) + def boundingRect(self): + return self.active_plot.boundingRect() # def paint(self, p, *args): # pass - def boundingRect(self): - return self.plots[0].boundingRect() - def _mk_lines_array(data: List, size: int) -> np.ndarray: """Create an ndarray to hold lines graphics objects. From 46c804db0b23291c6b00ed0a3714fdb281861844 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 20 Jul 2020 16:58:40 -0400 Subject: [PATCH 058/206] Support the `stream_quotes()` api in questrade backend --- piker/brokers/questrade.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 00e49587..f3708d3f 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -565,7 +565,6 @@ class Client: time_frame: str = '1m', count: float = 20e3, is_paid_feed: bool = False, - as_np: bool = False, ) -> List[Dict[str, Any]]: """Retreive OHLCV bars for a symbol over a range to the present. From cb8215c2030083a1a5e103d5774ef5f3898539f2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 31 Jul 2020 00:10:47 -0400 Subject: [PATCH 059/206] Also log the payload --- piker/brokers/data.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index 9580add8..dbb76772 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -142,8 +142,7 @@ async def stream_poll_requests( # to the last quote received new = set(quote.items()) - set(last.items()) if new: - log.info( - f"New quote {quote['symbol']}:\n{new}") + log.info(f"New quote {symbol}:\n{new}") _cache[symbol] = quote # only ship diff updates and other required fields @@ -171,6 +170,8 @@ async def stream_poll_requests( # XXX: very questrade specific payload['size'] = quote['lastTradeSize'] + log.info(f"New paylod {symbol}:\n{payload}") + # XXX: we append to a list for the options case where the # subscription topic (key) is the same for all # expiries even though this is uncessary for the From ec4f7476c536181ee1c6d118ce9d84d83eb1f74c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 1 Aug 2020 22:22:12 -0400 Subject: [PATCH 060/206] Drop "pipfiles"; pipenv is getting the hard boot --- Pipfile | 24 -- Pipfile.lock | 644 --------------------------------------------------- 2 files changed, 668 deletions(-) delete mode 100644 Pipfile delete mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 8e055400..00000000 --- a/Pipfile +++ /dev/null @@ -1,24 +0,0 @@ -[[source]] -url = "https://pypi.python.org/simple" -verify_ssl = true -name = "pypi" - -[packages] -e1839a8 = {path = ".",editable = true} -# use master branch for "guest mode" -trio = {git = "git://github.com/python-trio/trio.git"} -Cython = "*" -# use master branch kivy since wheels seem borked (due to cython stuff) -kivy = {git = "git://github.com/kivy/kivy.git"} -pdbpp = "*" -msgpack = "*" -tractor = {git = "git://github.com/goodboy/tractor.git"} -toml = "*" -pyqtgraph = "*" -qdarkstyle = "*" -pyqt5 = "*" - -[dev-packages] -pytest = "*" -pdbpp = "*" -piker = {editable = true,path = "."} diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index dc2ec297..00000000 --- a/Pipfile.lock +++ /dev/null @@ -1,644 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "4f280bbb01bd2a384bbe963ec012e2bfa89b852a45a009674e30a3e281cd6b04" - }, - "pipfile-spec": 6, - "requires": {}, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.python.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "anyio": { - "hashes": [ - "sha256:2b758634cf1adc3589937d91f6736b3696d091c1485a10c9cc3f1bd5e6d96813", - "sha256:78db97333af17381cadd2cc0dbd3d4e0258e4932368eab8895cb65a269db054e" - ], - "version": "==1.2.3" - }, - "asks": { - "hashes": [ - "sha256:d7289cb5b7a28614e4fecab63b3734e2a4296d3c323e315f8dc4b546d64f71b7" - ], - "version": "==2.3.6" - }, - "async-generator": { - "hashes": [ - "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b", - "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144" - ], - "version": "==1.10" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, - "click": { - "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" - ], - "version": "==7.1.1" - }, - "colorlog": { - "hashes": [ - "sha256:30aaef5ab2a1873dec5da38fd6ba568fa761c9fa10b40241027fa3edea47f3d2", - "sha256:732c191ebbe9a353ec160d043d02c64ddef9028de8caae4cfa8bd49b6afed53e" - ], - "version": "==4.1.0" - }, - "cython": { - "hashes": [ - "sha256:03f6bbb380ad0acb744fb06e42996ea217e9d00016ca0ff6f2e7d60f580d0360", - "sha256:05e8cfd3a3a6087aec49a1ae08a89171db991956209406d1e5576f9db70ece52", - "sha256:05eb79efc8029d487251c8a2702a909a8ba33c332e06d2f3980866541bd81253", - "sha256:094d28a34c3fa992ae02aea1edbe6ff89b3cc5870b6ee38b5baeb805dc57b013", - "sha256:0c70e842e52e2f50cc43bad43b5e5bc515f30821a374e544abb0e0746f2350ff", - "sha256:1dcdaa319558eb924294a554dcf6c12383ec947acc7e779e8d3622409a7f7d28", - "sha256:1fc5bdda28f25fec44e4721677458aa509d743cd350862270309d61aa148d6ff", - "sha256:280573a01d9348d44a42d6a9c651d9f7eb1fe9217df72555b2a118f902996a10", - "sha256:298ceca7b0f0da4205fcb0b7c9ac9e120e2dafffd5019ba1618e84ef89434b5a", - "sha256:4074a8bff0040035673cc6dd365a762476d6bff4d03d8ce6904e3e53f9a25dc8", - "sha256:41e7068e95fbf9ec94b41437f989caf9674135e770a39cdb9c00de459bafd1bc", - "sha256:47e5e1502d52ef03387cf9d3b3241007961a84a466e58a3b74028e1dd4957f8c", - "sha256:521340844cf388d109ceb61397f3fd5250ccb622a1a8e93559e8de76c80940a9", - "sha256:6c53338c1811f8c6d7f8cb7abd874810b15045e719e8207f957035c9177b4213", - "sha256:75c2dda47dcc3c77449712b1417bb6b89ec3b7b02e18c64262494dceffdf455e", - "sha256:773c5a98e463b52f7e8197254b39b703a5ea1972aef3a94b3b921515d77dd041", - "sha256:78c3068dcba300d473fef57cdf523e34b37de522f5a494ef9ee1ac9b4b8bbe3f", - "sha256:7bc18fc5a170f2c1cef5387a3d997c28942918bbee0f700e73fd2178ee8d474d", - "sha256:7f89eff20e4a7a64b55210dac17aea711ed8a3f2e78f2ff784c0e984302583dd", - "sha256:89458b49976b1dee5d89ab4ac943da3717b4292bf624367e862e4ee172fcce99", - "sha256:986f871c0fa649b293061236b93782d25c293a8dd8117c7ba05f8a61bdc261ae", - "sha256:a0f495a4fe5278aab278feee35e6102efecde5176a8a74dd28c28e3fc5c8d7c7", - "sha256:a14aa436586c41633339415de82a41164691d02d3e661038da533be5d40794a5", - "sha256:b8ab3ab38afc47d8f4fe629b836243544351cef681b6bdb1dc869028d6fdcbfb", - "sha256:bb487881608ebd293592553c618f0c83316f4f13a64cb18605b1d2fb9fd3da3e", - "sha256:c0b24bfe3431b3cb7ced323bca813dbd13aca973a1475b512d3331fd0de8ec60", - "sha256:c7894c06205166d360ab2915ae306d1f7403e9ce3d3aaeff4095eaf98e42ce66", - "sha256:d4039bb7f234ad32267c55e72fd49fb56078ea102f9d9d8559f6ec34d4887630", - "sha256:e4d6bb8703d0319eb04b7319b12ea41580df44fd84d83ccda13ea463c6801414", - "sha256:e8fab9911fd2fa8e5af407057cb8bdf87762f983cba483fa3234be20a9a0af77", - "sha256:f3818e578e687cdb21dc4aa4a3bc6278c656c9c393e9eda14dd04943f478863d", - "sha256:fe666645493d72712c46e4fbe8bec094b06aec3c337400479e9704439c9d9586" - ], - "index": "pypi", - "version": "==0.29.14" - }, - "e1839a8": { - "editable": true, - "path": "." - }, - "fancycompleter": { - "hashes": [ - "sha256:09e0feb8ae242abdfd7ef2ba55069a46f011814a80fe5476be48f51b00247272", - "sha256:dd076bca7d9d524cc7f25ec8f35ef95388ffef9ef46def4d3d25e9b044ad7080" - ], - "version": "==0.9.1" - }, - "h11": { - "hashes": [ - "sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1", - "sha256:4bc6d6a1238b7615b266ada57e0618568066f57dd6fa967d1290ec9309b2f2f1" - ], - "version": "==0.9.0" - }, - "idna": { - "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" - ], - "version": "==2.9" - }, - "kivy": { - "git": "git://github.com/kivy/kivy.git", - "ref": "9398c8d5d260c9f4d5dd0aadc7de5001bddaf984" - }, - "msgpack": { - "hashes": [ - "sha256:0cc7ca04e575ba34fea7cfcd76039f55def570e6950e4155a4174368142c8e1b", - "sha256:187794cd1eb73acccd528247e3565f6760bd842d7dc299241f830024a7dd5610", - "sha256:1904b7cb65342d0998b75908304a03cb004c63ef31e16c8c43fee6b989d7f0d7", - "sha256:229a0ccdc39e9b6c6d1033cd8aecd9c296823b6c87f0de3943c59b8bc7c64bee", - "sha256:24149a75643aeaa81ece4259084d11b792308a6cf74e796cbb35def94c89a25a", - "sha256:30b88c47e0cdb6062daed88ca283b0d84fa0d2ad6c273aa0788152a1c643e408", - "sha256:32fea0ea3cd1ef820286863a6202dcfd62a539b8ec3edcbdff76068a8c2cc6ce", - "sha256:355f7fd0f90134229eaeefaee3cf42e0afc8518e8f3cd4b25f541a7104dcb8f9", - "sha256:4abdb88a9b67e64810fb54b0c24a1fd76b12297b4f7a1467d85a14dd8367191a", - "sha256:757bd71a9b89e4f1db0622af4436d403e742506dbea978eba566815dc65ec895", - "sha256:76df51492bc6fa6cc8b65d09efdb67cbba3cbfe55004c3afc81352af92b4a43c", - "sha256:774f5edc3475917cd95fe593e625d23d8580f9b48b570d8853d06cac171cd170", - "sha256:8a3ada8401736df2bf497f65589293a86c56e197a80ae7634ec2c3150a2f5082", - "sha256:a06efd0482a1942aad209a6c18321b5e22d64eb531ea20af138b28172d8f35ba", - "sha256:b24afc52e18dccc8c175de07c1d680bdf315844566f4952b5bedb908894bec79", - "sha256:b8b4bd3dafc7b92608ae5462add1c8cc881851c2d4f5d8977fdea5b081d17f21", - "sha256:c6e5024fc0cdf7f83b6624850309ddd7e06c48a75fa0d1c5173de4d93300eb19", - "sha256:db7ff14abc73577b0bcbcf73ecff97d3580ecaa0fc8724babce21fdf3fe08ef6", - "sha256:dedf54d72d9e7b6d043c244c8213fe2b8bbfe66874b9a65b39c4cc892dd99dd4", - "sha256:ea3c2f859346fcd55fc46e96885301d9c2f7a36d453f5d8f2967840efa1e1830", - "sha256:f0f47bafe9c9b8ed03e19a100a743662dd8c6d0135e684feea720a0d0046d116" - ], - "index": "pypi", - "version": "==0.6.2" - }, - "numpy": { - "hashes": [ - "sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6", - "sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e", - "sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc", - "sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc", - "sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a", - "sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa", - "sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3", - "sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121", - "sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971", - "sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26", - "sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd", - "sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480", - "sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec", - "sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77", - "sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57", - "sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07", - "sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572", - "sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73", - "sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca", - "sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474", - "sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5" - ], - "version": "==1.18.1" - }, - "outcome": { - "hashes": [ - "sha256:ee46c5ce42780cde85d55a61819d0e6b8cb490f1dbd749ba75ff2629771dcd2d", - "sha256:fc7822068ba7dd0fc2532743611e8a73246708d3564e29a39f93d6ab3701b66f" - ], - "version": "==1.0.1" - }, - "pandas": { - "hashes": [ - "sha256:23e177d43e4bf68950b0f8788b6a2fef2f478f4ec94883acb627b9264522a98a", - "sha256:2530aea4fe46e8df7829c3f05e0a0f821c893885d53cb8ac9b89cc67c143448c", - "sha256:303827f0bb40ff610fbada5b12d50014811efcc37aaf6ef03202dc3054bfdda1", - "sha256:3b019e3ea9f5d0cfee0efabae2cfd3976874e90bcc3e97b29600e5a9b345ae3d", - "sha256:3c07765308f091d81b6735d4f2242bb43c332cc3461cae60543df6b10967fe27", - "sha256:5036d4009012a44aa3e50173e482b664c1fae36decd277c49e453463798eca4e", - "sha256:6f38969e2325056f9959efbe06c27aa2e94dd35382265ad0703681d993036052", - "sha256:74a470d349d52b9d00a2ba192ae1ee22155bb0a300fd1ccb2961006c3fa98ed3", - "sha256:7d77034e402165b947f43050a8a415aa3205abfed38d127ea66e57a2b7b5a9e0", - "sha256:7f9a509f6f11fa8b9313002ebdf6f690a7aa1dd91efd95d90185371a0d68220e", - "sha256:942b5d04762feb0e55b2ad97ce2b254a0ffdd344b56493b04a627266e24f2d82", - "sha256:a9fbe41663416bb70ed05f4e16c5f377519c0dc292ba9aa45f5356e37df03a38", - "sha256:d10e83866b48c0cdb83281f786564e2a2b51a7ae7b8a950c3442ad3c9e36b48c", - "sha256:e2140e1bbf9c46db9936ee70f4be6584d15ff8dc3dfff1da022d71227d53bad3" - ], - "version": "==1.0.1" - }, - "pdbpp": { - "hashes": [ - "sha256:73ff220d5006e0ecdc3e2705d8328d8aa5ac27fef95cc06f6e42cd7d22d55eb8" - ], - "index": "pypi", - "version": "==0.10.2" - }, - "psutil": { - "hashes": [ - "sha256:06660136ab88762309775fd47290d7da14094422d915f0466e0adf8e4b22214e", - "sha256:0c11adde31011a286197630ba2671e34651f004cc418d30ae06d2033a43c9e20", - "sha256:0c211eec4185725847cb6c28409646c7cfa56fdb531014b35f97b5dc7fe04ff9", - "sha256:0fc7a5619b47f74331add476fbc6022d7ca801c22865c7069ec0867920858963", - "sha256:3004361c6b93dbad71330d992c1ae409cb8314a6041a0b67507cc882357f583e", - "sha256:5e8dbf31871b0072bcba8d1f2861c0ec6c84c78f13c723bb6e981bce51b58f12", - "sha256:6d81b9714791ef9a3a00b2ca846ee547fc5e53d259e2a6258c3d2054928039ff", - "sha256:724390895cff80add7a1c4e7e0a04d9c94f3ee61423a2dcafd83784fabbd1ee9", - "sha256:ad21281f7bd6c57578dd53913d2d44218e9e29fd25128d10ff7819ef16fa46e7", - "sha256:f21a7bb4b207e4e7c60b3c40ffa89d790997619f04bbecec9db8e3696122bc78", - "sha256:f60042bef7dc50a78c06334ca8e25580455948ba2fa98f240d034a4fed9141a5" - ], - "index": "pypi", - "version": "==5.6.6" - }, - "pygments": { - "hashes": [ - "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", - "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" - ], - "version": "==2.6.1" - }, - "pyqtgraph": { - "hashes": [ - "sha256:4c08ab34881fae5ecf9ddfe6c1220b9e41e6d3eb1579a7d8ef501abb8e509251" - ], - "index": "pypi", - "version": "==0.10.0" - }, - "pyrepl": { - "hashes": [ - "sha256:292570f34b5502e871bbb966d639474f2b57fbfcd3373c2d6a2f3d56e681a775" - ], - "version": "==0.9.0" - }, - "pyside2": { - "hashes": [ - "sha256:589b90944c24046d31bf76694590a600d59d20130015086491b793a81753629a", - "sha256:63cc845434388b398b79b65f7b5312b9b5348fbc772d84092c9245efbf341197", - "sha256:7c57fe60ed57a3a8b95d9163abca9caa803a1470f29b40bff8ef4103b97a96c8", - "sha256:7c61a6883f3474939097b9dabc80f028887046be003ce416da1b3565a08d1f92", - "sha256:ed6d22c7a3a99f480d4c9348bcced97ef7bc0c9d353ad3665ae705e8eb61feb5", - "sha256:ede8ed6e7021232184829d16614eb153f188ea250862304eac35e04b2bd0120c" - ], - "index": "pypi", - "version": "==5.13.2" - }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "version": "==2.8.1" - }, - "pytz": { - "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" - ], - "version": "==2019.3" - }, - "shiboken2": { - "hashes": [ - "sha256:5e84a4b4e7ab08bb5db0a8168e5d0316fbf3c25b788012701a82079faadfb19b", - "sha256:7c766c4160636a238e0e4430e2f40b504b13bcc4951902eb78cd5c971f26c898", - "sha256:81fa9b288c6c4b4c91220fcca2002eadb48fc5c3238e8bd88e982e00ffa77c53", - "sha256:ca08a3c95b1b20ac2b243b7b06379609bd73929dbc27b28c01415feffe3bcea1", - "sha256:e2f72b5cfdb8b48bdb55bda4b42ec7d36d1bce0be73d6d7d4a358225d6fb5f25", - "sha256:e6543506cb353d417961b9ec3c6fc726ec2f72eeab609dc88943c2e5cb6d6408" - ], - "version": "==5.13.2" - }, - "six": { - "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" - ], - "version": "==1.14.0" - }, - "sniffio": { - "hashes": [ - "sha256:20ed6d5b46f8ae136d00b9dcb807615d83ed82ceea6b2058cecb696765246da5", - "sha256:8e3810100f69fe0edd463d02ad407112542a11ffdc29f67db2bf3771afb87a21" - ], - "version": "==1.1.0" - }, - "sortedcontainers": { - "hashes": [ - "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", - "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" - ], - "version": "==2.1.0" - }, - "toml": { - "hashes": [ - "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" - ], - "index": "pypi", - "version": "==0.10.0" - }, - "tractor": { - "git": "git://github.com/goodboy/tractor.git", - "ref": "ab349cdb8d8cbf2e3d48c0589cb710a43483f233" - }, - "trio": { - "hashes": [ - "sha256:a6d83c0cb4a177ec0f5179ce88e27914d5c8e6fd01c4285176b949e6ddc88c6c", - "sha256:f1cf00054ad974c86d9b7afa187a65d79fd5995340abe01e8e4784d86f4acb30" - ], - "index": "pypi", - "version": "==0.13.0" - }, - "wmctrl": { - "hashes": [ - "sha256:d806f65ac1554366b6e31d29d7be2e8893996c0acbb2824bbf2b1f49cf628a13" - ], - "version": "==0.3" - } - }, - "develop": { - "anyio": { - "hashes": [ - "sha256:2b758634cf1adc3589937d91f6736b3696d091c1485a10c9cc3f1bd5e6d96813", - "sha256:78db97333af17381cadd2cc0dbd3d4e0258e4932368eab8895cb65a269db054e" - ], - "version": "==1.2.3" - }, - "asks": { - "hashes": [ - "sha256:d7289cb5b7a28614e4fecab63b3734e2a4296d3c323e315f8dc4b546d64f71b7" - ], - "version": "==2.3.6" - }, - "async-generator": { - "hashes": [ - "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b", - "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144" - ], - "version": "==1.10" - }, - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, - "click": { - "hashes": [ - "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", - "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" - ], - "version": "==7.1.1" - }, - "colorlog": { - "hashes": [ - "sha256:30aaef5ab2a1873dec5da38fd6ba568fa761c9fa10b40241027fa3edea47f3d2", - "sha256:732c191ebbe9a353ec160d043d02c64ddef9028de8caae4cfa8bd49b6afed53e" - ], - "version": "==4.1.0" - }, - "cython": { - "hashes": [ - "sha256:03f6bbb380ad0acb744fb06e42996ea217e9d00016ca0ff6f2e7d60f580d0360", - "sha256:05e8cfd3a3a6087aec49a1ae08a89171db991956209406d1e5576f9db70ece52", - "sha256:05eb79efc8029d487251c8a2702a909a8ba33c332e06d2f3980866541bd81253", - "sha256:094d28a34c3fa992ae02aea1edbe6ff89b3cc5870b6ee38b5baeb805dc57b013", - "sha256:0c70e842e52e2f50cc43bad43b5e5bc515f30821a374e544abb0e0746f2350ff", - "sha256:1dcdaa319558eb924294a554dcf6c12383ec947acc7e779e8d3622409a7f7d28", - "sha256:1fc5bdda28f25fec44e4721677458aa509d743cd350862270309d61aa148d6ff", - "sha256:280573a01d9348d44a42d6a9c651d9f7eb1fe9217df72555b2a118f902996a10", - "sha256:298ceca7b0f0da4205fcb0b7c9ac9e120e2dafffd5019ba1618e84ef89434b5a", - "sha256:4074a8bff0040035673cc6dd365a762476d6bff4d03d8ce6904e3e53f9a25dc8", - "sha256:41e7068e95fbf9ec94b41437f989caf9674135e770a39cdb9c00de459bafd1bc", - "sha256:47e5e1502d52ef03387cf9d3b3241007961a84a466e58a3b74028e1dd4957f8c", - "sha256:521340844cf388d109ceb61397f3fd5250ccb622a1a8e93559e8de76c80940a9", - "sha256:6c53338c1811f8c6d7f8cb7abd874810b15045e719e8207f957035c9177b4213", - "sha256:75c2dda47dcc3c77449712b1417bb6b89ec3b7b02e18c64262494dceffdf455e", - "sha256:773c5a98e463b52f7e8197254b39b703a5ea1972aef3a94b3b921515d77dd041", - "sha256:78c3068dcba300d473fef57cdf523e34b37de522f5a494ef9ee1ac9b4b8bbe3f", - "sha256:7bc18fc5a170f2c1cef5387a3d997c28942918bbee0f700e73fd2178ee8d474d", - "sha256:7f89eff20e4a7a64b55210dac17aea711ed8a3f2e78f2ff784c0e984302583dd", - "sha256:89458b49976b1dee5d89ab4ac943da3717b4292bf624367e862e4ee172fcce99", - "sha256:986f871c0fa649b293061236b93782d25c293a8dd8117c7ba05f8a61bdc261ae", - "sha256:a0f495a4fe5278aab278feee35e6102efecde5176a8a74dd28c28e3fc5c8d7c7", - "sha256:a14aa436586c41633339415de82a41164691d02d3e661038da533be5d40794a5", - "sha256:b8ab3ab38afc47d8f4fe629b836243544351cef681b6bdb1dc869028d6fdcbfb", - "sha256:bb487881608ebd293592553c618f0c83316f4f13a64cb18605b1d2fb9fd3da3e", - "sha256:c0b24bfe3431b3cb7ced323bca813dbd13aca973a1475b512d3331fd0de8ec60", - "sha256:c7894c06205166d360ab2915ae306d1f7403e9ce3d3aaeff4095eaf98e42ce66", - "sha256:d4039bb7f234ad32267c55e72fd49fb56078ea102f9d9d8559f6ec34d4887630", - "sha256:e4d6bb8703d0319eb04b7319b12ea41580df44fd84d83ccda13ea463c6801414", - "sha256:e8fab9911fd2fa8e5af407057cb8bdf87762f983cba483fa3234be20a9a0af77", - "sha256:f3818e578e687cdb21dc4aa4a3bc6278c656c9c393e9eda14dd04943f478863d", - "sha256:fe666645493d72712c46e4fbe8bec094b06aec3c337400479e9704439c9d9586" - ], - "index": "pypi", - "version": "==0.29.14" - }, - "fancycompleter": { - "hashes": [ - "sha256:09e0feb8ae242abdfd7ef2ba55069a46f011814a80fe5476be48f51b00247272", - "sha256:dd076bca7d9d524cc7f25ec8f35ef95388ffef9ef46def4d3d25e9b044ad7080" - ], - "version": "==0.9.1" - }, - "h11": { - "hashes": [ - "sha256:33d4bca7be0fa039f4e84d50ab00531047e53d6ee8ffbc83501ea602c169cae1", - "sha256:4bc6d6a1238b7615b266ada57e0618568066f57dd6fa967d1290ec9309b2f2f1" - ], - "version": "==0.9.0" - }, - "idna": { - "hashes": [ - "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", - "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" - ], - "version": "==2.9" - }, - "more-itertools": { - "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" - ], - "version": "==8.2.0" - }, - "msgpack": { - "hashes": [ - "sha256:0cc7ca04e575ba34fea7cfcd76039f55def570e6950e4155a4174368142c8e1b", - "sha256:187794cd1eb73acccd528247e3565f6760bd842d7dc299241f830024a7dd5610", - "sha256:1904b7cb65342d0998b75908304a03cb004c63ef31e16c8c43fee6b989d7f0d7", - "sha256:229a0ccdc39e9b6c6d1033cd8aecd9c296823b6c87f0de3943c59b8bc7c64bee", - "sha256:24149a75643aeaa81ece4259084d11b792308a6cf74e796cbb35def94c89a25a", - "sha256:30b88c47e0cdb6062daed88ca283b0d84fa0d2ad6c273aa0788152a1c643e408", - "sha256:32fea0ea3cd1ef820286863a6202dcfd62a539b8ec3edcbdff76068a8c2cc6ce", - "sha256:355f7fd0f90134229eaeefaee3cf42e0afc8518e8f3cd4b25f541a7104dcb8f9", - "sha256:4abdb88a9b67e64810fb54b0c24a1fd76b12297b4f7a1467d85a14dd8367191a", - "sha256:757bd71a9b89e4f1db0622af4436d403e742506dbea978eba566815dc65ec895", - "sha256:76df51492bc6fa6cc8b65d09efdb67cbba3cbfe55004c3afc81352af92b4a43c", - "sha256:774f5edc3475917cd95fe593e625d23d8580f9b48b570d8853d06cac171cd170", - "sha256:8a3ada8401736df2bf497f65589293a86c56e197a80ae7634ec2c3150a2f5082", - "sha256:a06efd0482a1942aad209a6c18321b5e22d64eb531ea20af138b28172d8f35ba", - "sha256:b24afc52e18dccc8c175de07c1d680bdf315844566f4952b5bedb908894bec79", - "sha256:b8b4bd3dafc7b92608ae5462add1c8cc881851c2d4f5d8977fdea5b081d17f21", - "sha256:c6e5024fc0cdf7f83b6624850309ddd7e06c48a75fa0d1c5173de4d93300eb19", - "sha256:db7ff14abc73577b0bcbcf73ecff97d3580ecaa0fc8724babce21fdf3fe08ef6", - "sha256:dedf54d72d9e7b6d043c244c8213fe2b8bbfe66874b9a65b39c4cc892dd99dd4", - "sha256:ea3c2f859346fcd55fc46e96885301d9c2f7a36d453f5d8f2967840efa1e1830", - "sha256:f0f47bafe9c9b8ed03e19a100a743662dd8c6d0135e684feea720a0d0046d116" - ], - "index": "pypi", - "version": "==0.6.2" - }, - "numpy": { - "hashes": [ - "sha256:1786a08236f2c92ae0e70423c45e1e62788ed33028f94ca99c4df03f5be6b3c6", - "sha256:17aa7a81fe7599a10f2b7d95856dc5cf84a4eefa45bc96123cbbc3ebc568994e", - "sha256:20b26aaa5b3da029942cdcce719b363dbe58696ad182aff0e5dcb1687ec946dc", - "sha256:2d75908ab3ced4223ccba595b48e538afa5ecc37405923d1fea6906d7c3a50bc", - "sha256:39d2c685af15d3ce682c99ce5925cc66efc824652e10990d2462dfe9b8918c6a", - "sha256:56bc8ded6fcd9adea90f65377438f9fea8c05fcf7c5ba766bef258d0da1554aa", - "sha256:590355aeade1a2eaba17617c19edccb7db8d78760175256e3cf94590a1a964f3", - "sha256:70a840a26f4e61defa7bdf811d7498a284ced303dfbc35acb7be12a39b2aa121", - "sha256:77c3bfe65d8560487052ad55c6998a04b654c2fbc36d546aef2b2e511e760971", - "sha256:9537eecf179f566fd1c160a2e912ca0b8e02d773af0a7a1120ad4f7507cd0d26", - "sha256:9acdf933c1fd263c513a2df3dceecea6f3ff4419d80bf238510976bf9bcb26cd", - "sha256:ae0975f42ab1f28364dcda3dde3cf6c1ddab3e1d4b2909da0cb0191fa9ca0480", - "sha256:b3af02ecc999c8003e538e60c89a2b37646b39b688d4e44d7373e11c2debabec", - "sha256:b6ff59cee96b454516e47e7721098e6ceebef435e3e21ac2d6c3b8b02628eb77", - "sha256:b765ed3930b92812aa698a455847141869ef755a87e099fddd4ccf9d81fffb57", - "sha256:c98c5ffd7d41611407a1103ae11c8b634ad6a43606eca3e2a5a269e5d6e8eb07", - "sha256:cf7eb6b1025d3e169989416b1adcd676624c2dbed9e3bcb7137f51bfc8cc2572", - "sha256:d92350c22b150c1cae7ebb0ee8b5670cc84848f6359cf6b5d8f86617098a9b73", - "sha256:e422c3152921cece8b6a2fb6b0b4d73b6579bd20ae075e7d15143e711f3ca2ca", - "sha256:e840f552a509e3380b0f0ec977e8124d0dc34dc0e68289ca28f4d7c1d0d79474", - "sha256:f3d0a94ad151870978fb93538e95411c83899c9dc63e6fb65542f769568ecfa5" - ], - "version": "==1.18.1" - }, - "outcome": { - "hashes": [ - "sha256:ee46c5ce42780cde85d55a61819d0e6b8cb490f1dbd749ba75ff2629771dcd2d", - "sha256:fc7822068ba7dd0fc2532743611e8a73246708d3564e29a39f93d6ab3701b66f" - ], - "version": "==1.0.1" - }, - "packaging": { - "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" - ], - "version": "==20.3" - }, - "pandas": { - "hashes": [ - "sha256:23e177d43e4bf68950b0f8788b6a2fef2f478f4ec94883acb627b9264522a98a", - "sha256:2530aea4fe46e8df7829c3f05e0a0f821c893885d53cb8ac9b89cc67c143448c", - "sha256:303827f0bb40ff610fbada5b12d50014811efcc37aaf6ef03202dc3054bfdda1", - "sha256:3b019e3ea9f5d0cfee0efabae2cfd3976874e90bcc3e97b29600e5a9b345ae3d", - "sha256:3c07765308f091d81b6735d4f2242bb43c332cc3461cae60543df6b10967fe27", - "sha256:5036d4009012a44aa3e50173e482b664c1fae36decd277c49e453463798eca4e", - "sha256:6f38969e2325056f9959efbe06c27aa2e94dd35382265ad0703681d993036052", - "sha256:74a470d349d52b9d00a2ba192ae1ee22155bb0a300fd1ccb2961006c3fa98ed3", - "sha256:7d77034e402165b947f43050a8a415aa3205abfed38d127ea66e57a2b7b5a9e0", - "sha256:7f9a509f6f11fa8b9313002ebdf6f690a7aa1dd91efd95d90185371a0d68220e", - "sha256:942b5d04762feb0e55b2ad97ce2b254a0ffdd344b56493b04a627266e24f2d82", - "sha256:a9fbe41663416bb70ed05f4e16c5f377519c0dc292ba9aa45f5356e37df03a38", - "sha256:d10e83866b48c0cdb83281f786564e2a2b51a7ae7b8a950c3442ad3c9e36b48c", - "sha256:e2140e1bbf9c46db9936ee70f4be6584d15ff8dc3dfff1da022d71227d53bad3" - ], - "version": "==1.0.1" - }, - "pdbpp": { - "hashes": [ - "sha256:73ff220d5006e0ecdc3e2705d8328d8aa5ac27fef95cc06f6e42cd7d22d55eb8" - ], - "index": "pypi", - "version": "==0.10.2" - }, - "piker": { - "editable": true, - "path": "." - }, - "pluggy": { - "hashes": [ - "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", - "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" - ], - "version": "==0.13.1" - }, - "py": { - "hashes": [ - "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", - "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" - ], - "version": "==1.8.1" - }, - "pygments": { - "hashes": [ - "sha256:647344a061c249a3b74e230c739f434d7ea4d8b1d5f3721bc0f3558049b38f44", - "sha256:ff7a40b4860b727ab48fad6360eb351cc1b33cbf9b15a0f689ca5353e9463324" - ], - "version": "==2.6.1" - }, - "pyparsing": { - "hashes": [ - "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", - "sha256:c342dccb5250c08d45fd6f8b4a559613ca603b57498511740e65cd11a2e7dcec" - ], - "version": "==2.4.6" - }, - "pyrepl": { - "hashes": [ - "sha256:292570f34b5502e871bbb966d639474f2b57fbfcd3373c2d6a2f3d56e681a775" - ], - "version": "==0.9.0" - }, - "pytest": { - "hashes": [ - "sha256:8e256fe71eb74e14a4d20a5987bb5e1488f0511ee800680aaedc62b9358714e8", - "sha256:ff0090819f669aaa0284d0f4aad1a6d9d67a6efdc6dd4eb4ac56b704f890a0d6" - ], - "index": "pypi", - "version": "==5.2.4" - }, - "python-dateutil": { - "hashes": [ - "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", - "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" - ], - "version": "==2.8.1" - }, - "pytz": { - "hashes": [ - "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", - "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" - ], - "version": "==2019.3" - }, - "six": { - "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" - ], - "version": "==1.14.0" - }, - "sniffio": { - "hashes": [ - "sha256:20ed6d5b46f8ae136d00b9dcb807615d83ed82ceea6b2058cecb696765246da5", - "sha256:8e3810100f69fe0edd463d02ad407112542a11ffdc29f67db2bf3771afb87a21" - ], - "version": "==1.1.0" - }, - "sortedcontainers": { - "hashes": [ - "sha256:974e9a32f56b17c1bac2aebd9dcf197f3eb9cd30553c5852a3187ad162e1a03a", - "sha256:d9e96492dd51fae31e60837736b38fe42a187b5404c16606ff7ee7cd582d4c60" - ], - "version": "==2.1.0" - }, - "trio": { - "hashes": [ - "sha256:a6d83c0cb4a177ec0f5179ce88e27914d5c8e6fd01c4285176b949e6ddc88c6c", - "sha256:f1cf00054ad974c86d9b7afa187a65d79fd5995340abe01e8e4784d86f4acb30" - ], - "index": "pypi", - "version": "==0.13.0" - }, - "wcwidth": { - "hashes": [ - "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", - "sha256:f28b3e8a6483e5d49e7f8949ac1a78314e740333ae305b4ba5defd3e74fb37a8" - ], - "version": "==0.1.8" - }, - "wmctrl": { - "hashes": [ - "sha256:d806f65ac1554366b6e31d29d7be2e8893996c0acbb2824bbf2b1f49cf628a13" - ], - "version": "==0.3" - } - } -} From dc919fa67658e1f8a6ed09eee3f7f558d01955cc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 1 Aug 2020 22:23:19 -0400 Subject: [PATCH 061/206] Set tractor loglevel in cli config --- piker/cli/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/piker/cli/__init__.py b/piker/cli/__init__.py index 097f9ad5..03dd87ae 100644 --- a/piker/cli/__init__.py +++ b/piker/cli/__init__.py @@ -48,9 +48,10 @@ def pikerd(loglevel, host, tl): @click.option('--broker', '-b', default=DEFAULT_BROKER, help='Broker backend to use') @click.option('--loglevel', '-l', default='warning', help='Logging level') +@click.option('--tl', is_flag=True, help='Enable tractor logging') @click.option('--configdir', '-c', help='Configuration directory') @click.pass_context -def cli(ctx, broker, loglevel, configdir): +def cli(ctx, broker, loglevel, tl, configdir): if configdir is not None: assert os.path.isdir(configdir), f"`{configdir}` is not a valid path" config._override_config_dir(configdir) @@ -60,11 +61,16 @@ def cli(ctx, broker, loglevel, configdir): 'broker': broker, 'brokermod': get_brokermod(broker), 'loglevel': loglevel, + 'tractorloglevel': None, 'log': get_console_log(loglevel), 'confdir': _config_dir, 'wl_path': _watchlists_data_path, }) + # allow enabling same loglevel in ``tractor`` machinery + if tl: + ctx.obj.update({'tractorloglevel': loglevel}) + def _load_clis() -> None: from ..data import marketstore as _ From 7a78c3a1c7ccdb3e68e6e46d464046915997485a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 1 Aug 2020 22:24:51 -0400 Subject: [PATCH 062/206] Add a couple more deps --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 66208cea..fcd812e8 100755 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ setup( # async 'trio', + 'trio-websocket', # 'tractor', # from github currently 'async_generator', @@ -69,6 +70,7 @@ setup( # UI 'PyQt5', 'pyqtgraph', + 'qdarkstyle', # tsdbs 'pymarketstore', From 1a143f6b162dc10e99e9be889948580cbbc41b3e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 00:18:54 -0400 Subject: [PATCH 063/206] Pass piker log level through to tractor for chart app --- piker/ui/cli.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index cc81af85..5161dbbf 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -63,7 +63,7 @@ def monitor(config, rate, name, dhost, test, tl): name='monitor', loglevel=loglevel if tl else None, rpc_module_paths=['piker.ui.kivy.monitor'], - start_method='forkserver', + # start_method='trio', ) @@ -101,18 +101,17 @@ def optschain(config, symbol, date, tl, rate, test): partial(main, tries=1), name='kivy-options-chain', loglevel=loglevel if tl else None, - start_method='forkserver', + # start_method='forkserver', ) @cli.command() -@click.option('--tl', is_flag=True, help='Enable tractor logging') @click.option('--date', '-d', help='Contracts expiry date') @click.option('--test', '-t', help='Test quote stream file') @click.option('--rate', '-r', default=1, help='Logging level') @click.argument('symbol', required=True) @click.pass_obj -def chart(config, symbol, date, tl, rate, test): +def chart(config, symbol, date, rate, test): """Start an option chain UI """ from ._chart import _main @@ -120,5 +119,6 @@ def chart(config, symbol, date, tl, rate, test): # global opts loglevel = config['loglevel'] brokername = config['broker'] + tractorloglevel = config['tractorloglevel'] - _main(sym=symbol, brokername=brokername, loglevel=loglevel) + _main(sym=symbol, brokername=brokername, loglevel=tractorloglevel) From 2eea946e5b8c3782cd36b901ed7e637d6954b52b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 01:42:04 -0400 Subject: [PATCH 064/206] Drop forkserver usage. We've got the sweet and realable `trio` spawner now :) --- piker/ui/cli.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 5161dbbf..c1052733 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -63,7 +63,6 @@ def monitor(config, rate, name, dhost, test, tl): name='monitor', loglevel=loglevel if tl else None, rpc_module_paths=['piker.ui.kivy.monitor'], - # start_method='trio', ) @@ -101,7 +100,6 @@ def optschain(config, symbol, date, tl, rate, test): partial(main, tries=1), name='kivy-options-chain', loglevel=loglevel if tl else None, - # start_method='forkserver', ) @@ -117,7 +115,6 @@ def chart(config, symbol, date, rate, test): from ._chart import _main # global opts - loglevel = config['loglevel'] brokername = config['broker'] tractorloglevel = config['tractorloglevel'] From d81f6620f5c2a7dc37cf3f198ab9d4a148e79bb1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 12:17:38 -0400 Subject: [PATCH 065/206] Passthrough loglevel from qtractor --- piker/ui/_exec.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 06944d0e..9a2c4ce7 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -16,7 +16,6 @@ import tractor from outcome import Error -# Taken from Quantdom class MainWindow(QtGui.QMainWindow): size = (800, 500) @@ -89,8 +88,6 @@ def run_qtractor( 'main': instance, } - # force mp since we may spawn an ib asyncio based backend - tractor._spawn.try_set_start_method('forkserver') # setup tractor entry point args args = ( # async_fn @@ -98,7 +95,7 @@ def run_qtractor( # args (widgets,), # kwargs - {'loglevel': 'info'}, + {'loglevel': loglevel}, # arbiter_addr ( tractor._default_arbiter_host, From b2506b04f6a234c2f105fd192504103b0dbbc829 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 12:18:53 -0400 Subject: [PATCH 066/206] Attempt more reliable chart startup Wait for a first actual real-time quote before starting graphics update tasks. Use the new normalized tick format brokers are expected to emit as a `quotes['ticks']` list. Auto detect time frame from historical bars. --- piker/ui/_chart.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 68439ecb..37ce6823 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -333,7 +333,7 @@ class ChartPlotWidget(pg.PlotWidget): # XXX: label setting doesn't seem to work? # likely custom graphics need special handling - # label = pg.LabelItem(justify='left') + # label = pg.LabelItem(justify='right') # self.addItem(label) # label.setText("Yo yoyo") # label.setText("x=") @@ -480,7 +480,7 @@ class ChartPlotWidget(pg.PlotWidget): bars = self.parent._array[lbar:rbar] if not len(bars): # likely no data loaded yet - print(f"WTF bars_range = {lbar}:{rbar}") + log.error(f"WTF bars_range = {lbar}:{rbar}") return elif lbar < 0: breakpoint() @@ -604,6 +604,7 @@ def _main( # historical data fetch brokermod = brokers.get_brokermod(brokername) + async with brokermod.get_client() as client: # figure out the exact symbol bars = await client.bars(symbol=sym) @@ -614,11 +615,18 @@ def _main( # determine ohlc delay between bars times = bars['time'] - delay = times[-1] - times[-2] + + # find expected time step between datums + delay = times[-1] - times[times != times[-1]][-1] async def add_new_bars(delay_s): """Task which inserts new bars into the ohlc every ``delay_s`` seconds. """ + # TODO: right now we'll spin printing bars if the last time + # stamp is before a large period of no market activity. + # Likely the best way to solve this is to make this task + # aware of the instrument's tradable hours? + # adjust delay to compensate for trio processing time ad = delay_s - 0.002 @@ -675,7 +683,7 @@ def _main( func, brokername=brokermod.name, sym=sym, - loglevel='info', + # loglevel='info', ) stream = await portal.result() @@ -691,23 +699,27 @@ def _main( async with trio.open_nursery() as n: from piker import fsp - async with data.open_feed(brokername, [sym]) as stream: - # start graphics tasks - n.start_soon(add_new_bars, delay) + async with data.open_feed( + brokername, + [sym], + loglevel=qtractor_kwargs['loglevel'], + ) as (fquote, stream): + # start downstream processor n.start_soon(stream_to_chart, fsp.broker_latency) + # wait for a first quote before we start any update tasks + quote = await stream.__anext__() + + # start graphics tasks after receiving first live quote + n.start_soon(add_new_bars, delay) + async for quote in stream: - # XXX: why are we getting both of these again? ticks = quote.get('ticks') if ticks: for tick in ticks: - if tick['tickType'] in (48, 77): + if tick.get('type') == 'trade': linked_charts.update_from_quote( {'last': tick['price']} ) - # else: - # linked_charts.update_from_quote( - # {'last': quote['close']} - # ) run_qtractor(_main, (), ChartSpace, **qtractor_kwargs) From 971b8716472a58d4220a2afd4187100e39770d06 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 15:23:20 -0400 Subject: [PATCH 067/206] Handle "mouse-not-on-plot" edge cases --- piker/ui/_graphics.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index e3b8fdc3..0eae1f81 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -18,7 +18,7 @@ from ._axes import YAxisLabel, XAxisLabel # TODO: # - checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 50 +_mouse_rate_limit = 30 class CrossHair(pg.GraphicsObject): @@ -104,7 +104,11 @@ class CrossHair(pg.GraphicsObject): pos = evt[0] # find position inside active plot - mouse_point = self.active_plot.mapToView(pos) + try: + mouse_point = self.active_plot.mapToView(pos) + except AttributeError: + # mouse was not on active plot + return self.graphics[self.active_plot]['hl'].setY( mouse_point.y() @@ -120,10 +124,10 @@ class CrossHair(pg.GraphicsObject): self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) def boundingRect(self): - return self.active_plot.boundingRect() - - # def paint(self, p, *args): - # pass + try: + return self.active_plot.boundingRect() + except AttributeError: + return self.plots[0].boundingRect() def _mk_lines_array(data: List, size: int) -> np.ndarray: From 6b572eb0ef5a595f9f5b4059e879c9f231fdb632 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 20:09:27 -0400 Subject: [PATCH 068/206] Add ravel() reference link --- piker/ui/_graphics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 0eae1f81..4f8cc355 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -134,7 +134,7 @@ def _mk_lines_array(data: List, size: int) -> np.ndarray: """Create an ndarray to hold lines graphics objects. """ # TODO: might want to just make this a 2d array to be faster at - # flattening using .ravel()? + # flattening using .ravel(): https://stackoverflow.com/a/60089929 return np.zeros_like( data, shape=(int(size),), From 65fb92eaffa869bec5293515a430862b58f2132b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Aug 2020 20:10:06 -0400 Subject: [PATCH 069/206] Flatten out chart tasks --- piker/ui/_chart.py | 289 ++++++++++++++++++++++++--------------------- piker/ui/_exec.py | 13 +- 2 files changed, 162 insertions(+), 140 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 37ce6823..af7f52b6 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,7 +1,7 @@ """ High level Qt chart widgets. """ -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Dict, Any import time from PyQt5 import QtCore, QtGui @@ -21,6 +21,9 @@ from ._source import Symbol from .. import brokers from .. import data from ..log import get_logger +from ._exec import run_qtractor +from ._source import ohlc_dtype +from .. import fsp log = get_logger(__name__) @@ -166,13 +169,13 @@ class LinkedSplitCharts(QtGui.QWidget): # add crosshairs self._ch = CrossHair( - parent=self, #.chart, + parent=self, # subplots=[plot for plot, d in self.subplots], digits=self.digits ) self.chart = self.add_plot( name='main', - array=array, #['close'], + array=array, xaxis=self.xaxis, ohlc=True, ) @@ -586,140 +589,158 @@ class ChartView(pg.ViewBox): self.sigRangeChangedManually.emit(mask) +async def add_new_bars(delay_s, linked_charts): + """Task which inserts new bars into the ohlc every ``delay_s`` seconds. + """ + # TODO: right now we'll spin printing bars if the last time + # stamp is before a large period of no market activity. + # Likely the best way to solve this is to make this task + # aware of the instrument's tradable hours? + + # adjust delay to compensate for trio processing time + ad = delay_s - 0.002 + + ohlc = linked_charts._array + + async def sleep(): + """Sleep until next time frames worth has passed from last bar. + """ + last_ts = ohlc[-1]['time'] + delay = max((last_ts + ad) - time.time(), 0) + await trio.sleep(delay) + + # sleep for duration of current bar + await sleep() + + while True: + # TODO: bunch of stuff: + # - I'm starting to think all this logic should be + # done in one place and "graphics update routines" + # should not be doing any length checking and array diffing. + # - don't keep appending, but instead increase the + # underlying array's size less frequently + # - handle odd lot orders + # - update last open price correctly instead + # of copying it from last bar's close + # - 5 sec bar lookback-autocorrection like tws does? + (index, t, close) = ohlc[-1][['index', 'time', 'close']] + new = np.append( + ohlc, + np.array( + [(index + 1, t + delay_s, close, close, + close, close, 0)], + dtype=ohlc.dtype + ), + ) + ohlc = linked_charts._array = new + last_quote = ohlc[-1] + + # we **don't** update the bar right now + # since the next quote that arrives should + await sleep() + + # if the last bar has not changed print a flat line and + # move to the next + if last_quote == ohlc[-1]: + log.debug("Printing flat line for {sym}") + linked_charts.update_from_array(ohlc) + + +async def _async_main( + sym: str, + brokername: str, + + # implicit required argument provided by ``qtractor_run()`` + widgets: Dict[str, Any], + + # all kwargs are passed through from the CLI entrypoint + loglevel: str = None, +) -> None: + """Main Qt-trio routine invoked by the Qt loop with + the widgets ``dict``. + """ + chart_app = widgets['main'] + + # historical data fetch + brokermod = brokers.get_brokermod(brokername) + + async with brokermod.get_client() as client: + # figure out the exact symbol + bars = await client.bars(symbol=sym) + + # remember, msgpack-numpy's ``from_buffer` returns read-only array + bars = np.array(bars[list(ohlc_dtype.names)]) + linked_charts = chart_app.load_symbol(sym, bars) + + # determine ohlc delay between bars + times = bars['time'] + + # find expected time step between datums + delay = times[-1] - times[times != times[-1]][-1] + + async def stream_to_chart(func): + + async with tractor.open_nursery() as n: + portal = await n.run_in_actor( + f'fsp_{func.__name__}', + func, + brokername=brokermod.name, + sym=sym, + # loglevel='info', + ) + stream = await portal.result() + + # retreive named layout and style instructions + layout = await stream.__anext__() + + async for quote in stream: + ticks = quote.get('ticks') + if ticks: + for tick in ticks: + print(tick) + + async with trio.open_nursery() as n: + + async with data.open_feed( + brokername, + [sym], + loglevel=loglevel, + ) as (fquote, stream): + # start downstream processor + n.start_soon(stream_to_chart, fsp.broker_latency) + + # wait for a first quote before we start any update tasks + quote = await stream.__anext__() + + # start graphics tasks after receiving first live quote + n.start_soon(add_new_bars, delay, linked_charts) + + async for quote in stream: + ticks = quote.get('ticks') + if ticks: + for tick in ticks: + if tick.get('type') == 'trade': + linked_charts.update_from_quote( + {'last': tick['price']} + ) + + def _main( sym: str, brokername: str, **qtractor_kwargs, ) -> None: - """Entry point to spawn a chart app. + """Sync entry point to start a chart app. """ - from ._exec import run_qtractor - from ._source import ohlc_dtype - - async def _main(widgets): - """Main Qt-trio routine invoked by the Qt loop with - the widgets ``dict``. - """ - chart_app = widgets['main'] - - # historical data fetch - brokermod = brokers.get_brokermod(brokername) - - async with brokermod.get_client() as client: - # figure out the exact symbol - bars = await client.bars(symbol=sym) - - # remember, msgpack-numpy's ``from_buffer` returns read-only array - bars = np.array(bars[list(ohlc_dtype.names)]) - linked_charts = chart_app.load_symbol(sym, bars) - - # determine ohlc delay between bars - times = bars['time'] - - # find expected time step between datums - delay = times[-1] - times[times != times[-1]][-1] - - async def add_new_bars(delay_s): - """Task which inserts new bars into the ohlc every ``delay_s`` seconds. - """ - # TODO: right now we'll spin printing bars if the last time - # stamp is before a large period of no market activity. - # Likely the best way to solve this is to make this task - # aware of the instrument's tradable hours? - - # adjust delay to compensate for trio processing time - ad = delay_s - 0.002 - - ohlc = linked_charts._array - - async def sleep(): - """Sleep until next time frames worth has passed from last bar. - """ - last_ts = ohlc[-1]['time'] - delay = max((last_ts + ad) - time.time(), 0) - await trio.sleep(delay) - - # sleep for duration of current bar - await sleep() - - while True: - # TODO: bunch of stuff: - # - I'm starting to think all this logic should be - # done in one place and "graphics update routines" - # should not be doing any length checking and array diffing. - # - don't keep appending, but instead increase the - # underlying array's size less frequently - # - handle odd lot orders - # - update last open price correctly instead - # of copying it from last bar's close - # - 5 sec bar lookback-autocorrection like tws does? - (index, t, close) = ohlc[-1][['index', 'time', 'close']] - new = np.append( - ohlc, - np.array( - [(index + 1, t + delay, close, close, - close, close, 0)], - dtype=ohlc.dtype - ), - ) - ohlc = linked_charts._array = new - last_quote = ohlc[-1] - - # we **don't** update the bar right now - # since the next quote that arrives should - await sleep() - - # if the last bar has not changed print a flat line and - # move to the next - if last_quote == ohlc[-1]: - log.debug("Printing flat line for {sym}") - linked_charts.update_from_array(ohlc) - - async def stream_to_chart(func): - - async with tractor.open_nursery() as n: - portal = await n.run_in_actor( - f'fsp_{func.__name__}', - func, - brokername=brokermod.name, - sym=sym, - # loglevel='info', - ) - stream = await portal.result() - - # retreive named layout and style instructions - layout = await stream.__anext__() - - async for quote in stream: - ticks = quote.get('ticks') - if ticks: - for tick in ticks: - print(tick) - - async with trio.open_nursery() as n: - from piker import fsp - - async with data.open_feed( - brokername, - [sym], - loglevel=qtractor_kwargs['loglevel'], - ) as (fquote, stream): - # start downstream processor - n.start_soon(stream_to_chart, fsp.broker_latency) - - # wait for a first quote before we start any update tasks - quote = await stream.__anext__() - - # start graphics tasks after receiving first live quote - n.start_soon(add_new_bars, delay) - - async for quote in stream: - ticks = quote.get('ticks') - if ticks: - for tick in ticks: - if tick.get('type') == 'trade': - linked_charts.update_from_quote( - {'last': tick['price']} - ) - - run_qtractor(_main, (), ChartSpace, **qtractor_kwargs) + # Qt entry point + run_qtractor( + # func + _async_main, + # args, + (sym, brokername), + # kwargs passed through + qtractor_kwargs, + # main widget + ChartSpace, + # **qtractor_kwargs + ) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 9a2c4ce7..7a137fed 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -5,6 +5,7 @@ Run ``trio`` in guest mode on top of the Qt event loop. All global Qt runtime settings are mostly defined here. """ import traceback +from typing import Tuple, Callable, Dict import PyQt5 # noqa from pyqtgraph import QtGui @@ -28,11 +29,11 @@ class MainWindow(QtGui.QMainWindow): def run_qtractor( - func, - args, + func: Callable, + args: Tuple, + kwargs: Dict, main_widget: QtGui.QWidget, window_type: QtGui.QMainWindow = MainWindow, - loglevel: str = None, ) -> None: # avoids annoying message when entering debugger from qt loop pyqtRemoveInputHook() @@ -92,10 +93,10 @@ def run_qtractor( args = ( # async_fn func, - # args - (widgets,), + # args (always append widgets list) + args + (widgets,), # kwargs - {'loglevel': loglevel}, + kwargs, # arbiter_addr ( tractor._default_arbiter_host, From 81fb327fe1181119a8c0192c80e4e70bfd1f1ec5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 3 Aug 2020 21:31:56 -0400 Subject: [PATCH 070/206] Add `services` cmd for monitoring actors --- piker/brokers/questrade.py | 2 +- piker/cli/__init__.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index f3708d3f..62fd1773 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -1134,7 +1134,7 @@ async def stream_quotes( loglevel: str = None, # feed_type: str = 'stock', ) -> AsyncGenerator[str, Dict[str, Any]]: - # XXX: why do we need this again? + # XXX: required to propagate ``tractor`` loglevel to piker logging get_console_log(tractor.current_actor().loglevel) async with get_cached_client('questrade') as client: diff --git a/piker/cli/__init__.py b/piker/cli/__init__.py index 03dd87ae..b43f52b1 100644 --- a/piker/cli/__init__.py +++ b/piker/cli/__init__.py @@ -6,7 +6,7 @@ import os import click import tractor -from ..log import get_console_log, get_logger +from ..log import get_console_log, get_logger, colorize_json from ..brokers import get_brokermod, config @@ -72,8 +72,36 @@ def cli(ctx, broker, loglevel, tl, configdir): ctx.obj.update({'tractorloglevel': loglevel}) +@cli.command() +@click.option('--tl', is_flag=True, help='Enable tractor logging') +@click.argument('names', nargs=-1, required=False) +@click.pass_obj +def services(config, tl, names): + + async def list_services(): + async with tractor.get_arbiter( + *tractor.current_actor()._arb_addr + ) as portal: + registry = await portal.run('self', 'get_registry') + json_d = {} + for uid, socket in registry.items(): + name, uuid = uid + host, port = socket + json_d[f'{name}.{uuid}'] = f'{host}:{port}' + click.echo( + f"Available `piker` services:\n{colorize_json(json_d)}" + ) + + tractor.run( + list_services, + name='service_query', + loglevel=config['loglevel'] if tl else None, + ) + + def _load_clis() -> None: from ..data import marketstore as _ + from ..data import cli as _ from ..brokers import cli as _ # noqa from ..ui import cli as _ # noqa from ..watchlists import cli as _ # noqa From 9bbf0e0d7a4dc2a2372584d766f66c024bef8bb7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 9 Aug 2020 00:03:09 -0400 Subject: [PATCH 071/206] Add a normalizer routine which emits quote differentials/ticks --- piker/brokers/questrade.py | 115 +++++++++++++++++++++++++++++++------ 1 file changed, 98 insertions(+), 17 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 62fd1773..98989741 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -9,6 +9,7 @@ from datetime import datetime from functools import partial import itertools import configparser +from pprint import pformat from typing import ( List, Tuple, Dict, Any, Iterator, NamedTuple, AsyncGenerator, @@ -837,7 +838,8 @@ _qt_stock_keys = { # 'low52w': 'low52w', # put in info widget # 'high52w': 'high52w', # "lastTradePriceTrHrs": 7.99, - 'lastTradeTime': ('fill_time', datetime.fromisoformat), + # 'lastTradeTime': ('fill_time', datetime.fromisoformat), + 'lastTradeTime': 'fill_time', "lastTradeTick": 'tick', # ("Equal", "Up", "Down") # "symbolId": 3575753, # "tier": "", @@ -913,6 +915,7 @@ def format_stock_quote( new[new_key] = value displayable[new_key] = display_value + new['displayable'] = displayable return new, displayable @@ -973,6 +976,7 @@ def format_option_quote( quote: dict, symbol_data: dict, keymap: dict = _qt_option_keys, + include_displayables: bool = True, ) -> Tuple[dict, dict]: """Remap a list of quote dicts ``quotes`` using the mapping of old keys -> new keys ``keymap`` returning 2 dicts: one with raw data and the other @@ -1060,7 +1064,10 @@ async def get_cached_client( await client._exit_stack.aclose() -async def smoke_quote(get_quotes, tickers): # , broker): +async def smoke_quote( + get_quotes, + tickers +): """Do an initial "smoke" request for symbols in ``tickers`` filtering out any symbols not supported by the broker queried in the call to ``get_quotes()``. @@ -1099,6 +1106,7 @@ async def smoke_quote(get_quotes, tickers): # , broker): log.error( f"{symbol} seems to be defunct") + quote['symbol'] = symbol payload[symbol] = quote return payload @@ -1107,20 +1115,90 @@ async def smoke_quote(get_quotes, tickers): # , broker): ########################################### +# unbounded, shared between streaming tasks +_symbol_info_cache = {} + + # function to format packets delivered to subscribers def packetizer( topic: str, quotes: Dict[str, Any], - formatter: Callable, - symbol_data: Dict[str, Any], ) -> Dict[str, Any]: """Normalize quotes by name into dicts using broker-specific processing. """ - new = {} - for quote in quotes: - new[quote['symbol']], _ = formatter(quote, symbol_data) + # repack into symbol keyed dict + return {q['symbol']: q for q in quotes} + +def normalize( + quotes: Dict[str, Any], + _cache: Dict[str, Any], # dict held in scope of the streaming loop + formatter: Callable, +) -> Dict[str, Any]: + """Deliver normalized quotes by name into dicts using + broker-specific processing; only emit changes differeing from the + last quote sample creating a psuedo-tick type datum. + """ + new = {} + # XXX: this is effectively emitting "sampled ticks" + # useful for polling setups but obviously should be + # disabled if you're already rx-ing per-tick data. + for quote in quotes: + symbol = quote['symbol'] + + # look up last quote from cache + last = _cache.setdefault(symbol, {}) + _cache[symbol] = quote + + # compute volume difference + last_volume = last.get('volume', 0) + current_volume = quote['volume'] + volume_diff = current_volume - last_volume + + # find all keys that have match to a new value compared + # to the last quote received + changed = set(quote.items()) - set(last.items()) + if changed: + log.info(f"New quote {symbol}:\n{changed}") + + # TODO: can we reduce the # of iterations here and in + # called funcs? + payload = {k: quote[k] for k, v in changed} + payload['symbol'] = symbol # required by formatter + + # TODO: we should probaby do the "computed" fields + # processing found inside this func in a downstream actor? + fquote, _ = formatter(payload, _symbol_info_cache) + fquote['key'] = fquote['symbol'] = symbol + + # if there was volume likely the last size of + # shares traded is useful info and it's possible + # that the set difference from above will disregard + # a "size" value since the same # of shares were traded + # volume = payload.get('volume') + if volume_diff: + if volume_diff < 0: + log.error(f"Uhhh {symbol} volume: {volume_diff} ?") + + fquote['volume_delta'] = volume_diff + + # TODO: We can emit 2 ticks here: + # - one for the volume differential + # - one for the last known trade size + # The first in theory can be unwound and + # interpolated assuming the broker passes an + # accurate daily VWAP value. + # To make this work we need a universal ``size`` + # field that is normalized before hitting this logic. + fquote['size'] = quote.get('lastTradeSize', 0) + if 'last' not in fquote: + fquote['last'] = quote.get('lastTradePrice', float('nan')) + + new[symbol] = fquote + + if new: + log.info(f"New quotes:\n{pformat(new)}") return new @@ -1129,13 +1207,12 @@ async def stream_quotes( ctx: tractor.Context, # marks this as a streaming func symbols: List[str], feed_type: str = 'stock', - diff_cached: bool = True, rate: int = 3, loglevel: str = None, # feed_type: str = 'stock', ) -> AsyncGenerator[str, Dict[str, Any]]: # XXX: required to propagate ``tractor`` loglevel to piker logging - get_console_log(tractor.current_actor().loglevel) + get_console_log(loglevel) async with get_cached_client('questrade') as client: if feed_type == 'stock': @@ -1144,17 +1221,25 @@ async def stream_quotes( # do a smoke quote (note this mutates the input list and filters # out bad symbols for now) - payload = await smoke_quote(get_quotes, list(symbols)) + first_quotes = await smoke_quote(get_quotes, list(symbols)) else: formatter = format_option_quote get_quotes = await option_quoter(client, symbols) # packetize - payload = { + first_quotes = { quote['symbol']: quote for quote in await get_quotes(symbols) } + # update global symbol data state sd = await client.symbol_info(symbols) + _symbol_info_cache.update(sd) + + # pre-process first set of quotes + payload = {} + for sym, quote in first_quotes.items(): + fquote, _ = formatter(quote, sd) + payload[sym] = fquote # push initial smoke quote response for client initialization await ctx.send_yield(payload) @@ -1167,15 +1252,11 @@ async def stream_quotes( task_name=feed_type, ctx=ctx, topics=symbols, - packetizer=partial( - packetizer, - formatter=formatter, - symbol_data=sd, - ), + packetizer=packetizer, # actual target "streaming func" args get_quotes=get_quotes, - diff_cached=diff_cached, + normalizer=partial(normalize, formatter=formatter), rate=rate, ) log.info("Terminating stream quoter task") From 241b2374e863867310478f1c580ad3273c3c3edb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Aug 2020 15:23:35 -0400 Subject: [PATCH 072/206] Port `DataFeed` api to broker specific normalizer routine --- piker/brokers/data.py | 139 ++++++++---------------------------------- 1 file changed, 24 insertions(+), 115 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index dbb76772..ea2076bb 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -81,10 +81,10 @@ class BrokerFeed: @tractor.msg.pub(tasks=['stock', 'option']) async def stream_poll_requests( - get_topics: typing.Callable, + get_topics: Callable, get_quotes: Coroutine, + normalizer: Callable, rate: int = 3, # delay between quote requests - diff_cached: bool = True, # only deliver "new" quotes to the queue ) -> None: """Stream requests for quotes for a set of symbols at the given ``rate`` (per second). @@ -129,59 +129,15 @@ async def stream_poll_requests( quotes = await wait_for_network(request_quotes) new_quotes = {} - if diff_cached: - # If cache is enabled then only deliver "new" changes. - # Useful for polling setups but obviously should be - # disabled if you're rx-ing per-tick data. - for quote in quotes: - symbol = quote['symbol'] - last = _cache.setdefault(symbol, {}) - last_volume = last.get('volume', 0) - # find all keys that have match to a new value compared - # to the last quote received - new = set(quote.items()) - set(last.items()) - if new: - log.info(f"New quote {symbol}:\n{new}") - _cache[symbol] = quote - - # only ship diff updates and other required fields - payload = {k: quote[k] for k, v in new} - payload['symbol'] = symbol - - # if there was volume likely the last size of - # shares traded is useful info and it's possible - # that the set difference from above will disregard - # a "size" value since the same # of shares were traded - volume = payload.get('volume') - if volume: - volume_since_last_quote = volume - last_volume - assert volume_since_last_quote > 0 - payload['volume_delta'] = volume_since_last_quote - - # TODO: We can emit 2 ticks here: - # - one for the volume differential - # - one for the last known trade size - # The first in theory can be unwound and - # interpolated assuming the broker passes an - # accurate daily VWAP value. - # To make this work we need a universal ``size`` - # field that is normalized before hitting this logic. - # XXX: very questrade specific - payload['size'] = quote['lastTradeSize'] - - log.info(f"New paylod {symbol}:\n{payload}") - - # XXX: we append to a list for the options case where the - # subscription topic (key) is the same for all - # expiries even though this is uncessary for the - # stock case (different topic [i.e. symbol] for each - # quote). - new_quotes.setdefault(quote['key'], []).append(payload) - else: - # log.debug(f"Delivering quotes:\n{quotes}") - for quote in quotes: - new_quotes.setdefault(quote['key'], []).append(quote) + normalized = normalizer(quotes, _cache) + for symbol, quote in normalized.items(): + # XXX: we append to a list for the options case where the + # subscription topic (key) is the same for all + # expiries even though this is uncessary for the + # stock case (different topic [i.e. symbol] for each + # quote). + new_quotes.setdefault(quote['key'], []).append(quote) if new_quotes: yield new_quotes @@ -208,53 +164,6 @@ async def symbol_data(broker: str, tickers: List[str]): return await feed.client.symbol_info(tickers) -async def smoke_quote(get_quotes, tickers, broker): - """Do an initial "smoke" request for symbols in ``tickers`` filtering - out any symbols not supported by the broker queried in the call to - ``get_quotes()``. - """ - # TODO: trim out with #37 - ################################################# - # get a single quote filtering out any bad tickers - # NOTE: this code is always run for every new client - # subscription even when a broker quoter task is already running - # since the new client needs to know what symbols are accepted - log.warn(f"Retrieving smoke quote for symbols {tickers}") - quotes = await get_quotes(tickers) - - # report any tickers that aren't returned in the first quote - invalid_tickers = set(tickers) - set(map(itemgetter('key'), quotes)) - for symbol in invalid_tickers: - tickers.remove(symbol) - log.warn( - f"Symbol `{symbol}` not found by broker `{broker}`" - ) - - # pop any tickers that return "empty" quotes - payload = {} - for quote in quotes: - symbol = quote['symbol'] - if quote is None: - log.warn( - f"Symbol `{symbol}` not found by broker" - f" `{broker}`") - # XXX: not this mutates the input list (for now) - tickers.remove(symbol) - continue - - # report any unknown/invalid symbols (QT specific) - if quote.get('low52w', False) is None: - log.error( - f"{symbol} seems to be defunct") - - payload[symbol] = quote - - return payload - - # end of section to be trimmed out with #37 - ########################################### - - @asynccontextmanager async def get_cached_feed( brokername: str, @@ -297,7 +206,6 @@ async def start_quote_stream( broker: str, symbols: List[Any], feed_type: str = 'stock', - diff_cached: bool = True, rate: int = 3, ) -> None: """Handle per-broker quote stream subscriptions using a "lazy" pub-sub @@ -316,8 +224,6 @@ async def start_quote_stream( f"{ctx.chan.uid} subscribed to {broker} for symbols {symbols}") # another actor task may have already created it async with get_cached_feed(broker) as feed: - # function to format packets delivered to subscribers - packetizer = None if feed_type == 'stock': get_quotes = feed.quoters.setdefault( @@ -326,7 +232,7 @@ async def start_quote_stream( ) # do a smoke quote (note this mutates the input list and filters # out bad symbols for now) - payload = await smoke_quote(get_quotes, symbols, broker) + first_quotes = await feed.mod.smoke_quote(get_quotes, symbols) formatter = feed.mod.format_stock_quote elif feed_type == 'option': @@ -338,22 +244,27 @@ async def start_quote_stream( await feed.mod.option_quoter(feed.client, symbols) ) # packetize - payload = { + first_quotes = { quote['symbol']: quote for quote in await get_quotes(symbols) } formatter = feed.mod.format_option_quote sd = await feed.client.symbol_info(symbols) - # formatter = partial(formatter, symbol_data=sd) + feed.mod._symbol_info_cache.update(sd) - packetizer = partial( - feed.mod.packetizer, + normalize = partial( + feed.mod.normalize, formatter=formatter, - symbol_data=sd, ) - # push initial smoke quote response for client initialization + # pre-process first set of quotes + payload = {} + for sym, quote in first_quotes.items(): + fquote, _ = formatter(quote, sd) + assert fquote['displayable'] + payload[sym] = fquote + await ctx.send_yield(payload) await stream_poll_requests( @@ -362,11 +273,11 @@ async def start_quote_stream( task_name=feed_type, ctx=ctx, topics=symbols, - packetizer=packetizer, + packetizer=feed.mod.packetizer, # actual func args get_quotes=get_quotes, - diff_cached=diff_cached, + normalizer=normalize, rate=rate, ) log.info( @@ -401,7 +312,6 @@ class DataFeed: symbols: Sequence[str], feed_type: str, rate: int = 1, - diff_cached: bool = True, test: str = '', ) -> (AsyncGenerator, dict): if feed_type not in self._allowed: @@ -445,7 +355,6 @@ class DataFeed: broker=self.brokermod.name, symbols=symbols, feed_type=feed_type, - diff_cached=diff_cached, rate=rate, ) From 8a46f8d6eddad5876f78088f043c108f431eeb39 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Aug 2020 15:48:57 -0400 Subject: [PATCH 073/206] Port monitor to normalized streams --- piker/ui/kivy/monitor.py | 51 ++++++++++++++++++++-------------------- piker/ui/kivy/tabular.py | 10 +++++--- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/piker/ui/kivy/monitor.py b/piker/ui/kivy/monitor.py index 932b581e..47e52605 100644 --- a/piker/ui/kivy/monitor.py +++ b/piker/ui/kivy/monitor.py @@ -69,7 +69,6 @@ async def update_quotes( chngcell.color = color hdrcell.color = color - # briefly highlight bg of certain cells on each trade execution unflash = set() tick_color = None @@ -105,39 +104,37 @@ async def update_quotes( # initial coloring to_sort = set() - for sym, quote in first_quotes.items(): - row = table.get_row(sym) - record, displayable = formatter( - quote, symbol_data=symbol_data) - row.update(record, displayable) - color_row(row, record, {}) + for quote in first_quotes: + row = table.get_row(quote['symbol']) + row.update(quote) + color_row(row, quote, {}) to_sort.add(row.widget) table.render_rows(to_sort) log.debug("Finished initializing update loop") task_status.started() + # real-time cell update loop async for quotes in agen: # new quotes data only to_sort = set() for symbol, quote in quotes.items(): row = table.get_row(symbol) - record, displayable = formatter( - quote, symbol_data=symbol_data) # don't red/green the header cell in ``row.update()`` - record.pop('symbol') + quote.pop('symbol') + quote.pop('key') # determine if sorting should happen sort_key = table.sort_key last = row.get_field(sort_key) - new = record.get(sort_key, last) + new = quote.get(sort_key, last) if new != last: to_sort.add(row.widget) # update and color - cells = row.update(record, displayable) - color_row(row, record, cells) + cells = row.update(quote) + color_row(row, quote, cells) if to_sort: table.render_rows(to_sort) @@ -179,18 +176,14 @@ async def _async_main( This is started with cli cmd `piker monitor`. ''' feed = DataFeed(portal, brokermod) - quote_gen, quotes = await feed.open_stream( + quote_gen, first_quotes = await feed.open_stream( symbols, 'stock', rate=rate, test=test, ) - - first_quotes, _ = feed.format_quotes(quotes) - - if first_quotes[0].get('last') is None: - log.error("Broker API is down temporarily") - return + first_quotes_list = list(first_quotes.copy().values()) + quotes = list(first_quotes.copy().values()) # build out UI Window.set_title(f"monitor: {name}\t(press ? for help)") @@ -202,7 +195,9 @@ async def _async_main( bidasks = brokermod._stock_bidasks # add header row - headers = first_quotes[0].keys() + headers = list(first_quotes_list[0].keys()) + headers.remove('displayable') + header = Row( {key: key for key in headers}, headers=headers, @@ -217,11 +212,17 @@ async def _async_main( cols=1, size_hint=(1, None), ) - for ticker_record in first_quotes: + for ticker_record in first_quotes_list: + symbol = ticker_record['symbol'] table.append_row( - ticker_record['symbol'], - Row(ticker_record, headers=('symbol',), - bidasks=bidasks, table=table) + symbol, + Row( + ticker_record, + headers=('symbol',), + bidasks=bidasks, + no_cell=('displayable',), + table=table + ) ) table.last_clicked_row = next(iter(table.symbols2rows.values())) diff --git a/piker/ui/kivy/tabular.py b/piker/ui/kivy/tabular.py index 5a5a6b3a..c11d6514 100644 --- a/piker/ui/kivy/tabular.py +++ b/piker/ui/kivy/tabular.py @@ -300,10 +300,10 @@ class Row(HoverBehavior, GridLayout): # handle bidask cells if key in layouts: self.add_widget(layouts[key]) - elif key in children_flat: + elif key in children_flat or key in no_cell: # these cells have already been added to the `BidAskLayout` continue - elif key not in no_cell: + else: cell = self._append_cell(val, key, header=header) cell.key = key self._cell_widgets[key] = cell @@ -329,7 +329,7 @@ class Row(HoverBehavior, GridLayout): self.add_widget(cell) return cell - def update(self, record, displayable): + def update(self, record): """Update this row's cells with new values from a quote ``record``. @@ -341,7 +341,11 @@ class Row(HoverBehavior, GridLayout): fgreen = colorcode('forestgreen') red = colorcode('red2') + displayable = record['displayable'] + for key, val in record.items(): + if key not in displayable: + continue last = self.get_field(key) color = gray try: From a29b7d9be5ec5fff1ea6f0ab06f023054fdaf3af Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 14 Aug 2020 22:17:57 -0400 Subject: [PATCH 074/206] Start "interaction" module --- piker/ui/_interaction.py | 77 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 piker/ui/_interaction.py diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py new file mode 100644 index 00000000..bde6a4c6 --- /dev/null +++ b/piker/ui/_interaction.py @@ -0,0 +1,77 @@ +""" +UX interaction customs. +""" +import pyqtgraph as pg +from pyqtgraph import functions as fn + +from ..log import get_logger +from ._style import _min_points_to_show + + +log = get_logger(__name__) + + +class ChartView(pg.ViewBox): + """Price chart view box with interaction behaviors you'd expect from + any interactive platform: + + - zoom on mouse scroll that auto fits y-axis + - no vertical scrolling + - zoom to a "fixed point" on the y-axis + """ + def __init__( + self, + parent=None, + **kwargs, + ): + super().__init__(parent=parent, **kwargs) + # disable vertical scrolling + self.setMouseEnabled(x=True, y=False) + self.linked_charts = None + + def wheelEvent(self, ev, axis=None): + """Override "center-point" location for scrolling. + + This is an override of the ``ViewBox`` method simply changing + the center of the zoom to be the y-axis. + + TODO: PR a method into ``pyqtgraph`` to make this configurable + """ + + if axis in (0, 1): + mask = [False, False] + mask[axis] = self.state['mouseEnabled'][axis] + else: + mask = self.state['mouseEnabled'][:] + + # don't zoom more then the min points setting + l, lbar, rbar, r = self.linked_charts.chart.bars_range() + vl = r - l + + if ev.delta() > 0 and vl <= _min_points_to_show: + log.trace("Max zoom bruh...") + return + if ev.delta() < 0 and vl >= len(self.linked_charts._array): + log.trace("Min zoom bruh...") + return + + # actual scaling factor + s = 1.015 ** (ev.delta() * -1 / 20) # self.state['wheelScaleFactor']) + s = [(None if m is False else s) for m in mask] + + # center = pg.Point( + # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) + # ) + + # XXX: scroll "around" the right most element in the view + furthest_right_coord = self.boundingRect().topRight() + center = pg.Point( + fn.invertQTransform( + self.childGroup.transform() + ).map(furthest_right_coord) + ) + + self._resetTarget() + self.scaleBy(s, center) + ev.accept() + self.sigRangeChangedManually.emit(mask) From 04e21a96da5cf3cbf58fc92158f97abacbc769fa Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 Aug 2020 07:41:18 -0400 Subject: [PATCH 075/206] Use partial, pass kwargs to `tractor._main()` --- piker/ui/_exec.py | 26 +++++++++++--------------- piker/ui/cli.py | 10 +++++++++- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 7a137fed..c169640d 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -4,8 +4,9 @@ Trio - Qt integration Run ``trio`` in guest mode on top of the Qt event loop. All global Qt runtime settings are mostly defined here. """ +from functools import partial import traceback -from typing import Tuple, Callable, Dict +from typing import Tuple, Callable, Dict, Any import PyQt5 # noqa from pyqtgraph import QtGui @@ -31,8 +32,8 @@ class MainWindow(QtGui.QMainWindow): def run_qtractor( func: Callable, args: Tuple, - kwargs: Dict, main_widget: QtGui.QWidget, + tractor_kwargs: Dict[str, Any] = {}, window_type: QtGui.QMainWindow = MainWindow, ) -> None: # avoids annoying message when entering debugger from qt loop @@ -90,26 +91,21 @@ def run_qtractor( } # setup tractor entry point args - args = ( - # async_fn - func, - # args (always append widgets list) - args + (widgets,), - # kwargs - kwargs, - # arbiter_addr - ( + main = partial( + tractor._main, + async_fn=func, + args=args + (widgets,), + arbiter_addr=( tractor._default_arbiter_host, tractor._default_arbiter_port, ), - # name - 'qtractor', + name='qtractor', + **tractor_kwargs, ) # guest mode trio.lowlevel.start_guest_run( - tractor._main, - *args, + main, run_sync_soon_threadsafe=run_sync_soon_threadsafe, done_callback=done_callback, ) diff --git a/piker/ui/cli.py b/piker/ui/cli.py index c1052733..74a3cae3 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -63,6 +63,7 @@ def monitor(config, rate, name, dhost, test, tl): name='monitor', loglevel=loglevel if tl else None, rpc_module_paths=['piker.ui.kivy.monitor'], + debug_mode=True, ) @@ -118,4 +119,11 @@ def chart(config, symbol, date, rate, test): brokername = config['broker'] tractorloglevel = config['tractorloglevel'] - _main(sym=symbol, brokername=brokername, loglevel=tractorloglevel) + _main( + sym=symbol, + brokername=brokername, + tractor_kwargs={ + 'debug_mode': True, + 'loglevel': tractorloglevel, + }, + ) From edb32e8c2b1027613cd81247e4bc20723c800929 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 Aug 2020 07:52:17 -0400 Subject: [PATCH 076/206] Drop weird chart type enum --- piker/ui/_graphics.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 4f8cc355..54cbcd43 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -370,11 +370,3 @@ class BarItems(pg.GraphicsObject): # p.setBrush(self.bear_brush) # p.drawRects(*rects[Quotes.close < Quotes.open]) - - -class ChartType(Enum): - """Bar type to graphics class map. - """ - BAR = BarItems - # CANDLESTICK = CandlestickItems - LINE = pg.PlotDataItem From fd21f4b0fe35b737ab29867843f69f8d7c0aac63 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 Aug 2020 15:32:09 -0400 Subject: [PATCH 077/206] WIP initial draft of FSP subsystem This is a first attempt at a financial signal processing subsystem which utilizes async generators for streaming frames of numpy array data between actors. In this initial attempt the focus is on processing price data and relaying it to the chart app for real-time display. So far this seems to work (with decent latency) but much more work is likely needed around improving the data model for even better latency and less data duplication. Surprisingly (or not?) a lot of simplifications to the charting code came out of this in terms of conducting graphics updates in streaming tasks instead of hiding them inside the obfuscated mess that is the Qt-style-inheritance-OO-90s-trash. The goal from here on wards will be to enforce strict semantics around reading and writing of data such that state is kept outside "object trees" as much as possible and streaming function semantics guide our flow model. Unsurprisingly, this reduction in "instance state" is happening wherever we use `trio` ;) A little summary on the technical changes: - not going to explain the fsp system yet; it's too nascent and probably going to get some heavy editing. - drop any "update" methods from the `LinkedCharts` type since each sub-chart will have it's own update task and thus a separate update loop; further individual graphics (per chart) may eventually require this same design. - delete `ChartView`; moved into separate mod. - add "stream from fsp" task to start our foray into real-time actor processed numpy streaming. --- piker/fsp.py | 161 +++++++++++++++++ piker/ui/_chart.py | 437 +++++++++++++++++++++------------------------ piker/ui/_style.py | 10 +- 3 files changed, 366 insertions(+), 242 deletions(-) create mode 100644 piker/fsp.py diff --git a/piker/fsp.py b/piker/fsp.py new file mode 100644 index 00000000..fff90a48 --- /dev/null +++ b/piker/fsp.py @@ -0,0 +1,161 @@ +""" +Financial signal processing for the peeps. +""" +from typing import AsyncIterator, List + +import numpy as np + +from .log import get_logger +from . import data + +log = get_logger(__name__) + + +def rec2array( + rec: np.ndarray, + fields: List[str] = None +) -> np.ndarray: + """Convert record array to std array. + + Taken from: + https://github.com/scikit-hep/root_numpy/blob/master/root_numpy/_utils.py#L20 + """ + simplify = False + + if fields is None: + fields = rec.dtype.names + elif isinstance(fields, str): + fields = [fields] + simplify = True + + # Creates a copy and casts all data to the same type + arr = np.dstack([rec[field] for field in fields]) + + # Check for array-type fields. If none, then remove outer dimension. + # Only need to check first field since np.dstack will anyway raise an + # exception if the shapes don't match + # np.dstack will also fail if fields is an empty list + if not rec.dtype[fields[0]].shape: + arr = arr[0] + + if simplify: + # remove last dimension (will be of size 1) + arr = arr.reshape(arr.shape[:-1]) + + return arr + + +async def pull_and_process( + bars: np.ndarray, + brokername: str, + # symbols: List[str], + symbol: str, + fsp_func_name: str, +) -> AsyncIterator[dict]: + + # async def _yield_bars(): + # yield bars + + # hist_out: np.ndarray = None + + func = latency + + # Conduct a single iteration of fsp with historical bars input + # async for hist_out in func(_yield_bars(), bars): + # yield {symbol: hist_out} + + # open a data feed stream with requested broker + async with data.open_feed( + brokername, + [symbol], + ) as (fquote, stream): + + # TODO: load appropriate fsp with input args + + async def filter_by_sym(sym, stream): + async for quotes in stream: + for symbol, quotes in quotes.items(): + if symbol == sym: + yield quotes + + async for processed in func( + filter_by_sym(symbol, stream), + bars, + ): + print(f"{fsp_func_name}: {processed}") + yield processed + + +# TODO: things to figure the fuck out: +# - how to handle non-plottable values +# - composition of fsps / implicit chaining + +async def latency( + source: 'TickStream[Dict[str, float]]', + ohlcv: np.ndarray +) -> AsyncIterator[np.ndarray]: + """Compute High-Low midpoint value. + """ + # TODO: do we want to offer yielding this async + # before the rt data connection comes up? + + # deliver zeros for all prior history + yield np.zeros(len(ohlcv)) + + _last = None + async for quote in source: + fill_time = quote.get('rtTime_s') + if fill_time and fill_time != _last: + value = quote['brokerd_ts'] - fill_time + print(f"latency: {value}") + yield value + + _last = fill_time + # ticks = quote.get('ticks', ()) + # for tick in ticks: + # if tick.get('type') == 'trade': + + +async def last( + source: 'TickStream[Dict[str, float]]', + ohlcv: np.ndarray +) -> AsyncIterator[np.ndarray]: + """Compute High-Low midpoint value. + """ + # first_frame = (await source.__anext__()) + + # deliver historical processed data first + yield ohlcv['close'] + + async for quote in source: + yield quote['close'] + + +async def wma( + source, #: AsyncStream[np.ndarray], + ohlcv: np.ndarray, # price time-frame "aware" + lookback: np.ndarray, # price time-frame "aware" + weights: np.ndarray, +) -> AsyncIterator[np.ndarray]: # i like FinSigStream + """Weighted moving average. + + ``weights`` is a sequence of already scaled values. As an example + for the WMA often found in "techincal analysis": + ``weights = np.arange(1, N) * N*(N-1)/2``. + """ + length = len(weights) + _lookback = np.zeros(length - 1) + + ohlcv.from_tf('5m') + + # async for frame_len, frame in source: + async for frame in source: + wma = np.convolve( + ohlcv[-length:]['close'], + # np.concatenate((_lookback, frame)), + weights, + 'valid' + ) + # todo: handle case where frame_len < length - 1 + _lookback = frame[-(length-1):] + yield wma diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index af7f52b6..cc5f1515 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,11 +1,10 @@ """ High level Qt chart widgets. """ -from typing import List, Optional, Tuple, Dict, Any +from typing import Optional, Tuple, Dict, Any import time from PyQt5 import QtCore, QtGui -from pyqtgraph import functions as fn import numpy as np import pyqtgraph as pg import tractor @@ -15,14 +14,15 @@ from ._axes import ( DynamicDateAxis, PriceAxis, ) -from ._graphics import CrossHair, ChartType -from ._style import _xaxis_at +from ._graphics import CrossHair, BarItems +from ._style import _xaxis_at, _min_points_to_show from ._source import Symbol from .. import brokers from .. import data from ..log import get_logger from ._exec import run_qtractor from ._source import ohlc_dtype +from ._interaction import ChartView from .. import fsp @@ -50,7 +50,7 @@ class ChartSpace(QtGui.QWidget): self.v_layout.addLayout(self.toolbar_layout) self.v_layout.addLayout(self.h_layout) - self._plot_cache = {} + self._chart_cache = {} def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() @@ -81,16 +81,19 @@ class ChartSpace(QtGui.QWidget): """ # XXX: let's see if this causes mem problems self.window.setWindowTitle(f'piker chart {symbol}') - self.chart = self._plot_cache.setdefault(symbol, LinkedSplitCharts()) + linkedcharts = self._chart_cache.setdefault( + symbol, + LinkedSplitCharts() + ) s = Symbol(key=symbol) # remove any existing plots if not self.h_layout.isEmpty(): - self.h_layout.removeWidget(self.chart) + self.h_layout.removeWidget(linkedcharts) - self.chart.plot(s, data) - self.h_layout.addWidget(self.chart) - return self.chart + main_chart = linkedcharts.plot_main(s, data) + self.h_layout.addWidget(linkedcharts) + return linkedcharts, main_chart # TODO: add signalling painter system # def add_signals(self): @@ -118,7 +121,7 @@ class LinkedSplitCharts(QtGui.QWidget): self._array: np.ndarray = None # main data source self._ch: CrossHair = None # crosshair graphics self.chart: ChartPlotWidget = None # main (ohlc) chart - self.subplots: List[ChartPlotWidget] = [] + self.subplots: Dict[Tuple[str, ...], ChartPlotWidget] = {} self.xaxis = DynamicDateAxis( orientation='bottom', @@ -148,29 +151,28 @@ class LinkedSplitCharts(QtGui.QWidget): """Set the proportion of space allocated for linked subcharts. """ major = 1 - prop - min_h_ind = int(self.height() * prop / len(self.subplots)) + min_h_ind = int((self.height() * prop) / len(self.subplots)) sizes = [int(self.height() * major)] sizes.extend([min_h_ind] * len(self.subplots)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) - def plot( + def plot_main( self, symbol: Symbol, array: np.ndarray, ohlc: bool = True, - ) -> None: + ) -> 'ChartPlotWidget': """Start up and show main (price) chart and all linked subcharts. """ self.digits = symbol.digits() - # XXX: this will eventually be a view onto shared mem - # or some higher level type / API + # TODO: this should eventually be a view onto shared mem or some + # higher level type / API self._array = array # add crosshairs self._ch = CrossHair( parent=self, - # subplots=[plot for plot, d in self.subplots], digits=self.digits ) self.chart = self.add_plot( @@ -179,26 +181,13 @@ class LinkedSplitCharts(QtGui.QWidget): xaxis=self.xaxis, ohlc=True, ) + # add crosshair graphic self.chart.addItem(self._ch) + + # style? self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) - # TODO: this is where we would load an indicator chain - # XXX: note, if this isn't index aligned with - # the source data the chart will go haywire. - inds = [('open', lambda a: a['close'])] - - for name, func in inds: - - # compute historical subchart values from input array - data = func(array) - - # create sub-plot - ind_chart = self.add_plot(name=name, array=data) - - self.subplots.append((ind_chart, func)) - - # scale split regions - self.set_split_sizes() + return self.chart def add_plot( self, @@ -211,25 +200,28 @@ class LinkedSplitCharts(QtGui.QWidget): If ``name`` == ``"main"`` the chart will be the the primary view. """ + if self.chart is None and name != 'main': + raise RuntimeError( + "A main plot must be created first with `.plot_main()`") + + # source of our custom interactions cv = ChartView() + cv.linked_charts = self + # use "indicator axis" by default xaxis = self.xaxis_ind if xaxis is None else xaxis cpw = ChartPlotWidget( - linked_charts=self, + array=array, parent=self.splitter, axisItems={'bottom': xaxis, 'right': PriceAxis()}, - # axisItems={'top': self.xaxis_ind, 'right': PriceAxis()}, viewBox=cv, ) # this name will be used to register the primary # graphics curve managed by the subchart cpw.name = name - cv.linked_charts = self cpw.plotItem.vb.linked_charts = self - cpw.setFrameStyle( - QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain - ) + cpw.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) # self.splitter.addWidget(cpw) @@ -240,68 +232,20 @@ class LinkedSplitCharts(QtGui.QWidget): if ohlc: cpw.draw_ohlc(array) else: - cpw.draw_curve(array, name) + cpw.draw_curve(array) # add to cross-hair's known plots self._ch.add_plot(cpw) + if name != "main": + # track by name + self.subplots[name] = cpw + + # scale split regions + self.set_split_sizes() + return cpw - def update_from_quote( - self, - quote: dict - ) -> List[pg.GraphicsObject]: - """Update all linked chart graphics with a new quote - datum. - - Return the modified graphics objects in a list. - """ - # TODO: eventually we'll want to update bid/ask labels and other - # data as subscribed by underlying UI consumers. - last = quote.get('last') or quote['close'] - index, time, open, high, low, close, volume = self._array[-1] - - # update ohlc (I guess we're enforcing this for now?) - # overwrite from quote - self._array[-1] = ( - index, - time, - open, - max(high, last), - min(low, last), - last, - volume, - ) - self.update_from_array(self._array) - - def update_from_array( - self, - array: np.ndarray, - **kwargs, - ) -> None: - """Update all linked chart graphics with a new input array. - - Return the modified graphics objects in a list. - """ - # update the ohlc sequence graphics chart - # we send a reference to the whole updated array - self.chart.update_from_array(array, **kwargs) - - # TODO: the "data" here should really be a function - # and it should be managed and computed outside of this UI - graphics = [] - for chart, func in self.subplots: - # process array in entirely every update - # TODO: change this for streaming - data = func(array) - graphic = chart.update_from_array(data, name=chart.name, **kwargs) - graphics.append(graphic) - - return graphics - - -_min_points_to_show = 3 - class ChartPlotWidget(pg.PlotWidget): """``GraphicsView`` subtype containing a single ``PlotItem``. @@ -323,7 +267,8 @@ class ChartPlotWidget(pg.PlotWidget): def __init__( self, - linked_charts, + # the data view we generate graphics from + array: np.ndarray, **kwargs, # parent=None, # background='default', @@ -332,7 +277,8 @@ class ChartPlotWidget(pg.PlotWidget): """Configure chart display settings. """ super().__init__(**kwargs) - self.parent = linked_charts + self._array = array # readonly view of data + self._graphics = {} # registry of underlying graphics # XXX: label setting doesn't seem to work? # likely custom graphics need special handling @@ -341,9 +287,6 @@ class ChartPlotWidget(pg.PlotWidget): # label.setText("Yo yoyo") # label.setText("x=") - # to be filled in when graphics are rendered by name - self._graphics = {} - # show only right side axes self.hideAxis('left') self.showAxis('right') @@ -383,28 +326,30 @@ class ChartPlotWidget(pg.PlotWidget): """ l, r = self.view_range() lbar = max(l, 0) - rbar = min(r, len(self.parent._array)) + rbar = min(r, len(self._array)) return l, lbar, rbar, r def draw_ohlc( self, data: np.ndarray, # XXX: pretty sure this is dumb and we don't need an Enum - style: ChartType = ChartType.BAR, - ) -> None: + style: pg.GraphicsObject = BarItems, + ) -> pg.GraphicsObject: """Draw OHLC datums to chart. """ # remember it's an enum type.. - graphics = style.value() + graphics = style() # adds all bar/candle graphics objects for each data point in # the np array buffer to be drawn on next render cycle graphics.draw_from_data(data) - self._graphics['main'] = graphics self.addItem(graphics) + self._graphics['ohlc_main'] = graphics + # set xrange limits xlast = data[-1]['index'] + # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) @@ -413,15 +358,20 @@ class ChartPlotWidget(pg.PlotWidget): def draw_curve( self, data: np.ndarray, - name: Optional[str] = None, - ) -> None: + name: Optional[str] = 'line_main', + ) -> pg.PlotDataItem: # draw the indicator as a plain curve - curve = pg.PlotDataItem(data, antialias=True) + curve = pg.PlotDataItem( + data, + antialias=True, + # TODO: see how this handles with custom ohlcv bars graphics + clipToView=True, + ) self.addItem(curve) # register overlay curve with name if not self._graphics and name is None: - name = 'main' + name = 'line_main' self._graphics[name] = curve @@ -439,16 +389,14 @@ class ChartPlotWidget(pg.PlotWidget): def update_from_array( self, + name: str, array: np.ndarray, - name: str = 'main', **kwargs, ) -> pg.GraphicsObject: + graphics = self._graphics[name] graphics.update_from_array(array, **kwargs) - # update view - self._set_yrange() - return graphics def _set_yrange( @@ -470,7 +418,7 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: logic to check if end of bars in view extra = view_len - _min_points_to_show begin = 0 - extra - end = len(self.parent._array) - 1 + extra + end = len(self._array) - 1 + extra log.trace( f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" @@ -480,7 +428,7 @@ class ChartPlotWidget(pg.PlotWidget): self._set_xlimits(begin, end) # TODO: this should be some kind of numpy view api - bars = self.parent._array[lbar:rbar] + bars = self._array[lbar:rbar] if not len(bars): # likely no data loaded yet log.error(f"WTF bars_range = {lbar}:{rbar}") @@ -523,72 +471,6 @@ class ChartPlotWidget(pg.PlotWidget): self.scene().leaveEvent(ev) -class ChartView(pg.ViewBox): - """Price chart view box with interaction behaviors you'd expect from - any interactive platform: - - - zoom on mouse scroll that auto fits y-axis - - no vertical scrolling - - zoom to a "fixed point" on the y-axis - """ - def __init__( - self, - parent=None, - **kwargs, - ): - super().__init__(parent=parent, **kwargs) - # disable vertical scrolling - self.setMouseEnabled(x=True, y=False) - self.linked_charts = None - - def wheelEvent(self, ev, axis=None): - """Override "center-point" location for scrolling. - - This is an override of the ``ViewBox`` method simply changing - the center of the zoom to be the y-axis. - - TODO: PR a method into ``pyqtgraph`` to make this configurable - """ - - if axis in (0, 1): - mask = [False, False] - mask[axis] = self.state['mouseEnabled'][axis] - else: - mask = self.state['mouseEnabled'][:] - - # don't zoom more then the min points setting - l, lbar, rbar, r = self.linked_charts.chart.bars_range() - vl = r - l - - if ev.delta() > 0 and vl <= _min_points_to_show: - log.trace("Max zoom bruh...") - return - if ev.delta() < 0 and vl >= len(self.linked_charts._array): - log.trace("Min zoom bruh...") - return - - # actual scaling factor - s = 1.015 ** (ev.delta() * -1 / 20) # self.state['wheelScaleFactor']) - s = [(None if m is False else s) for m in mask] - - # center = pg.Point( - # fn.invertQTransform(self.childGroup.transform()).map(ev.pos()) - # ) - - # XXX: scroll "around" the right most element in the view - furthest_right_coord = self.boundingRect().topRight() - center = pg.Point( - fn.invertQTransform( - self.childGroup.transform() - ).map(furthest_right_coord) - ) - - self._resetTarget() - self.scaleBy(s, center) - ev.accept() - self.sigRangeChangedManually.emit(mask) - - async def add_new_bars(delay_s, linked_charts): """Task which inserts new bars into the ohlc every ``delay_s`` seconds. """ @@ -600,7 +482,8 @@ async def add_new_bars(delay_s, linked_charts): # adjust delay to compensate for trio processing time ad = delay_s - 0.002 - ohlc = linked_charts._array + price_chart = linked_charts.chart + ohlc = price_chart._array async def sleep(): """Sleep until next time frames worth has passed from last bar. @@ -623,27 +506,54 @@ async def add_new_bars(delay_s, linked_charts): # - update last open price correctly instead # of copying it from last bar's close # - 5 sec bar lookback-autocorrection like tws does? - (index, t, close) = ohlc[-1][['index', 'time', 'close']] - new = np.append( - ohlc, - np.array( - [(index + 1, t + delay_s, close, close, - close, close, 0)], - dtype=ohlc.dtype - ), - ) - ohlc = linked_charts._array = new + + def incr_ohlc_array(array: np.ndarray): + (index, t, close) = array[-1][['index', 'time', 'close']] + new_array = np.append( + array, + np.array( + [(index + 1, t + delay_s, close, close, + close, close, 0)], + dtype=array.dtype + ), + ) + return new_array + + # add new increment/bar + ohlc = price_chart._array = incr_ohlc_array(ohlc) + + # TODO: generalize this increment logic + for name, chart in linked_charts.subplots.items(): + data = chart._array + chart._array = np.append( + data, + np.array(data[-1], dtype=data.dtype) + ) + + # read value at "open" of bar last_quote = ohlc[-1] - # we **don't** update the bar right now - # since the next quote that arrives should + # We **don't** update the bar right now + # since the next quote that arrives should in the + # tick streaming task await sleep() - # if the last bar has not changed print a flat line and - # move to the next + # XXX: If the last bar has not changed print a flat line and + # move to the next. This is a "animation" choice that we may not + # keep. if last_quote == ohlc[-1]: log.debug("Printing flat line for {sym}") - linked_charts.update_from_array(ohlc) + price_chart.update_from_array('ohlc_main', ohlc) + + # resize view + price_chart._set_yrange() + + + for name, chart in linked_charts.subplots.items(): + chart.update_from_array('line_main', chart._array) + + # resize view + chart._set_yrange() async def _async_main( @@ -670,7 +580,9 @@ async def _async_main( # remember, msgpack-numpy's ``from_buffer` returns read-only array bars = np.array(bars[list(ohlc_dtype.names)]) - linked_charts = chart_app.load_symbol(sym, bars) + + # load in symbol's ohlc data + linked_charts, chart = chart_app.load_symbol(sym, bars) # determine ohlc delay between bars times = bars['time'] @@ -678,69 +590,120 @@ async def _async_main( # find expected time step between datums delay = times[-1] - times[times != times[-1]][-1] - async def stream_to_chart(func): - - async with tractor.open_nursery() as n: - portal = await n.run_in_actor( - f'fsp_{func.__name__}', - func, - brokername=brokermod.name, - sym=sym, - # loglevel='info', - ) - stream = await portal.result() - - # retreive named layout and style instructions - layout = await stream.__anext__() - - async for quote in stream: - ticks = quote.get('ticks') - if ticks: - for tick in ticks: - print(tick) - async with trio.open_nursery() as n: + # load initial fsp chain (otherwise known as "indicators") + n.start_soon( + chart_from_fsp, + linked_charts, + fsp.latency, + sym, + bars, + brokermod, + loglevel, + ) + + # graphics update loop + async with data.open_feed( brokername, [sym], loglevel=loglevel, ) as (fquote, stream): - # start downstream processor - n.start_soon(stream_to_chart, fsp.broker_latency) # wait for a first quote before we start any update tasks quote = await stream.__anext__() + print(f'RECEIVED FIRST QUOTE {quote}') # start graphics tasks after receiving first live quote n.start_soon(add_new_bars, delay, linked_charts) - async for quote in stream: - ticks = quote.get('ticks') - if ticks: + async for quotes in stream: + for sym, quote in quotes.items(): + ticks = quote.get('ticks', ()) for tick in ticks: if tick.get('type') == 'trade': - linked_charts.update_from_quote( - {'last': tick['price']} + + # TODO: eventually we'll want to update + # bid/ask labels and other data as + # subscribed by underlying UI consumers. + # last = quote.get('last') or quote['close'] + last = tick['price'] + + # update ohlc (I guess we're enforcing this + # for now?) overwrite from quote + high, low = chart._array[-1][['high', 'low']] + chart._array[['high', 'low', 'close']][-1] = ( + max(high, last), + min(low, last), + last, ) + chart.update_from_array( + 'ohlc_main', + chart._array, + ) + + +async def chart_from_fsp( + linked_charts, + fsp_func, + sym, + bars, + brokermod, + loglevel, +) -> None: + """Start financial signal processing in subactor. + + Pass target entrypoint and historical data. + """ + func_name = fsp_func.__name__ + + async with tractor.open_nursery() as n: + portal = await n.run_in_actor( + f'fsp.{func_name}', # name as title of sub-chart + + # subactor entrypoint + fsp.pull_and_process, + bars=bars, + brokername=brokermod.name, + symbol=sym, + fsp_func_name=func_name, + + # tractor config + loglevel=loglevel, + ) + + stream = await portal.result() + + # receive processed historical data-array as first message + history = (await stream.__anext__()) + + # TODO: enforce type checking here + newbars = np.array(history) + + chart = linked_charts.add_plot( + name=func_name, + array=newbars, + ) + + # update sub-plot graphics + async for value in stream: + chart._array[-1] = value + chart.update_from_array('line_main', chart._array) + chart._set_yrange() def _main( sym: str, brokername: str, - **qtractor_kwargs, + tractor_kwargs, ) -> None: """Sync entry point to start a chart app. """ # Qt entry point run_qtractor( - # func - _async_main, - # args, - (sym, brokername), - # kwargs passed through - qtractor_kwargs, - # main widget - ChartSpace, - # **qtractor_kwargs + func=_async_main, + args=(sym, brokername), + main_widget=ChartSpace, + tractor_kwargs=tractor_kwargs, ) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index a5e3b64e..ea497a73 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -1,11 +1,10 @@ """ -Qt styling. +Qt UI styling. """ import pyqtgraph as pg from PyQt5 import QtGui - # chart-wide font _font = QtGui.QFont("Hack", 4) _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) @@ -15,6 +14,10 @@ _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) _xaxis_at = 'bottom' +# charting config +_min_points_to_show = 3 + + _tina_mode = False @@ -22,8 +25,5 @@ def enable_tina_mode() -> None: """Enable "tina mode" to make everything look "conventional" like your pet hedgehog always wanted. """ - - _tina_mode = True - # white background (for tinas like our pal xb) pg.setConfigOption('background', 'w') From 61e460a4223b2c60072bf95a75e70924f516645d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 21 Aug 2020 14:28:02 -0400 Subject: [PATCH 078/206] Start brokers.api module --- piker/brokers/api.py | 53 ++++++++++++++++++++++++++++++++++++++ piker/brokers/questrade.py | 44 +++---------------------------- 2 files changed, 56 insertions(+), 41 deletions(-) create mode 100644 piker/brokers/api.py diff --git a/piker/brokers/api.py b/piker/brokers/api.py new file mode 100644 index 00000000..29fe5577 --- /dev/null +++ b/piker/brokers/api.py @@ -0,0 +1,53 @@ +""" +Actor-aware broker agnostic interface. +""" +from contextlib import asynccontextmanager, AsyncExitStack + +import trio +import tractor + +from . import get_brokermod +from ..log import get_logger + + +log = get_logger(__name__) + + +@asynccontextmanager +async def get_cached_client( + brokername: str, + *args, + **kwargs, +) -> 'Client': # noqa + """Get a cached broker client from the current actor's local vars. + + If one has not been setup do it and cache it. + """ + # check if a cached client is in the local actor's statespace + ss = tractor.current_actor().statespace + clients = ss.setdefault('clients', {'_lock': trio.Lock()}) + lock = clients['_lock'] + client = None + try: + log.info(f"Loading existing `{brokername}` daemon") + async with lock: + client = clients[brokername] + client._consumers += 1 + yield client + except KeyError: + log.info(f"Creating new client for broker {brokername}") + async with lock: + brokermod = get_brokermod(brokername) + exit_stack = AsyncExitStack() + client = await exit_stack.enter_async_context( + brokermod.get_client() + ) + client._consumers = 0 + client._exit_stack = exit_stack + clients[brokername] = client + yield client + finally: + client._consumers -= 1 + if client._consumers <= 0: + # teardown the client + await client._exit_stack.aclose() diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 98989741..6063b9f6 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -31,6 +31,8 @@ from ._util import resproc, BrokerError, SymbolNotFound from ..log import get_logger, colorize_json, get_console_log from .._async_utils import async_lifo_cache from . import get_brokermod +from . import api + log = get_logger(__name__) @@ -1024,46 +1026,6 @@ def format_option_quote( return new, displayable -@asynccontextmanager -async def get_cached_client( - brokername: str, - *args, - **kwargs, -) -> 'Client': - """Get a cached broker client from the current actor's local vars. - - If one has not been setup do it and cache it. - """ - # check if a cached client is in the local actor's statespace - ss = tractor.current_actor().statespace - clients = ss.setdefault('clients', {'_lock': trio.Lock()}) - lock = clients['_lock'] - client = None - try: - log.info(f"Loading existing `{brokername}` daemon") - async with lock: - client = clients[brokername] - client._consumers += 1 - yield client - except KeyError: - log.info(f"Creating new client for broker {brokername}") - async with lock: - brokermod = get_brokermod(brokername) - exit_stack = contextlib.AsyncExitStack() - client = await exit_stack.enter_async_context( - brokermod.get_client() - ) - client._consumers = 0 - client._exit_stack = exit_stack - clients[brokername] = client - yield client - finally: - client._consumers -= 1 - if client._consumers <= 0: - # teardown the client - await client._exit_stack.aclose() - - async def smoke_quote( get_quotes, tickers @@ -1214,7 +1176,7 @@ async def stream_quotes( # XXX: required to propagate ``tractor`` loglevel to piker logging get_console_log(loglevel) - async with get_cached_client('questrade') as client: + async with api.get_cached_client('questrade') as client: if feed_type == 'stock': formatter = format_stock_quote get_quotes = await stock_quoter(client, symbols) From f46fa99a6e52946f6e428d688cbab5fd401b70df Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 Aug 2020 14:15:52 -0400 Subject: [PATCH 079/206] Add "contents" labels to charts Add a default "contents label" (eg. OHLC values for bar charts) to each chart and update on crosshair interaction. Few technical changes to make this happen: - adjust bar graphics to have the HL line be in the "middle" of the underlying arrays' "index range" in the containing view. - add a label dict each chart's graphics name to a label + update routine - use symbol names instead of this "main" identifier crap for referring to particular price curves/graphics --- piker/ui/_chart.py | 125 ++++++++++++++++++++++++++++-------------- piker/ui/_graphics.py | 54 ++++++++++++------ 2 files changed, 120 insertions(+), 59 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index cc5f1515..b92103ae 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,7 +1,7 @@ """ High level Qt chart widgets. """ -from typing import Optional, Tuple, Dict, Any +from typing import Tuple, Dict, Any import time from PyQt5 import QtCore, QtGui @@ -172,14 +172,15 @@ class LinkedSplitCharts(QtGui.QWidget): # add crosshairs self._ch = CrossHair( - parent=self, + linkedsplitcharts=self, digits=self.digits ) self.chart = self.add_plot( - name='main', + name=symbol.key, array=array, xaxis=self.xaxis, ohlc=True, + _is_main=True, ) # add crosshair graphic self.chart.addItem(self._ch) @@ -195,12 +196,13 @@ class LinkedSplitCharts(QtGui.QWidget): array: np.ndarray, xaxis: DynamicDateAxis = None, ohlc: bool = False, + _is_main: bool = False, ) -> 'ChartPlotWidget': """Add (sub)plots to chart widget by name. If ``name`` == ``"main"`` the chart will be the the primary view. """ - if self.chart is None and name != 'main': + if self.chart is None and not _is_main: raise RuntimeError( "A main plot must be created first with `.plot_main()`") @@ -230,14 +232,14 @@ class LinkedSplitCharts(QtGui.QWidget): # draw curve graphics if ohlc: - cpw.draw_ohlc(array) + cpw.draw_ohlc(name, array) else: - cpw.draw_curve(array) + cpw.draw_curve(name, array) # add to cross-hair's known plots self._ch.add_plot(cpw) - if name != "main": + if not _is_main: # track by name self.subplots[name] = cpw @@ -279,13 +281,7 @@ class ChartPlotWidget(pg.PlotWidget): super().__init__(**kwargs) self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics - - # XXX: label setting doesn't seem to work? - # likely custom graphics need special handling - # label = pg.LabelItem(justify='right') - # self.addItem(label) - # label.setText("Yo yoyo") - # label.setText("x=") + self._labels = {} # registry of underlying graphics # show only right side axes self.hideAxis('left') @@ -303,6 +299,11 @@ class ChartPlotWidget(pg.PlotWidget): # based on ohlc contents self.sigXRangeChanged.connect(self._set_yrange) + def _update_contents_label(self, index: int) -> None: + if index > 0 and index < len(self._array): + for name, (label, update) in self._labels.items(): + update(index) + def _set_xlimits( self, xfirst: int, @@ -331,6 +332,7 @@ class ChartPlotWidget(pg.PlotWidget): def draw_ohlc( self, + name: str, data: np.ndarray, # XXX: pretty sure this is dumb and we don't need an Enum style: pg.GraphicsObject = BarItems, @@ -345,7 +347,25 @@ class ChartPlotWidget(pg.PlotWidget): graphics.draw_from_data(data) self.addItem(graphics) - self._graphics['ohlc_main'] = graphics + self._graphics[name] = graphics + + # XXX: How to stack labels vertically? + label = pg.LabelItem( + justify='left', + size='5pt', + ) + self.scene().addItem(label) + + def update(index: int) -> None: + label.setText( + "{name} O:{} H:{} L:{} C:{} V:{}".format( + *self._array[index].item()[2:], + name=name, + ) + ) + + self._labels[name] = (label, update) + self._update_contents_label(index=-1) # set xrange limits xlast = data[-1]['index'] @@ -357,8 +377,8 @@ class ChartPlotWidget(pg.PlotWidget): def draw_curve( self, + name: str, data: np.ndarray, - name: Optional[str] = 'line_main', ) -> pg.PlotDataItem: # draw the indicator as a plain curve curve = pg.PlotDataItem( @@ -371,10 +391,24 @@ class ChartPlotWidget(pg.PlotWidget): # register overlay curve with name if not self._graphics and name is None: - name = 'line_main' + name = 'a_line_bby' self._graphics[name] = curve + # XXX: How to stack labels vertically? + label = pg.LabelItem( + justify='left', + size='5pt', + ) + self.scene().addItem(label) + + def update(index: int) -> None: + data = self._array[index] + label.setText(f"{name}: {index} {data}") + + self._labels[name] = (label, update) + self._update_contents_label(index=-1) + # set a "startup view" xlast = len(data) - 1 @@ -531,29 +565,28 @@ async def add_new_bars(delay_s, linked_charts): ) # read value at "open" of bar - last_quote = ohlc[-1] + # last_quote = ohlc[-1] + # XXX: If the last bar has not changed print a flat line and + # move to the next. This is a "animation" choice that we may not + # keep. + # if last_quote == ohlc[-1]: + # log.debug("Printing flat line for {sym}") + + # update chart graphics and resize view + price_chart.update_from_array(price_chart.name, ohlc) + price_chart._set_yrange() + + for name, chart in linked_charts.subplots.items(): + chart.update_from_array(chart.name, chart._array) + chart._set_yrange() # We **don't** update the bar right now # since the next quote that arrives should in the # tick streaming task await sleep() - # XXX: If the last bar has not changed print a flat line and - # move to the next. This is a "animation" choice that we may not - # keep. - if last_quote == ohlc[-1]: - log.debug("Printing flat line for {sym}") - price_chart.update_from_array('ohlc_main', ohlc) - - # resize view - price_chart._set_yrange() - - - for name, chart in linked_charts.subplots.items(): - chart.update_from_array('line_main', chart._array) - - # resize view - chart._set_yrange() + # TODO: should we update a graphics again time here? + # Think about race conditions with data update task. async def _async_main( @@ -639,9 +672,10 @@ async def _async_main( last, ) chart.update_from_array( - 'ohlc_main', + chart.name, chart._array, ) + chart._set_yrange() async def chart_from_fsp( @@ -676,7 +710,7 @@ async def chart_from_fsp( stream = await portal.result() # receive processed historical data-array as first message - history = (await stream.__anext__()) + history: np.ndarray = (await stream.__anext__()) # TODO: enforce type checking here newbars = np.array(history) @@ -686,10 +720,19 @@ async def chart_from_fsp( array=newbars, ) - # update sub-plot graphics + # check for data length mis-allignment and fill missing values + diff = len(chart._array) - len(linked_charts.chart._array) + if diff < 0: + data = chart._array + chart._array = np.append( + data, + np.full(abs(diff), data[-1], dtype=data.dtype) + ) + + # update chart graphics async for value in stream: chart._array[-1] = value - chart.update_from_array('line_main', chart._array) + chart.update_from_array(chart.name, chart._array) chart._set_yrange() @@ -703,7 +746,7 @@ def _main( # Qt entry point run_qtractor( func=_async_main, - args=(sym, brokername), - main_widget=ChartSpace, - tractor_kwargs=tractor_kwargs, + args=(sym, brokername), + main_widget=ChartSpace, + tractor_kwargs=tractor_kwargs, ) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 54cbcd43..edd75987 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -2,7 +2,6 @@ Chart graphics for displaying a slew of different data types. """ from typing import List -from enum import Enum from itertools import chain import numpy as np @@ -23,10 +22,14 @@ _mouse_rate_limit = 30 class CrossHair(pg.GraphicsObject): - def __init__(self, parent, digits: int = 0): + def __init__( + self, + linkedsplitcharts: 'LinkedSplitCharts', + digits: int = 0 + ) -> None: super().__init__() self.pen = pg.mkPen('#a9a9a9') # gray? - self.parent = parent + self.lsc = linkedsplitcharts self.graphics = {} self.plots = [] self.active_plot = None @@ -110,18 +113,27 @@ class CrossHair(pg.GraphicsObject): # mouse was not on active plot return - self.graphics[self.active_plot]['hl'].setY( - mouse_point.y() - ) + x, y = mouse_point.x(), mouse_point.y() + + plot = self.active_plot + + self.graphics[plot]['hl'].setY(y) + self.graphics[self.active_plot]['yl'].update_label( evt_post=pos, point_view=mouse_point ) - # move the vertical line to the current x coordinate in all charts - for opts in self.graphics.values(): - opts['vl'].setX(mouse_point.x()) + for plot, opts in self.graphics.items(): + # move the vertical line to the current x + opts['vl'].setX(x) + + # update the chart's "contents" label + plot._update_contents_label(int(x)) # update the label on the bottom of the crosshair - self.xaxis_label.update_label(evt_post=pos, point_view=mouse_point) + self.xaxis_label.update_label( + evt_post=pos, + point_view=mouse_point + ) def boundingRect(self): try: @@ -149,8 +161,8 @@ def _mk_lines_array(data: List, size: int) -> np.ndarray: def bars_from_ohlc( data: np.ndarray, + w: float, start: int = 0, - w: float = 0.43, ) -> np.ndarray: """Generate an array of lines objects from input ohlc data. """ @@ -160,9 +172,15 @@ def bars_from_ohlc( open, high, low, close, index = q[ ['open', 'high', 'low', 'close', 'index']] + # place the x-coord start as "middle" of the drawing range such + # that the open arm line-graphic is at the left-most-side of + # the indexe's range according to the view mapping. + index_start = index + w + # high - low line if low != high: - hl = QLineF(index, low, index, high) + # hl = QLineF(index, low, index, high) + hl = QLineF(index_start, low, index_start, high) else: # XXX: if we don't do it renders a weird rectangle? # see below too for handling this later... @@ -170,9 +188,9 @@ def bars_from_ohlc( hl._flat = True # open line - o = QLineF(index - w, open, index, open) + o = QLineF(index_start - w, open, index_start, open) # close line - c = QLineF(index + w, close, index, close) + c = QLineF(index_start + w, close, index_start, close) # indexing here is as per the below comments # lines[3*i:3*i+3] = (hl, o, c) @@ -207,7 +225,7 @@ class BarItems(pg.GraphicsObject): sigPlotChanged = QtCore.Signal(object) # 0.5 is no overlap between arms, 1.0 is full overlap - w: float = 0.4 + w: float = 0.43 bull_pen = pg.mkPen('#808080') # XXX: tina mode, see below @@ -234,7 +252,7 @@ class BarItems(pg.GraphicsObject): ): """Draw OHLC datum graphics from a ``np.recarray``. """ - lines = bars_from_ohlc(data, start=start) + lines = bars_from_ohlc(data, self.w, start=start) # save graphics for later reference and keep track # of current internal "last index" @@ -280,7 +298,7 @@ class BarItems(pg.GraphicsObject): if extra > 0: # generate new graphics to match provided array new = array[index:index + extra] - lines = bars_from_ohlc(new) + lines = bars_from_ohlc(new, self.w) bars_added = len(lines) self.lines[index:index + bars_added] = lines self.index += bars_added @@ -311,7 +329,7 @@ class BarItems(pg.GraphicsObject): # if the bar was flat it likely does not have # the index set correctly due to a rendering bug # see above - body.setLine(i, low, i, high) + body.setLine(i + self.w, low, i + self.w, high) body._flat = False else: body.setLine(body.x1(), low, body.x2(), high) From 58b2e7e395b094915463679c8ff3b2a899c8c1da Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 Aug 2020 21:45:13 -0400 Subject: [PATCH 080/206] Refer to main chart's data for date axis --- piker/ui/_axes.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index b64b72ec..d586bbda 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -12,17 +12,25 @@ from ._style import _font class PriceAxis(pg.AxisItem): - def __init__(self): + def __init__( + self, + # chart: 'ChartPlotWidget', + ) -> None: super().__init__(orientation='right') self.setStyle(**{ - 'textFillLimits': [(0, 0.8)], - # 'tickTextWidth': 5, - # 'tickTextHeight': 5, + 'textFillLimits': [(0, 1)], + # 'tickTextWidth': 10, + # 'tickTextHeight': 25, # 'autoExpandTextSpace': True, # 'maxTickLength': -20, + # 'stopAxisAtTick': (True, True), }) self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font) + self.setWidth(150) + # self.chart = chart + # accesed normally via + # .getAxis('right') # XXX: drop for now since it just eats up h space @@ -56,7 +64,7 @@ class DynamicDateAxis(pg.AxisItem): # strings = super().tickStrings(values, scale, spacing) s_period = 'D1' strings = [] - bars = self.linked_charts._array + bars = self.linked_charts.chart._array quotes_count = len(bars) - 1 for ibar in values: @@ -143,7 +151,7 @@ class XAxisLabel(AxisLabel): def tick_to_string(self, tick_pos): # TODO: change to actual period tpl = self.parent.tick_tpl['D1'] - bars = self.parent.linked_charts._array + bars = self.parent.linked_charts.chart._array if tick_pos > len(bars): return 'Unknown Time' return fromtimestamp(bars[round(tick_pos)]['time']).strftime(tpl) From 8d2933817448a181fa7d5618f664b67d1bc6deb9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 Aug 2020 21:45:34 -0400 Subject: [PATCH 081/206] Cleanup latency tracker --- piker/fsp.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/piker/fsp.py b/piker/fsp.py index fff90a48..40f14f3a 100644 --- a/piker/fsp.py +++ b/piker/fsp.py @@ -102,19 +102,16 @@ async def latency( # deliver zeros for all prior history yield np.zeros(len(ohlcv)) - _last = None async for quote in source: - fill_time = quote.get('rtTime_s') - if fill_time and fill_time != _last: - value = quote['brokerd_ts'] - fill_time - print(f"latency: {value}") + ts = quote.get('broker_ts') + if ts: + print( + f"broker time: {quote['broker_ts']}" + f"brokerd time: {quote['brokerd_ts']}" + ) + value = quote['brokerd_ts'] - quote['broker_ts'] yield value - _last = fill_time - # ticks = quote.get('ticks', ()) - # for tick in ticks: - # if tick.get('type') == 'trade': - async def last( source: 'TickStream[Dict[str, float]]', @@ -122,8 +119,6 @@ async def last( ) -> AsyncIterator[np.ndarray]: """Compute High-Low midpoint value. """ - # first_frame = (await source.__anext__()) - # deliver historical processed data first yield ohlcv['close'] From d7466a58b41ec53ebda7c74e2810789d7eb6bc38 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Aug 2020 12:27:41 -0400 Subject: [PATCH 082/206] Add updateable y-sticky label --- piker/ui/_axes.py | 127 +++++++++++++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 36 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index d586bbda..64c5a007 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -3,11 +3,12 @@ Chart axes graphics and behavior. """ import pyqtgraph as pg from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import QPointF # from .quantdom.base import Quotes from .quantdom.utils import fromtimestamp -from ._style import _font +from ._style import _font, hcolor class PriceAxis(pg.AxisItem): @@ -18,7 +19,7 @@ class PriceAxis(pg.AxisItem): ) -> None: super().__init__(orientation='right') self.setStyle(**{ - 'textFillLimits': [(0, 1)], + 'textFillLimits': [(0, 0.5)], # 'tickTextWidth': 10, # 'tickTextHeight': 25, # 'autoExpandTextSpace': True, @@ -52,10 +53,11 @@ class DynamicDateAxis(pg.AxisItem): # default styling self.setStyle( tickTextOffset=7, - textFillLimits=[(0, 0.90)], + textFillLimits=[(0, 0.70)], # TODO: doesn't seem to work -> bug in pyqtgraph? # tickTextHeight=2, ) + # self.setHeight(35) def tickStrings(self, values, scale, spacing): # if len(values) > 1 or not values: @@ -80,13 +82,13 @@ class DynamicDateAxis(pg.AxisItem): class AxisLabel(pg.GraphicsObject): # bg_color = pg.mkColor('#a9a9a9') - bg_color = pg.mkColor('#808080') - fg_color = pg.mkColor('#000000') + bg_color = pg.mkColor(hcolor('gray')) + fg_color = pg.mkColor(hcolor('black')) def __init__( self, parent=None, - digits=0, + digits=1, color=None, opacity=1, **kwargs @@ -96,33 +98,17 @@ class AxisLabel(pg.GraphicsObject): self.opacity = opacity self.label_str = '' self.digits = digits - # self.quotes_count = len(Quotes) - 1 + # some weird color convertion logic? if isinstance(color, QtGui.QPen): self.bg_color = color.color() - self.fg_color = pg.mkColor('#ffffff') + self.fg_color = pg.mkColor(hcolor('black')) elif isinstance(color, list): self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} - self.fg_color = pg.mkColor('#ffffff') + self.fg_color = pg.mkColor(hcolor('white')) self.setFlag(self.ItemIgnoresTransformations) - def tick_to_string(self, tick_pos): - raise NotImplementedError() - - def boundingRect(self): # noqa - raise NotImplementedError() - - def update_label(self, evt_post, point_view): - raise NotImplementedError() - - def update_label_test(self, ypos=0, ydata=0): - self.label_str = self.tick_to_string(ydata) - height = self.boundingRect().height() - offset = 0 # if have margins - new_pos = QtCore.QPointF(0, ypos - height / 2 - offset) - self.setPos(new_pos) - def paint(self, p, option, widget): p.setRenderHint(p.TextAntialiasing, True) p.setPen(self.fg_color) @@ -141,12 +127,38 @@ class AxisLabel(pg.GraphicsObject): p.drawText(option.rect, self.text_flags, self.label_str) + # uggggghhhh + def tick_to_string(self, tick_pos): + raise NotImplementedError() + + def boundingRect(self): # noqa + raise NotImplementedError() + + def update_label(self, evt_post, point_view): + raise NotImplementedError() + + # end uggggghhhh + + +# _common_text_flags = ( +# QtCore.Qt.TextDontClip | +# QtCore.Qt.AlignCenter | +# QtCore.Qt.AlignTop | +# QtCore.Qt.AlignHCenter | +# QtCore.Qt.AlignVCenter +# ) + class XAxisLabel(AxisLabel): text_flags = ( - QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter | QtCore.Qt.AlignTop + QtCore.Qt.TextDontClip + | QtCore.Qt.AlignCenter + # | QtCore.Qt.AlignTop + | QtCore.Qt.AlignVCenter + # | QtCore.Qt.AlignHCenter ) + # text_flags = _common_text_flags def tick_to_string(self, tick_pos): # TODO: change to actual period @@ -157,34 +169,77 @@ class XAxisLabel(AxisLabel): return fromtimestamp(bars[round(tick_pos)]['time']).strftime(tpl) def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 145, 50) + return QtCore.QRectF(0, 0, 145, 40) - def update_label(self, evt_post, point_view): - ibar = point_view.x() + def update_label(self, abs_pos, data): + # ibar = view_pos.x() # if ibar > self.quotes_count: # return - self.label_str = self.tick_to_string(ibar) + self.label_str = self.tick_to_string(data) width = self.boundingRect().width() offset = 0 # if have margins - new_pos = QtCore.QPointF(evt_post.x() - width / 2 - offset, 0) + new_pos = QPointF(abs_pos.x() - width / 2 - offset, 0) self.setPos(new_pos) class YAxisLabel(AxisLabel): + # text_flags = _common_text_flags text_flags = ( - QtCore.Qt.TextDontClip | QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter + QtCore.Qt.AlignLeft + | QtCore.Qt.TextDontClip + | QtCore.Qt.AlignVCenter ) def tick_to_string(self, tick_pos): + # WTF IS THIS FORMAT? return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 100, 40) + return QtCore.QRectF(0, 0, 120, 30) - def update_label(self, evt_post, point_view): - self.label_str = self.tick_to_string(point_view.y()) + def update_label( + self, + abs_pos: QPointF, # scene coords + data: float, # data for text + ) -> None: + self.label_str = self.tick_to_string(data) height = self.boundingRect().height() offset = 0 # if have margins - new_pos = QtCore.QPointF(0, evt_post.y() - height / 2 - offset) + new_pos = QPointF(0, abs_pos.y() - height / 2 - offset) self.setPos(new_pos) + + +class YSticky(YAxisLabel): + """Y-axis label that sticks to where it's placed despite chart resizing. + """ + def __init__( + self, + chart, + *args, + **kwargs + ) -> None: + super().__init__(*args, **kwargs) + self._chart = chart + + # XXX: not sure why this wouldn't work with a proxy? + # pg.SignalProxy( + # delay=0, + # rateLimit=60, + # slot=last.update_on_resize, + # ) + chart.sigRangeChanged.connect(self.update_on_resize) + + def update_on_resize(self, vr, r): + # TODO: figure out how to generalize across data schema + self.update_from_data(*self._chart._array[-1][['index', 'close']]) + + def update_from_data( + self, + index: int, + last: float, + ) -> None: + self.update_label( + self._chart.mapFromView(QPointF(index, last)), + last + ) From 363d4cf609402faa1cd7afee94414fd3666f8c97 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Aug 2020 12:28:38 -0400 Subject: [PATCH 083/206] Start color map --- piker/ui/_style.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index ea497a73..18e2c94e 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -27,3 +27,19 @@ def enable_tina_mode() -> None: """ # white background (for tinas like our pal xb) pg.setConfigOption('background', 'w') + + +def hcolor(name: str): + """Hex color codes by hipster speak. + """ + return '#' + { + 'black': '000000', # lives matter + 'white': 'ffffff', # for tinas and sunbathers + 'gray': '808080', # like the kick + 'dad_blue': '326693', # like his shirt + 'vwap_blue': '0582fb', + + # traditional + 'tina_green': '00cc00', + 'tina_red': 'fa0000', + }[name] From 7a245ddda443b585e011f93710e596728c83deb4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Aug 2020 12:29:29 -0400 Subject: [PATCH 084/206] Add and update y-sticky labels on new price data --- piker/ui/_chart.py | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b92103ae..22e86578 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -15,7 +15,8 @@ from ._axes import ( PriceAxis, ) from ._graphics import CrossHair, BarItems -from ._style import _xaxis_at, _min_points_to_show +from ._axes import YSticky +from ._style import _xaxis_at, _min_points_to_show, hcolor from ._source import Symbol from .. import brokers from .. import data @@ -225,7 +226,6 @@ class LinkedSplitCharts(QtGui.QWidget): cpw.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) - # self.splitter.addWidget(cpw) # link chart x-axis to main quotes chart cpw.setXLink(self.chart) @@ -246,6 +246,9 @@ class LinkedSplitCharts(QtGui.QWidget): # scale split regions self.set_split_sizes() + # XXX: we need this right? + # self.splitter.addWidget(cpw) + return cpw @@ -282,6 +285,7 @@ class ChartPlotWidget(pg.PlotWidget): self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics self._labels = {} # registry of underlying graphics + self._ysticks = {} # registry of underlying graphics # show only right side axes self.hideAxis('left') @@ -350,6 +354,7 @@ class ChartPlotWidget(pg.PlotWidget): self._graphics[name] = graphics # XXX: How to stack labels vertically? + # Ogi says: " label = pg.LabelItem( justify='left', size='5pt', @@ -373,6 +378,8 @@ class ChartPlotWidget(pg.PlotWidget): # show last 50 points on startup self.plotItem.vb.setXRange(xlast - 50, xlast + 50) + self._add_sticky(name) + return graphics def draw_curve( @@ -421,6 +428,21 @@ class ChartPlotWidget(pg.PlotWidget): return curve + def _add_sticky( + self, + name: str, + # retreive: Callable[None, np.ndarray], + ) -> YSticky: + # add y-axis "last" value label + last = self._ysticks['last'] = YSticky( + chart=self, + parent=self.getAxis('right'), + # digits=0, + opacity=1, + color=pg.mkPen(hcolor('gray')) + ) + return last + def update_from_array( self, name: str, @@ -636,6 +658,10 @@ async def _async_main( loglevel, ) + # update last price sticky + last = chart._ysticks['last'] + last.update_from_data(*chart._array[-1][['index', 'close']]) + # graphics update loop async with data.open_feed( @@ -646,7 +672,7 @@ async def _async_main( # wait for a first quote before we start any update tasks quote = await stream.__anext__() - print(f'RECEIVED FIRST QUOTE {quote}') + log.info(f'RECEIVED FIRST QUOTE {quote}') # start graphics tasks after receiving first live quote n.start_soon(add_new_bars, delay, linked_charts) @@ -675,6 +701,10 @@ async def _async_main( chart.name, chart._array, ) + # update sticky(s) + last = chart._ysticks['last'] + last.update_from_data( + *chart._array[-1][['index', 'close']]) chart._set_yrange() From fca6257152cd307e34a7499cc94c4b778280345b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Aug 2020 12:32:14 -0400 Subject: [PATCH 085/206] Use dashed lines for crosshair --- piker/ui/_graphics.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index edd75987..a07b3efc 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -10,7 +10,7 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF from .quantdom.utils import timeit -from ._style import _xaxis_at # , _tina_mode +from ._style import _xaxis_at, hcolor from ._axes import YAxisLabel, XAxisLabel @@ -18,17 +18,21 @@ from ._axes import YAxisLabel, XAxisLabel # - checkout pyqtgraph.PlotCurveItem.setCompositionMode _mouse_rate_limit = 30 +_debounce_delay = 10 class CrossHair(pg.GraphicsObject): def __init__( self, - linkedsplitcharts: 'LinkedSplitCharts', + linkedsplitcharts: 'LinkedSplitCharts', # noqa digits: int = 0 ) -> None: super().__init__() - self.pen = pg.mkPen('#a9a9a9') # gray? + self.pen = pg.mkPen( + color='#a9a9a9', # gray? + style=QtCore.Qt.DashLine, + ) self.lsc = linkedsplitcharts self.graphics = {} self.plots = [] @@ -47,7 +51,7 @@ class CrossHair(pg.GraphicsObject): yl = YAxisLabel( parent=plot.getAxis('right'), digits=digits or self.digits, - opacity=1 + opacity=0.7, ) # TODO: checkout what ``.sigDelayed`` can be used for @@ -55,17 +59,20 @@ class CrossHair(pg.GraphicsObject): px_moved = pg.SignalProxy( plot.scene().sigMouseMoved, rateLimit=_mouse_rate_limit, - slot=self.mouseMoved + slot=self.mouseMoved, + delay=_debounce_delay, ) px_enter = pg.SignalProxy( plot.sig_mouse_enter, rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Enter', plot), + delay=_debounce_delay, ) px_leave = pg.SignalProxy( plot.sig_mouse_leave, rateLimit=_mouse_rate_limit, slot=lambda: self.mouseAction('Leave', plot), + delay=_debounce_delay, ) self.graphics[plot] = { 'vl': vl, @@ -80,13 +87,16 @@ class CrossHair(pg.GraphicsObject): # place below the last plot self.xaxis_label = XAxisLabel( parent=self.plots[-1].getAxis('bottom'), - opacity=1 + opacity=0.7 ) else: # keep x-axis right below main chart first = self.plots[0] xaxis = first.getAxis('bottom') - self.xaxis_label = XAxisLabel(parent=xaxis, opacity=1) + self.xaxis_label = XAxisLabel( + parent=xaxis, + opacity=0.7, + ) def mouseAction(self, action, plot): # noqa if action == 'Enter': @@ -100,7 +110,10 @@ class CrossHair(pg.GraphicsObject): self.graphics[plot]['yl'].hide() self.active_plot = None - def mouseMoved(self, evt): # noqa + def mouseMoved( + self, + evt: 'Tuple[QMouseEvent]', # noqa + ) -> None: # noqa """Update horizonal and vertical lines when mouse moves inside either the main chart or any indicator subplot. """ @@ -108,6 +121,7 @@ class CrossHair(pg.GraphicsObject): # find position inside active plot try: + # map to view coordinate system mouse_point = self.active_plot.mapToView(pos) except AttributeError: # mouse was not on active plot @@ -120,7 +134,7 @@ class CrossHair(pg.GraphicsObject): self.graphics[plot]['hl'].setY(y) self.graphics[self.active_plot]['yl'].update_label( - evt_post=pos, point_view=mouse_point + abs_pos=pos, data=y ) for plot, opts in self.graphics.items(): # move the vertical line to the current x @@ -131,8 +145,8 @@ class CrossHair(pg.GraphicsObject): # update the label on the bottom of the crosshair self.xaxis_label.update_label( - evt_post=pos, - point_view=mouse_point + abs_pos=pos, + data=x ) def boundingRect(self): @@ -226,7 +240,7 @@ class BarItems(pg.GraphicsObject): # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 - bull_pen = pg.mkPen('#808080') + bull_pen = pg.mkPen(hcolor('gray')) # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') @@ -244,6 +258,9 @@ class BarItems(pg.GraphicsObject): # track the current length of drawable lines within the larger array self.index: int = 0 + def last_value(self) -> QLineF: + return self.lines[self.index - 1]['rarm'] + @timeit def draw_from_data( self, From 387a696232309f842e32749ebd1c4aaa642aa633 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 31 Aug 2020 17:16:44 -0400 Subject: [PATCH 086/206] Even more colors --- piker/ui/_style.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 18e2c94e..fcaeacf6 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -3,6 +3,7 @@ Qt UI styling. """ import pyqtgraph as pg from PyQt5 import QtGui +from qdarkstyle.palette import DarkPalette # chart-wide font @@ -29,17 +30,34 @@ def enable_tina_mode() -> None: pg.setConfigOption('background', 'w') -def hcolor(name: str): +def hcolor(name: str) -> str: """Hex color codes by hipster speak. """ - return '#' + { - 'black': '000000', # lives matter - 'white': 'ffffff', # for tinas and sunbathers - 'gray': '808080', # like the kick - 'dad_blue': '326693', # like his shirt - 'vwap_blue': '0582fb', + return { + # lives matter + 'black': '#000000', + 'erie_black': '#1B1B1B', + 'licorice': '#1A1110', + 'papas_special': '#06070c', + + # fifty shades + 'gray': '#808080', # like the kick + 'jet': '#343434', + 'charcoal': '#36454F', + + # palette + 'default': DarkPalette.COLOR_BACKGROUND_NORMAL, + + 'white': '#ffffff', # for tinas and sunbathers + + # blue zone + 'dad_blue': '#326693', # like his shirt + 'vwap_blue': '#0582fb', + 'dodger_blue': '#1e90ff', # like the team? + 'panasonic_blue': '#0040be', # from japan # traditional - 'tina_green': '00cc00', - 'tina_red': 'fa0000', + 'tina_green': '#00cc00', + 'tina_red': '#fa0000', + }[name] From ea2a675adfe224efa78f562e0a6801e29dc1afa8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 31 Aug 2020 17:17:20 -0400 Subject: [PATCH 087/206] Use dashed crosshair, simplify x-axis alloc --- piker/ui/_graphics.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index a07b3efc..032f79eb 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -19,6 +19,7 @@ from ._axes import YAxisLabel, XAxisLabel _mouse_rate_limit = 30 _debounce_delay = 10 +_ch_label_opac = 1 class CrossHair(pg.GraphicsObject): @@ -29,7 +30,13 @@ class CrossHair(pg.GraphicsObject): digits: int = 0 ) -> None: super().__init__() + # XXX: not sure why these are instance variables? + # It's not like we can change them on the fly..? self.pen = pg.mkPen( + color=hcolor('default'), + style=QtCore.Qt.DashLine, + ) + self.lines_pen = pg.mkPen( color='#a9a9a9', # gray? style=QtCore.Qt.DashLine, ) @@ -46,12 +53,13 @@ class CrossHair(pg.GraphicsObject): ) -> None: # add ``pg.graphicsItems.InfiniteLine``s # vertical and horizonal lines and a y-axis label - vl = plot.addLine(x=0, pen=self.pen, movable=False) - hl = plot.addLine(y=0, pen=self.pen, movable=False) + vl = plot.addLine(x=0, pen=self.lines_pen, movable=False) + hl = plot.addLine(y=0, pen=self.lines_pen, movable=False) yl = YAxisLabel( parent=plot.getAxis('right'), digits=digits or self.digits, - opacity=0.7, + opacity=_ch_label_opac, + color=self.pen, ) # TODO: checkout what ``.sigDelayed`` can be used for @@ -82,21 +90,16 @@ class CrossHair(pg.GraphicsObject): } self.plots.append(plot) - # determine where to place x-axis label - if _xaxis_at == 'bottom': - # place below the last plot - self.xaxis_label = XAxisLabel( - parent=self.plots[-1].getAxis('bottom'), - opacity=0.7 - ) - else: - # keep x-axis right below main chart - first = self.plots[0] - xaxis = first.getAxis('bottom') - self.xaxis_label = XAxisLabel( - parent=xaxis, - opacity=0.7, - ) + # Determine where to place x-axis label. + # Place below the last plot by default, ow + # keep x-axis right below main chart + plot_index = -1 if _xaxis_at == 'bottom' else 0 + + self.xaxis_label = XAxisLabel( + parent=self.plots[plot_index].getAxis('bottom'), + opacity=_ch_label_opac, + color=self.pen, + ) def mouseAction(self, action, plot): # noqa if action == 'Enter': From 17d205f773e9e7a3bad79cab4d638cd11ba3e02e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 31 Aug 2020 17:18:02 -0400 Subject: [PATCH 088/206] Add proper x-axis time-stamping --- piker/ui/_axes.py | 94 ++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 64c5a007..2a705a53 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -1,12 +1,17 @@ """ Chart axes graphics and behavior. """ +import time +from functools import partial +from typing import List + + +# import numpy as np +import pandas as pd import pyqtgraph as pg from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QPointF - -# from .quantdom.base import Quotes from .quantdom.utils import fromtimestamp from ._style import _font, hcolor @@ -15,7 +20,6 @@ class PriceAxis(pg.AxisItem): def __init__( self, - # chart: 'ChartPlotWidget', ) -> None: super().__init__(orientation='right') self.setStyle(**{ @@ -28,10 +32,7 @@ class PriceAxis(pg.AxisItem): }) self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font) - self.setWidth(150) - # self.chart = chart - # accesed normally via - # .getAxis('right') + self.setWidth(125) # XXX: drop for now since it just eats up h space @@ -43,9 +44,20 @@ class PriceAxis(pg.AxisItem): class DynamicDateAxis(pg.AxisItem): - tick_tpl = {'D1': '%Y-%b-%d'} + # time formats mapped by seconds between bars + tick_tpl = { + 60*60*24: '%Y-%b-%d', + 60: '%H:%M', + 30: '%H:%M:%S', + 5: '%H:%M:%S', + } - def __init__(self, linked_charts, *args, **kwargs): + def __init__( + self, + linked_charts, + *args, + **kwargs + ) -> None: super().__init__(*args, **kwargs) self.linked_charts = linked_charts self.setTickFont(_font) @@ -59,24 +71,25 @@ class DynamicDateAxis(pg.AxisItem): ) # self.setHeight(35) - def tickStrings(self, values, scale, spacing): - # if len(values) > 1 or not values: - # values = Quotes.time - - # strings = super().tickStrings(values, scale, spacing) - s_period = 'D1' - strings = [] + def _indexes_to_timestrs( + self, + indexes: List[int], + ) -> List[str]: bars = self.linked_charts.chart._array - quotes_count = len(bars) - 1 + times = bars['time'] + bars_len = len(bars) + delay = times[-1] - times[times != times[-1]][-1] - for ibar in values: - if ibar > quotes_count: - return strings - dt_tick = fromtimestamp(bars[int(ibar)]['time']) - strings.append( - dt_tick.strftime(self.tick_tpl[s_period]) - ) - return strings + epochs = times[list( + map(int, filter(lambda i: i < bars_len, indexes)) + )] + # TODO: **don't** have this hard coded shift to EST + dts = pd.to_datetime(epochs, unit='s') - 4*pd.offsets.Hour() + return dts.strftime(self.tick_tpl[delay]) + + + def tickStrings(self, values: List[float], scale, spacing): + return self._indexes_to_timestrs(values) class AxisLabel(pg.GraphicsObject): @@ -88,7 +101,7 @@ class AxisLabel(pg.GraphicsObject): def __init__( self, parent=None, - digits=1, + digits=2, color=None, opacity=1, **kwargs @@ -128,6 +141,7 @@ class AxisLabel(pg.GraphicsObject): p.drawText(option.rect, self.text_flags, self.label_str) # uggggghhhh + def tick_to_string(self, tick_pos): raise NotImplementedError() @@ -160,24 +174,20 @@ class XAxisLabel(AxisLabel): ) # text_flags = _common_text_flags - def tick_to_string(self, tick_pos): - # TODO: change to actual period - tpl = self.parent.tick_tpl['D1'] - bars = self.parent.linked_charts.chart._array - if tick_pos > len(bars): - return 'Unknown Time' - return fromtimestamp(bars[round(tick_pos)]['time']).strftime(tpl) - def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 145, 40) + # TODO: we need to get the parent axe's dimensions transformed + # to abs coords to be 100% correct here: + # self.parent.boundingRect() + return QtCore.QRectF(0, 0, 100, 31) - def update_label(self, abs_pos, data): - # ibar = view_pos.x() - # if ibar > self.quotes_count: - # return - self.label_str = self.tick_to_string(data) + def update_label( + self, + abs_pos: QPointF, # scene coords + data: float, # data for text + offset: int = 0 # if have margins, k? + ) -> None: + self.label_str = self.parent._indexes_to_timestrs([int(data)])[0] width = self.boundingRect().width() - offset = 0 # if have margins new_pos = QPointF(abs_pos.x() - width / 2 - offset, 0) self.setPos(new_pos) @@ -202,10 +212,10 @@ class YAxisLabel(AxisLabel): self, abs_pos: QPointF, # scene coords data: float, # data for text + offset: int = 0 # if have margins, k? ) -> None: self.label_str = self.tick_to_string(data) height = self.boundingRect().height() - offset = 0 # if have margins new_pos = QPointF(0, abs_pos.y() - height / 2 - offset) self.setPos(new_pos) From f1b72dfd6b9eb86212a5007c0289b4eb8bea2db2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 31 Aug 2020 17:18:35 -0400 Subject: [PATCH 089/206] Better bg color, tweak margins. --- piker/ui/_chart.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 22e86578..d52de12c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -30,7 +30,7 @@ from .. import fsp log = get_logger(__name__) # margins -CHART_MARGINS = (0, 0, 10, 3) +CHART_MARGINS = (0, 0, 5, 3) class ChartSpace(QtGui.QWidget): @@ -43,7 +43,7 @@ class ChartSpace(QtGui.QWidget): self.v_layout = QtGui.QVBoxLayout(self) self.v_layout.setContentsMargins(0, 0, 0, 0) self.toolbar_layout = QtGui.QHBoxLayout() - self.toolbar_layout.setContentsMargins(10, 10, 15, 0) + self.toolbar_layout.setContentsMargins(5, 5, 10, 0) self.h_layout = QtGui.QHBoxLayout() # self.init_timeframes_ui() @@ -275,13 +275,15 @@ class ChartPlotWidget(pg.PlotWidget): # the data view we generate graphics from array: np.ndarray, **kwargs, - # parent=None, - # background='default', - # plotItem=None, ): """Configure chart display settings. """ - super().__init__(**kwargs) + super().__init__( + background=hcolor('papas_special'), + # parent=None, + # plotItem=None, + **kwargs + ) self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics self._labels = {} # registry of underlying graphics From 0e513599ebf9972e3fe5a7b952b6a098a28a1933 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 1 Sep 2020 13:24:08 -0400 Subject: [PATCH 090/206] Info log fsp output for now --- piker/fsp.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/piker/fsp.py b/piker/fsp.py index 40f14f3a..f1418c2a 100644 --- a/piker/fsp.py +++ b/piker/fsp.py @@ -82,7 +82,7 @@ async def pull_and_process( filter_by_sym(symbol, stream), bars, ): - print(f"{fsp_func_name}: {processed}") + log.info(f"{fsp_func_name}: {processed}") yield processed @@ -105,27 +105,13 @@ async def latency( async for quote in source: ts = quote.get('broker_ts') if ts: - print( - f"broker time: {quote['broker_ts']}" - f"brokerd time: {quote['brokerd_ts']}" - ) + # This is codified in the per-broker normalization layer + # TODO: Add more measure points and diffs for full system + # stack tracing. value = quote['brokerd_ts'] - quote['broker_ts'] yield value -async def last( - source: 'TickStream[Dict[str, float]]', - ohlcv: np.ndarray -) -> AsyncIterator[np.ndarray]: - """Compute High-Low midpoint value. - """ - # deliver historical processed data first - yield ohlcv['close'] - - async for quote in source: - yield quote['close'] - - async def wma( source, #: AsyncStream[np.ndarray], ohlcv: np.ndarray, # price time-frame "aware" From 48c70e8ae499465b63b5b93eaaeffa8b91ff177b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 2 Sep 2020 12:36:24 -0400 Subject: [PATCH 091/206] better readme --- README.rst | 91 ++++++++++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/README.rst b/README.rst index f4851c2d..7631c095 100644 --- a/README.rst +++ b/README.rst @@ -1,26 +1,28 @@ piker ----- -Trading gear for hackers. +trading gear for hackers. |travis| -``piker`` is an attempt at a pro-grade, broker agnostic, next-gen FOSS toolset for real-time -trading and financial analysis targetted at hardcore Linux users. +``piker`` is an attempt at a pro-grade, broker agnostic, next-gen FOSS +toolset for real-time trading and financial analysis targetted at +hardcore Linux users. -It tries to use as much bleeding edge tech as possible including (but not limited to): +it tries to use as much bleeding edge tech as possible including (but not limited to): -- Python 3.7+ for glue_ and business logic -- trio_ for structured concurrency +- latest Python for glue_ and business logic +- trio_ for `structured concurrency`_ - tractor_ for distributed, multi-core, real-time streaming - marketstore_ for historical and real-time tick data persistence and sharing - techtonicdb_ for L2 book storage - Qt_ for pristine high performance UIs - pyqtgraph_ for real-time charting -- ``numpy`` for `fast numerics`_ +- ``numpy`` and ``numba`` for `fast numerics`_ .. |travis| image:: https://img.shields.io/travis/pikers/piker/master.svg :target: https://travis-ci.org/pikers/piker .. _trio: https://github.com/python-trio/trio +.. _structured concurrency: https://trio.discourse.group/ .. _tractor: https://github.com/goodboy/tractor .. _marketstore: https://github.com/alpacahq/marketstore .. _techtonicdb: https://github.com/0b01/tectonicdb @@ -31,35 +33,26 @@ It tries to use as much bleeding edge tech as possible including (but not limite Focus and Features: ******************* -- 100% federated: running your code on your hardware with your - broker's data feeds, privately, **is the point** (this is not a web-based *I - don't know how to run my own system* project). -- Asset class, broker, exchange agnostic. -- Built on a highly reliable `structured concurrent actor model - `_ with built in async streaming and scalability protocols - allowing for a distributed architecture from the ground up. -- Privacy: your orders, indicators, algos are all run client side and - are shared only with the (groups of) traders you specify. -- Production grade, highly attractive native UIs that feel and fit like - a proper pair of skinny jeans; only meant to be used with a proper - tiling window manager (no, we are not ignorant enough to roll our own). -- Sophisticated charting capable of processing large data sets in real-time - while sanely displaying complex models and strategy systems. -- Built-in support for *hipstery* indicators and studies that you - probably haven't heard of but that the authors **know** generate alpha - when paired with the right strategies. -- Emphasis on collaboration through sharing of data, ideas, and processing - power. We will not host your code in the cloud nor ask you to - participate in any lame "alpha competitions". -- Adoption is very low priority, especially if you're not an experienced - trader; the system is not built for sale it is built for *people*. -- No, we will never have a "corporation friendly license"; if you intend to use - this code base we must know about it. +- zero web +- zero pump +- zero "backtesting" (aka yabf) +- zero "cloud" +- 100% federated: your code, your hardware, your broker's data feeds +- privacy +- broker/exchange agnostic +- built on a `structured concurrent actor model `_ +- production grade, highly attractive native UIs +- expected to be used from tiling wms +- sophisticated rt charting +- emphasis on collaboration through UI and data sharing +- zero interest in adoption by suits +- not built for *sale*; built for *people* +- no corporate friendly license, ever. -Fitting with these tenets, we're always open to new framework suggestions and ideas. +fitting with these tenets, we're always open to new framework suggestions and ideas. -Building the best looking, most reliable, keyboard friendly trading platform is the dream. -Feel free to pipe in with your ideas and quiffs. +building the best looking, most reliable, keyboard friendly trading platform is the dream. +feel free to pipe in with your ideas and quiffs. Install @@ -80,9 +73,10 @@ For a development install:: Broker Support ************** -For live data feeds the set of supported brokers is: +For live data feeds the in-progress set of supported brokers is: + - Questrade_ which comes with effectively free L1 -- IB_ via ib_insync +- IB_ via ``ib_insync`` - Webull_ via the reverse engineered public API - Kraken_ for crypto over their public websocket API @@ -94,22 +88,17 @@ If you want your broker supported and they have an API let us know. .. _Kraken: https://www.kraken.com/features/api#public-market-data -Check out some charts -********************* -Bet you weren't expecting this from the foss:: +Check out our charts +******************** +bet you weren't expecting this from the foss bby:: - piker chart spy.arca + piker chart -b kraken XBTUSD -It is also possible to run a specific broker's data feed as a top -level micro-service daemon:: +If anyone asks you what this project is about +********************************************* +tell them *it's a broken crypto trading platform that doesn't scale*. - pikerd -l info -b ib - - -Then start the client app as normal:: - - piker chart -b ib ES.GLOBEX - - -.. _pipenv: https://docs.pipenv.org/ +How do i get involved? +********************** +coming soon. From aed310dd8bac81edf635c4f75cddbdecaa3d66c4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 2 Sep 2020 12:45:24 -0400 Subject: [PATCH 092/206] Kill pipenv --- README.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 7631c095..4ec7456f 100644 --- a/README.rst +++ b/README.rst @@ -22,11 +22,12 @@ it tries to use as much bleeding edge tech as possible including (but not limite .. |travis| image:: https://img.shields.io/travis/pikers/piker/master.svg :target: https://travis-ci.org/pikers/piker .. _trio: https://github.com/python-trio/trio -.. _structured concurrency: https://trio.discourse.group/ .. _tractor: https://github.com/goodboy/tractor +.. _structured concurrency: https://trio.discourse.group/ .. _marketstore: https://github.com/alpacahq/marketstore .. _techtonicdb: https://github.com/0b01/tectonicdb .. _Qt: https://www.qt.io/ +.. _pyqtgraph: https://github.com/pyqtgraph/pyqtgraph .. _glue: https://numpy.org/doc/stable/user/c-info.python-as-glue.html#using-python-as-glue .. _fast numerics: https://zerowithdot.com/python-numpy-and-pandas-performance/ @@ -40,7 +41,7 @@ Focus and Features: - 100% federated: your code, your hardware, your broker's data feeds - privacy - broker/exchange agnostic -- built on a `structured concurrent actor model `_ +- built on a structured concurrent actor model - production grade, highly attractive native UIs - expected to be used from tiling wms - sophisticated rt charting @@ -49,10 +50,11 @@ Focus and Features: - not built for *sale*; built for *people* - no corporate friendly license, ever. -fitting with these tenets, we're always open to new framework suggestions and ideas. +fitting with these tenets, we're always open to new framework +suggestions and ideas. -building the best looking, most reliable, keyboard friendly trading platform is the dream. -feel free to pipe in with your ideas and quiffs. +building the best looking, most reliable, keyboard friendly trading +platform is the dream. feel free to pipe in with your ideas and quiffs. Install @@ -67,8 +69,7 @@ For a development install:: git clone git@github.com:pikers/piker.git cd piker - pipenv install --pre -e . - pipenv shell + pip install -e . Broker Support From b1591e3ee158a450b0264fedffedbb9443df61f3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 6 Sep 2020 11:32:06 -0400 Subject: [PATCH 093/206] Start mucking with faster bars updates Use a ``rec2array`` struct array converter to generate lines sequence faster. Start looking into using a `QPixmap` to avoid redrawing all bars every update. --- piker/ui/_graphics.py | 123 +++++++++++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 24 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 032f79eb..3fc87f07 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -1,6 +1,7 @@ """ Chart graphics for displaying a slew of different data types. """ +import time from typing import List from itertools import chain @@ -14,6 +15,40 @@ from ._style import _xaxis_at, hcolor from ._axes import YAxisLabel, XAxisLabel +def rec2array( + rec: np.ndarray, + fields: List[str] = None +) -> np.ndarray: + """Convert record array to std array. + + Taken from: + https://github.com/scikit-hep/root_numpy/blob/master/root_numpy/_utils.py#L20 + """ + simplify = False + + if fields is None: + fields = rec.dtype.names + elif isinstance(fields, str): + fields = [fields] + simplify = True + + # Creates a copy and casts all data to the same type + arr = np.dstack([rec[field] for field in fields]) + + # Check for array-type fields. If none, then remove outer dimension. + # Only need to check first field since np.dstack will anyway raise an + # exception if the shapes don't match + # np.dstack will also fail if fields is an empty list + if not rec.dtype[fields[0]].shape: + arr = arr[0] + + if simplify: + # remove last dimension (will be of size 1) + arr = arr.reshape(arr.shape[:-1]) + + return arr + + # TODO: # - checkout pyqtgraph.PlotCurveItem.setCompositionMode @@ -243,7 +278,7 @@ class BarItems(pg.GraphicsObject): # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 - bull_pen = pg.mkPen(hcolor('gray')) + bars_pen = pg.mkPen(hcolor('gray')) # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') @@ -252,12 +287,21 @@ class BarItems(pg.GraphicsObject): def __init__(self): super().__init__() self.picture = QtGui.QPicture() + # TODO: implement updateable pixmap solution + # self.picture = QtGui.QPixmap() + + # cache bounds dimensions + self._boundingRect = None # XXX: not sure this actually needs to be an array other # then for the old tina mode calcs for up/down bars below? # lines container self.lines = _mk_lines_array([], 50e3) + # can't use this until we aren't copying the array passed + # to ``QPainter.drawLines()``. + self._cached_lines = None + # track the current length of drawable lines within the larger array self.index: int = 0 @@ -273,6 +317,7 @@ class BarItems(pg.GraphicsObject): """Draw OHLC datum graphics from a ``np.recarray``. """ lines = bars_from_ohlc(data, self.w, start=start) + # xmx, xmn, ymx, ymn = # save graphics for later reference and keep track # of current internal "last index" @@ -281,23 +326,49 @@ class BarItems(pg.GraphicsObject): self.index = index self.draw_lines() - def draw_lines(self): + def draw_lines( + self, + istart=0, + use_cached=False, + ) -> None: """Draw the current line set using the painter. """ - to_draw = self.lines[ - ['body', 'larm', 'rarm']][:self.index] - - # pre-computing a QPicture object allows paint() to run much - # more quickly, rather than re-drawing the shapes every time. - p = QtGui.QPainter(self.picture) - p.setPen(self.bull_pen) + start = time.time() + # istart = self.index - 100 + # if self._cached_lines is not None and ( + # not (self.index > len(self._cached_lines)) + # or use_cached + # ): + # to_draw = self._cached_lines + # else: # TODO: might be better to use 2d array? # try our fsp.rec2array() and a np.ravel() for speedup # otherwise we might just have to go 2d ndarray of objects. - # see conlusion on speed here: # https://stackoverflow.com/a/60089929 - p.drawLines(*chain.from_iterable(to_draw)) - p.end() + # see conlusion on speed: # https://stackoverflow.com/a/60089929 + self._cached_lines = to_draw = np.ravel(rec2array( + self.lines[['body', 'larm', 'rarm']][istart:self.index] + )) + + # pre-computing a QPicture object allows paint() to run much + # more quickly, rather than re-drawing the shapes every time. + p = QtGui.QPainter(self.picture) + p.setPen(self.bars_pen) + + # TODO: is there any way to not have to pass all the lines every + # iteration? It seems they won't draw unless it's done this way.. + p.drawLines(*to_draw) + # p.end() + + # trigger re-render + # https://doc.qt.io/qt-5/qgraphicsitem.html#update + self.update() + + diff = time.time() - start + + # print(f'cached: {use_cached} update took {diff}') + # print(f"start: {istart} -> drawing {len(to_draw)} lines") + # print(self.picture.data()) def update_from_array( self, @@ -315,6 +386,9 @@ class BarItems(pg.GraphicsObject): index = self.index length = len(array) extra = length - index + + # start_bar_to_update = index - 1 + if extra > 0: # generate new graphics to match provided array new = array[index:index + extra] @@ -323,14 +397,14 @@ class BarItems(pg.GraphicsObject): self.lines[index:index + bars_added] = lines self.index += bars_added - # else: # current bar update - # do we really need to verify the entire past data set? - # index, time, open, high, low, close, volume - i, time, open, _, _, close, _ = array[-1] + # start_bar_to_update = index - bars_added + + # self.draw_lines(istart=self.index - 1) + + # current bar update + i, time, open, close, = array[-1][['index', 'time', 'open', 'close']] last = close i, body, larm, rarm = self.lines[index-1] - if not rarm: - breakpoint() # XXX: is there a faster way to modify this? # update right arm @@ -354,11 +428,8 @@ class BarItems(pg.GraphicsObject): else: body.setLine(body.x1(), low, body.x2(), high) - # draw the pic - self.draw_lines() - - # trigger re-render - self.update() + # self.draw_lines(use_cached=extra == 0) + self.draw_lines() #istart=self.index - 1) # be compat with ``pg.PlotCurveItem`` setData = update_from_array @@ -366,14 +437,18 @@ class BarItems(pg.GraphicsObject): # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() def paint(self, p, opt, widget): + # import pdb; pdb.set_trace() + # p.setCompositionMode(0) p.drawPicture(0, 0, self.picture) def boundingRect(self): + # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect # boundingRect _must_ indicate the entire area that will be # drawn on or else we will get artifacts and possibly crashing. # (in this case, QPicture does all the work of computing the # bounding rect for us) - return QtCore.QRectF(self.picture.boundingRect()) + br = self._boundingRect = QtCore.QRectF(self.picture.boundingRect()) + return br # XXX: when we get back to enabling tina mode for xb From 9d8a867767d1a3b2021d89aafdd0a0a1c9c54810 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 7 Sep 2020 16:41:11 -0400 Subject: [PATCH 094/206] Bar graphics update speed improvements Use two separate `QPicture` instances: - one for the 3 lines for the last bar - one for all the historical bars lines On price changes update the last bar and only update historical bars when the current bar's period expires (when a new bar is "added"). Add a flag `just_history` for this `BarItems.draw_lines()`. Also, switch the internal lines array/buffer to a 2D numpy array to avoid the type-cast step and instead just flatten using `numpy.ravel()`. Overall this should avoid the problem of draws getting slower over time as new bars are added to the history since price updates only redraw a single bar to the "last" `QPicture` instance. Ideally in the future we can make the `history` `QPicture` a `QPixmap` but it looks like this will require some internal work in `pyqtgraph` to support it. --- piker/ui/_chart.py | 21 ++++-- piker/ui/_graphics.py | 155 +++++++++++++++++++++++++----------------- 2 files changed, 105 insertions(+), 71 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index d52de12c..5f075ed2 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -345,18 +345,17 @@ class ChartPlotWidget(pg.PlotWidget): ) -> pg.GraphicsObject: """Draw OHLC datums to chart. """ - # remember it's an enum type.. - graphics = style() - + graphics = style(self.plotItem) # adds all bar/candle graphics objects for each data point in # the np array buffer to be drawn on next render cycle - graphics.draw_from_data(data) self.addItem(graphics) + # draw after to allow self.scene() to work... + graphics.draw_from_data(data) self._graphics[name] = graphics # XXX: How to stack labels vertically? - # Ogi says: " + # Ogi says: "use ..." label = pg.LabelItem( justify='left', size='5pt', @@ -596,8 +595,13 @@ async def add_new_bars(delay_s, linked_charts): # if last_quote == ohlc[-1]: # log.debug("Printing flat line for {sym}") - # update chart graphics and resize view - price_chart.update_from_array(price_chart.name, ohlc) + # update chart historical bars graphics + price_chart.update_from_array( + price_chart.name, + ohlc, + just_history=True + ) + # resize view price_chart._set_yrange() for name, chart in linked_charts.subplots.items(): @@ -611,6 +615,9 @@ async def add_new_bars(delay_s, linked_charts): # TODO: should we update a graphics again time here? # Think about race conditions with data update task. + # UPDATE: don't think this should matter know since the last bar + # and the prior historical bars are being updated in 2 separate + # steps now. async def _async_main( diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 3fc87f07..25ea7bdd 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -1,9 +1,8 @@ """ Chart graphics for displaying a slew of different data types. """ -import time +# import time from typing import List -from itertools import chain import numpy as np import pyqtgraph as pg @@ -197,17 +196,10 @@ class CrossHair(pg.GraphicsObject): def _mk_lines_array(data: List, size: int) -> np.ndarray: """Create an ndarray to hold lines graphics objects. """ - # TODO: might want to just make this a 2d array to be faster at - # flattening using .ravel(): https://stackoverflow.com/a/60089929 return np.zeros_like( data, - shape=(int(size),), - dtype=[ - ('index', int), - ('body', object), - ('rarm', object), - ('larm', object) - ], + shape=(int(size), 3), + dtype=object, ) @@ -245,10 +237,13 @@ def bars_from_ohlc( c = QLineF(index_start + w, close, index_start, close) # indexing here is as per the below comments - # lines[3*i:3*i+3] = (hl, o, c) - lines[i] = (index, hl, o, c) + lines[i] = (hl, o, c) - # if not _tina_mode: # piker mode + # XXX: in theory we could get a further speedup by using a flat + # array and avoiding the call to `np.ravel()` below? + # lines[3*i:3*i+3] = (hl, o, c) + + # if not _tina_mode: # else _tina_mode: # self.lines = lines = np.concatenate( # [high_to_low, open_sticks, close_sticks]) @@ -284,30 +279,30 @@ class BarItems(pg.GraphicsObject): # bull_brush = pg.mkPen('#00cc00') # bear_brush = pg.mkPen('#fa0000') - def __init__(self): + def __init__( + self, + # scene: 'QGraphicsScene', # noqa + plotitem: 'pg.PlotItem', # noqa + ) -> None: super().__init__() - self.picture = QtGui.QPicture() + self.last = QtGui.QPicture() + self.history = QtGui.QPicture() # TODO: implement updateable pixmap solution - # self.picture = QtGui.QPixmap() - - # cache bounds dimensions - self._boundingRect = None + self._pi = plotitem + # self._scene = plotitem.vb.scene() + # self.picture = QtGui.QPixmap(1000, 300) + # plotitem.addItem(self.picture) + # self._pmi = None + # self._pmi = self._scene.addPixmap(self.picture) # XXX: not sure this actually needs to be an array other # then for the old tina mode calcs for up/down bars below? # lines container self.lines = _mk_lines_array([], 50e3) - # can't use this until we aren't copying the array passed - # to ``QPainter.drawLines()``. - self._cached_lines = None - # track the current length of drawable lines within the larger array self.index: int = 0 - def last_value(self) -> QLineF: - return self.lines[self.index - 1]['rarm'] - @timeit def draw_from_data( self, @@ -317,62 +312,69 @@ class BarItems(pg.GraphicsObject): """Draw OHLC datum graphics from a ``np.recarray``. """ lines = bars_from_ohlc(data, self.w, start=start) - # xmx, xmn, ymx, ymn = # save graphics for later reference and keep track # of current internal "last index" index = len(lines) self.lines[:index] = lines self.index = index - self.draw_lines() + self.draw_lines(just_history=True, iend=self.index) def draw_lines( self, istart=0, - use_cached=False, + iend=None, + just_history=False, + # TODO: could get even fancier and only update the single close line? + lines=None, ) -> None: """Draw the current line set using the painter. """ - start = time.time() - # istart = self.index - 100 - # if self._cached_lines is not None and ( - # not (self.index > len(self._cached_lines)) - # or use_cached - # ): - # to_draw = self._cached_lines - # else: + # start = time.time() - # TODO: might be better to use 2d array? - # try our fsp.rec2array() and a np.ravel() for speedup - # otherwise we might just have to go 2d ndarray of objects. - # see conlusion on speed: # https://stackoverflow.com/a/60089929 - self._cached_lines = to_draw = np.ravel(rec2array( - self.lines[['body', 'larm', 'rarm']][istart:self.index] - )) + if just_history: + istart = 0 + iend = iend or self.index - 1 + pic = self.history + else: + istart = self.index - 1 + iend = self.index + pic = self.last + + if iend is not None: + iend = iend + + # use 2d array of lines objects, see conlusion on speed: + # https://stackoverflow.com/a/60089929 + to_draw = np.ravel(self.lines[istart:iend]) # pre-computing a QPicture object allows paint() to run much # more quickly, rather than re-drawing the shapes every time. - p = QtGui.QPainter(self.picture) + p = QtGui.QPainter(pic) p.setPen(self.bars_pen) # TODO: is there any way to not have to pass all the lines every # iteration? It seems they won't draw unless it's done this way.. p.drawLines(*to_draw) - # p.end() + p.end() + + # XXX: if we ever try using `QPixmap` again... + # if self._pmi is None: + # self._pmi = self.scene().addPixmap(self.picture) + # else: + # self._pmi.setPixmap(self.picture) # trigger re-render # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - diff = time.time() - start - - # print(f'cached: {use_cached} update took {diff}') - # print(f"start: {istart} -> drawing {len(to_draw)} lines") - # print(self.picture.data()) + # diff = time.time() - start + # print(f'{len(to_draw)} lines update took {diff}') def update_from_array( self, array: np.ndarray, + just_history=False, ) -> None: """Update the last datum's bar graphic from input data array. @@ -387,7 +389,7 @@ class BarItems(pg.GraphicsObject): length = len(array) extra = length - index - # start_bar_to_update = index - 1 + # start_bar_to_update = index - 100 if extra > 0: # generate new graphics to match provided array @@ -398,13 +400,15 @@ class BarItems(pg.GraphicsObject): self.index += bars_added # start_bar_to_update = index - bars_added - - # self.draw_lines(istart=self.index - 1) + if just_history: + self.draw_lines(just_history=True) # istart=self.index - 1) + return # current bar update - i, time, open, close, = array[-1][['index', 'time', 'open', 'close']] + i, open, close, = array[-1][['index', 'open', 'close']] last = close - i, body, larm, rarm = self.lines[index-1] + # i, body, larm, rarm = self.lines[index-1] + body, larm, rarm = self.lines[index-1] # XXX: is there a faster way to modify this? # update right arm @@ -428,8 +432,7 @@ class BarItems(pg.GraphicsObject): else: body.setLine(body.x1(), low, body.x2(), high) - # self.draw_lines(use_cached=extra == 0) - self.draw_lines() #istart=self.index - 1) + self.draw_lines(just_history=False) # be compat with ``pg.PlotCurveItem`` setData = update_from_array @@ -437,18 +440,42 @@ class BarItems(pg.GraphicsObject): # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() def paint(self, p, opt, widget): - # import pdb; pdb.set_trace() + # start = time.time() + + # TODO: use to avoid drawing artefacts? + # self.prepareGeometryChange() + # p.setCompositionMode(0) - p.drawPicture(0, 0, self.picture) + + p.drawPicture(0, 0, self.history) + p.drawPicture(0, 0, self.last) + + # TODO: if we can ever make pixmaps work... + # p.drawPixmap(0, 0, self.picture) + # self._pmi.setPixmap(self.picture) + # print(self.scene()) + + # diff = time.time() - start + # print(f'draw time {diff}') def boundingRect(self): + # TODO: can we do rect caching to make this faster? + # Qt docs: https://doc.qt.io/qt-5/qgraphicsitem.html#boundingRect # boundingRect _must_ indicate the entire area that will be # drawn on or else we will get artifacts and possibly crashing. # (in this case, QPicture does all the work of computing the - # bounding rect for us) - br = self._boundingRect = QtCore.QRectF(self.picture.boundingRect()) - return br + # bounding rect for us). + + # compute aggregate bounding rectangle + lb = self.last.boundingRect() + hb = self.history.boundingRect() + return QtCore.QRectF( + # top left + QtCore.QPointF(hb.topLeft()), + # total size + QtCore.QSizeF(lb.size() + hb.size()) + ) # XXX: when we get back to enabling tina mode for xb From 7d24e8eeb0edaf894210cd14453cb2f906a52f1e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 8 Sep 2020 09:59:29 -0400 Subject: [PATCH 095/206] First draft of real-time rsi using numba --- piker/fsp.py | 198 +++++++++++++++++++++++++++++++++------------ piker/ui/_chart.py | 2 +- 2 files changed, 149 insertions(+), 51 deletions(-) diff --git a/piker/fsp.py b/piker/fsp.py index f1418c2a..25014155 100644 --- a/piker/fsp.py +++ b/piker/fsp.py @@ -1,9 +1,11 @@ """ Financial signal processing for the peeps. """ -from typing import AsyncIterator, List +from typing import AsyncIterator, List, Callable +import tractor import numpy as np +from numba import jit, float64, int64, void, optional from .log import get_logger from . import data @@ -11,58 +13,49 @@ from . import data log = get_logger(__name__) -def rec2array( - rec: np.ndarray, - fields: List[str] = None -) -> np.ndarray: - """Convert record array to std array. - - Taken from: - https://github.com/scikit-hep/root_numpy/blob/master/root_numpy/_utils.py#L20 +async def latency( + source: 'TickStream[Dict[str, float]]', + ohlcv: np.ndarray +) -> AsyncIterator[np.ndarray]: + """Compute High-Low midpoint value. """ - simplify = False + # TODO: do we want to offer yielding this async + # before the rt data connection comes up? - if fields is None: - fields = rec.dtype.names - elif isinstance(fields, str): - fields = [fields] - simplify = True + # deliver zeros for all prior history + yield np.zeros(len(ohlcv)) - # Creates a copy and casts all data to the same type - arr = np.dstack([rec[field] for field in fields]) - - # Check for array-type fields. If none, then remove outer dimension. - # Only need to check first field since np.dstack will anyway raise an - # exception if the shapes don't match - # np.dstack will also fail if fields is an empty list - if not rec.dtype[fields[0]].shape: - arr = arr[0] - - if simplify: - # remove last dimension (will be of size 1) - arr = arr.reshape(arr.shape[:-1]) - - return arr + async for quote in source: + ts = quote.get('broker_ts') + if ts: + # This is codified in the per-broker normalization layer + # TODO: Add more measure points and diffs for full system + # stack tracing. + value = quote['brokerd_ts'] - quote['broker_ts'] + yield value -async def pull_and_process( +async def stream_and_process( bars: np.ndarray, brokername: str, # symbols: List[str], symbol: str, fsp_func_name: str, + func: Callable = latency, ) -> AsyncIterator[dict]: + # remember, msgpack-numpy's ``from_buffer` returns read-only array + # bars = np.array(bars[list(ohlc_dtype.names)]) + # async def _yield_bars(): # yield bars # hist_out: np.ndarray = None - func = latency - # Conduct a single iteration of fsp with historical bars input # async for hist_out in func(_yield_bars(), bars): # yield {symbol: hist_out} + func = _rsi # open a data feed stream with requested broker async with data.open_feed( @@ -90,26 +83,131 @@ async def pull_and_process( # - how to handle non-plottable values # - composition of fsps / implicit chaining -async def latency( - source: 'TickStream[Dict[str, float]]', - ohlcv: np.ndarray -) -> AsyncIterator[np.ndarray]: - """Compute High-Low midpoint value. - """ - # TODO: do we want to offer yielding this async - # before the rt data connection comes up? +@jit( + float64[:]( + float64[:], + optional(float64), + optional(float64) + ), + nopython=True, + nogil=True +) +def ema( + y: 'np.ndarray[float64]', + alpha: optional(float64) = None, + ylast: optional(float64) = None, +) -> 'np.ndarray[float64]': + r"""Exponential weighted moving average owka 'Exponential smoothing'. - # deliver zeros for all prior history - yield np.zeros(len(ohlcv)) + - https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + - https://en.wikipedia.org/wiki/Exponential_smoothing + + Fun facts: + A geometric progression is the discrete version of an + exponential function, that is where the name for this + smoothing method originated according to statistics lore. In + signal processing parlance, an EMA is a first order IIR filter. + + .. math:: + + .tex + {S_{t}={\begin{cases}Y_{1},&t=1 + \\\alpha Y_{t}+(1-\alpha )\cdot S_{t-1},&t>1\end{cases}}} + + .nerd + (2) s = { + s[0] = y[0]; t = 0 + s[t] = a*y[t] + (1-a)*s[t-1], t > 0. + } + """ + n = y.shape[0] + + if alpha is None: + # https://en.wikipedia.org/wiki/Moving_average#Relationship_between_SMA_and_EMA + # use the "center of mass" convention making an ema compare + # directly to the com of a SMA or WMA: + alpha = 2 / float(n + 1) + + s = np.empty(n, dtype=float64) + + if n == 1: + s[0] = y[0] * alpha + ylast * (1 - alpha) + + else: + if ylast is None: + s[0] = y[0] + else: + s[0] = ylast + + for i in range(1, n): + s[i] = y[i] * alpha + s[i-1] * (1 - alpha) + + return s + + +# @jit( +# float64[:]( +# float64[:], +# int64, +# ), +# # nopython=True, +# nogil=True +# ) +def rsi( + signal: 'np.ndarray[float64]', + period: int = 14, + up_ema_last: float64 = None, + down_ema_last: float64 = None, +) -> 'np.ndarray[float64]': + alpha = 1/period + print(signal) + + df = np.diff(signal) + up, down = np.where(df > 0, df, 0), np.where(df < 0, -df, 0) + up_ema = ema(up, alpha, up_ema_last) + down_ema = ema(down, alpha, down_ema_last) + rs = up_ema / down_ema + print(f'up_ema: {up_ema}\ndown_ema: {down_ema}') + print(f'rs: {rs}') + # map rs through sigmoid (with range [0, 100]) + rsi = 100 - 100 / (1 + rs) + # rsi = 100 * (up_ema / (up_ema + down_ema)) + # also return the last ema state for next iteration + return rsi, up_ema[-1], down_ema[-1] + + +# @piker.fsp( +# aggregates=['30s', '1m', '5m', '1H', '4H', '1D'], +# ) +async def _rsi( + source: 'QuoteStream[Dict[str, Any]]', # noqa + ohlcv: np.ndarray, + period: int = 14, +) -> AsyncIterator[np.ndarray]: + """Multi-timeframe streaming RSI. + + https://en.wikipedia.org/wiki/Relative_strength_index + """ + sig = ohlcv['close'] + rsi_h, up_ema_last, down_ema_last = rsi(sig, period, None, None) + + # deliver history + yield rsi_h async for quote in source: - ts = quote.get('broker_ts') - if ts: - # This is codified in the per-broker normalization layer - # TODO: Add more measure points and diffs for full system - # stack tracing. - value = quote['brokerd_ts'] - quote['broker_ts'] - yield value + # tick based updates + for tick in quote.get('ticks', ()): + if tick.get('type') == 'trade': + last = np.array([sig[-1], tick['price']]) + # await tractor.breakpoint() + rsi_out, up_ema_last, down_ema_last = rsi( + last, + period=period, + up_ema_last=up_ema_last, + down_ema_last=down_ema_last, + ) + print(f'last: {last}\n rsi: {rsi_out}') + yield rsi_out[-1] async def wma( diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 5f075ed2..a41d7546 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -736,7 +736,7 @@ async def chart_from_fsp( f'fsp.{func_name}', # name as title of sub-chart # subactor entrypoint - fsp.pull_and_process, + fsp.stream_and_process, bars=bars, brokername=brokermod.name, symbol=sym, From ee6e4d2207c6afd30d54704dd8fdbc13efcff289 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 8 Sep 2020 09:59:54 -0400 Subject: [PATCH 096/206] Add colorama for numba tracebacks --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index fcd812e8..99effd91 100755 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( 'colorlog', 'attrs', 'pygments', + 'colorama', # numba traceback coloring # async 'trio', From 9a59f2408dccdc13a3aba548e04265ecabffbf85 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Sep 2020 10:46:33 -0400 Subject: [PATCH 097/206] Start fsp subpackage, separate momo stuff --- piker/fsp/__init__.py | 80 +++++++++++++++++++++++++++++ piker/{fsp.py => fsp/_momo.py} | 94 +++++----------------------------- 2 files changed, 93 insertions(+), 81 deletions(-) create mode 100644 piker/fsp/__init__.py rename piker/{fsp.py => fsp/_momo.py} (64%) diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py new file mode 100644 index 00000000..bf2ea127 --- /dev/null +++ b/piker/fsp/__init__.py @@ -0,0 +1,80 @@ +""" +Financial signal processing for the peeps. +""" +from typing import AsyncIterator, Callable + +import numpy as np + +from ..log import get_logger +from .. import data +from ._momo import _rsi + +log = get_logger(__name__) + + +_fsps = {'rsi': _rsi} + + +async def latency( + source: 'TickStream[Dict[str, float]]', # noqa + ohlcv: np.ndarray +) -> AsyncIterator[np.ndarray]: + """Compute High-Low midpoint value. + """ + # TODO: do we want to offer yielding this async + # before the rt data connection comes up? + + # deliver zeros for all prior history + yield np.zeros(len(ohlcv)) + + async for quote in source: + ts = quote.get('broker_ts') + if ts: + # This is codified in the per-broker normalization layer + # TODO: Add more measure points and diffs for full system + # stack tracing. + value = quote['brokerd_ts'] - quote['broker_ts'] + yield value + + +async def stream_and_process( + bars: np.ndarray, + brokername: str, + # symbols: List[str], + symbol: str, + fsp_func_name: str, +) -> AsyncIterator[dict]: + + # remember, msgpack-numpy's ``from_buffer` returns read-only array + # bars = np.array(bars[list(ohlc_dtype.names)]) + + # async def _yield_bars(): + # yield bars + + # hist_out: np.ndarray = None + + # Conduct a single iteration of fsp with historical bars input + # async for hist_out in func(_yield_bars(), bars): + # yield {symbol: hist_out} + func: Callable = _fsps[fsp_func_name] + + # open a data feed stream with requested broker + async with data.open_feed( + brokername, + [symbol], + ) as (fquote, stream): + + # TODO: load appropriate fsp with input args + + async def filter_by_sym(sym, stream): + async for quotes in stream: + for symbol, quotes in quotes.items(): + if symbol == sym: + yield quotes + + async for processed in func( + filter_by_sym(symbol, stream), + bars, + ): + log.info(f"{fsp_func_name}: {processed}") + yield processed diff --git a/piker/fsp.py b/piker/fsp/_momo.py similarity index 64% rename from piker/fsp.py rename to piker/fsp/_momo.py index 25014155..6f1a4c0a 100644 --- a/piker/fsp.py +++ b/piker/fsp/_momo.py @@ -1,82 +1,10 @@ """ -Financial signal processing for the peeps. +Momentum bby. """ -from typing import AsyncIterator, List, Callable +from typing import AsyncIterator -import tractor import numpy as np -from numba import jit, float64, int64, void, optional - -from .log import get_logger -from . import data - -log = get_logger(__name__) - - -async def latency( - source: 'TickStream[Dict[str, float]]', - ohlcv: np.ndarray -) -> AsyncIterator[np.ndarray]: - """Compute High-Low midpoint value. - """ - # TODO: do we want to offer yielding this async - # before the rt data connection comes up? - - # deliver zeros for all prior history - yield np.zeros(len(ohlcv)) - - async for quote in source: - ts = quote.get('broker_ts') - if ts: - # This is codified in the per-broker normalization layer - # TODO: Add more measure points and diffs for full system - # stack tracing. - value = quote['brokerd_ts'] - quote['broker_ts'] - yield value - - -async def stream_and_process( - bars: np.ndarray, - brokername: str, - # symbols: List[str], - symbol: str, - fsp_func_name: str, - func: Callable = latency, -) -> AsyncIterator[dict]: - - # remember, msgpack-numpy's ``from_buffer` returns read-only array - # bars = np.array(bars[list(ohlc_dtype.names)]) - - # async def _yield_bars(): - # yield bars - - # hist_out: np.ndarray = None - - # Conduct a single iteration of fsp with historical bars input - # async for hist_out in func(_yield_bars(), bars): - # yield {symbol: hist_out} - func = _rsi - - # open a data feed stream with requested broker - async with data.open_feed( - brokername, - [symbol], - ) as (fquote, stream): - - # TODO: load appropriate fsp with input args - - async def filter_by_sym(sym, stream): - async for quotes in stream: - for symbol, quotes in quotes.items(): - if symbol == sym: - yield quotes - - async for processed in func( - filter_by_sym(symbol, stream), - bars, - ): - log.info(f"{fsp_func_name}: {processed}") - yield processed +from numba import jit, float64, optional # TODO: things to figure the fuck out: @@ -160,15 +88,15 @@ def rsi( down_ema_last: float64 = None, ) -> 'np.ndarray[float64]': alpha = 1/period - print(signal) + # print(signal) df = np.diff(signal) up, down = np.where(df > 0, df, 0), np.where(df < 0, -df, 0) up_ema = ema(up, alpha, up_ema_last) down_ema = ema(down, alpha, down_ema_last) rs = up_ema / down_ema - print(f'up_ema: {up_ema}\ndown_ema: {down_ema}') - print(f'rs: {rs}') + # print(f'up_ema: {up_ema}\ndown_ema: {down_ema}') + # print(f'rs: {rs}') # map rs through sigmoid (with range [0, 100]) rsi = 100 - 100 / (1 + rs) # rsi = 100 * (up_ema / (up_ema + down_ema)) @@ -194,11 +122,14 @@ async def _rsi( # deliver history yield rsi_h + _last = sig[-1] + async for quote in source: # tick based updates for tick in quote.get('ticks', ()): if tick.get('type') == 'trade': - last = np.array([sig[-1], tick['price']]) + curr = tick['price'] + last = np.array([_last, curr]) # await tractor.breakpoint() rsi_out, up_ema_last, down_ema_last = rsi( last, @@ -206,7 +137,8 @@ async def _rsi( up_ema_last=up_ema_last, down_ema_last=down_ema_last, ) - print(f'last: {last}\n rsi: {rsi_out}') + _last = curr + # print(f'last: {last}\n rsi: {rsi_out}') yield rsi_out[-1] @@ -236,5 +168,5 @@ async def wma( 'valid' ) # todo: handle case where frame_len < length - 1 - _lookback = frame[-(length-1):] + _lookback = frame[-(length-1):] # noqa yield wma From fc0a03d597b386abb2d822183730725a7df8bd00 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Sep 2020 10:47:08 -0400 Subject: [PATCH 098/206] Handle OHLC *and* signal indexing --- piker/ui/_axes.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 2a705a53..46f64b5d 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -186,7 +186,10 @@ class XAxisLabel(AxisLabel): data: float, # data for text offset: int = 0 # if have margins, k? ) -> None: - self.label_str = self.parent._indexes_to_timestrs([int(data)])[0] + timestrs = self.parent._indexes_to_timestrs([int(data)]) + if not timestrs.any(): + return + self.label_str = timestrs[0] width = self.boundingRect().width() new_pos = QPointF(abs_pos.x() - width / 2 - offset, 0) self.setPos(new_pos) @@ -241,8 +244,20 @@ class YSticky(YAxisLabel): chart.sigRangeChanged.connect(self.update_on_resize) def update_on_resize(self, vr, r): - # TODO: figure out how to generalize across data schema - self.update_from_data(*self._chart._array[-1][['index', 'close']]) + # TODO: add an `.index` to the array data-buffer layer + # and make this way less shitty... + a = self._chart._array + fields = a.dtype.fields + if fields and 'index' in fields: + index, last = a[-1][['index', 'close']] + else: + # non-ohlc case + index = len(a) - 1 + last = a[-1] + self.update_from_data( + index, + last, + ) def update_from_data( self, From 80f191c57daf551ec5340caeb7461e5c2703b1c0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Sep 2020 10:47:44 -0400 Subject: [PATCH 099/206] Adjust range logic to avoid overlap with labels By mapping any in view "contents labels" to the range of the ``ViewBox``'s data we can avoid having graphics overlap with labels. Take this approach instead of specifying a min y-range using the std and activate the range compute on resize and mouser scrolling. Also, add y-sticky update for signal plots. --- piker/ui/_chart.py | 79 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index a41d7546..dd100fb9 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,7 +1,7 @@ """ High level Qt chart widgets. """ -from typing import Tuple, Dict, Any +from typing import Tuple, Dict, Any, Optional import time from PyQt5 import QtCore, QtGui @@ -274,6 +274,7 @@ class ChartPlotWidget(pg.PlotWidget): self, # the data view we generate graphics from array: np.ndarray, + yrange: Optional[Tuple[float, float]] = None, **kwargs, ): """Configure chart display settings. @@ -282,12 +283,15 @@ class ChartPlotWidget(pg.PlotWidget): background=hcolor('papas_special'), # parent=None, # plotItem=None, + # useOpenGL=True, **kwargs ) self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics self._labels = {} # registry of underlying graphics self._ysticks = {} # registry of underlying graphics + self._yrange = yrange + self._vb = self.plotItem.vb # show only right side axes self.hideAxis('left') @@ -301,10 +305,16 @@ class ChartPlotWidget(pg.PlotWidget): # use cross-hair for cursor self.setCursor(QtCore.Qt.CrossCursor) - # assign callback for rescaling y-axis automatically - # based on ohlc contents + # Assign callback for rescaling y-axis automatically + # based on data contents and ``ViewBox`` state. self.sigXRangeChanged.connect(self._set_yrange) + vb = self._vb + # for mouse wheel which doesn't seem to emit XRangeChanged + vb.sigRangeChangedManually.connect(self._set_yrange) + # for when the splitter(s) are resized + vb.sigResized.connect(self._set_yrange) + def _update_contents_label(self, index: int) -> None: if index > 0 and index < len(self._array): for name, (label, update) in self._labels.items(): @@ -364,9 +374,10 @@ class ChartPlotWidget(pg.PlotWidget): def update(index: int) -> None: label.setText( - "{name} O:{} H:{} L:{} C:{} V:{}".format( + "{name}[{index}] -> O:{} H:{} L:{} C:{} V:{}".format( *self._array[index].item()[2:], name=name, + index=index, ) ) @@ -412,7 +423,7 @@ class ChartPlotWidget(pg.PlotWidget): def update(index: int) -> None: data = self._array[index] - label.setText(f"{name}: {index} {data}") + label.setText(f"{name} -> {data}") self._labels[name] = (label, update) self._update_contents_label(index=-1) @@ -427,6 +438,8 @@ class ChartPlotWidget(pg.PlotWidget): # "only update with new items" on the pg.PlotDataItem curve.update_from_array = curve.setData + self._add_sticky(name) + return curve def _add_sticky( @@ -435,7 +448,7 @@ class ChartPlotWidget(pg.PlotWidget): # retreive: Callable[None, np.ndarray], ) -> YSticky: # add y-axis "last" value label - last = self._ysticks['last'] = YSticky( + last = self._ysticks[name] = YSticky( chart=self, parent=self.getAxis('right'), # digits=0, @@ -490,33 +503,43 @@ class ChartPlotWidget(pg.PlotWidget): # likely no data loaded yet log.error(f"WTF bars_range = {lbar}:{rbar}") return - elif lbar < 0: - breakpoint() # TODO: should probably just have some kinda attr mark # that determines this behavior based on array type try: ylow = bars['low'].min() yhigh = bars['high'].max() - std = np.std(bars['close']) + # std = np.std(bars['close']) except IndexError: # must be non-ohlc array? ylow = bars.min() yhigh = bars.max() - std = np.std(bars) + # std = np.std(bars) # view margins: stay within 10% of the "true range" diff = yhigh - ylow - ylow = ylow - (diff * 0.1) - yhigh = yhigh + (diff * 0.1) + ylow = ylow - (diff * 0.04) + yhigh = yhigh + (diff * 0.01) + + # compute contents label "height" in view terms + if self._labels: + label = self._labels[self.name][0] + rect = label.itemRect() + tl, br = rect.topLeft(), rect.bottomRight() + vb = self.plotItem.vb + top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y()) + label_h = top - bottom + # print(f'label height {self.name}: {label_h}') + else: + label_h = 0 chart = self chart.setLimits( yMin=ylow, - yMax=yhigh, - minYRange=std + yMax=yhigh + label_h, + # minYRange=std ) - chart.setYRange(ylow, yhigh) + chart.setYRange(ylow, yhigh + label_h) def enterEvent(self, ev): # noqa # pg.PlotWidget.enterEvent(self, ev) @@ -577,7 +600,10 @@ async def add_new_bars(delay_s, linked_charts): return new_array # add new increment/bar + start = time.time() ohlc = price_chart._array = incr_ohlc_array(ohlc) + diff = time.time() - start + print(f'array append took {diff}') # TODO: generalize this increment logic for name, chart in linked_charts.subplots.items(): @@ -660,7 +686,7 @@ async def _async_main( n.start_soon( chart_from_fsp, linked_charts, - fsp.latency, + 'rsi', sym, bars, brokermod, @@ -668,8 +694,10 @@ async def _async_main( ) # update last price sticky - last = chart._ysticks['last'] - last.update_from_data(*chart._array[-1][['index', 'close']]) + last_price_sticky = chart._ysticks[chart.name] + last_price_sticky.update_from_data( + *chart._array[-1][['index', 'close']] + ) # graphics update loop @@ -711,15 +739,14 @@ async def _async_main( chart._array, ) # update sticky(s) - last = chart._ysticks['last'] - last.update_from_data( + last_price_sticky.update_from_data( *chart._array[-1][['index', 'close']]) chart._set_yrange() async def chart_from_fsp( linked_charts, - fsp_func, + func_name, sym, bars, brokermod, @@ -729,8 +756,6 @@ async def chart_from_fsp( Pass target entrypoint and historical data. """ - func_name = fsp_func.__name__ - async with tractor.open_nursery() as n: portal = await n.run_in_actor( f'fsp.{func_name}', # name as title of sub-chart @@ -749,7 +774,7 @@ async def chart_from_fsp( stream = await portal.result() # receive processed historical data-array as first message - history: np.ndarray = (await stream.__anext__()) + history = (await stream.__anext__()) # TODO: enforce type checking here newbars = np.array(history) @@ -768,9 +793,15 @@ async def chart_from_fsp( np.full(abs(diff), data[-1], dtype=data.dtype) ) + value = chart._array[-1] + last_val_sticky = chart._ysticks[chart.name] + last_val_sticky.update_from_data(-1, value) + # update chart graphics async for value in stream: chart._array[-1] = value + last_val_sticky.update_from_data(-1, value) + chart._set_yrange() chart.update_from_array(chart.name, chart._array) chart._set_yrange() From da2325239ce3ea1ce30bfdb36bbd779c39928109 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 9 Sep 2020 21:19:36 -0400 Subject: [PATCH 100/206] Copy non-base dtype fields on bar increment --- piker/ui/_axes.py | 7 ++----- piker/ui/_chart.py | 28 +++++++++++++++------------- piker/ui/_source.py | 6 +++--- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 46f64b5d..24389f77 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -1,8 +1,6 @@ """ Chart axes graphics and behavior. """ -import time -from functools import partial from typing import List @@ -12,7 +10,6 @@ import pyqtgraph as pg from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QPointF -from .quantdom.utils import fromtimestamp from ._style import _font, hcolor @@ -78,7 +75,8 @@ class DynamicDateAxis(pg.AxisItem): bars = self.linked_charts.chart._array times = bars['time'] bars_len = len(bars) - delay = times[-1] - times[times != times[-1]][-1] + # delay = times[-1] - times[times != times[-1]][-1] + delay = times[-1] - times[-2] epochs = times[list( map(int, filter(lambda i: i < bars_len, indexes)) @@ -87,7 +85,6 @@ class DynamicDateAxis(pg.AxisItem): dts = pd.to_datetime(epochs, unit='s') - 4*pd.offsets.Hour() return dts.strftime(self.tick_tpl[delay]) - def tickStrings(self, values: List[float], scale, spacing): return self._indexes_to_timestrs(values) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index dd100fb9..13071f1a 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -22,7 +22,7 @@ from .. import brokers from .. import data from ..log import get_logger from ._exec import run_qtractor -from ._source import ohlc_dtype +from ._source import base_ohlc_dtype from ._interaction import ChartView from .. import fsp @@ -507,13 +507,13 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: should probably just have some kinda attr mark # that determines this behavior based on array type try: - ylow = bars['low'].min() - yhigh = bars['high'].max() + ylow = np.nanmin(bars['low']) + yhigh = np.nanmax(bars['high']) # std = np.std(bars['close']) except IndexError: # must be non-ohlc array? - ylow = bars.min() - yhigh = bars.max() + ylow = np.nanmin(bars) + yhigh = np.nanmax(bars) # std = np.std(bars) # view margins: stay within 10% of the "true range" @@ -589,14 +589,13 @@ async def add_new_bars(delay_s, linked_charts): def incr_ohlc_array(array: np.ndarray): (index, t, close) = array[-1][['index', 'time', 'close']] - new_array = np.append( - array, - np.array( - [(index + 1, t + delay_s, close, close, - close, close, 0)], - dtype=array.dtype - ), - ) + + # this copies non-std fields (eg. vwap) from the last datum + _next = np.array(array[-1], dtype=array.dtype) + _next[ + ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] + ] = (index + 1, t + delay_s, 0, close, close, close, close) + new_array = np.append(array, _next,) return new_array # add new increment/bar @@ -668,6 +667,9 @@ async def _async_main( # figure out the exact symbol bars = await client.bars(symbol=sym) + # allow broker to declare historical data fields + ohlc_dtype = getattr(brokermod, 'ohlc_dtype', base_ohlc_dtype) + # remember, msgpack-numpy's ``from_buffer` returns read-only array bars = np.array(bars[list(ohlc_dtype.names)]) diff --git a/piker/ui/_source.py b/piker/ui/_source.py index 0224538b..b8816636 100644 --- a/piker/ui/_source.py +++ b/piker/ui/_source.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -ohlc_dtype = np.dtype( +base_ohlc_dtype = np.dtype( [ ('index', int), ('time', float), @@ -38,7 +38,7 @@ def ohlc_zeros(length: int) -> np.ndarray: For "why a structarray" see here: https://stackoverflow.com/a/52443038 Bottom line, they're faster then ``np.recarray``. """ - return np.zeros(length, dtype=ohlc_dtype) + return np.zeros(length, dtype=base_ohlc_dtype) @dataclass @@ -88,7 +88,7 @@ def from_df( df = df.rename(columns=columns) for name in df.columns: - if name not in ohlc_dtype.names[1:]: + if name not in base_ohlc_dtype.names[1:]: del df[name] # TODO: it turns out column access on recarrays is actually slower: From a4a5bff3fa4a202a5fff97a7a1f1cf71b5ac985b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 10 Sep 2020 16:16:21 -0400 Subject: [PATCH 101/206] Yes, even more grays --- piker/ui/_style.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index fcaeacf6..04edf6e8 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -43,10 +43,16 @@ def hcolor(name: str) -> str: # fifty shades 'gray': '#808080', # like the kick 'jet': '#343434', + 'cadet': '#91A3B0', + 'marengo': '#91A3B0', 'charcoal': '#36454F', + 'gunmetal': '#91A3B0', + 'battleship': '#848482', + 'davies': '#555555', # palette 'default': DarkPalette.COLOR_BACKGROUND_NORMAL, + 'default_light': DarkPalette.COLOR_BACKGROUND_LIGHT, 'white': '#ffffff', # for tinas and sunbathers From e91ba55d685019c7a9592e8cf2ce06fd0e04e8ce Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Sep 2020 12:25:06 -0400 Subject: [PATCH 102/206] Always draw any history bars on update --- piker/ui/_graphics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 25ea7bdd..18badc2c 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -286,6 +286,7 @@ class BarItems(pg.GraphicsObject): ) -> None: super().__init__() self.last = QtGui.QPicture() + # self.buffer = QtGui.QPicture() self.history = QtGui.QPicture() # TODO: implement updateable pixmap solution self._pi = plotitem @@ -400,8 +401,8 @@ class BarItems(pg.GraphicsObject): self.index += bars_added # start_bar_to_update = index - bars_added + self.draw_lines(just_history=True) if just_history: - self.draw_lines(just_history=True) # istart=self.index - 1) return # current bar update From eb5d64ceefb60ae199e6e7d0ed8d083962155ecb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Sep 2020 13:16:11 -0400 Subject: [PATCH 103/206] Add support for overlay curves and fixed y-range Allow passing a fixed ylow, yhigh tuple to `._set_yrange()` which avoids recomputing the range from data if desired (eg. rsi-like bounded signals). Add support for overlay curves to the OHLC chart and add basic support to brokers which provide a historical 'vwap`. The data array increment logic had to be tweaked to copy the vwap from the last bar. Oh, and hack the subchart curves with two extra prepended datums to make them align "better" with the ohlc main chart; need to talk to `pyqtgraph` core about how to do this more correctly. --- piker/ui/_chart.py | 152 +++++++++++++++++++++++++++++++++------------ 1 file changed, 111 insertions(+), 41 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 13071f1a..f15a8d53 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -292,6 +292,7 @@ class ChartPlotWidget(pg.PlotWidget): self._ysticks = {} # registry of underlying graphics self._yrange = yrange self._vb = self.plotItem.vb + self._static_yrange = None # show only right side axes self.hideAxis('left') @@ -368,7 +369,7 @@ class ChartPlotWidget(pg.PlotWidget): # Ogi says: "use ..." label = pg.LabelItem( justify='left', - size='5pt', + size='4pt', ) self.scene().addItem(label) @@ -383,6 +384,7 @@ class ChartPlotWidget(pg.PlotWidget): self._labels[name] = (label, update) self._update_contents_label(index=-1) + label.show() # set xrange limits xlast = data[-1]['index'] @@ -398,13 +400,22 @@ class ChartPlotWidget(pg.PlotWidget): self, name: str, data: np.ndarray, + overlay: bool = False, + **pdi_kwargs, ) -> pg.PlotDataItem: # draw the indicator as a plain curve + _pdi_defaults = { + 'pen': pg.mkPen(hcolor('default_light')), + } + pdi_kwargs.update(_pdi_defaults) + curve = pg.PlotDataItem( data, antialias=True, + name=name, # TODO: see how this handles with custom ohlcv bars graphics clipToView=True, + **pdi_kwargs, ) self.addItem(curve) @@ -417,8 +428,15 @@ class ChartPlotWidget(pg.PlotWidget): # XXX: How to stack labels vertically? label = pg.LabelItem( justify='left', - size='5pt', + size='4pt', ) + label.setParentItem(self._vb) + if overlay: + # position bottom left if an overlay + label.anchor(itemPos=(0, 1), parentPos=(0, 1), offset=(0, 25)) + + label.show() + self.scene().addItem(label) def update(index: int) -> None: @@ -471,6 +489,8 @@ class ChartPlotWidget(pg.PlotWidget): def _set_yrange( self, + *, + yrange: Optional[Tuple[float, float]] = None, ) -> None: """Set the viewable y-range based on embedded data. @@ -483,38 +503,47 @@ class ChartPlotWidget(pg.PlotWidget): # figure out x-range in view such that user can scroll "off" the data # set up to the point where ``_min_points_to_show`` are left. # if l < lbar or r > rbar: - bars_len = rbar - lbar view_len = r - l # TODO: logic to check if end of bars in view extra = view_len - _min_points_to_show begin = 0 - extra end = len(self._array) - 1 + extra - log.trace( - f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" - f"view_len: {view_len}, bars_len: {bars_len}\n" - f"begin: {begin}, end: {end}, extra: {extra}" - ) + # bars_len = rbar - lbar + # log.trace( + # f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" + # f"view_len: {view_len}, bars_len: {bars_len}\n" + # f"begin: {begin}, end: {end}, extra: {extra}" + # ) self._set_xlimits(begin, end) - # TODO: this should be some kind of numpy view api - bars = self._array[lbar:rbar] - if not len(bars): - # likely no data loaded yet - log.error(f"WTF bars_range = {lbar}:{rbar}") - return + # yrange + if self._static_yrange is not None: + yrange = self._static_yrange - # TODO: should probably just have some kinda attr mark - # that determines this behavior based on array type - try: - ylow = np.nanmin(bars['low']) - yhigh = np.nanmax(bars['high']) - # std = np.std(bars['close']) - except IndexError: - # must be non-ohlc array? - ylow = np.nanmin(bars) - yhigh = np.nanmax(bars) - # std = np.std(bars) + if yrange is not None: + ylow, yhigh = yrange + self._static_yrange = yrange + else: + + # TODO: this should be some kind of numpy view api + bars = self._array[lbar:rbar] + if not len(bars): + # likely no data loaded yet + log.error(f"WTF bars_range = {lbar}:{rbar}") + return + + # TODO: should probably just have some kinda attr mark + # that determines this behavior based on array type + try: + ylow = np.nanmin(bars['low']) + yhigh = np.nanmax(bars['high']) + # std = np.std(bars['close']) + except IndexError: + # must be non-ohlc array? + ylow = np.nanmin(bars) + yhigh = np.nanmax(bars) + # std = np.std(bars) # view margins: stay within 10% of the "true range" diff = yhigh - ylow @@ -527,8 +556,12 @@ class ChartPlotWidget(pg.PlotWidget): rect = label.itemRect() tl, br = rect.topLeft(), rect.bottomRight() vb = self.plotItem.vb - top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y()) - label_h = top - bottom + try: + # on startup labels might not yet be rendered + top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y()) + label_h = top - bottom + except np.linalg.LinAlgError: + label_h = 0 # print(f'label height {self.name}: {label_h}') else: label_h = 0 @@ -575,6 +608,17 @@ async def add_new_bars(delay_s, linked_charts): # sleep for duration of current bar await sleep() + def incr_ohlc_array(array: np.ndarray): + (index, t, close) = array[-1][['index', 'time', 'close']] + + # this copies non-std fields (eg. vwap) from the last datum + _next = np.array(array[-1], dtype=array.dtype) + _next[ + ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] + ] = (index + 1, t + delay_s, 0, close, close, close, close) + new_array = np.append(array, _next,) + return new_array + while True: # TODO: bunch of stuff: # - I'm starting to think all this logic should be @@ -587,17 +631,6 @@ async def add_new_bars(delay_s, linked_charts): # of copying it from last bar's close # - 5 sec bar lookback-autocorrection like tws does? - def incr_ohlc_array(array: np.ndarray): - (index, t, close) = array[-1][['index', 'time', 'close']] - - # this copies non-std fields (eg. vwap) from the last datum - _next = np.array(array[-1], dtype=array.dtype) - _next[ - ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] - ] = (index + 1, t + delay_s, 0, close, close, close, close) - new_array = np.append(array, _next,) - return new_array - # add new increment/bar start = time.time() ohlc = price_chart._array = incr_ohlc_array(ohlc) @@ -624,7 +657,11 @@ async def add_new_bars(delay_s, linked_charts): price_chart.update_from_array( price_chart.name, ohlc, - just_history=True + # When appending a new bar, in the time between the insert + # here and the Qt render call the underlying price data may + # have already been updated, thus make sure to also update + # the last bar if necessary on this render cycle. + # just_history=True ) # resize view price_chart._set_yrange() @@ -676,6 +713,16 @@ async def _async_main( # load in symbol's ohlc data linked_charts, chart = chart_app.load_symbol(sym, bars) + # plot historical vwap if available + vwap_in_history = False + if 'vwap' in bars.dtype.fields: + vwap_in_history = True + chart.draw_curve( + name='vwap', + data=bars['vwap'], + overlay=True, + ) + # determine ohlc delay between bars times = bars['time'] @@ -745,6 +792,16 @@ async def _async_main( *chart._array[-1][['index', 'close']]) chart._set_yrange() + vwap = quote.get('vwap') + if vwap and vwap_in_history: + chart._array['vwap'][-1] = vwap + print(f"vwap: {quote['vwap']}") + # update vwap overlay line + chart.update_from_array( + 'vwap', + chart._array['vwap'], + ) + async def chart_from_fsp( linked_charts, @@ -781,6 +838,18 @@ async def chart_from_fsp( # TODO: enforce type checking here newbars = np.array(history) + # XXX: hack to get curves aligned with bars graphics: prepend a copy of + # the first datum.. + # TODO: talk to ``pyqtgraph`` core about proper way to solve + newbars = np.append( + np.array(newbars[0], dtype=newbars.dtype), + newbars + ) + newbars = np.append( + np.array(newbars[0], dtype=newbars.dtype), + newbars + ) + chart = linked_charts.add_plot( name=func_name, array=newbars, @@ -799,13 +868,14 @@ async def chart_from_fsp( last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) + chart._set_yrange(yrange=(0, 100)) + # update chart graphics async for value in stream: chart._array[-1] = value last_val_sticky.update_from_data(-1, value) - chart._set_yrange() chart.update_from_array(chart.name, chart._array) - chart._set_yrange() + # chart._set_yrange() def _main( From 2cc2b3280501105aa3726e413ba4f0a5d5e07b7a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Sep 2020 19:32:07 -0400 Subject: [PATCH 104/206] Fix dbz with `np.divide()` --- piker/fsp/_momo.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 6f1a4c0a..557c0614 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -94,7 +94,12 @@ def rsi( up, down = np.where(df > 0, df, 0), np.where(df < 0, -df, 0) up_ema = ema(up, alpha, up_ema_last) down_ema = ema(down, alpha, down_ema_last) - rs = up_ema / down_ema + rs = np.divide( + up_ema, + down_ema, + out=np.zeros_like(up_ema), + where=down_ema!=0 + ) # print(f'up_ema: {up_ema}\ndown_ema: {down_ema}') # print(f'rs: {rs}') # map rs through sigmoid (with range [0, 100]) @@ -117,6 +122,9 @@ async def _rsi( https://en.wikipedia.org/wiki/Relative_strength_index """ sig = ohlcv['close'] + + # TODO: the emas here should be seeded with a period SMA as per + # wilder's original formula.. rsi_h, up_ema_last, down_ema_last = rsi(sig, period, None, None) # deliver history From 07beec59bfef0aac7e60cdc5309c0b12b6526b89 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 11 Sep 2020 19:33:05 -0400 Subject: [PATCH 105/206] Hopefully fix datum offset on signal sub-plots Added a comment to clarify, ish. Add `ChartPlotWidget._overlays` as registry of curves added on top of main graphics. Hackishly (ad-hoc-ishly?) update the curve assuming the data resides in the same `._array` for now (which it does for historical vwap). --- piker/ui/_chart.py | 44 +++++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index f15a8d53..b33a613c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -288,6 +288,7 @@ class ChartPlotWidget(pg.PlotWidget): ) self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics + self._overlays = {} # registry of overlay curves self._labels = {} # registry of underlying graphics self._ysticks = {} # registry of underlying graphics self._yrange = yrange @@ -317,7 +318,7 @@ class ChartPlotWidget(pg.PlotWidget): vb.sigResized.connect(self._set_yrange) def _update_contents_label(self, index: int) -> None: - if index > 0 and index < len(self._array): + if index >= 0 and index < len(self._array): for name, (label, update) in self._labels.items(): update(index) @@ -421,7 +422,7 @@ class ChartPlotWidget(pg.PlotWidget): # register overlay curve with name if not self._graphics and name is None: - name = 'a_line_bby' + name = 'a_stupid_line_bby' self._graphics[name] = curve @@ -431,12 +432,13 @@ class ChartPlotWidget(pg.PlotWidget): size='4pt', ) label.setParentItem(self._vb) + if overlay: # position bottom left if an overlay label.anchor(itemPos=(0, 1), parentPos=(0, 1), offset=(0, 25)) + self._overlays[name] = curve label.show() - self.scene().addItem(label) def update(index: int) -> None: @@ -637,6 +639,7 @@ async def add_new_bars(delay_s, linked_charts): diff = time.time() - start print(f'array append took {diff}') + # TODO: generalize this increment logic for name, chart in linked_charts.subplots.items(): data = chart._array @@ -666,6 +669,15 @@ async def add_new_bars(delay_s, linked_charts): # resize view price_chart._set_yrange() + for name, curve in price_chart._overlays.items(): + # TODO: standard api for signal lookups per plot + if name in price_chart._array.dtype.fields: + # should have already been incremented above + price_chart.update_from_array( + name, + price_chart._array[name], + ) + for name, chart in linked_charts.subplots.items(): chart.update_from_array(chart.name, chart._array) chart._set_yrange() @@ -835,21 +847,9 @@ async def chart_from_fsp( # receive processed historical data-array as first message history = (await stream.__anext__()) - # TODO: enforce type checking here + # TODO: enforce type checking here? newbars = np.array(history) - # XXX: hack to get curves aligned with bars graphics: prepend a copy of - # the first datum.. - # TODO: talk to ``pyqtgraph`` core about proper way to solve - newbars = np.append( - np.array(newbars[0], dtype=newbars.dtype), - newbars - ) - newbars = np.append( - np.array(newbars[0], dtype=newbars.dtype), - newbars - ) - chart = linked_charts.add_plot( name=func_name, array=newbars, @@ -857,16 +857,26 @@ async def chart_from_fsp( # check for data length mis-allignment and fill missing values diff = len(chart._array) - len(linked_charts.chart._array) - if diff < 0: + if diff <= 0: data = chart._array chart._array = np.append( data, np.full(abs(diff), data[-1], dtype=data.dtype) ) + # XXX: hack to get curves aligned with bars graphics: prepend + # a copy of the first datum.. + # TODO: talk to ``pyqtgraph`` core about proper way to solve this + data = chart._array + chart._array = np.append( + np.array(data[0], dtype=data.dtype), + data, + ) + value = chart._array[-1] last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) + chart.update_from_array(chart.name, chart._array) chart._set_yrange(yrange=(0, 100)) From 2f1e9ee7601acdad97c21717d613252486ec41fd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 12 Sep 2020 11:52:59 -0400 Subject: [PATCH 106/206] Update dev deps to current state of things.. --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 99effd91..db505cfd 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from setuptools import setup +from setuptools import setup, find_packages with open('README.rst', encoding='utf-8') as f: readme = f.read() @@ -32,12 +32,7 @@ setup( maintainer='Tyler Goodlet', url='https://github.com/pikers/piker', platforms=['linux'], - packages=[ - 'piker', - 'piker.brokers', - 'piker.ui', - 'piker.testing', - ], + packages=find_packages(), entry_points={ 'console_scripts': [ 'piker = piker.cli:cli', @@ -58,13 +53,14 @@ setup( 'async_generator', # brokers - 'asks', + 'asks==2.4.8', 'ib_insync', # numerics 'arrow', # better datetimes 'cython', 'numpy', + 'numba', 'pandas', 'msgpack-numpy', From 712e36b9d57c7406ab4ef6c2649407faca7f944c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 16 Sep 2020 09:25:11 -0400 Subject: [PATCH 107/206] First draft of a shared numpy array sub-system This adds a shared memory "incrementing array" sub-sys interface for single writer, multi-reader style data passing. The main motivation is to avoid multiple copies of the same `numpy` array across actors (plus now we can start being fancy like ray). There still seems to be some odd issues with the "resource tracker" complaining at teardown (likely partially to do with SIGINT stuff) so some further digging in the stdlib code is likely coming. Pertains to #107 and #98 --- piker/data/_sharedmem.py | 200 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 piker/data/_sharedmem.py diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py new file mode 100644 index 00000000..cb565b26 --- /dev/null +++ b/piker/data/_sharedmem.py @@ -0,0 +1,200 @@ +""" +NumPy based shared memory for real-time FSP. +""" +from sys import byteorder +from contextlib import contextmanager +from typing import Tuple, Optional +from multiprocessing import shared_memory + +import numpy as np +# from numpy.lib import recfunctions as rfn + + +base_ohlc_dtype = np.dtype( + [ + ('index', int), + ('time', float), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', int), + ] +) + + +class SharedInt: + def __init__( + self, + token: str, + create: bool = False, + ) -> None: + # create a single entry array for storing an index counter + self._shm = shared_memory.SharedMemory( + name=token, + create=create, + size=4, # std int + ) + self._token = self._shm.name + + @property + def value(self) -> int: + return int.from_bytes(self._shm.buf, byteorder) + + @value.setter + def value(self, value) -> None: + self._shm.buf[:] = value.to_bytes(4, byteorder) + + +class SharedArray: + def __init__( + self, + shmarr: np.ndarray, + counter: SharedInt, + shm: shared_memory.SharedMemory, + readonly: bool = True, + ) -> None: + self._array = shmarr + self._i = counter + self._len = len(shmarr) + self._shm = shm + self._readonly = readonly + + # TODO: ringbuf api? + + @property + def token(self) -> Tuple[str, str]: + return (self._shm.name, self._i._token) + + @property + def name(self) -> str: + return self._shm.name + + @property + def index(self) -> int: + return self._i.value % self._len + + @property + def array(self) -> np.ndarray: + return self._array[:self._i.value] + + def last( + self, + length: int = 1, + ) -> np.ndarray: + return self.array[-length:] + + def push( + self, + data: np.ndarray, + ) -> int: + # push array data and return updated index + length = len(data) + self._array[self._i.value:length] = data + self._i.value += length + + def close(self) -> None: + self._i._shm.close() + self._shm.close() + + def destroy(self) -> None: + self._i._shm.unlink() + self._shm.unlink() + + # def flush(self) -> None: + # # flush to storage backend? + # ... + + +@contextmanager +def open_shared_array( + name: Optional[str] = None, + create: bool = True, + # approx number of 5s bars in a "day" + size: int = int(60*60*10/5), + dtype: np.dtype = base_ohlc_dtype, + readonly: bool = False, +) -> SharedArray: + """Open a memory shared ``numpy`` using the standard library. + + This call unlinks (aka permanently destroys) the buffer on teardown + and thus should be used from the parent-most accessor (process). + """ + # create new shared mem segment for which we + # have write permission + a = np.zeros(size, dtype=dtype) + shm = shared_memory.SharedMemory(name=name, create=True, size=a.nbytes) + shmarr = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) + shmarr[:] = a[:] + shmarr.setflags(write=int(not readonly)) + + counter = SharedInt( + token=shm.name + "_counter", + create=True, + ) + counter.value = 0 + + sha = SharedArray( + shmarr, + counter, + shm, + readonly=readonly, + ) + try: + yield sha + finally: + sha.close() + print(f"UNLINKING {sha.token}") + sha.destroy() + + +@contextmanager +def attach_shared_array( + token: Tuple[str, str], + size: int = int(60*60*10/5), + dtype: np.dtype = base_ohlc_dtype, + readonly: bool = True, +) -> SharedArray: + """Load and attach to an existing shared memory array. + """ + array_name, counter_name = token + + shm = shared_memory.SharedMemory(name=array_name) + shmarr = np.ndarray((size,), dtype=dtype, buffer=shm.buf) + shmarr.setflags(write=int(not readonly)) + + counter = SharedInt(token=counter_name) + # make sure we can read + counter.value + + sha = SharedArray( + shmarr, + counter, + shm, + readonly=readonly, + ) + sha.array + try: + yield sha + finally: + pass + sha.close() + + +@contextmanager +def maybe_open_shared_array( + name: str, + **kwargs, +) -> SharedArray: + try: + with open_shared_array( + name=name, + **kwargs, + ) as shm: + yield shm + except FileExistsError: + with attach_shared_array( + token=(name, name + '_counter'), + **kwargs, + ) as shm: + yield shm From 17491ba8194d10420ff2b1f2245aa7bc1a171702 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 17 Sep 2020 09:03:11 -0400 Subject: [PATCH 108/206] Disconnect stdlib's resource_tracker, fix .push() Logic in `SharedArray.push()` was totally wrong. Remove all the `multiprocessing.resource_tracker` crap such that we aren't loading an extra process at every layer and we don't get tons of errors when cleaning on in an SC way. --- piker/data/_sharedmem.py | 55 ++++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index cb565b26..cfe162a7 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -1,13 +1,36 @@ """ -NumPy based shared memory for real-time FSP. +NumPy compatible shared memory buffers for real-time FSP. """ from sys import byteorder from contextlib import contextmanager from typing import Tuple, Optional from multiprocessing import shared_memory +from multiprocessing import resource_tracker as mantracker +from _posixshmem import shm_unlink import numpy as np -# from numpy.lib import recfunctions as rfn + + +# Tell the "resource tracker" thing to fuck off. +class ManTracker(mantracker.ResourceTracker): + def register(self, name, rtype): + pass + + def unregister(self, name, rtype): + pass + + def ensure_running(self): + pass + + +# "know your land and know your prey" +# https://www.dailymotion.com/video/x6ozzco +mantracker._resource_tracker = ManTracker() +mantracker.register = mantracker._resource_tracker.register +mantracker.ensure_running = mantracker._resource_tracker.ensure_running +ensure_running = mantracker._resource_tracker.ensure_running +mantracker.unregister = mantracker._resource_tracker.unregister +mantracker.getfd = mantracker._resource_tracker.getfd base_ohlc_dtype = np.dtype( @@ -88,22 +111,31 @@ class SharedArray: self, data: np.ndarray, ) -> int: - # push array data and return updated index + """Ring buffer like "push" to append data + into the buffer and return updated index. + """ length = len(data) - self._array[self._i.value:length] = data - self._i.value += length + # TODO: use .index for actual ring logic? + index = self._i.value + end = index + length + self._array[index:end] = data[:] + self._i.value = end + return end def close(self) -> None: self._i._shm.close() self._shm.close() def destroy(self) -> None: - self._i._shm.unlink() - self._shm.unlink() + if shared_memory._USE_POSIX: + # We manually unlink to bypass all the "resource tracker" + # nonsense meant for non-SC systems. + shm_unlink(self._i._shm.name) + shm_unlink(self._shm.name) - # def flush(self) -> None: - # # flush to storage backend? - # ... + def flush(self) -> None: + # TODO: flush to storage backend like markestore? + ... @contextmanager @@ -155,7 +187,8 @@ def attach_shared_array( dtype: np.dtype = base_ohlc_dtype, readonly: bool = True, ) -> SharedArray: - """Load and attach to an existing shared memory array. + """Load and attach to an existing shared memory array previously + created by another process using ``open_shared_array``. """ array_name, counter_name = token From f872fbecf89c6f992faf98415b64f4ffccdd9f44 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 17 Sep 2020 09:12:05 -0400 Subject: [PATCH 109/206] Hook IB up to shared memory system Adjust the `data.open_feed()` api to take a shm token so the broker-daemon can attach a previously created (by the parent actor) mem buf and push real-time tick data. There's still some sloppiness here in terms of ensuring only one mem buf per symbol (can be seen in `stream_quotes()`) which should really managed at the data api level. Add a bar incrementing stream-task which delivers increment msgs to any consumers. --- piker/brokers/ib.py | 111 +++++++++++++++++++++++++++-------------- piker/data/__init__.py | 73 +++++++++++++++++++-------- piker/data/_buffer.py | 74 +++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 58 deletions(-) create mode 100644 piker/data/_buffer.py diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 67c3e799..b0d89d1b 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -25,7 +25,10 @@ import trio import tractor from ..log import get_logger, get_console_log -from ..data import maybe_spawn_brokerd +from ..data import ( + maybe_spawn_brokerd, iterticks, attach_shared_array, + incr_buffer, +) from ..ui._source import from_df @@ -104,7 +107,6 @@ _adhoc_cmdty_data_map = { # NOTE: cmdtys don't have trade data: # https://groups.io/g/twsapi/message/44174 'XAUUSD': ({'conId': 69067924}, {'whatToShow': 'MIDPOINT'}), - 'XAUUSD': ({'conId': 69067924}, {'whatToShow': 'MIDPOINT'}), } @@ -143,7 +145,7 @@ class Client: # durationStr='1 D', # time length calcs - durationStr='{count} S'.format(count=3000 * 5), + durationStr='{count} S'.format(count=5000 * 5), barSizeSetting='5 secs', # always use extended hours @@ -487,6 +489,7 @@ def normalize( # @tractor.msg.pub async def stream_quotes( symbols: List[str], + shared_array_token: Tuple[str, str], loglevel: str = None, # compat for @tractor.msg.pub topics: Any = None, @@ -508,26 +511,25 @@ async def stream_quotes( method='stream_ticker', symbol=sym, ) + + async with get_client() as client: + bars = await client.bars(symbol=sym) + async with aclosing(stream): # first quote can be ignored as a 2nd with newer data is sent? first_ticker = await stream.__anext__() + quote = normalize(first_ticker) + # ugh, clear ticks since we've consumed them + # (ahem, ib_insync is stateful trash) + first_ticker.ticks = [] + + log.debug(f"First ticker received {quote}") if type(first_ticker.contract) not in (ibis.Commodity, ibis.Forex): suffix = 'exchange' calc_price = False # should be real volume for contract - quote = normalize(first_ticker) - log.debug(f"First ticker received {quote}") - - con = quote['contract'] - topic = '.'.join((con['symbol'], con[suffix])).lower() - yield {topic: quote} - - # ugh, clear ticks since we've consumed them - # (ahem, ib_insync is stateful trash) - first_ticker.ticks = [] - async for ticker in stream: # spin consuming tickers until we get a real market datum if not ticker.rtTime: @@ -535,10 +537,6 @@ async def stream_quotes( continue else: log.debug("Received first real volume tick") - quote = normalize(ticker) - topic = '.'.join((con['symbol'], con[suffix])).lower() - yield {topic: quote} - # ugh, clear ticks since we've consumed them # (ahem, ib_insync is stateful trash) ticker.ticks = [] @@ -550,28 +548,65 @@ async def stream_quotes( # commodities don't have an exchange name for some reason? suffix = 'secType' calc_price = True + ticker = first_ticker - async for ticker in stream: - quote = normalize( - ticker, - calc_price=calc_price - ) - con = quote['contract'] - topic = '.'.join((con['symbol'], con[suffix])).lower() - yield {topic: quote} + con = quote['contract'] + quote = normalize(ticker, calc_price=calc_price) + topic = '.'.join((con['symbol'], con[suffix])).lower() + first_quote = {topic: quote} + ticker.ticks = [] - # ugh, clear ticks since we've consumed them - ticker.ticks = [] + # load historical ohlcv in to shared mem + ss = tractor.current_actor().statespace + existing_shm = ss.get(f'ib_shm.{sym}') + if not existing_shm: + readonly = False + else: + readonly = True + shm = existing_shm + with attach_shared_array( + token=shared_array_token, + readonly=readonly + ) as shm: + if not existing_shm: + shm.push(bars) + ss[f'ib_shm.{sym}'] = shm -if __name__ == '__main__': - import sys - sym = sys.argv[1] + yield (first_quote, shm.token) + else: + yield (first_quote, None) - contract = asyncio.run( - _aio_run_client_method( - 'find_contract', - symbol=sym, - ) - ) - print(contract) + async for ticker in stream: + quote = normalize( + ticker, + calc_price=calc_price + ) + # TODO: in theory you can send the IPC msg *before* + # writing to the sharedmem array to decrease latency, + # however, that will require `tractor.msg.pub` support + # here or at least some way to prevent task switching + # at the yield such that the array write isn't delayed + # while another consumer is serviced.. + + # if we are the lone tick writer + if not existing_shm: + for tick in iterticks(quote, type='trade'): + last = tick['price'] + # print(f'broker last: {tick}') + + # update last entry + # benchmarked in the 4-5 us range + high, low = shm.array[-1][['high', 'low']] + shm.array[['high', 'low', 'close']][-1] = ( + max(high, last), + min(low, last), + last, + ) + + con = quote['contract'] + topic = '.'.join((con['symbol'], con[suffix])).lower() + yield {topic: quote} + + # ugh, clear ticks since we've consumed them + ticker.ticks = [] diff --git a/piker/data/__init__.py b/piker/data/__init__.py index 25efe088..3ef85c96 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -18,6 +18,20 @@ import tractor from ..brokers import get_brokermod from ..log import get_logger, get_console_log +from ._normalize import iterticks +from ._sharedmem import ( + maybe_open_shared_array, attach_shared_array, open_shared_array, +) +from ._buffer import incr_buffer + + +__all__ = [ + 'maybe_open_shared_array', + 'attach_shared_array', + 'open_shared_array', + 'iterticks', + 'incr_buffer', +] log = get_logger(__name__) @@ -27,7 +41,7 @@ __ingestors__ = [ ] -def get_ingestor(name: str) -> ModuleType: +def get_ingestormod(name: str) -> ModuleType: """Return the imported ingestor module by name. """ module = import_module('.' + name, 'piker.data') @@ -39,6 +53,7 @@ def get_ingestor(name: str) -> ModuleType: _data_mods = [ 'piker.brokers.core', 'piker.brokers.data', + 'piker.data', ] @@ -100,22 +115,40 @@ async def open_feed( if loglevel is None: loglevel = tractor.current_actor().loglevel - async with maybe_spawn_brokerd( - mod.name, - loglevel=loglevel, - ) as portal: - stream = await portal.run( - mod.__name__, - 'stream_quotes', - symbols=symbols, - topics=symbols, - ) - # Feed is required to deliver an initial quote asap. - # TODO: should we timeout and raise a more explicit error? - # with trio.fail_after(5): - with trio.fail_after(float('inf')): - # Retreive initial quote for each symbol - # such that consumer code can know the data layout - first_quote = await stream.__anext__() - log.info(f"Received first quote {first_quote}") - yield (first_quote, stream) + with maybe_open_shared_array( + name=f'{name}.{symbols[0]}.buf', + readonly=True, # we expect the sub-actor to write + ) as shmarr: + async with maybe_spawn_brokerd( + mod.name, + loglevel=loglevel, + ) as portal: + stream = await portal.run( + mod.__name__, + 'stream_quotes', + symbols=symbols, + shared_array_token=shmarr.token, + topics=symbols, + ) + # Feed is required to deliver an initial quote asap. + # TODO: should we timeout and raise a more explicit error? + # with trio.fail_after(5): + with trio.fail_after(float('inf')): + # Retreive initial quote for each symbol + # such that consumer code can know the data layout + first_quote, child_shmarr_token = await stream.__anext__() + log.info(f"Received first quote {first_quote}") + + if child_shmarr_token is not None: + # we are the buffer writer task + increment_stream = await portal.run( + 'piker.data', + 'incr_buffer', + shm_token=child_shmarr_token, + ) + + assert child_shmarr_token == shmarr.token + else: + increment_stream = None + + yield (first_quote, stream, increment_stream, shmarr) diff --git a/piker/data/_buffer.py b/piker/data/_buffer.py new file mode 100644 index 00000000..44a21d3f --- /dev/null +++ b/piker/data/_buffer.py @@ -0,0 +1,74 @@ +""" +Data buffers for fast shared humpy. +""" +import time + +import tractor +import numpy as np +import trio + +from ._sharedmem import SharedArray, attach_shared_array + + +@tractor.stream +async def incr_buffer( + ctx: tractor.Context, + shm_token: str, + # delay_s: Optional[float] = None, +): + """Task which inserts new bars into the provide shared memory array + every ``delay_s`` seconds. + """ + # TODO: right now we'll spin printing bars if the last time + # stamp is before a large period of no market activity. + # Likely the best way to solve this is to make this task + # aware of the instrument's tradable hours? + + with attach_shared_array( + token=shm_token, + readonly=False, + ) as shm: + + # determine ohlc delay between bars + # to determine time step between datums + times = shm.array['time'] + delay_s = times[-1] - times[times != times[-1]][-1] + + # adjust delay to compensate for trio processing time + ad = delay_s - 0.002 + + async def sleep(): + """Sleep until next time frames worth has passed from last bar. + """ + # last_ts = shm.array[-1]['time'] + # delay = max((last_ts + ad) - time.time(), 0) + # await trio.sleep(delay) + await trio.sleep(ad) + + while True: + # sleep for duration of current bar + await sleep() + + # append new entry to buffer thus "incrementing" the bar + array = shm.array + last = array[-1:].copy() + # last = np.array(last, dtype=array.dtype) + # shm.push(last) + # array = shm.array + # last = array[-1].copy() + (index, t, close) = last[0][['index', 'time', 'close']] + + # new = np.array(last, dtype=array.dtype) + + # this copies non-std fields (eg. vwap) from the last datum + last[ + ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] + ][0] = (index + 1, t + delay_s, 0, close, close, close, close) + + # write to the buffer + print('incrementing array') + # await tractor.breakpoint() + shm.push(last) + + # yield the new buffer index value + await ctx.send_yield(shm._i.value) From 6fa4f6e9433d044d1806c920771161e30c5ef999 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 17 Sep 2020 09:25:30 -0400 Subject: [PATCH 110/206] Port charts to new shm arrays --- piker/ui/_chart.py | 240 ++++++++++++++++++++++++--------------------- 1 file changed, 130 insertions(+), 110 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b33a613c..780fe502 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -20,6 +20,7 @@ from ._style import _xaxis_at, _min_points_to_show, hcolor from ._source import Symbol from .. import brokers from .. import data +from ..data._normalize import iterticks from ..log import get_logger from ._exec import run_qtractor from ._source import base_ohlc_dtype @@ -483,6 +484,7 @@ class ChartPlotWidget(pg.PlotWidget): array: np.ndarray, **kwargs, ) -> pg.GraphicsObject: + self._array = array graphics = self._graphics[name] graphics.update_from_array(array, **kwargs) @@ -586,8 +588,9 @@ class ChartPlotWidget(pg.PlotWidget): self.scene().leaveEvent(ev) -async def add_new_bars(delay_s, linked_charts): - """Task which inserts new bars into the ohlc every ``delay_s`` seconds. +async def check_for_new_bars(delay_s, ohlcv, linked_charts): + """Task which updates from new bars in the shared ohlcv buffer every + ``delay_s`` seconds. """ # TODO: right now we'll spin printing bars if the last time # stamp is before a large period of no market activity. @@ -598,29 +601,19 @@ async def add_new_bars(delay_s, linked_charts): ad = delay_s - 0.002 price_chart = linked_charts.chart - ohlc = price_chart._array + # ohlc = price_chart._array async def sleep(): """Sleep until next time frames worth has passed from last bar. """ - last_ts = ohlc[-1]['time'] - delay = max((last_ts + ad) - time.time(), 0) - await trio.sleep(delay) + # last_ts = ohlcv.array[-1]['time'] + # delay = max((last_ts + ad) - time.time(), 0) + # await trio.sleep(delay) + await trio.sleep(ad) # sleep for duration of current bar await sleep() - def incr_ohlc_array(array: np.ndarray): - (index, t, close) = array[-1][['index', 'time', 'close']] - - # this copies non-std fields (eg. vwap) from the last datum - _next = np.array(array[-1], dtype=array.dtype) - _next[ - ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] - ] = (index + 1, t + delay_s, 0, close, close, close, close) - new_array = np.append(array, _next,) - return new_array - while True: # TODO: bunch of stuff: # - I'm starting to think all this logic should be @@ -633,12 +626,6 @@ async def add_new_bars(delay_s, linked_charts): # of copying it from last bar's close # - 5 sec bar lookback-autocorrection like tws does? - # add new increment/bar - start = time.time() - ohlc = price_chart._array = incr_ohlc_array(ohlc) - diff = time.time() - start - print(f'array append took {diff}') - # TODO: generalize this increment logic for name, chart in linked_charts.subplots.items(): @@ -655,11 +642,12 @@ async def add_new_bars(delay_s, linked_charts): # keep. # if last_quote == ohlc[-1]: # log.debug("Printing flat line for {sym}") + # print(ohlcv.array) # update chart historical bars graphics price_chart.update_from_array( price_chart.name, - ohlc, + ohlcv.array, # When appending a new bar, in the time between the insert # here and the Qt render call the underlying price data may # have already been updated, thus make sure to also update @@ -712,107 +700,134 @@ async def _async_main( # historical data fetch brokermod = brokers.get_brokermod(brokername) - async with brokermod.get_client() as client: - # figure out the exact symbol - bars = await client.bars(symbol=sym) + async with data.open_feed( + brokername, + [sym], + loglevel=loglevel, + ) as (fquote, stream, incr_stream, ohlcv): - # allow broker to declare historical data fields - ohlc_dtype = getattr(brokermod, 'ohlc_dtype', base_ohlc_dtype) + bars = ohlcv.array - # remember, msgpack-numpy's ``from_buffer` returns read-only array - bars = np.array(bars[list(ohlc_dtype.names)]) + # load in symbol's ohlc data + linked_charts, chart = chart_app.load_symbol(sym, bars) - # load in symbol's ohlc data - linked_charts, chart = chart_app.load_symbol(sym, bars) + # plot historical vwap if available + vwap_in_history = False + if 'vwap' in bars.dtype.fields: + vwap_in_history = True + chart.draw_curve( + name='vwap', + data=bars['vwap'], + overlay=True, + ) - # plot historical vwap if available - vwap_in_history = False - if 'vwap' in bars.dtype.fields: - vwap_in_history = True - chart.draw_curve( - name='vwap', - data=bars['vwap'], - overlay=True, - ) + # determine ohlc delay between bars + # to determine time step between datums + times = bars['time'] + delay = times[-1] - times[times != times[-1]][-1] - # determine ohlc delay between bars - times = bars['time'] + async with trio.open_nursery() as n: - # find expected time step between datums - delay = times[-1] - times[times != times[-1]][-1] + # load initial fsp chain (otherwise known as "indicators") + n.start_soon( + chart_from_fsp, + linked_charts, + 'rsi', # eventually will be n-compose syntax + sym, + bars, + brokermod, + loglevel, + ) - async with trio.open_nursery() as n: - - # load initial fsp chain (otherwise known as "indicators") - n.start_soon( - chart_from_fsp, - linked_charts, - 'rsi', - sym, - bars, - brokermod, - loglevel, - ) - - # update last price sticky - last_price_sticky = chart._ysticks[chart.name] - last_price_sticky.update_from_data( - *chart._array[-1][['index', 'close']] - ) - - # graphics update loop - - async with data.open_feed( - brokername, - [sym], - loglevel=loglevel, - ) as (fquote, stream): + # update last price sticky + last_price_sticky = chart._ysticks[chart.name] + last_price_sticky.update_from_data( + *ohlcv.array[-1][['index', 'close']] + ) # wait for a first quote before we start any update tasks quote = await stream.__anext__() - log.info(f'RECEIVED FIRST QUOTE {quote}') + log.info(f'Received first quote {quote}') - # start graphics tasks after receiving first live quote - n.start_soon(add_new_bars, delay, linked_charts) + # start graphics update loop(s)after receiving first live quote + n.start_soon( + chart_from_quotes, + chart, + stream, + ohlcv, + vwap_in_history, + ) + n.start_soon( + check_for_new_bars, + delay, + ohlcv, + linked_charts + ) - async for quotes in stream: - for sym, quote in quotes.items(): - ticks = quote.get('ticks', ()) - for tick in ticks: - if tick.get('type') == 'trade': + # probably where we'll eventually start the user input loop + await trio.sleep_forever() - # TODO: eventually we'll want to update - # bid/ask labels and other data as - # subscribed by underlying UI consumers. - # last = quote.get('last') or quote['close'] - last = tick['price'] - # update ohlc (I guess we're enforcing this - # for now?) overwrite from quote - high, low = chart._array[-1][['high', 'low']] - chart._array[['high', 'low', 'close']][-1] = ( - max(high, last), - min(low, last), - last, - ) - chart.update_from_array( - chart.name, - chart._array, - ) - # update sticky(s) - last_price_sticky.update_from_data( - *chart._array[-1][['index', 'close']]) - chart._set_yrange() +async def chart_from_quotes( + chart: ChartPlotWidget, + stream, + ohlcv: np.ndarray, + vwap_in_history: bool = False, +) -> None: + """The 'main' (price) chart real-time update loop. + """ - vwap = quote.get('vwap') - if vwap and vwap_in_history: - chart._array['vwap'][-1] = vwap - print(f"vwap: {quote['vwap']}") - # update vwap overlay line - chart.update_from_array( - 'vwap', - chart._array['vwap'], - ) + last_price_sticky = chart._ysticks[chart.name] + + print_next = False + async for quotes in stream: + for sym, quote in quotes.items(): + for tick in iterticks(quote, type='trade'): + # TODO: eventually we'll want to update + # bid/ask labels and other data as + # subscribed by underlying UI consumers. + # last_close = ohlcv.array[-1]['close'] + + # last = quote.get('last') or quote['close'] + # last = tick['price'] + + # if print_next: + # print(f"next last: {last}") + # print_next = False + + # if last_close != last: + # log.error(f"array last_close: {last_close}\nlast: {last}") + # print_next = True + + # update ohlc (I guess we're enforcing this + # for now?) overwrite from quote + # high, low = chart._array[-1][['high', 'low']] + # chart._array[['high', 'low', 'close']][-1] = ( + # max(high, last), + # min(low, last), + # last, + # ) + last = ohlcv.array[-1] + chart.update_from_array( + chart.name, + ohlcv.array, + ) + # update sticky(s) + last_price_sticky.update_from_data( + *last[['index', 'close']]) + chart._set_yrange() + + vwap = quote.get('vwap') + if vwap and vwap_in_history: + # chart._array['vwap'][-1] = vwap + last['vwap'] = vwap + print(f"vwap: {quote['vwap']}") + # update vwap overlay line + chart.update_from_array( + 'vwap', + # chart._array['vwap'], + ohlcv.array['vwap'], + ) async def chart_from_fsp( @@ -882,7 +897,12 @@ async def chart_from_fsp( # update chart graphics async for value in stream: + + # start = time.time() chart._array[-1] = value + # diff = time.time() - start + # print(f'FSP array append took {diff}') + last_val_sticky.update_from_data(-1, value) chart.update_from_array(chart.name, chart._array) # chart._set_yrange() From cd540fd07ea41a544e323f6c714a2bd6aab57a6a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 17 Sep 2020 13:22:01 -0400 Subject: [PATCH 111/206] Cleanups --- piker/data/_buffer.py | 25 ++++----- piker/data/_sharedmem.py | 22 +++++--- piker/ui/_chart.py | 112 +++++++++------------------------------ 3 files changed, 52 insertions(+), 107 deletions(-) diff --git a/piker/data/_buffer.py b/piker/data/_buffer.py index 44a21d3f..0c25309f 100644 --- a/piker/data/_buffer.py +++ b/piker/data/_buffer.py @@ -4,10 +4,9 @@ Data buffers for fast shared humpy. import time import tractor -import numpy as np import trio -from ._sharedmem import SharedArray, attach_shared_array +from ._sharedmem import attach_shared_array @tractor.stream @@ -40,35 +39,33 @@ async def incr_buffer( async def sleep(): """Sleep until next time frames worth has passed from last bar. """ - # last_ts = shm.array[-1]['time'] - # delay = max((last_ts + ad) - time.time(), 0) - # await trio.sleep(delay) - await trio.sleep(ad) + last_ts = shm.array[-1]['time'] + delay = max((last_ts + ad) - time.time(), 0) + await trio.sleep(delay) + # await trio.sleep(ad) while True: # sleep for duration of current bar await sleep() + # TODO: in theory we could make this faster by copying the + # "last" readable value into the underlying larger buffer's + # next value and then incrementing the counter instead of + # using ``.push()``? + # append new entry to buffer thus "incrementing" the bar array = shm.array last = array[-1:].copy() - # last = np.array(last, dtype=array.dtype) - # shm.push(last) - # array = shm.array - # last = array[-1].copy() (index, t, close) = last[0][['index', 'time', 'close']] - # new = np.array(last, dtype=array.dtype) - # this copies non-std fields (eg. vwap) from the last datum last[ ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] ][0] = (index + 1, t + delay_s, 0, close, close, close, close) # write to the buffer - print('incrementing array') - # await tractor.breakpoint() shm.push(last) + # print('incrementing array') # yield the new buffer index value await ctx.send_yield(shm._i.value) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index cfe162a7..6747ed9d 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -58,7 +58,6 @@ class SharedInt: create=create, size=4, # std int ) - self._token = self._shm.name @property def value(self) -> int: @@ -68,6 +67,12 @@ class SharedInt: def value(self, value) -> None: self._shm.buf[:] = value.to_bytes(4, byteorder) + def destroy(self) -> None: + if shared_memory._USE_POSIX: + # We manually unlink to bypass all the "resource tracker" + # nonsense meant for non-SC systems. + shm_unlink(self._shm.name) + class SharedArray: def __init__( @@ -87,7 +92,7 @@ class SharedArray: @property def token(self) -> Tuple[str, str]: - return (self._shm.name, self._i._token) + return (self._shm.name, self._i._shm.name) @property def name(self) -> str: @@ -130,8 +135,8 @@ class SharedArray: if shared_memory._USE_POSIX: # We manually unlink to bypass all the "resource tracker" # nonsense meant for non-SC systems. - shm_unlink(self._i._shm.name) shm_unlink(self._shm.name) + self._i.destroy() def flush(self) -> None: # TODO: flush to storage backend like markestore? @@ -141,9 +146,8 @@ class SharedArray: @contextmanager def open_shared_array( name: Optional[str] = None, - create: bool = True, - # approx number of 5s bars in a "day" - size: int = int(60*60*10/5), + # approx number of 5s bars in a "day" x2 + size: int = int(2*60*60*10/5), dtype: np.dtype = base_ohlc_dtype, readonly: bool = False, ) -> SharedArray: @@ -155,7 +159,11 @@ def open_shared_array( # create new shared mem segment for which we # have write permission a = np.zeros(size, dtype=dtype) - shm = shared_memory.SharedMemory(name=name, create=True, size=a.nbytes) + shm = shared_memory.SharedMemory( + name=name, + create=True, + size=a.nbytes + ) shmarr = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) shmarr[:] = a[:] shmarr.setflags(write=int(not readonly)) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 780fe502..7695d89e 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -23,7 +23,6 @@ from .. import data from ..data._normalize import iterticks from ..log import get_logger from ._exec import run_qtractor -from ._source import base_ohlc_dtype from ._interaction import ChartView from .. import fsp @@ -588,7 +587,7 @@ class ChartPlotWidget(pg.PlotWidget): self.scene().leaveEvent(ev) -async def check_for_new_bars(delay_s, ohlcv, linked_charts): +async def check_for_new_bars(incr_stream, ohlcv, linked_charts): """Task which updates from new bars in the shared ohlcv buffer every ``delay_s`` seconds. """ @@ -597,36 +596,9 @@ async def check_for_new_bars(delay_s, ohlcv, linked_charts): # Likely the best way to solve this is to make this task # aware of the instrument's tradable hours? - # adjust delay to compensate for trio processing time - ad = delay_s - 0.002 - price_chart = linked_charts.chart - # ohlc = price_chart._array - - async def sleep(): - """Sleep until next time frames worth has passed from last bar. - """ - # last_ts = ohlcv.array[-1]['time'] - # delay = max((last_ts + ad) - time.time(), 0) - # await trio.sleep(delay) - await trio.sleep(ad) - - # sleep for duration of current bar - await sleep() - - while True: - # TODO: bunch of stuff: - # - I'm starting to think all this logic should be - # done in one place and "graphics update routines" - # should not be doing any length checking and array diffing. - # - don't keep appending, but instead increase the - # underlying array's size less frequently - # - handle odd lot orders - # - update last open price correctly instead - # of copying it from last bar's close - # - 5 sec bar lookback-autocorrection like tws does? - + async for index in incr_stream: # TODO: generalize this increment logic for name, chart in linked_charts.subplots.items(): data = chart._array @@ -635,15 +607,6 @@ async def check_for_new_bars(delay_s, ohlcv, linked_charts): np.array(data[-1], dtype=data.dtype) ) - # read value at "open" of bar - # last_quote = ohlc[-1] - # XXX: If the last bar has not changed print a flat line and - # move to the next. This is a "animation" choice that we may not - # keep. - # if last_quote == ohlc[-1]: - # log.debug("Printing flat line for {sym}") - # print(ohlcv.array) - # update chart historical bars graphics price_chart.update_from_array( price_chart.name, @@ -670,17 +633,6 @@ async def check_for_new_bars(delay_s, ohlcv, linked_charts): chart.update_from_array(chart.name, chart._array) chart._set_yrange() - # We **don't** update the bar right now - # since the next quote that arrives should in the - # tick streaming task - await sleep() - - # TODO: should we update a graphics again time here? - # Think about race conditions with data update task. - # UPDATE: don't think this should matter know since the last bar - # and the prior historical bars are being updated in 2 separate - # steps now. - async def _async_main( sym: str, @@ -721,11 +673,6 @@ async def _async_main( overlay=True, ) - # determine ohlc delay between bars - # to determine time step between datums - times = bars['time'] - delay = times[-1] - times[times != times[-1]][-1] - async with trio.open_nursery() as n: # load initial fsp chain (otherwise known as "indicators") @@ -759,7 +706,8 @@ async def _async_main( ) n.start_soon( check_for_new_bars, - delay, + incr_stream, + # delay, ohlcv, linked_charts ) @@ -776,41 +724,35 @@ async def chart_from_quotes( ) -> None: """The 'main' (price) chart real-time update loop. """ - + # TODO: bunch of stuff: + # - I'm starting to think all this logic should be + # done in one place and "graphics update routines" + # should not be doing any length checking and array diffing. + # - handle odd lot orders + # - update last open price correctly instead + # of copying it from last bar's close + # - 5 sec bar lookback-autocorrection like tws does? last_price_sticky = chart._ysticks[chart.name] - print_next = False async for quotes in stream: for sym, quote in quotes.items(): for tick in iterticks(quote, type='trade'): - # TODO: eventually we'll want to update - # bid/ask labels and other data as - # subscribed by underlying UI consumers. - # last_close = ohlcv.array[-1]['close'] - - # last = quote.get('last') or quote['close'] - # last = tick['price'] - - # if print_next: - # print(f"next last: {last}") - # print_next = False - - # if last_close != last: - # log.error(f"array last_close: {last_close}\nlast: {last}") - # print_next = True - - # update ohlc (I guess we're enforcing this - # for now?) overwrite from quote - # high, low = chart._array[-1][['high', 'low']] - # chart._array[['high', 'low', 'close']][-1] = ( - # max(high, last), - # min(low, last), - # last, - # ) - last = ohlcv.array[-1] + # TODO: + # - eventually we'll want to update bid/ask labels and + # other data as subscribed by underlying UI consumers. + # - in theory we should be able to read buffer data + # faster then msgs arrive.. needs some tinkering and + # testing + start = time.time() + array = ohlcv.array + diff = time.time() - start + print(f'read time: {diff}') + # last = ohlcv.array[-1] + last = array[-1] chart.update_from_array( chart.name, - ohlcv.array, + # ohlcv.array, + array, ) # update sticky(s) last_price_sticky.update_from_data( @@ -819,13 +761,11 @@ async def chart_from_quotes( vwap = quote.get('vwap') if vwap and vwap_in_history: - # chart._array['vwap'][-1] = vwap last['vwap'] = vwap print(f"vwap: {quote['vwap']}") # update vwap overlay line chart.update_from_array( 'vwap', - # chart._array['vwap'], ohlcv.array['vwap'], ) From 38469bd6ef5c28ac903623dc0a4b54436645a79f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 08:31:47 -0400 Subject: [PATCH 112/206] Slight rework: shm API Add an internal `_Token` to do interchange (un)packing for passing "references" to shm blocks between actors. Part of the token involves providing the `numpy.dtype` in a cross-actor format. Add a module variable for caching "known tokens" per actor. Drop use of context managers since they tear down shm blocks too soon in debug mode and there seems to be no reason to unlink/close shm before the process has terminated; if code needs it torn down explicitly, it can. --- piker/data/_sharedmem.py | 209 ++++++++++++++++++++++++++++++--------- 1 file changed, 160 insertions(+), 49 deletions(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 6747ed9d..d6c53a95 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -1,16 +1,24 @@ """ NumPy compatible shared memory buffers for real-time FSP. """ +from typing import List +from dataclasses import dataclass, asdict from sys import byteorder -from contextlib import contextmanager from typing import Tuple, Optional from multiprocessing import shared_memory from multiprocessing import resource_tracker as mantracker from _posixshmem import shm_unlink +import tractor import numpy as np +from ..log import get_logger + + +log = get_logger(__name__) + + # Tell the "resource tracker" thing to fuck off. class ManTracker(mantracker.ResourceTracker): def register(self, name, rtype): @@ -74,6 +82,58 @@ class SharedInt: shm_unlink(self._shm.name) +@dataclass +class _Token: + """Internal represenation of a shared memory "token" + which can be used to key a system wide post shm entry. + """ + shm_name: str # this servers as a "key" value + shm_counter_name: str + dtype_descr: List[Tuple[str]] + + def __post_init__(self): + # np.array requires a list for dtype + self.dtype_descr = np.dtype( + list(self.dtype_descr)).descr + + def as_msg(self): + return asdict(self) + + @classmethod + def from_msg(self, msg: dict) -> '_Token': + return msg if isinstance(msg, _Token) else _Token(**msg) + + +# TODO: this api? +# _known_tokens = tractor.ActorVar('_shm_tokens', {}) +# _known_tokens = tractor.ContextStack('_known_tokens', ) +# _known_tokens = trio.RunVar('shms', {}) + +# process-local store of keys to tokens +_known_tokens = {} + + +def get_shm_token(key: str) -> _Token: + """Convenience func to check if a token + for the provided key is known by this process. + """ + return _known_tokens.get(key) + + +def _make_token( + key: str, + dtype: np.dtype = base_ohlc_dtype, +) -> _Token: + """Create a serializable token that can be used + to access a shared array. + """ + return _Token( + key, + key + "_counter", + dtype.descr + ) + + class SharedArray: def __init__( self, @@ -91,12 +151,19 @@ class SharedArray: # TODO: ringbuf api? @property - def token(self) -> Tuple[str, str]: - return (self._shm.name, self._i._shm.name) + def _token(self) -> _Token: + return _Token( + self._shm.name, + self._i._shm.name, + self._array.dtype.descr, + ) @property - def name(self) -> str: - return self._shm.name + def token(self) -> dict: + """Shared memory token that can be serialized + and used by another process to attach to this array. + """ + return self._token.as_msg() @property def index(self) -> int: @@ -143,9 +210,8 @@ class SharedArray: ... -@contextmanager -def open_shared_array( - name: Optional[str] = None, +def open_shm_array( + key: Optional[str] = None, # approx number of 5s bars in a "day" x2 size: int = int(2*60*60*10/5), dtype: np.dtype = base_ohlc_dtype, @@ -160,51 +226,66 @@ def open_shared_array( # have write permission a = np.zeros(size, dtype=dtype) shm = shared_memory.SharedMemory( - name=name, + name=key, create=True, size=a.nbytes ) - shmarr = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) - shmarr[:] = a[:] - shmarr.setflags(write=int(not readonly)) + array = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) + array[:] = a[:] + array.setflags(write=int(not readonly)) + + token = _make_token( + key=key, + dtype=dtype + ) counter = SharedInt( - token=shm.name + "_counter", + token=token.shm_counter_name, create=True, ) counter.value = 0 - sha = SharedArray( - shmarr, + shmarr = SharedArray( + array, counter, shm, readonly=readonly, ) - try: - yield sha - finally: - sha.close() - print(f"UNLINKING {sha.token}") - sha.destroy() + + assert shmarr._token == token + _known_tokens[key] = shmarr.token + + # "unlink" created shm on process teardown by + # pushing teardown calls onto actor context stack + actor = tractor.current_actor() + actor._lifetime_stack.callback(shmarr.close) + actor._lifetime_stack.callback(shmarr.destroy) + return shmarr -@contextmanager -def attach_shared_array( - token: Tuple[str, str], +def attach_shm_array( + token: Tuple[str, str, Tuple[str, str]], size: int = int(60*60*10/5), - dtype: np.dtype = base_ohlc_dtype, + # dtype: np.dtype = base_ohlc_dtype, readonly: bool = True, ) -> SharedArray: """Load and attach to an existing shared memory array previously created by another process using ``open_shared_array``. """ - array_name, counter_name = token + token = _Token.from_msg(token) + key = token.shm_name + if key in _known_tokens: + assert _known_tokens[key] == token, "WTF" - shm = shared_memory.SharedMemory(name=array_name) - shmarr = np.ndarray((size,), dtype=dtype, buffer=shm.buf) + shm = shared_memory.SharedMemory(name=key) + shmarr = np.ndarray( + (size,), + dtype=token.dtype_descr, + buffer=shm.buf + ) shmarr.setflags(write=int(not readonly)) - counter = SharedInt(token=counter_name) + counter = SharedInt(token=token.shm_counter_name) # make sure we can read counter.value @@ -214,28 +295,58 @@ def attach_shared_array( shm, readonly=readonly, ) + # read test sha.array - try: - yield sha - finally: - pass - sha.close() + + # Stash key -> token knowledge for future queries + # via `maybe_opepn_shm_array()` but only after we know + # we can attach. + if key not in _known_tokens: + _known_tokens[key] = token + + # "close" attached shm on process teardown + actor = tractor.current_actor() + actor._lifetime_stack.callback(sha.close) + return sha -@contextmanager -def maybe_open_shared_array( - name: str, +def maybe_open_shm_array( + key: str, + dtype: np.dtype = base_ohlc_dtype, **kwargs, -) -> SharedArray: +) -> Tuple[SharedArray, bool]: + """Attempt to attach to a shared memory block by a + "key" determined by the users overall "system" + (presumes you don't have the block's explicit token). + + This function is meant to solve the problem of + discovering whether a shared array token has been + allocated or discovered by the actor running in + **this** process. Systems where multiple actors + may seek to access a common block can use this + function to attempt to acquire a token as discovered + by the actors who have previously stored a + "key" -> ``_Token`` map in an actor local variable. + + If you know the explicit ``_Token`` for your memory + instead use ``attach_shm_array``. + """ try: - with open_shared_array( - name=name, - **kwargs, - ) as shm: - yield shm - except FileExistsError: - with attach_shared_array( - token=(name, name + '_counter'), - **kwargs, - ) as shm: - yield shm + # see if we already know this key + token = _known_tokens[key] + return attach_shm_array(token=token, **kwargs), False + except KeyError: + log.debug(f"Could not find {key} in shms cache") + if dtype: + token = _make_token(key, dtype) + try: + return attach_shm_array(token=token, **kwargs), False + except FileNotFoundError: + log.debug(f"Could not attach to shm with token {token}") + + # This actor does not know about memory + # associated with the provided "key". + # Attempt to open a block and expect + # to fail if a block has been allocated + # on the OS by someone else. + return open_shm_array(key=key, dtype=dtype, **kwargs), True From b1093dc71d1c77ab1e0da012767f52e86cff5a7c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 12:14:24 -0400 Subject: [PATCH 113/206] Add a `data.Feed` type Wraps the growing tuple of items being delivered by `open_feed()`. Add lazy loading of the broker's signal step stream with a `Feed.index_stream()` method. --- piker/data/__init__.py | 125 +++++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 41 deletions(-) diff --git a/piker/data/__init__.py b/piker/data/__init__.py index 3ef85c96..5459b4ba 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -5,6 +5,7 @@ We provide tsdb integrations for retrieving and storing data from your brokers as well as sharing your feeds with other fellow pikers. """ +from dataclasses import dataclass from contextlib import asynccontextmanager from importlib import import_module from types import ModuleType @@ -13,24 +14,28 @@ from typing import ( Sequence, AsyncIterator, Optional ) -import trio import tractor from ..brokers import get_brokermod from ..log import get_logger, get_console_log from ._normalize import iterticks from ._sharedmem import ( - maybe_open_shared_array, attach_shared_array, open_shared_array, + maybe_open_shm_array, + attach_shm_array, + open_shm_array, + SharedArray, + get_shm_token, ) from ._buffer import incr_buffer __all__ = [ - 'maybe_open_shared_array', - 'attach_shared_array', - 'open_shared_array', - 'iterticks', - 'incr_buffer', + 'iterticks', + 'maybe_open_shm_array', + 'attach_shm_array', + 'open_shm_array', + 'get_shm_token', + 'incr_buffer', ] @@ -99,6 +104,46 @@ async def maybe_spawn_brokerd( await nursery.cancel() +@dataclass +class Feed: + """A data feed for client-side interaction with far-process + real-time data sources. + + This is an thin abstraction on top of ``tractor``'s portals for + interacting with IPC streams and conducting automatic + memory buffer orchestration. + """ + name: str + stream: AsyncIterator[Dict[str, Any]] + shm: SharedArray + _broker_portal: tractor._portal.Portal + _index_stream: Optional[AsyncIterator[Dict[str, Any]]] = None + + async def receive(self) -> dict: + return await self.stream.__anext__() + + async def index_stream(self) -> AsyncIterator[int]: + if not self._index_stream: + # XXX: this should be singleton on a host, + # a lone broker-daemon per provider should be + # created for all practical purposes + self._index_stream = await self._broker_portal.run( + 'piker.data', + 'incr_buffer', + shm_token=self.shm.token, + topics=['index'], + ) + + return self._index_stream + + +def sym_to_shm_key( + broker: str, + symbol: str, +) -> str: + return f'{broker}.{symbol}' + + @asynccontextmanager async def open_feed( name: str, @@ -115,40 +160,38 @@ async def open_feed( if loglevel is None: loglevel = tractor.current_actor().loglevel - with maybe_open_shared_array( - name=f'{name}.{symbols[0]}.buf', - readonly=True, # we expect the sub-actor to write - ) as shmarr: - async with maybe_spawn_brokerd( - mod.name, - loglevel=loglevel, - ) as portal: - stream = await portal.run( - mod.__name__, - 'stream_quotes', - symbols=symbols, - shared_array_token=shmarr.token, - topics=symbols, - ) - # Feed is required to deliver an initial quote asap. - # TODO: should we timeout and raise a more explicit error? - # with trio.fail_after(5): - with trio.fail_after(float('inf')): - # Retreive initial quote for each symbol - # such that consumer code can know the data layout - first_quote, child_shmarr_token = await stream.__anext__() - log.info(f"Received first quote {first_quote}") + # attempt to allocate (or attach to) shm array for this + # broker/symbol + shm, opened = maybe_open_shm_array( + key=sym_to_shm_key(name, symbols[0]), - if child_shmarr_token is not None: - # we are the buffer writer task - increment_stream = await portal.run( - 'piker.data', - 'incr_buffer', - shm_token=child_shmarr_token, - ) + # we expect the sub-actor to write + readonly=True, + ) - assert child_shmarr_token == shmarr.token - else: - increment_stream = None + async with maybe_spawn_brokerd( + mod.name, + loglevel=loglevel, + ) as portal: + stream = await portal.run( + mod.__name__, + 'stream_quotes', + symbols=symbols, + shm_token=shm.token, - yield (first_quote, stream, increment_stream, shmarr) + # compat with eventual ``tractor.msg.pub`` + topics=symbols, + ) + shm_token, is_writer = await stream.receive() + shm_token['dtype_descr'] = list(shm_token['dtype_descr']) + assert shm_token == shm.token # sanity + + if is_writer: + log.info("Started shared mem bar writer") + + yield Feed( + name=name, + stream=stream, + shm=shm, + _broker_portal=portal, + ) From d93ce84a992bc63d939fd167bc9149d46b5eb57b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 12:24:02 -0400 Subject: [PATCH 114/206] Variety of IB backend improvements - Move to new shared mem system only writing on the first (by process) entry to `stream_quotes()`. - Deliver bars before first quote arrives so that chart can populate and then wait for initial arrival. - Allow caching clients per actor. - Load bars using the same (cached) client that starts the quote stream thus speeding up initialization. --- piker/brokers/ib.py | 184 +++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 88 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index b0d89d1b..77a1018c 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -26,8 +26,10 @@ import tractor from ..log import get_logger, get_console_log from ..data import ( - maybe_spawn_brokerd, iterticks, attach_shared_array, - incr_buffer, + maybe_spawn_brokerd, + iterticks, + attach_shm_array, + get_shm_token ) from ..ui._source import from_df @@ -145,7 +147,7 @@ class Client: # durationStr='1 D', # time length calcs - durationStr='{count} S'.format(count=5000 * 5), + durationStr='{count} S'.format(count=1000 * 5), barSizeSetting='5 secs', # always use extended hours @@ -311,6 +313,8 @@ class Client: _tws_port: int = 7497 _gw_port: int = 4002 _try_ports = [_tws_port, _gw_port] +_client_ids = itertools.count() +_client_cache = {} @asynccontextmanager @@ -321,36 +325,39 @@ async def _aio_get_client( ) -> Client: """Return an ``ib_insync.IB`` instance wrapped in our client API. """ - if client_id is None: - # if this is a persistent brokerd, try to allocate a new id for - # each client - try: - ss = tractor.current_actor().statespace - client_id = next(ss.setdefault('client_ids', itertools.count())) - # TODO: in case the arbiter has no record - # of existing brokerd we need to broadcase for one. - except RuntimeError: - # tractor likely isn't running - client_id = 1 - - ib = NonShittyIB() - ports = _try_ports if port is None else [port] - _err = None - for port in ports: - try: - await ib.connectAsync(host, port, clientId=client_id) - break - except ConnectionRefusedError as ce: - _err = ce - log.warning(f'Failed to connect on {port}') - else: - raise ConnectionRefusedError(_err) + # first check cache for existing client try: - yield Client(ib) - except BaseException: - ib.disconnect() - raise + yield _client_cache[(host, port)] + except KeyError: + # TODO: in case the arbiter has no record + # of existing brokerd we need to broadcast for one. + + if client_id is None: + # if this is a persistent brokerd, try to allocate a new id for + # each client + client_id = next(_client_ids) + + ib = NonShittyIB() + ports = _try_ports if port is None else [port] + _err = None + for port in ports: + try: + await ib.connectAsync(host, port, clientId=client_id) + break + except ConnectionRefusedError as ce: + _err = ce + log.warning(f'Failed to connect on {port}') + else: + raise ConnectionRefusedError(_err) + + try: + client = Client(ib) + _client_cache[(host, port)] = client + yield client + except BaseException: + ib.disconnect() + raise async def _aio_run_client_method( @@ -489,7 +496,7 @@ def normalize( # @tractor.msg.pub async def stream_quotes( symbols: List[str], - shared_array_token: Tuple[str, str], + shm_token: Tuple[str, str, List[tuple]], loglevel: str = None, # compat for @tractor.msg.pub topics: Any = None, @@ -506,16 +513,35 @@ async def stream_quotes( # TODO: support multiple subscriptions sym = symbols[0] - stream = await tractor.to_asyncio.run_task( - _trio_run_client_method, + stream = await _trio_run_client_method( method='stream_ticker', symbol=sym, ) - async with get_client() as client: - bars = await client.bars(symbol=sym) - async with aclosing(stream): + + # maybe load historical ohlcv in to shared mem + # check if shm has already been created by previous + # feed initialization + writer_exists = get_shm_token(shm_token['shm_name']) + + if not writer_exists: + shm = attach_shm_array( + token=shm_token, + # we are writer + readonly=False, + ) + bars = await _trio_run_client_method( + method='bars', + symbol=sym, + ) + + shm.push(bars) + shm_token = shm.token + + # pass back token, and bool, signalling if we're the writer + yield shm_token, not writer_exists + # first quote can be ignored as a 2nd with newer data is sent? first_ticker = await stream.__anext__() quote = normalize(first_ticker) @@ -538,7 +564,7 @@ async def stream_quotes( else: log.debug("Received first real volume tick") # ugh, clear ticks since we've consumed them - # (ahem, ib_insync is stateful trash) + # (ahem, ib_insync is truly stateful trash) ticker.ticks = [] # XXX: this works because we don't use @@ -555,58 +581,40 @@ async def stream_quotes( topic = '.'.join((con['symbol'], con[suffix])).lower() first_quote = {topic: quote} ticker.ticks = [] + # yield first quote asap + yield first_quote - # load historical ohlcv in to shared mem - ss = tractor.current_actor().statespace - existing_shm = ss.get(f'ib_shm.{sym}') - if not existing_shm: - readonly = False - else: - readonly = True - shm = existing_shm + async for ticker in stream: + quote = normalize( + ticker, + calc_price=calc_price + ) + # TODO: in theory you can send the IPC msg *before* + # writing to the sharedmem array to decrease latency, + # however, that will require `tractor.msg.pub` support + # here or at least some way to prevent task switching + # at the yield such that the array write isn't delayed + # while another consumer is serviced.. - with attach_shared_array( - token=shared_array_token, - readonly=readonly - ) as shm: - if not existing_shm: - shm.push(bars) - ss[f'ib_shm.{sym}'] = shm + # if we are the lone tick writer start writing + # the buffer with appropriate trade data + if not writer_exists: + for tick in iterticks(quote, type='trade'): + last = tick['price'] + # print(f'broker last: {tick}') - yield (first_quote, shm.token) - else: - yield (first_quote, None) + # update last entry + # benchmarked in the 4-5 us range + high, low = shm.array[-1][['high', 'low']] + shm.array[['high', 'low', 'close']][-1] = ( + max(high, last), + min(low, last), + last, + ) - async for ticker in stream: - quote = normalize( - ticker, - calc_price=calc_price - ) - # TODO: in theory you can send the IPC msg *before* - # writing to the sharedmem array to decrease latency, - # however, that will require `tractor.msg.pub` support - # here or at least some way to prevent task switching - # at the yield such that the array write isn't delayed - # while another consumer is serviced.. + con = quote['contract'] + topic = '.'.join((con['symbol'], con[suffix])).lower() + yield {topic: quote} - # if we are the lone tick writer - if not existing_shm: - for tick in iterticks(quote, type='trade'): - last = tick['price'] - # print(f'broker last: {tick}') - - # update last entry - # benchmarked in the 4-5 us range - high, low = shm.array[-1][['high', 'low']] - shm.array[['high', 'low', 'close']][-1] = ( - max(high, last), - min(low, last), - last, - ) - - con = quote['contract'] - topic = '.'.join((con['symbol'], con[suffix])).lower() - yield {topic: quote} - - # ugh, clear ticks since we've consumed them - ticker.ticks = [] + # ugh, clear ticks since we've consumed them + ticker.ticks = [] From 373ff902290ff163c808c12573830b56a4892040 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 12:31:08 -0400 Subject: [PATCH 115/206] Only need UTC offset hacking if time w broker is messed.. --- piker/ui/_axes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 24389f77..53d90c73 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -82,7 +82,7 @@ class DynamicDateAxis(pg.AxisItem): map(int, filter(lambda i: i < bars_len, indexes)) )] # TODO: **don't** have this hard coded shift to EST - dts = pd.to_datetime(epochs, unit='s') - 4*pd.offsets.Hour() + dts = pd.to_datetime(epochs, unit='s') # - 4*pd.offsets.Hour() return dts.strftime(self.tick_tpl[delay]) def tickStrings(self, values: List[float], scale, spacing): From efb52f22921d688ed686b270704ec96bf6dde9ea Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 14:30:50 -0400 Subject: [PATCH 116/206] Make shared array buffer incrementer a message pub Drop ctx manager api and use `tractor.msg.pub`. --- piker/data/_buffer.py | 81 ++++++++++++++++++++++--------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/piker/data/_buffer.py b/piker/data/_buffer.py index 0c25309f..43be7dc0 100644 --- a/piker/data/_buffer.py +++ b/piker/data/_buffer.py @@ -1,18 +1,19 @@ """ Data buffers for fast shared humpy. """ +from typing import Tuple, Callable import time import tractor import trio -from ._sharedmem import attach_shared_array +from ._sharedmem import attach_shm_array -@tractor.stream +@tractor.msg.pub async def incr_buffer( - ctx: tractor.Context, - shm_token: str, + shm_token: dict, + get_topics: Callable[..., Tuple[str]], # delay_s: Optional[float] = None, ): """Task which inserts new bars into the provide shared memory array @@ -23,49 +24,51 @@ async def incr_buffer( # Likely the best way to solve this is to make this task # aware of the instrument's tradable hours? - with attach_shared_array( + shm = attach_shm_array( token=shm_token, readonly=False, - ) as shm: + ) - # determine ohlc delay between bars - # to determine time step between datums - times = shm.array['time'] - delay_s = times[-1] - times[times != times[-1]][-1] + # determine ohlc delay between bars + # to determine time step between datums + times = shm.array['time'] + delay_s = times[-1] - times[times != times[-1]][-1] - # adjust delay to compensate for trio processing time - ad = delay_s - 0.002 + # adjust delay to compensate for trio processing time + ad = delay_s - 0.002 - async def sleep(): - """Sleep until next time frames worth has passed from last bar. - """ - last_ts = shm.array[-1]['time'] - delay = max((last_ts + ad) - time.time(), 0) - await trio.sleep(delay) - # await trio.sleep(ad) + async def sleep(): + """Sleep until next time frames worth has passed from last bar. + """ + # last_ts = shm.array[-1]['time'] + # delay = max((last_ts + ad) - time.time(), 0) + # await trio.sleep(delay) + await trio.sleep(ad) - while True: - # sleep for duration of current bar - await sleep() + while True: + # sleep for duration of current bar + await sleep() - # TODO: in theory we could make this faster by copying the - # "last" readable value into the underlying larger buffer's - # next value and then incrementing the counter instead of - # using ``.push()``? + # TODO: in theory we could make this faster by copying the + # "last" readable value into the underlying larger buffer's + # next value and then incrementing the counter instead of + # using ``.push()``? - # append new entry to buffer thus "incrementing" the bar - array = shm.array - last = array[-1:].copy() - (index, t, close) = last[0][['index', 'time', 'close']] + # append new entry to buffer thus "incrementing" the bar + array = shm.array + last = array[-1:].copy() + (index, t, close) = last[0][['index', 'time', 'close']] - # this copies non-std fields (eg. vwap) from the last datum - last[ - ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] - ][0] = (index + 1, t + delay_s, 0, close, close, close, close) + # this copies non-std fields (eg. vwap) from the last datum + last[ + ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] + ][0] = (index + 1, t + delay_s, 0, close, close, close, close) - # write to the buffer - shm.push(last) - # print('incrementing array') + # write to the buffer + shm.push(last) + # print('incrementing array') - # yield the new buffer index value - await ctx.send_yield(shm._i.value) + # print(get_topics()) + + # broadcast the buffer index step + yield {'index': shm._i.value} From 561cafbe5588cc2bdc6f989fe6f7d86a09ca08ae Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 15:26:15 -0400 Subject: [PATCH 117/206] Another black --- piker/ui/_style.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 04edf6e8..8a80d12f 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -39,6 +39,7 @@ def hcolor(name: str) -> str: 'erie_black': '#1B1B1B', 'licorice': '#1A1110', 'papas_special': '#06070c', + 'svags': '#0a0e14', # fifty shades 'gray': '#808080', # like the kick From ba4261f97436310d975cba5e25cc3791c9e75e87 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 15:53:20 -0400 Subject: [PATCH 118/206] Add timeit prints --- piker/ui/_graphics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 18badc2c..3fe43079 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -304,7 +304,7 @@ class BarItems(pg.GraphicsObject): # track the current length of drawable lines within the larger array self.index: int = 0 - @timeit + # @timeit def draw_from_data( self, data: np.recarray, @@ -321,6 +321,7 @@ class BarItems(pg.GraphicsObject): self.index = index self.draw_lines(just_history=True, iend=self.index) + # @timeit def draw_lines( self, istart=0, @@ -440,6 +441,7 @@ class BarItems(pg.GraphicsObject): # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() + # @timeit def paint(self, p, opt, widget): # start = time.time() From 4383579cd084e8f6e5476e77179010bc8405a8eb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 20:13:14 -0400 Subject: [PATCH 119/206] Use shm array in chart-fsp task Just like for the source OHLC, we now have the chart parent actor create an fsp shm array and use it to read back signal data for plotting. Some tweaks to get the price chart (and sub-charts) to load historical datums immediately instead of waiting on an initial quote. --- piker/ui/_chart.py | 201 +++++++++++++++++++++------------------------ 1 file changed, 95 insertions(+), 106 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 7695d89e..23237d3f 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -2,7 +2,6 @@ High level Qt chart widgets. """ from typing import Tuple, Dict, Any, Optional -import time from PyQt5 import QtCore, QtGui import numpy as np @@ -20,7 +19,10 @@ from ._style import _xaxis_at, _min_points_to_show, hcolor from ._source import Symbol from .. import brokers from .. import data -from ..data._normalize import iterticks +from ..data import ( + iterticks, + maybe_open_shm_array, +) from ..log import get_logger from ._exec import run_qtractor from ._interaction import ChartView @@ -542,7 +544,7 @@ class ChartPlotWidget(pg.PlotWidget): ylow = np.nanmin(bars['low']) yhigh = np.nanmax(bars['high']) # std = np.std(bars['close']) - except IndexError: + except (IndexError, ValueError): # must be non-ohlc array? ylow = np.nanmin(bars) yhigh = np.nanmax(bars) @@ -587,53 +589,6 @@ class ChartPlotWidget(pg.PlotWidget): self.scene().leaveEvent(ev) -async def check_for_new_bars(incr_stream, ohlcv, linked_charts): - """Task which updates from new bars in the shared ohlcv buffer every - ``delay_s`` seconds. - """ - # TODO: right now we'll spin printing bars if the last time - # stamp is before a large period of no market activity. - # Likely the best way to solve this is to make this task - # aware of the instrument's tradable hours? - - price_chart = linked_charts.chart - - async for index in incr_stream: - # TODO: generalize this increment logic - for name, chart in linked_charts.subplots.items(): - data = chart._array - chart._array = np.append( - data, - np.array(data[-1], dtype=data.dtype) - ) - - # update chart historical bars graphics - price_chart.update_from_array( - price_chart.name, - ohlcv.array, - # When appending a new bar, in the time between the insert - # here and the Qt render call the underlying price data may - # have already been updated, thus make sure to also update - # the last bar if necessary on this render cycle. - # just_history=True - ) - # resize view - price_chart._set_yrange() - - for name, curve in price_chart._overlays.items(): - # TODO: standard api for signal lookups per plot - if name in price_chart._array.dtype.fields: - # should have already been incremented above - price_chart.update_from_array( - name, - price_chart._array[name], - ) - - for name, chart in linked_charts.subplots.items(): - chart.update_from_array(chart.name, chart._array) - chart._set_yrange() - - async def _async_main( sym: str, brokername: str, @@ -656,8 +611,9 @@ async def _async_main( brokername, [sym], loglevel=loglevel, - ) as (fquote, stream, incr_stream, ohlcv): + ) as feed: + ohlcv = feed.shm bars = ohlcv.array # load in symbol's ohlc data @@ -673,6 +629,8 @@ async def _async_main( overlay=True, ) + chart._set_yrange() + async with trio.open_nursery() as n: # load initial fsp chain (otherwise known as "indicators") @@ -681,7 +639,7 @@ async def _async_main( linked_charts, 'rsi', # eventually will be n-compose syntax sym, - bars, + ohlcv, brokermod, loglevel, ) @@ -692,21 +650,22 @@ async def _async_main( *ohlcv.array[-1][['index', 'close']] ) - # wait for a first quote before we start any update tasks - quote = await stream.__anext__() - log.info(f'Received first quote {quote}') - # start graphics update loop(s)after receiving first live quote n.start_soon( chart_from_quotes, chart, - stream, + feed.stream, ohlcv, vwap_in_history, ) + + # wait for a first quote before we start any update tasks + quote = await feed.receive() + log.info(f'Received first quote {quote}') + n.start_soon( check_for_new_bars, - incr_stream, + feed, # delay, ohlcv, linked_charts @@ -743,20 +702,14 @@ async def chart_from_quotes( # - in theory we should be able to read buffer data # faster then msgs arrive.. needs some tinkering and # testing - start = time.time() array = ohlcv.array - diff = time.time() - start - print(f'read time: {diff}') - # last = ohlcv.array[-1] last = array[-1] chart.update_from_array( chart.name, - # ohlcv.array, array, ) # update sticky(s) - last_price_sticky.update_from_data( - *last[['index', 'close']]) + last_price_sticky.update_from_data(*last[['index', 'close']]) chart._set_yrange() vwap = quote.get('vwap') @@ -764,17 +717,14 @@ async def chart_from_quotes( last['vwap'] = vwap print(f"vwap: {quote['vwap']}") # update vwap overlay line - chart.update_from_array( - 'vwap', - ohlcv.array['vwap'], - ) + chart.update_from_array('vwap', ohlcv.array['vwap']) async def chart_from_fsp( linked_charts, func_name, sym, - bars, + src_shm, brokermod, loglevel, ) -> None: @@ -782,14 +732,31 @@ async def chart_from_fsp( Pass target entrypoint and historical data. """ + name = f'fsp.{func_name}' + # TODO: load function here and introspect + # return stream type(s) + fsp_dtype = np.dtype([('index', int), (func_name, float)]) + async with tractor.open_nursery() as n: + key = f'{sym}.' + name + + shm, opened = maybe_open_shm_array( + key, + # TODO: create entry for each time frame + dtype=fsp_dtype, + readonly=True, + ) + assert opened + + # start fsp sub-actor portal = await n.run_in_actor( - f'fsp.{func_name}', # name as title of sub-chart + name, # name as title of sub-chart # subactor entrypoint - fsp.stream_and_process, - bars=bars, + fsp.cascade, brokername=brokermod.name, + src_shm_token=src_shm.token, + dst_shm_token=shm.token, symbol=sym, fsp_func_name=func_name, @@ -799,55 +766,77 @@ async def chart_from_fsp( stream = await portal.result() - # receive processed historical data-array as first message - history = (await stream.__anext__()) - - # TODO: enforce type checking here? - newbars = np.array(history) + # receive last index for processed historical + # data-array as first msg + _ = await stream.receive() chart = linked_charts.add_plot( name=func_name, - array=newbars, + + # TODO: enforce type checking here? + array=shm.array, ) - # check for data length mis-allignment and fill missing values - diff = len(chart._array) - len(linked_charts.chart._array) - if diff <= 0: - data = chart._array - chart._array = np.append( - data, - np.full(abs(diff), data[-1], dtype=data.dtype) - ) - - # XXX: hack to get curves aligned with bars graphics: prepend - # a copy of the first datum.. - # TODO: talk to ``pyqtgraph`` core about proper way to solve this - data = chart._array - chart._array = np.append( - np.array(data[0], dtype=data.dtype), - data, - ) - - value = chart._array[-1] + array = shm.array[func_name] + value = array[-1] last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, chart._array) + chart.update_from_array(chart.name, array) chart._set_yrange(yrange=(0, 100)) + chart._shm = shm + # update chart graphics async for value in stream: - - # start = time.time() - chart._array[-1] = value - # diff = time.time() - start - # print(f'FSP array append took {diff}') - + array = shm.array[func_name] + value = array[-1] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, chart._array) + chart.update_from_array(chart.name, array) # chart._set_yrange() +async def check_for_new_bars(feed, ohlcv, linked_charts): + """Task which updates from new bars in the shared ohlcv buffer every + ``delay_s`` seconds. + """ + # TODO: right now we'll spin printing bars if the last time + # stamp is before a large period of no market activity. + # Likely the best way to solve this is to make this task + # aware of the instrument's tradable hours? + + price_chart = linked_charts.chart + + async for index in await feed.index_stream(): + + # update chart historical bars graphics + price_chart.update_from_array( + price_chart.name, + ohlcv.array, + # When appending a new bar, in the time between the insert + # here and the Qt render call the underlying price data may + # have already been updated, thus make sure to also update + # the last bar if necessary on this render cycle which is + # why we **don't** set: + # just_history=True + ) + # resize view + price_chart._set_yrange() + + for name, curve in price_chart._overlays.items(): + # TODO: standard api for signal lookups per plot + if name in price_chart._array.dtype.fields: + # should have already been incremented above + price_chart.update_from_array( + name, + price_chart._array[name], + ) + + for name, chart in linked_charts.subplots.items(): + chart.update_from_array(chart.name, chart._shm.array[chart.name]) + chart._set_yrange() + + def _main( sym: str, brokername: str, From 2fcbefa6e125130389788275981cab162b5aaa91 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 20:23:48 -0400 Subject: [PATCH 120/206] Use shm in fsp cascading This kicks off what will be the beginning of hopefully a very nice (soft) real-time financial signal processing system. We're keeping the hack to "time align" curves (for now) with the bars for now by slapping in an extra datum at index 0. --- piker/fsp/__init__.py | 90 ++++++++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py index bf2ea127..0cb58867 100644 --- a/piker/fsp/__init__.py +++ b/piker/fsp/__init__.py @@ -1,13 +1,16 @@ """ Financial signal processing for the peeps. """ -from typing import AsyncIterator, Callable +from typing import AsyncIterator, Callable, Tuple +import trio +import tractor import numpy as np from ..log import get_logger from .. import data from ._momo import _rsi +from ..data import attach_shm_array, Feed log = get_logger(__name__) @@ -19,7 +22,7 @@ async def latency( source: 'TickStream[Dict[str, float]]', # noqa ohlcv: np.ndarray ) -> AsyncIterator[np.ndarray]: - """Compute High-Low midpoint value. + """Latency measurements, broker to piker. """ # TODO: do we want to offer yielding this async # before the rt data connection comes up? @@ -37,33 +40,40 @@ async def latency( yield value -async def stream_and_process( - bars: np.ndarray, +async def increment_signals( + feed: Feed, + dst_shm: 'SharedArray', # noqa +) -> None: + async for msg in await feed.index_stream(): + array = dst_shm.array + last = array[-1:].copy() + + # write new slot to the buffer + dst_shm.push(last) + + + +@tractor.stream +async def cascade( + ctx: tractor.Context, brokername: str, - # symbols: List[str], + src_shm_token: dict, + dst_shm_token: Tuple[str, np.dtype], symbol: str, fsp_func_name: str, ) -> AsyncIterator[dict]: + """Chain streaming signal processors and deliver output to + destination mem buf. + """ + src = attach_shm_array(token=src_shm_token) + dst = attach_shm_array(readonly=False, token=dst_shm_token) - # remember, msgpack-numpy's ``from_buffer` returns read-only array - # bars = np.array(bars[list(ohlc_dtype.names)]) - - # async def _yield_bars(): - # yield bars - - # hist_out: np.ndarray = None - - # Conduct a single iteration of fsp with historical bars input - # async for hist_out in func(_yield_bars(), bars): - # yield {symbol: hist_out} func: Callable = _fsps[fsp_func_name] # open a data feed stream with requested broker - async with data.open_feed( - brokername, - [symbol], - ) as (fquote, stream): + async with data.open_feed(brokername, [symbol]) as feed: + assert src.token == feed.shm.token # TODO: load appropriate fsp with input args async def filter_by_sym(sym, stream): @@ -72,9 +82,37 @@ async def stream_and_process( if symbol == sym: yield quotes - async for processed in func( - filter_by_sym(symbol, stream), - bars, - ): - log.info(f"{fsp_func_name}: {processed}") - yield processed + out_stream = func( + filter_by_sym(symbol, feed.stream), + feed.shm, + ) + + # Conduct a single iteration of fsp with historical bars input + # and get historical output + history = await out_stream.__anext__() + + + # TODO: talk to ``pyqtgraph`` core about proper way to solve this: + # XXX: hack to get curves aligned with bars graphics: prepend + # a copy of the first datum.. + dst.push(history[:1]) + + # check for data length mis-allignment and fill missing values + diff = len(src.array) - len(history) + if diff >= 0: + for _ in range(diff): + dst.push(history[:1]) + + # compare with source signal and time align + index = dst.push(history) + + yield index + + async with trio.open_nursery() as n: + n.start_soon(increment_signals, feed, dst) + + async for processed in out_stream: + log.info(f"{fsp_func_name}: {processed}") + index = src.index + dst.array[-1][fsp_func_name] = processed + await ctx.send_yield(index) From bfa7839370eef8fa5f93b06c319cf178bada8637 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 20:57:05 -0400 Subject: [PATCH 121/206] Adopt official color --- piker/ui/_axes.py | 2 +- piker/ui/_chart.py | 2 +- piker/ui/_graphics.py | 2 +- piker/ui/_style.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 53d90c73..973145c1 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -92,7 +92,7 @@ class DynamicDateAxis(pg.AxisItem): class AxisLabel(pg.GraphicsObject): # bg_color = pg.mkColor('#a9a9a9') - bg_color = pg.mkColor(hcolor('gray')) + bg_color = pg.mkColor(hcolor('pikers')) fg_color = pg.mkColor(hcolor('black')) def __init__( diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 23237d3f..3860c0f6 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -475,7 +475,7 @@ class ChartPlotWidget(pg.PlotWidget): parent=self.getAxis('right'), # digits=0, opacity=1, - color=pg.mkPen(hcolor('gray')) + color=pg.mkPen(hcolor('pikers')) ) return last diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 3fe43079..e0824dee 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -273,7 +273,7 @@ class BarItems(pg.GraphicsObject): # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 - bars_pen = pg.mkPen(hcolor('gray')) + bars_pen = pg.mkPen(hcolor('pikers')) # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 8a80d12f..d3fdaad5 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -50,6 +50,7 @@ def hcolor(name: str) -> str: 'gunmetal': '#91A3B0', 'battleship': '#848482', 'davies': '#555555', + 'pikers': '#666666', # like the cult # palette 'default': DarkPalette.COLOR_BACKGROUND_NORMAL, From 268e7484176de33f994de13aa898d2957fc58c58 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 22 Sep 2020 20:57:37 -0400 Subject: [PATCH 122/206] Drop extra prefix in logs --- piker/log.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/piker/log.py b/piker/log.py index 7d92d566..958de010 100644 --- a/piker/log.py +++ b/piker/log.py @@ -7,7 +7,9 @@ import json import tractor from pygments import highlight, lexers, formatters -_proj_name = 'piker' +# Makes it so we only see the full module name when using ``__name__`` +# without the extra "piker." prefix. +_proj_name = '' def get_logger(name: str = None) -> logging.Logger: From 3f0e17501116f5c8c31371b0d938faebbb3990ac Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 23 Sep 2020 13:15:27 -0400 Subject: [PATCH 123/206] Get bar oriented RSI working correctly --- piker/fsp/_momo.py | 132 +++++++++++++++++++++++++++++---------------- 1 file changed, 87 insertions(+), 45 deletions(-) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 557c0614..26e72b58 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -1,10 +1,13 @@ """ Momentum bby. """ -from typing import AsyncIterator +from typing import AsyncIterator, Optional import numpy as np -from numba import jit, float64, optional +from ringbuf import RingBuffer +from numba import jit, float64, optional, int64 + +from ..data._normalize import iterticks # TODO: things to figure the fuck out: @@ -47,6 +50,9 @@ def ema( s[0] = y[0]; t = 0 s[t] = a*y[t] + (1-a)*s[t-1], t > 0. } + + More discussion here: + https://stackoverflow.com/questions/42869495/numpy-version-of-exponential-weighted-moving-average-equivalent-to-pandas-ewm """ n = y.shape[0] @@ -67,6 +73,7 @@ def ema( else: s[0] = ylast + print(s) for i in range(1, n): s[i] = y[i] * alpha + s[i-1] * (1 - alpha) @@ -77,34 +84,40 @@ def ema( # float64[:]( # float64[:], # int64, +# float64, +# float64, # ), -# # nopython=True, +# nopython=True, # nogil=True # ) def rsi( signal: 'np.ndarray[float64]', - period: int = 14, + period: int64 = 14, up_ema_last: float64 = None, down_ema_last: float64 = None, ) -> 'np.ndarray[float64]': alpha = 1/period - # print(signal) df = np.diff(signal) - up, down = np.where(df > 0, df, 0), np.where(df < 0, -df, 0) + + up = np.where(df > 0, df, 0) up_ema = ema(up, alpha, up_ema_last) + + down = np.where(df < 0, -df, 0) down_ema = ema(down, alpha, down_ema_last) + + # avoid dbz errors rs = np.divide( up_ema, down_ema, out=np.zeros_like(up_ema), - where=down_ema!=0 + where=down_ema != 0 ) - # print(f'up_ema: {up_ema}\ndown_ema: {down_ema}') - # print(f'rs: {rs}') + # map rs through sigmoid (with range [0, 100]) rsi = 100 - 100 / (1 + rs) # rsi = 100 * (up_ema / (up_ema + down_ema)) + # also return the last ema state for next iteration return rsi, up_ema[-1], down_ema[-1] @@ -114,67 +127,96 @@ def rsi( # ) async def _rsi( source: 'QuoteStream[Dict[str, Any]]', # noqa - ohlcv: np.ndarray, + ohlcv: "ShmArray[T<'close'>]", period: int = 14, ) -> AsyncIterator[np.ndarray]: """Multi-timeframe streaming RSI. https://en.wikipedia.org/wiki/Relative_strength_index """ - sig = ohlcv['close'] + sig = ohlcv.array['close'] # TODO: the emas here should be seeded with a period SMA as per # wilder's original formula.. - rsi_h, up_ema_last, down_ema_last = rsi(sig, period, None, None) + rsi_h, last_up_ema_close, last_down_ema_close = rsi(sig, period, None, None) # deliver history yield rsi_h - _last = sig[-1] + index = ohlcv.index async for quote in source: # tick based updates - for tick in quote.get('ticks', ()): - if tick.get('type') == 'trade': - curr = tick['price'] - last = np.array([_last, curr]) - # await tractor.breakpoint() - rsi_out, up_ema_last, down_ema_last = rsi( - last, - period=period, - up_ema_last=up_ema_last, - down_ema_last=down_ema_last, - ) - _last = curr - # print(f'last: {last}\n rsi: {rsi_out}') - yield rsi_out[-1] + for tick in iterticks(quote): + # though incorrect below is interesting + # sig = ohlcv.last(period)['close'] + sig = ohlcv.last(2)['close'] + + # the ema needs to be computed from the "last bar" + if ohlcv.index > index: + last_up_ema_close = up_ema_last + last_down_ema_close = down_ema_last + index = ohlcv.index + + rsi_out, up_ema_last, down_ema_last = rsi( + sig, + period=period, + up_ema_last=last_up_ema_close, + down_ema_last=last_down_ema_close, + ) + print(f'rsi_out: {rsi_out}') + yield rsi_out[-1:] + +def wma( + signal: np.ndarray, +) -> np.ndarray: + if weights is None: + # default is a standard arithmetic mean + seq = np.full((length,), 1) + weights = seq / seq.sum() + + assert length == len(weights) async def wma( source, #: AsyncStream[np.ndarray], + length: int, ohlcv: np.ndarray, # price time-frame "aware" - lookback: np.ndarray, # price time-frame "aware" - weights: np.ndarray, -) -> AsyncIterator[np.ndarray]: # i like FinSigStream - """Weighted moving average. + weights: Optional[np.ndarray] = None, +) -> AsyncIterator[np.ndarray]: # maybe something like like FspStream? + """Streaming weighted moving average. ``weights`` is a sequence of already scaled values. As an example for the WMA often found in "techincal analysis": ``weights = np.arange(1, N) * N*(N-1)/2``. """ - length = len(weights) - _lookback = np.zeros(length - 1) + # deliver historical output as "first yield" + yield np.convolve(ohlcv.array['close'], weights, 'valid') - ohlcv.from_tf('5m') + # begin real-time section - # async for frame_len, frame in source: - async for frame in source: - wma = np.convolve( - ohlcv[-length:]['close'], - # np.concatenate((_lookback, frame)), - weights, - 'valid' - ) - # todo: handle case where frame_len < length - 1 - _lookback = frame[-(length-1):] # noqa - yield wma + # fill length samples as lookback history + # ringbuf = RingBuffer(format='f', capacity=2*length) + # overflow = ringbuf.push(ohlcv['close'][-length + 1:]) + # assert overflow is None + + # lookback = np.zeros((length,)) + # lookback[:-1] = ohlcv['close'][-length + 1:] + + # async for frame in atleast(length, source): + async for quote in source: + for tick in iterticks(quote, type='trade'): + # writes no matter what + overflow = ringbuf.push(np.array([tick['price']])) + assert overflow is None + + # history = np.concatenate(ringbuf.pop(length - 1), frame) + + sig = ohlcv.last(length) + history = ringbuf.pop(ringbuf.read_available) + yield np.convolve(history, weights, 'valid') + + # push back `length-1` datums as lookback in preparation + # for next minimum 1 datum arrival which will require + # another "window's worth" of history. + ringbuf.push(history[-length + 1:]) From 41e85ccaa997802fa973237dec194c6d0eed73a7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 24 Sep 2020 13:04:47 -0400 Subject: [PATCH 124/206] Break wma calc into sync func --- piker/fsp/__init__.py | 2 +- piker/fsp/_momo.py | 67 +++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 39 deletions(-) diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py index 0cb58867..0b73b071 100644 --- a/piker/fsp/__init__.py +++ b/piker/fsp/__init__.py @@ -112,7 +112,7 @@ async def cascade( n.start_soon(increment_signals, feed, dst) async for processed in out_stream: - log.info(f"{fsp_func_name}: {processed}") + log.debug(f"{fsp_func_name}: {processed}") index = src.index dst.array[-1][fsp_func_name] = processed await ctx.send_yield(index) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 26e72b58..6c701ef2 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -122,6 +122,22 @@ def rsi( return rsi, up_ema[-1], down_ema[-1] +def wma( + signal: np.ndarray, + length: int, + weights: Optional[np.ndarray] = None, +) -> np.ndarray: + if weights is None: + # default is a standard arithmetic mean + seq = np.full((length,), 1) + weights = seq / seq.sum() + + assert length == len(weights) + + # return np.convolve(ohlcv.array['close'], weights, 'valid') + return np.convolve(signal, weights, 'valid') + + # @piker.fsp( # aggregates=['30s', '1m', '5m', '1H', '4H', '1D'], # ) @@ -136,9 +152,14 @@ async def _rsi( """ sig = ohlcv.array['close'] + # wilder says to seed the RSI EMAs with the SMA for the "period" + seed = wma(ohlcv.last(period)['close'], period)[0] + # TODO: the emas here should be seeded with a period SMA as per # wilder's original formula.. - rsi_h, last_up_ema_close, last_down_ema_close = rsi(sig, period, None, None) + rsi_h, last_up_ema_close, last_down_ema_close = rsi(sig, period, seed, seed) + up_ema_last = last_up_ema_close + down_ema_last = last_down_ema_close # deliver history yield rsi_h @@ -150,9 +171,13 @@ async def _rsi( for tick in iterticks(quote): # though incorrect below is interesting # sig = ohlcv.last(period)['close'] + + # get only the last 2 "datums" which will be diffed to + # calculate the real-time RSI output datum sig = ohlcv.last(2)['close'] # the ema needs to be computed from the "last bar" + # TODO: how to make this cleaner if ohlcv.index > index: last_up_ema_close = up_ema_last last_down_ema_close = down_ema_last @@ -164,25 +189,13 @@ async def _rsi( up_ema_last=last_up_ema_close, down_ema_last=last_down_ema_close, ) - print(f'rsi_out: {rsi_out}') yield rsi_out[-1:] -def wma( - signal: np.ndarray, -) -> np.ndarray: - if weights is None: - # default is a standard arithmetic mean - seq = np.full((length,), 1) - weights = seq / seq.sum() - assert length == len(weights) - - -async def wma( +async def _wma( source, #: AsyncStream[np.ndarray], length: int, ohlcv: np.ndarray, # price time-frame "aware" - weights: Optional[np.ndarray] = None, ) -> AsyncIterator[np.ndarray]: # maybe something like like FspStream? """Streaming weighted moving average. @@ -191,32 +204,10 @@ async def wma( ``weights = np.arange(1, N) * N*(N-1)/2``. """ # deliver historical output as "first yield" - yield np.convolve(ohlcv.array['close'], weights, 'valid') + yield wma(ohlcv.array['close'], length) # begin real-time section - # fill length samples as lookback history - # ringbuf = RingBuffer(format='f', capacity=2*length) - # overflow = ringbuf.push(ohlcv['close'][-length + 1:]) - # assert overflow is None - - # lookback = np.zeros((length,)) - # lookback[:-1] = ohlcv['close'][-length + 1:] - - # async for frame in atleast(length, source): async for quote in source: for tick in iterticks(quote, type='trade'): - # writes no matter what - overflow = ringbuf.push(np.array([tick['price']])) - assert overflow is None - - # history = np.concatenate(ringbuf.pop(length - 1), frame) - - sig = ohlcv.last(length) - history = ringbuf.pop(ringbuf.read_available) - yield np.convolve(history, weights, 'valid') - - # push back `length-1` datums as lookback in preparation - # for next minimum 1 datum arrival which will require - # another "window's worth" of history. - ringbuf.push(history[-length + 1:]) + yield wma(ohlcv.last(length)) From e3e219aa4bcb1cf5b00380f8d33cbd8c0cee31e2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Sep 2020 15:16:58 -0400 Subject: [PATCH 125/206] Add multi-symbol-buffer increment support --- piker/data/__init__.py | 13 +++-- piker/data/_buffer.py | 113 ++++++++++++++++++++++++--------------- piker/data/_sharedmem.py | 12 ++--- 3 files changed, 83 insertions(+), 55 deletions(-) diff --git a/piker/data/__init__.py b/piker/data/__init__.py index 5459b4ba..e46db9b7 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -23,10 +23,13 @@ from ._sharedmem import ( maybe_open_shm_array, attach_shm_array, open_shm_array, - SharedArray, + ShmArray, get_shm_token, ) -from ._buffer import incr_buffer +from ._buffer import ( + increment_ohlc_buffer, + subscribe_ohlc_for_increment +) __all__ = [ @@ -35,7 +38,7 @@ __all__ = [ 'attach_shm_array', 'open_shm_array', 'get_shm_token', - 'incr_buffer', + 'subscribe_ohlc_for_increment', ] @@ -115,7 +118,7 @@ class Feed: """ name: str stream: AsyncIterator[Dict[str, Any]] - shm: SharedArray + shm: ShmArray _broker_portal: tractor._portal.Portal _index_stream: Optional[AsyncIterator[Dict[str, Any]]] = None @@ -129,7 +132,7 @@ class Feed: # created for all practical purposes self._index_stream = await self._broker_portal.run( 'piker.data', - 'incr_buffer', + 'increment_ohlc_buffer', shm_token=self.shm.token, topics=['index'], ) diff --git a/piker/data/_buffer.py b/piker/data/_buffer.py index 43be7dc0..5e1c3588 100644 --- a/piker/data/_buffer.py +++ b/piker/data/_buffer.py @@ -1,74 +1,99 @@ """ Data buffers for fast shared humpy. """ -from typing import Tuple, Callable -import time +from typing import Tuple, Callable, Dict +# import time import tractor import trio -from ._sharedmem import attach_shm_array +from ._sharedmem import ShmArray + + +_shms: Dict[int, ShmArray] = {} @tractor.msg.pub -async def incr_buffer( +async def increment_ohlc_buffer( shm_token: dict, get_topics: Callable[..., Tuple[str]], # delay_s: Optional[float] = None, ): """Task which inserts new bars into the provide shared memory array every ``delay_s`` seconds. + + This task fulfills 2 purposes: + - it takes the subscribed set of shm arrays and increments them + on a common time period + - broadcast of this increment "signal" message to other actor + subscribers + + Note that if **no** actor has initiated this task then **none** of + the underlying buffers will actually be incremented. """ - # TODO: right now we'll spin printing bars if the last time - # stamp is before a large period of no market activity. - # Likely the best way to solve this is to make this task - # aware of the instrument's tradable hours? - - shm = attach_shm_array( - token=shm_token, - readonly=False, - ) - - # determine ohlc delay between bars - # to determine time step between datums - times = shm.array['time'] - delay_s = times[-1] - times[times != times[-1]][-1] + # TODO: right now we'll spin printing bars if the last time stamp is + # before a large period of no market activity. Likely the best way + # to solve this is to make this task aware of the instrument's + # tradable hours? # adjust delay to compensate for trio processing time - ad = delay_s - 0.002 + ad = min(_shms.keys()) - 0.001 - async def sleep(): - """Sleep until next time frames worth has passed from last bar. - """ - # last_ts = shm.array[-1]['time'] - # delay = max((last_ts + ad) - time.time(), 0) - # await trio.sleep(delay) - await trio.sleep(ad) + # async def sleep(): + # """Sleep until next time frames worth has passed from last bar. + # """ + # # last_ts = shm.array[-1]['time'] + # # delay = max((last_ts + ad) - time.time(), 0) + # # await trio.sleep(delay) + # await trio.sleep(ad) + + total_s = 0 # total seconds counted + lowest = min(_shms.keys()) + ad = lowest - 0.001 while True: - # sleep for duration of current bar - await sleep() + # TODO: do we want to support dynamically + # adding a "lower" lowest increment period? + await trio.sleep(ad) + total_s += lowest - # TODO: in theory we could make this faster by copying the - # "last" readable value into the underlying larger buffer's - # next value and then incrementing the counter instead of - # using ``.push()``? + # # sleep for duration of current bar + # await sleep() - # append new entry to buffer thus "incrementing" the bar - array = shm.array - last = array[-1:].copy() - (index, t, close) = last[0][['index', 'time', 'close']] + # increment all subscribed shm arrays + # TODO: this in ``numba`` + for delay_s, shms in _shms.items(): + if total_s % delay_s != 0: + continue - # this copies non-std fields (eg. vwap) from the last datum - last[ - ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] - ][0] = (index + 1, t + delay_s, 0, close, close, close, close) + # TODO: numa this! + for shm in shms: + # TODO: in theory we could make this faster by copying the + # "last" readable value into the underlying larger buffer's + # next value and then incrementing the counter instead of + # using ``.push()``? - # write to the buffer - shm.push(last) - # print('incrementing array') + # append new entry to buffer thus "incrementing" the bar + array = shm.array + last = array[-1:].copy() + (index, t, close) = last[0][['index', 'time', 'close']] - # print(get_topics()) + # this copies non-std fields (eg. vwap) from the last datum + last[ + ['index', 'time', 'volume', 'open', 'high', 'low', 'close'] + ][0] = (index + 1, t + delay_s, 0, close, close, close, close) + + # write to the buffer + shm.push(last) # broadcast the buffer index step yield {'index': shm._i.value} + + +def subscribe_ohlc_for_increment( + shm: ShmArray, + delay: int, +) -> None: + """Add an OHLC ``ShmArray`` to the increment set. + """ + _shms.setdefault(delay, []).append(shm) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index d6c53a95..6c410423 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -134,7 +134,7 @@ def _make_token( ) -class SharedArray: +class ShmArray: def __init__( self, shmarr: np.ndarray, @@ -216,7 +216,7 @@ def open_shm_array( size: int = int(2*60*60*10/5), dtype: np.dtype = base_ohlc_dtype, readonly: bool = False, -) -> SharedArray: +) -> ShmArray: """Open a memory shared ``numpy`` using the standard library. This call unlinks (aka permanently destroys) the buffer on teardown @@ -245,7 +245,7 @@ def open_shm_array( ) counter.value = 0 - shmarr = SharedArray( + shmarr = ShmArray( array, counter, shm, @@ -268,7 +268,7 @@ def attach_shm_array( size: int = int(60*60*10/5), # dtype: np.dtype = base_ohlc_dtype, readonly: bool = True, -) -> SharedArray: +) -> ShmArray: """Load and attach to an existing shared memory array previously created by another process using ``open_shared_array``. """ @@ -289,7 +289,7 @@ def attach_shm_array( # make sure we can read counter.value - sha = SharedArray( + sha = ShmArray( shmarr, counter, shm, @@ -314,7 +314,7 @@ def maybe_open_shm_array( key: str, dtype: np.dtype = base_ohlc_dtype, **kwargs, -) -> Tuple[SharedArray, bool]: +) -> Tuple[ShmArray, bool]: """Attempt to attach to a shared memory block by a "key" determined by the users overall "system" (presumes you don't have the block's explicit token). From 8832804babdba0687ad2539b57aa6da3840700ee Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Sep 2020 15:19:33 -0400 Subject: [PATCH 126/206] Sub each new symbol to shm incrementing --- piker/brokers/ib.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 77a1018c..607d2e32 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -8,7 +8,7 @@ built on it) and thus actor aware API calls must be spawned with from contextlib import asynccontextmanager from dataclasses import asdict from functools import partial -from typing import List, Dict, Any, Tuple, Optional, AsyncGenerator, Callable +from typing import List, Dict, Any, Tuple, Optional, AsyncIterator, Callable import asyncio import logging import inspect @@ -29,7 +29,8 @@ from ..data import ( maybe_spawn_brokerd, iterticks, attach_shm_array, - get_shm_token + get_shm_token, + subscribe_ohlc_for_increment, ) from ..ui._source import from_df @@ -147,7 +148,7 @@ class Client: # durationStr='1 D', # time length calcs - durationStr='{count} S'.format(count=1000 * 5), + durationStr='{count} S'.format(count=100 * 5), barSizeSetting='5 secs', # always use extended hours @@ -494,14 +495,16 @@ def normalize( # TODO: figure out how to share quote feeds sanely despite # the wacky ``ib_insync`` api. # @tractor.msg.pub +@tractor.stream async def stream_quotes( + ctx: tractor.Context, symbols: List[str], shm_token: Tuple[str, str, List[tuple]], loglevel: str = None, # compat for @tractor.msg.pub topics: Any = None, get_topics: Callable = None, -) -> AsyncGenerator[str, Dict[str, Any]]: +) -> AsyncIterator[Dict[str, Any]]: """Stream symbol quotes. This is a ``trio`` callable routine meant to be invoked @@ -539,8 +542,12 @@ async def stream_quotes( shm.push(bars) shm_token = shm.token + times = shm.array['time'] + delay_s = times[-1] - times[times != times[-1]][-1] + subscribe_ohlc_for_increment(shm, delay_s) + # pass back token, and bool, signalling if we're the writer - yield shm_token, not writer_exists + await ctx.send_yield((shm_token, not writer_exists)) # first quote can be ignored as a 2nd with newer data is sent? first_ticker = await stream.__anext__() @@ -581,9 +588,11 @@ async def stream_quotes( topic = '.'.join((con['symbol'], con[suffix])).lower() first_quote = {topic: quote} ticker.ticks = [] - # yield first quote asap - yield first_quote + # yield first quote asap + await ctx.send_yield(first_quote) + + # real-time stream async for ticker in stream: quote = normalize( ticker, @@ -614,7 +623,8 @@ async def stream_quotes( con = quote['contract'] topic = '.'.join((con['symbol'], con[suffix])).lower() - yield {topic: quote} + + await ctx.send_yield({topic: quote}) # ugh, clear ticks since we've consumed them ticker.ticks = [] From 155c3eef2a384d3e295b05306db63d45d2c60954 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Sep 2020 16:03:58 -0400 Subject: [PATCH 127/206] Convert timeit thing to use print() --- piker/brokers/kraken.py | 12 ++++++++++-- piker/ui/quantdom/utils.py | 3 +-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 75333b9d..f2da2bb8 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -102,11 +102,18 @@ class Client: res.pop('last') bars = next(iter(res.values())) - # convert all fields to native types new_bars = [] - last_nz_vwap = None + + first = bars[0] + last_nz_vwap = first[-3] + if last_nz_vwap == 0: + # use close if vwap is zero + last_nz_vwap = first[-4] + + # convert all fields to native types for i, bar in enumerate(bars): # normalize weird zero-ed vwap values..cmon kraken.. + # indicates vwap didn't change since last bar vwap = float(bar[-3]) if vwap != 0: last_nz_vwap = vwap @@ -211,6 +218,7 @@ def normalize( @tractor.msg.pub async def stream_quotes( get_topics: Callable, + shared_array_token: Tuple[str, str, str], # These are the symbols not expected by the ws api # they are looked up inside this routine. symbols: List[str] = ['XBTUSD', 'XMRUSD'], diff --git a/piker/ui/quantdom/utils.py b/piker/ui/quantdom/utils.py index 0324b16e..af885cc4 100644 --- a/piker/ui/quantdom/utils.py +++ b/piker/ui/quantdom/utils.py @@ -50,8 +50,7 @@ def timeit(fn): def wrapper(*args, **kwargs): t = time.time() res = fn(*args, **kwargs) - logger = logging.getLogger('runtime') - logger.debug( + print( '%s.%s: %.4f sec' % (fn.__module__, fn.__qualname__, time.time() - t) ) From 8e2e695ba833c002c495a7b7cfe2148732d1b67d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 25 Sep 2020 16:07:29 -0400 Subject: [PATCH 128/206] Revert project name removal; breaks shit elsewhere somehow.. --- piker/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/log.py b/piker/log.py index 958de010..218a9fdb 100644 --- a/piker/log.py +++ b/piker/log.py @@ -9,7 +9,7 @@ from pygments import highlight, lexers, formatters # Makes it so we only see the full module name when using ``__name__`` # without the extra "piker." prefix. -_proj_name = '' +_proj_name = 'piker' def get_logger(name: str = None) -> logging.Logger: From 47d4ec5985414e56cac835ae9846ea7497985a53 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 26 Sep 2020 14:11:14 -0400 Subject: [PATCH 129/206] Move _source under data package --- piker/brokers/ib.py | 2 +- piker/{ui => data}/_source.py | 1 + piker/ui/_chart.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename piker/{ui => data}/_source.py (98%) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 607d2e32..01e30e3d 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -32,7 +32,7 @@ from ..data import ( get_shm_token, subscribe_ohlc_for_increment, ) -from ..ui._source import from_df +from ..data._source import from_df log = get_logger(__name__) diff --git a/piker/ui/_source.py b/piker/data/_source.py similarity index 98% rename from piker/ui/_source.py rename to piker/data/_source.py index b8816636..74238bb1 100644 --- a/piker/ui/_source.py +++ b/piker/data/_source.py @@ -8,6 +8,7 @@ import numpy as np import pandas as pd +# our minimum structured array layout for ohlc data base_ohlc_dtype = np.dtype( [ ('index', int), diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 3860c0f6..cab465d8 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -16,7 +16,7 @@ from ._axes import ( from ._graphics import CrossHair, BarItems from ._axes import YSticky from ._style import _xaxis_at, _min_points_to_show, hcolor -from ._source import Symbol +from ..data._source import Symbol from .. import brokers from .. import data from ..data import ( From 8a4528c006cb27db5066f40acb6172cc68bf7633 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 26 Sep 2020 14:11:55 -0400 Subject: [PATCH 130/206] Always ask backend for ohlc dtype --- piker/data/__init__.py | 11 +++++++++-- piker/data/_sharedmem.py | 25 ++++++------------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/piker/data/__init__.py b/piker/data/__init__.py index e46db9b7..153c1c8f 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -26,6 +26,7 @@ from ._sharedmem import ( ShmArray, get_shm_token, ) +from ._source import base_ohlc_dtype from ._buffer import ( increment_ohlc_buffer, subscribe_ohlc_for_increment @@ -163,10 +164,11 @@ async def open_feed( if loglevel is None: loglevel = tractor.current_actor().loglevel - # attempt to allocate (or attach to) shm array for this - # broker/symbol + # Attempt to allocate (or attach to) shm array for this broker/symbol shm, opened = maybe_open_shm_array( key=sym_to_shm_key(name, symbols[0]), + # use any broker defined ohlc dtype: + dtype=getattr(mod, '_ohlc_dtype', base_ohlc_dtype), # we expect the sub-actor to write readonly=True, @@ -185,7 +187,12 @@ async def open_feed( # compat with eventual ``tractor.msg.pub`` topics=symbols, ) + + # TODO: we can't do this **and** be compate with + # ``tractor.msg.pub``, should we maybe just drop this after + # tests are in? shm_token, is_writer = await stream.receive() + shm_token['dtype_descr'] = list(shm_token['dtype_descr']) assert shm_token == shm.token # sanity diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 6c410423..96526206 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -12,8 +12,8 @@ from _posixshmem import shm_unlink import tractor import numpy as np - from ..log import get_logger +from ._source import base_ohlc_dtype log = get_logger(__name__) @@ -41,19 +41,6 @@ mantracker.unregister = mantracker._resource_tracker.unregister mantracker.getfd = mantracker._resource_tracker.getfd -base_ohlc_dtype = np.dtype( - [ - ('index', int), - ('time', float), - ('open', float), - ('high', float), - ('low', float), - ('close', float), - ('volume', int), - ] -) - - class SharedInt: def __init__( self, @@ -122,15 +109,16 @@ def get_shm_token(key: str) -> _Token: def _make_token( key: str, - dtype: np.dtype = base_ohlc_dtype, + dtype: Optional[np.dtype] = None, ) -> _Token: """Create a serializable token that can be used to access a shared array. """ + dtype = base_ohlc_dtype if dtype is None else dtype return _Token( key, key + "_counter", - dtype.descr + np.dtype(dtype).descr ) @@ -214,7 +202,7 @@ def open_shm_array( key: Optional[str] = None, # approx number of 5s bars in a "day" x2 size: int = int(2*60*60*10/5), - dtype: np.dtype = base_ohlc_dtype, + dtype: Optional[np.dtype] = None, readonly: bool = False, ) -> ShmArray: """Open a memory shared ``numpy`` using the standard library. @@ -266,7 +254,6 @@ def open_shm_array( def attach_shm_array( token: Tuple[str, str, Tuple[str, str]], size: int = int(60*60*10/5), - # dtype: np.dtype = base_ohlc_dtype, readonly: bool = True, ) -> ShmArray: """Load and attach to an existing shared memory array previously @@ -312,7 +299,7 @@ def attach_shm_array( def maybe_open_shm_array( key: str, - dtype: np.dtype = base_ohlc_dtype, + dtype: Optional[np.dtype] = None, **kwargs, ) -> Tuple[ShmArray, bool]: """Attempt to attach to a shared memory block by a From d4eb5ccca48d9025498c3715d6149f54941e93bf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 26 Sep 2020 14:12:33 -0400 Subject: [PATCH 131/206] Handle vwap overlay with shm --- piker/ui/_chart.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index cab465d8..14b7c1ce 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -379,7 +379,7 @@ class ChartPlotWidget(pg.PlotWidget): def update(index: int) -> None: label.setText( "{name}[{index}] -> O:{} H:{} L:{} C:{} V:{}".format( - *self._array[index].item()[2:], + *self._array[index].item()[2:8], name=name, index=index, ) @@ -423,9 +423,6 @@ class ChartPlotWidget(pg.PlotWidget): self.addItem(curve) # register overlay curve with name - if not self._graphics and name is None: - name = 'a_stupid_line_bby' - self._graphics[name] = curve # XXX: How to stack labels vertically? @@ -485,7 +482,8 @@ class ChartPlotWidget(pg.PlotWidget): array: np.ndarray, **kwargs, ) -> pg.GraphicsObject: - self._array = array + if name not in self._overlays: + self._array = array graphics = self._graphics[name] graphics.update_from_array(array, **kwargs) @@ -712,10 +710,7 @@ async def chart_from_quotes( last_price_sticky.update_from_data(*last[['index', 'close']]) chart._set_yrange() - vwap = quote.get('vwap') - if vwap and vwap_in_history: - last['vwap'] = vwap - print(f"vwap: {quote['vwap']}") + if vwap_in_history: # update vwap overlay line chart.update_from_array('vwap', ohlcv.array['vwap']) From bc650406010e112adca9dd058b32fb9a4ae2fc5f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 26 Sep 2020 14:12:54 -0400 Subject: [PATCH 132/206] Add shm support to kraken backend --- piker/brokers/kraken.py | 191 ++++++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 74 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index f2da2bb8..a79a6399 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -3,8 +3,7 @@ Kraken backend. """ from contextlib import asynccontextmanager from dataclasses import dataclass, asdict, field -from itertools import starmap -from typing import List, Dict, Any, Callable +from typing import List, Dict, Any, Tuple, Optional import json import time @@ -18,6 +17,12 @@ import tractor from ._util import resproc, SymbolNotFound, BrokerError from ..log import get_logger, get_console_log +from ..data import ( + # iterticks, + attach_shm_array, + get_shm_token, + subscribe_ohlc_for_increment, +) log = get_logger(__name__) @@ -26,7 +31,7 @@ log = get_logger(__name__) _url = 'https://api.kraken.com/0' -# conversion to numpy worthy types +# Broker specific ohlc schema which includes a vwap field _ohlc_dtype = [ ('index', int), ('time', int), @@ -34,9 +39,9 @@ _ohlc_dtype = [ ('high', float), ('low', float), ('close', float), - ('vwap', float), ('volume', float), - ('count', int) + ('count', int), + ('vwap', float), ] # UI components allow this to be declared such that additional @@ -114,18 +119,24 @@ class Client: for i, bar in enumerate(bars): # normalize weird zero-ed vwap values..cmon kraken.. # indicates vwap didn't change since last bar - vwap = float(bar[-3]) + vwap = float(bar.pop(-3)) if vwap != 0: last_nz_vwap = vwap if vwap == 0: - bar[-3] = last_nz_vwap + vwap = last_nz_vwap + + # re-insert vwap as the last of the fields + bar.append(vwap) new_bars.append( (i,) + tuple( - ftype(bar[j]) for j, (name, ftype) in enumerate(_ohlc_dtype[1:]) + ftype(bar[j]) for j, (name, ftype) in enumerate( + _ohlc_dtype[1:] + ) ) ) - return np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + return array except KeyError: raise SymbolNotFound(json['error'][0] + f': {symbol}') @@ -215,15 +226,17 @@ def normalize( return topic, quote -@tractor.msg.pub +# @tractor.msg.pub async def stream_quotes( - get_topics: Callable, - shared_array_token: Tuple[str, str, str], + # get_topics: Callable, + shm_token: Tuple[str, str, List[tuple]], + symbols: List[str] = ['XBTUSD', 'XMRUSD'], # These are the symbols not expected by the ws api # they are looked up inside this routine. - symbols: List[str] = ['XBTUSD', 'XMRUSD'], sub_type: str = 'ohlc', loglevel: str = None, + # compat with eventual ``tractor.msg.pub`` + topics: Optional[List[str]] = None, ) -> None: """Subscribe for ohlc stream of quotes for ``pairs``. @@ -234,84 +247,114 @@ async def stream_quotes( ws_pairs = {} async with get_client() as client: + # keep client cached for real-time section for sym in symbols: ws_pairs[sym] = (await client.symbol_info(sym))['wsname'] - while True: - try: - async with trio_websocket.open_websocket_url( - 'wss://ws.kraken.com', - ) as ws: - # setup subs - # see: https://docs.kraken.com/websockets/#message-subscribe - subs = { - 'pair': list(ws_pairs.values()), - 'event': 'subscribe', - 'subscription': { - 'name': sub_type, - 'interval': 1, # 1 min - # 'name': 'ticker', - # 'name': 'openOrders', - # 'depth': '25', - }, - } - # TODO: we want to eventually allow unsubs which should - # be completely fine to request from a separate task - # since internally the ws methods appear to be FIFO - # locked. - await ws.send_message(json.dumps(subs)) + # maybe load historical ohlcv in to shared mem + # check if shm has already been created by previous + # feed initialization + writer_exists = get_shm_token(shm_token['shm_name']) - async def recv(): - return json.loads(await ws.get_message()) + symbol = symbols[0] - # pull a first quote and deliver - ohlc_gen = recv_ohlc(recv) - ohlc_last = await ohlc_gen.__anext__() + if not writer_exists: + shm = attach_shm_array( + token=shm_token, + # we are writer + readonly=False, + ) + bars = await client.bars(symbol=symbol) - topic, quote = normalize(ohlc_last) + shm.push(bars) + shm_token = shm.token - # packetize as {topic: quote} - yield {topic: quote} + times = shm.array['time'] + delay_s = times[-1] - times[times != times[-1]][-1] + subscribe_ohlc_for_increment(shm, delay_s) - # keep start of last interval for volume tracking - last_interval_start = ohlc_last.etime + yield shm_token, not writer_exists - # start streaming - async for ohlc in ohlc_gen: + while True: + try: + async with trio_websocket.open_websocket_url( + 'wss://ws.kraken.com', + ) as ws: + # setup subs + # https://docs.kraken.com/websockets/#message-subscribe + subs = { + 'pair': list(ws_pairs.values()), + 'event': 'subscribe', + 'subscription': { + 'name': sub_type, + 'interval': 1, # 1 min + # 'name': 'ticker', + # 'name': 'openOrders', + # 'depth': '25', + }, + } + # TODO: we want to eventually allow unsubs which should + # be completely fine to request from a separate task + # since internally the ws methods appear to be FIFO + # locked. + await ws.send_message(json.dumps(subs)) - # generate tick values to match time & sales pane: - # https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m - volume = ohlc.volume - if ohlc.etime > last_interval_start: # new interval - last_interval_start = ohlc.etime - tick_volume = volume - else: - # this is the tick volume *within the interval* - tick_volume = volume - ohlc_last.volume + async def recv(): + return json.loads(await ws.get_message()) - if tick_volume: - ohlc.ticks.append({ - 'type': 'trade', - 'price': ohlc.close, - 'size': tick_volume, - }) + # pull a first quote and deliver + ohlc_gen = recv_ohlc(recv) + ohlc_last = await ohlc_gen.__anext__() - topic, quote = normalize(ohlc) + topic, quote = normalize(ohlc_last) - # XXX: format required by ``tractor.msg.pub`` - # requires a ``Dict[topic: str, quote: dict]`` + # packetize as {topic: quote} yield {topic: quote} - ohlc_last = ohlc + # keep start of last interval for volume tracking + last_interval_start = ohlc_last.etime - except (ConnectionClosed, DisconnectionTimeout): - log.exception("Good job kraken...reconnecting") + # start streaming + async for ohlc in ohlc_gen: + # generate tick values to match time & sales pane: + # https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m + volume = ohlc.volume + if ohlc.etime > last_interval_start: # new interval + last_interval_start = ohlc.etime + tick_volume = volume + else: + # this is the tick volume *within the interval* + tick_volume = volume - ohlc_last.volume -if __name__ == '__main__': + last = ohlc.close + if tick_volume: + ohlc.ticks.append({ + 'type': 'trade', + 'price': last, + 'size': tick_volume, + }) - async def stream_ohlc(): - async for msg in stream_quotes(): - print(msg) + topic, quote = normalize(ohlc) - tractor.run(stream_ohlc) + # if we are the lone tick writer start writing + # the buffer with appropriate trade data + if not writer_exists: + # update last entry + # benchmarked in the 4-5 us range + high, low = shm.array[-1][['high', 'low']] + shm.array[['high', 'low', 'close', 'vwap']][-1] = ( + max(high, last), + min(low, last), + last, + ohlc.vwap, + ) + + # XXX: format required by ``tractor.msg.pub`` + # requires a ``Dict[topic: str, quote: dict]`` + yield {topic: quote} + + ohlc_last = ohlc + + except (ConnectionClosed, DisconnectionTimeout): + log.exception("Good job kraken...reconnecting") From acc8dd66f538541b1e3315969f3707c1cf14ddd6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 26 Sep 2020 14:20:55 -0400 Subject: [PATCH 133/206] Add data._normalize.py ... --- piker/data/_normalize.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 piker/data/_normalize.py diff --git a/piker/data/_normalize.py b/piker/data/_normalize.py new file mode 100644 index 00000000..9f73858d --- /dev/null +++ b/piker/data/_normalize.py @@ -0,0 +1,21 @@ +""" +Stream format enforcement. +""" +from typing import AsyncIterator, Optional + +import numpy as np + + +def iterticks( + quote: dict, + type: str = 'trade', +) -> AsyncIterator: + """Iterate through ticks delivered per quote cycle. + """ + # print(f"{quote}\n\n") + ticks = quote.get('ticks', ()) + if ticks: + for tick in ticks: + # print(tick) + if tick.get('type') == type: + yield tick From bceeaa56ffd54edc6d54c90f6633d645b0344d88 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 08:50:28 -0400 Subject: [PATCH 134/206] Drop ringbuf, didn't end up using --- piker/fsp/_momo.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 6c701ef2..8c5f386c 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -4,7 +4,6 @@ Momentum bby. from typing import AsyncIterator, Optional import numpy as np -from ringbuf import RingBuffer from numba import jit, float64, optional, int64 from ..data._normalize import iterticks From e524ee90456fb2eb5750aeb1884c839b22b713b9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 11:11:42 -0400 Subject: [PATCH 135/206] Resize everything with HiDPI scaling on --- piker/ui/_axes.py | 12 ++++++------ piker/ui/_exec.py | 15 ++++++++++++++- piker/ui/_graphics.py | 36 +++++++++++++++--------------------- piker/ui/_style.py | 3 ++- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 973145c1..43dc28ef 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -29,7 +29,7 @@ class PriceAxis(pg.AxisItem): }) self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font) - self.setWidth(125) + self.setWidth(50) # XXX: drop for now since it just eats up h space @@ -61,12 +61,12 @@ class DynamicDateAxis(pg.AxisItem): # default styling self.setStyle( - tickTextOffset=7, + tickTextOffset=4, textFillLimits=[(0, 0.70)], # TODO: doesn't seem to work -> bug in pyqtgraph? - # tickTextHeight=2, + # tickTextHeight=11, ) - # self.setHeight(35) + self.setHeight(10) def _indexes_to_timestrs( self, @@ -175,7 +175,7 @@ class XAxisLabel(AxisLabel): # TODO: we need to get the parent axe's dimensions transformed # to abs coords to be 100% correct here: # self.parent.boundingRect() - return QtCore.QRectF(0, 0, 100, 31) + return QtCore.QRectF(0, 2, 40, 10) def update_label( self, @@ -206,7 +206,7 @@ class YAxisLabel(AxisLabel): return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 120, 30) + return QtCore.QRectF(0, 0, 50, 11) def update_label( self, diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index c169640d..c1f247d9 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -8,16 +8,29 @@ from functools import partial import traceback from typing import Tuple, Callable, Dict, Any +# Qt specific import PyQt5 # noqa from pyqtgraph import QtGui from PyQt5 import QtCore -from PyQt5.QtCore import pyqtRemoveInputHook +from PyQt5.QtCore import ( + pyqtRemoveInputHook, Qt, QCoreApplication +) import qdarkstyle + import trio import tractor from outcome import Error +# Proper high DPI scaling is available in Qt >= 5.6.0. This attibute +# must be set before creating the application +if hasattr(Qt, 'AA_EnableHighDpiScaling'): + QCoreApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True) + +# if hasattr(Qt, 'AA_UseHighDpiPixmaps'): +# QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) + + class MainWindow(QtGui.QMainWindow): size = (800, 500) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index e0824dee..623c0e14 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -9,7 +9,7 @@ import pyqtgraph as pg from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF -from .quantdom.utils import timeit +# from .quantdom.utils import timeit from ._style import _xaxis_at, hcolor from ._axes import YAxisLabel, XAxisLabel @@ -286,7 +286,6 @@ class BarItems(pg.GraphicsObject): ) -> None: super().__init__() self.last = QtGui.QPicture() - # self.buffer = QtGui.QPicture() self.history = QtGui.QPicture() # TODO: implement updateable pixmap solution self._pi = plotitem @@ -307,10 +306,12 @@ class BarItems(pg.GraphicsObject): # @timeit def draw_from_data( self, - data: np.recarray, + data: np.ndarray, start: int = 0, ): - """Draw OHLC datum graphics from a ``np.recarray``. + """Draw OHLC datum graphics from a ``np.ndarray``. + + This routine is usually only called to draw the initial history. """ lines = bars_from_ohlc(data, self.w, start=start) @@ -332,20 +333,16 @@ class BarItems(pg.GraphicsObject): ) -> None: """Draw the current line set using the painter. """ - # start = time.time() - if just_history: - istart = 0 + # draw bars for the "history" picture iend = iend or self.index - 1 pic = self.history else: + # draw the last bar istart = self.index - 1 - iend = self.index + iend = iend or self.index pic = self.last - if iend is not None: - iend = iend - # use 2d array of lines objects, see conlusion on speed: # https://stackoverflow.com/a/60089929 to_draw = np.ravel(self.lines[istart:iend]) @@ -370,9 +367,6 @@ class BarItems(pg.GraphicsObject): # https://doc.qt.io/qt-5/qgraphicsitem.html#update self.update() - # diff = time.time() - start - # print(f'{len(to_draw)} lines update took {diff}') - def update_from_array( self, array: np.ndarray, @@ -407,9 +401,7 @@ class BarItems(pg.GraphicsObject): return # current bar update - i, open, close, = array[-1][['index', 'open', 'close']] - last = close - # i, body, larm, rarm = self.lines[index-1] + i, open, last, = array[-1][['index', 'open', 'close']] body, larm, rarm = self.lines[index-1] # XXX: is there a faster way to modify this? @@ -443,13 +435,18 @@ class BarItems(pg.GraphicsObject): # The only required methods are paint() and boundingRect() # @timeit def paint(self, p, opt, widget): - # start = time.time() # TODO: use to avoid drawing artefacts? # self.prepareGeometryChange() # p.setCompositionMode(0) + # TODO: one thing we could try here is pictures being drawn of + # a fixed count of bars such that based on the viewbox indices we + # only draw the "rounded up" number of "pictures worth" of bars + # as is necesarry for what's in "view". Not sure if this will + # lead to any perf gains other then when zoomed in to less bars + # in view. p.drawPicture(0, 0, self.history) p.drawPicture(0, 0, self.last) @@ -458,9 +455,6 @@ class BarItems(pg.GraphicsObject): # self._pmi.setPixmap(self.picture) # print(self.scene()) - # diff = time.time() - start - # print(f'draw time {diff}') - def boundingRect(self): # TODO: can we do rect caching to make this faster? diff --git a/piker/ui/_style.py b/piker/ui/_style.py index d3fdaad5..8d8f5099 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -10,13 +10,14 @@ from qdarkstyle.palette import DarkPalette _font = QtGui.QFont("Hack", 4) _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) - # splitter widget config _xaxis_at = 'bottom' # charting config _min_points_to_show = 3 +CHART_MARGINS = (0, 0, 2, 2) + _tina_mode = False From 8276b02f92db70cfd5a6441ec6872fd96861df92 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 12:28:54 -0400 Subject: [PATCH 136/206] Further label and axis sizing tweaks for hidpi --- piker/ui/_axes.py | 2 +- piker/ui/_chart.py | 40 +++++++++++++++++++++++++++------------- piker/ui/_graphics.py | 35 ----------------------------------- piker/ui/_style.py | 1 - 4 files changed, 28 insertions(+), 50 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 43dc28ef..1a8a65a3 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -29,7 +29,7 @@ class PriceAxis(pg.AxisItem): }) self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font) - self.setWidth(50) + self.setWidth(40) # XXX: drop for now since it just eats up h space diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 14b7c1ce..2446fd5e 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -15,7 +15,10 @@ from ._axes import ( ) from ._graphics import CrossHair, BarItems from ._axes import YSticky -from ._style import _xaxis_at, _min_points_to_show, hcolor +from ._style import ( + _xaxis_at, _min_points_to_show, hcolor, + CHART_MARGINS, +) from ..data._source import Symbol from .. import brokers from .. import data @@ -31,9 +34,6 @@ from .. import fsp log = get_logger(__name__) -# margins -CHART_MARGINS = (0, 0, 5, 3) - class ChartSpace(QtGui.QWidget): """High level widget which contains layouts for organizing @@ -135,13 +135,16 @@ class LinkedSplitCharts(QtGui.QWidget): linked_charts=self ) - if _xaxis_at == 'bottom': - self.xaxis.setStyle(showValues=False) - else: - self.xaxis_ind.setStyle(showValues=False) + # if _xaxis_at == 'bottom': + # self.xaxis.setStyle(showValues=False) + # # self.xaxis.hide() + # else: + # self.xaxis_ind.setStyle(showValues=False) + # self.xaxis.hide() self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) - self.splitter.setHandleWidth(5) + self.splitter.setMidLineWidth(3) + self.splitter.setHandleWidth(0) self.layout = QtGui.QVBoxLayout(self) self.layout.setContentsMargins(0, 0, 0, 0) @@ -187,6 +190,8 @@ class LinkedSplitCharts(QtGui.QWidget): ) # add crosshair graphic self.chart.addItem(self._ch) + if _xaxis_at == 'bottom': + self.chart.hideAxis('bottom') # style? self.chart.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) @@ -226,8 +231,9 @@ class LinkedSplitCharts(QtGui.QWidget): cpw.name = name cpw.plotItem.vb.linked_charts = self - cpw.setFrameStyle(QtGui.QFrame.StyledPanel | QtGui.QFrame.Plain) + cpw.setFrameStyle(QtGui.QFrame.StyledPanel) # | QtGui.QFrame.Plain) cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) + cpw.hideButtons() # link chart x-axis to main quotes chart cpw.setXLink(self.chart) @@ -285,7 +291,6 @@ class ChartPlotWidget(pg.PlotWidget): background=hcolor('papas_special'), # parent=None, # plotItem=None, - # useOpenGL=True, **kwargs ) self._array = array # readonly view of data @@ -431,10 +436,11 @@ class ChartPlotWidget(pg.PlotWidget): size='4pt', ) label.setParentItem(self._vb) + # label.setParentItem(self.getPlotItem()) if overlay: # position bottom left if an overlay - label.anchor(itemPos=(0, 1), parentPos=(0, 1), offset=(0, 25)) + label.anchor(itemPos=(0, 1), parentPos=(0, 1), offset=(0, 14)) self._overlays[name] = curve label.show() @@ -512,6 +518,14 @@ class ChartPlotWidget(pg.PlotWidget): begin = 0 - extra end = len(self._array) - 1 + extra + # XXX: test code for only rendering lines for the bars in view. + # This turns out to be very very poor perf when scaling out to + # many bars (think > 1k) on screen. + # name = self.name + # bars = self._graphics[self.name] + # bars.draw_lines( + # istart=max(lbar, l), iend=min(rbar, r), just_history=True) + # bars_len = rbar - lbar # log.trace( # f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" @@ -701,12 +715,12 @@ async def chart_from_quotes( # faster then msgs arrive.. needs some tinkering and # testing array = ohlcv.array - last = array[-1] chart.update_from_array( chart.name, array, ) # update sticky(s) + last = array[-1] last_price_sticky.update_from_data(*last[['index', 'close']]) chart._set_yrange() diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 623c0e14..9dee4564 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -13,41 +13,6 @@ from PyQt5.QtCore import QLineF from ._style import _xaxis_at, hcolor from ._axes import YAxisLabel, XAxisLabel - -def rec2array( - rec: np.ndarray, - fields: List[str] = None -) -> np.ndarray: - """Convert record array to std array. - - Taken from: - https://github.com/scikit-hep/root_numpy/blob/master/root_numpy/_utils.py#L20 - """ - simplify = False - - if fields is None: - fields = rec.dtype.names - elif isinstance(fields, str): - fields = [fields] - simplify = True - - # Creates a copy and casts all data to the same type - arr = np.dstack([rec[field] for field in fields]) - - # Check for array-type fields. If none, then remove outer dimension. - # Only need to check first field since np.dstack will anyway raise an - # exception if the shapes don't match - # np.dstack will also fail if fields is an empty list - if not rec.dtype[fields[0]].shape: - arr = arr[0] - - if simplify: - # remove last dimension (will be of size 1) - arr = arr.reshape(arr.shape[:-1]) - - return arr - - # TODO: # - checkout pyqtgraph.PlotCurveItem.setCompositionMode diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 8d8f5099..8f8dce60 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -13,7 +13,6 @@ _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config _xaxis_at = 'bottom' - # charting config _min_points_to_show = 3 CHART_MARGINS = (0, 0, 2, 2) From 6d5ccc6c3f0e9c76ebc96d9506c6d5f5824938c3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 14:18:14 -0400 Subject: [PATCH 137/206] Specify font size in pixels --- piker/ui/_style.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 8f8dce60..0378f079 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -7,7 +7,10 @@ from qdarkstyle.palette import DarkPalette # chart-wide font -_font = QtGui.QFont("Hack", 4) +_font = QtGui.QFont("Hack") +# use pixel size to be cross-resolution compatible +_font.setPixelSize(6) + _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config From 8d9a6845c6012fe98c3578e155b069dde6595552 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 16:18:21 -0400 Subject: [PATCH 138/206] Add a naive maxmin calc to avoid frequent resizes If we know the max and min in view then on datum updates we can avoid resizing the y-range when a new max/min has not yet arrived. This adds a very naive numpy calc in the drawing thread which we can likely improve with a more efficient streaming alternative which can also likely be run in a fsp subactor. Also, since this same calc is essentially done inside `._set_yrange()` we will likely want to allow passing the result into the method to avoid duplicate work. --- piker/ui/_chart.py | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 2446fd5e..8ee6cf25 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -137,10 +137,10 @@ class LinkedSplitCharts(QtGui.QWidget): # if _xaxis_at == 'bottom': # self.xaxis.setStyle(showValues=False) - # # self.xaxis.hide() + # self.xaxis.hide() # else: # self.xaxis_ind.setStyle(showValues=False) - # self.xaxis.hide() + # self.xaxis.hide() self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) self.splitter.setMidLineWidth(3) @@ -231,7 +231,7 @@ class LinkedSplitCharts(QtGui.QWidget): cpw.name = name cpw.plotItem.vb.linked_charts = self - cpw.setFrameStyle(QtGui.QFrame.StyledPanel) # | QtGui.QFrame.Plain) + cpw.setFrameStyle(QtGui.QFrame.StyledPanel) # | QtGui.QFrame.Plain) cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) cpw.hideButtons() @@ -705,15 +705,19 @@ async def chart_from_quotes( # - 5 sec bar lookback-autocorrection like tws does? last_price_sticky = chart._ysticks[chart.name] + def maxmin(): + array = chart._array + last_bars_range = chart.bars_range() + l, lbar, rbar, r = last_bars_range + in_view = array[lbar:rbar] + mx, mn = np.nanmax(in_view['high']), np.nanmin(in_view['low']) + return last_bars_range, mx, mn + + last_bars_range, last_mx, last_mn = maxmin() + async for quotes in stream: for sym, quote in quotes.items(): for tick in iterticks(quote, type='trade'): - # TODO: - # - eventually we'll want to update bid/ask labels and - # other data as subscribed by underlying UI consumers. - # - in theory we should be able to read buffer data - # faster then msgs arrive.. needs some tinkering and - # testing array = ohlcv.array chart.update_from_array( chart.name, @@ -722,12 +726,27 @@ async def chart_from_quotes( # update sticky(s) last = array[-1] last_price_sticky.update_from_data(*last[['index', 'close']]) - chart._set_yrange() + + # TODO: we need a streaming minmax algorithm here to + brange, mx, mn = maxmin() + if brange != last_bars_range: + if mx > last_mx or mn < last_mn: + # avoid running this every cycle. + chart._set_yrange() + + # save for next cycle + last_mx, last_mn = mx, mn if vwap_in_history: # update vwap overlay line chart.update_from_array('vwap', ohlcv.array['vwap']) + # TODO: + # - eventually we'll want to update bid/ask labels and + # other data as subscribed by underlying UI consumers. + # - in theory we should be able to read buffer data + # faster then msgs arrive.. needs some tinkering and testing + async def chart_from_fsp( linked_charts, @@ -830,7 +849,7 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): # just_history=True ) # resize view - price_chart._set_yrange() + # price_chart._set_yrange() for name, curve in price_chart._overlays.items(): # TODO: standard api for signal lookups per plot @@ -843,7 +862,7 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): for name, chart in linked_charts.subplots.items(): chart.update_from_array(chart.name, chart._shm.array[chart.name]) - chart._set_yrange() + # chart._set_yrange() def _main( From 2302e59f1d2854d60c7d1e5d3e84fbba76099f6b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 16:24:32 -0400 Subject: [PATCH 139/206] 3k bars for now, ignore rtTime --- piker/brokers/ib.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 01e30e3d..020bfecc 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -148,7 +148,7 @@ class Client: # durationStr='1 D', # time length calcs - durationStr='{count} S'.format(count=100 * 5), + durationStr='{count} S'.format(count=3000 * 5), barSizeSetting='5 secs', # always use extended hours @@ -486,8 +486,13 @@ def normalize( # add time stamps for downstream latency measurements data['brokerd_ts'] = time.time() - if ticker.rtTime: - data['broker_ts'] = data['rtTime_s'] = float(ticker.rtTime) / 1000. + + # stupid stupid shit...don't even care any more + # leave it until we do a proper latency study + # if ticker.rtTime is not None: + # data['broker_ts'] = data['rtTime_s'] = float( + # ticker.rtTime.timestamp) / 1000. + data.pop('rtTime') return data From 18097fc33b05aaf45a6bcb70f54f68ea16789b91 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 29 Sep 2020 16:24:59 -0400 Subject: [PATCH 140/206] Scale for hidpi pixmaps too --- piker/ui/_exec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index c1f247d9..1270d8ac 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -27,8 +27,8 @@ from outcome import Error if hasattr(Qt, 'AA_EnableHighDpiScaling'): QCoreApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True) -# if hasattr(Qt, 'AA_UseHighDpiPixmaps'): -# QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +if hasattr(Qt, 'AA_UseHighDpiPixmaps'): + QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) class MainWindow(QtGui.QMainWindow): From db273e1cd72da0d6de174be9ff1836fbba149435 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 12 Oct 2020 09:36:46 -0400 Subject: [PATCH 141/206] Up the mouse re-draw rate limit --- piker/ui/_graphics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 9dee4564..0297028f 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -16,8 +16,8 @@ from ._axes import YAxisLabel, XAxisLabel # TODO: # - checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 30 -_debounce_delay = 10 +_mouse_rate_limit = 60 +_debounce_delay = 20 _ch_label_opac = 1 From e0613675c789deb5bfbc75fed5fab3f0c20f776b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 15 Oct 2020 14:30:48 -0400 Subject: [PATCH 142/206] Fix chart command derp --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4ec7456f..ac8dfb02 100644 --- a/README.rst +++ b/README.rst @@ -93,7 +93,7 @@ Check out our charts ******************** bet you weren't expecting this from the foss bby:: - piker chart -b kraken XBTUSD + piker -b kraken chart XBTUSD If anyone asks you what this project is about From 454b445b4b737d9c62ac8bb0c573854cf99362b7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 15 Oct 2020 15:02:42 -0400 Subject: [PATCH 143/206] Add better shared mem writer task checking If you have a common broker feed daemon then likely you don't want to create superfluous shared mem buffers for the same symbol. This adds an ad hoc little context manger which keeps a bool state of whether a buffer writer task currently is running in this process. Before we were checking the shared array token cache and **not** clearing it when the writer task exited, resulting in incorrect writer/loader logic on the next entry.. Really, we need a better set of SC semantics around the shared mem stuff presuming there's only ever one writer per shared buffer at given time. Hopefully that will come soon! --- piker/brokers/ib.py | 202 ++++++++++++++++++++++------------------- piker/data/__init__.py | 8 +- 2 files changed, 116 insertions(+), 94 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 020bfecc..9f40a7d9 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -5,7 +5,7 @@ Note the client runs under an ``asyncio`` loop (since ``ib_insync`` is built on it) and thus actor aware API calls must be spawned with ``infected_aio==True``. """ -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, contextmanager from dataclasses import asdict from functools import partial from typing import List, Dict, Any, Tuple, Optional, AsyncIterator, Callable @@ -292,7 +292,7 @@ class Client: ticker: Ticker = self.ib.reqMktData(contract, ','.join(opts)) def push(t): - log.debug(t) + # log.debug(t) try: to_trio.send_nowait(t) except trio.BrokenResourceError: @@ -497,6 +497,21 @@ def normalize( return data +_local_buffer_writers = {} + + +@contextmanager +def activate_writer(key: str): + try: + writer_already_exists = _local_buffer_writers.get(key, False) + if not writer_already_exists: + _local_buffer_writers[key] = True + + yield writer_already_exists + finally: + _local_buffer_writers.pop(key, None) + + # TODO: figure out how to share quote feeds sanely despite # the wacky ``ib_insync`` api. # @tractor.msg.pub @@ -528,108 +543,113 @@ async def stream_quotes( async with aclosing(stream): - # maybe load historical ohlcv in to shared mem - # check if shm has already been created by previous - # feed initialization - writer_exists = get_shm_token(shm_token['shm_name']) + # check if a writer already is alive in a streaming task, + # otherwise start one and mark it as now existing + with activate_writer(shm_token['shm_name']) as writer_already_exists: - if not writer_exists: - shm = attach_shm_array( - token=shm_token, - # we are writer - readonly=False, - ) - bars = await _trio_run_client_method( - method='bars', - symbol=sym, - ) + # maybe load historical ohlcv in to shared mem + # check if shm has already been created by previous + # feed initialization + if not writer_already_exists: - shm.push(bars) - shm_token = shm.token + shm = attach_shm_array( + token=shm_token, - times = shm.array['time'] - delay_s = times[-1] - times[times != times[-1]][-1] - subscribe_ohlc_for_increment(shm, delay_s) + # we are the buffer writer + readonly=False, + ) + bars = await _trio_run_client_method( + method='bars', + symbol=sym, + ) - # pass back token, and bool, signalling if we're the writer - await ctx.send_yield((shm_token, not writer_exists)) + # write historical data to buffer + shm.push(bars) + shm_token = shm.token - # first quote can be ignored as a 2nd with newer data is sent? - first_ticker = await stream.__anext__() - quote = normalize(first_ticker) - # ugh, clear ticks since we've consumed them - # (ahem, ib_insync is stateful trash) - first_ticker.ticks = [] + times = shm.array['time'] + delay_s = times[-1] - times[times != times[-1]][-1] + subscribe_ohlc_for_increment(shm, delay_s) - log.debug(f"First ticker received {quote}") + # pass back token, and bool, signalling if we're the writer + await ctx.send_yield((shm_token, not writer_already_exists)) - if type(first_ticker.contract) not in (ibis.Commodity, ibis.Forex): - suffix = 'exchange' + # first quote can be ignored as a 2nd with newer data is sent? + first_ticker = await stream.__anext__() + quote = normalize(first_ticker) + # ugh, clear ticks since we've consumed them + # (ahem, ib_insync is stateful trash) + first_ticker.ticks = [] - calc_price = False # should be real volume for contract + log.debug(f"First ticker received {quote}") - async for ticker in stream: - # spin consuming tickers until we get a real market datum - if not ticker.rtTime: - log.debug(f"New unsent ticker: {ticker}") - continue - else: - log.debug("Received first real volume tick") - # ugh, clear ticks since we've consumed them - # (ahem, ib_insync is truly stateful trash) - ticker.ticks = [] + if type(first_ticker.contract) not in (ibis.Commodity, ibis.Forex): + suffix = 'exchange' - # XXX: this works because we don't use - # ``aclosing()`` above? - break - else: - # commodities don't have an exchange name for some reason? - suffix = 'secType' - calc_price = True - ticker = first_ticker + calc_price = False # should be real volume for contract - con = quote['contract'] - quote = normalize(ticker, calc_price=calc_price) - topic = '.'.join((con['symbol'], con[suffix])).lower() - first_quote = {topic: quote} - ticker.ticks = [] + async for ticker in stream: + # spin consuming tickers until we get a real market datum + if not ticker.rtTime: + log.debug(f"New unsent ticker: {ticker}") + continue + else: + log.debug("Received first real volume tick") + # ugh, clear ticks since we've consumed them + # (ahem, ib_insync is truly stateful trash) + ticker.ticks = [] - # yield first quote asap - await ctx.send_yield(first_quote) - - # real-time stream - async for ticker in stream: - quote = normalize( - ticker, - calc_price=calc_price - ) - # TODO: in theory you can send the IPC msg *before* - # writing to the sharedmem array to decrease latency, - # however, that will require `tractor.msg.pub` support - # here or at least some way to prevent task switching - # at the yield such that the array write isn't delayed - # while another consumer is serviced.. - - # if we are the lone tick writer start writing - # the buffer with appropriate trade data - if not writer_exists: - for tick in iterticks(quote, type='trade'): - last = tick['price'] - # print(f'broker last: {tick}') - - # update last entry - # benchmarked in the 4-5 us range - high, low = shm.array[-1][['high', 'low']] - shm.array[['high', 'low', 'close']][-1] = ( - max(high, last), - min(low, last), - last, - ) + # XXX: this works because we don't use + # ``aclosing()`` above? + break + else: + # commodities don't have an exchange name for some reason? + suffix = 'secType' + calc_price = True + ticker = first_ticker con = quote['contract'] + quote = normalize(ticker, calc_price=calc_price) topic = '.'.join((con['symbol'], con[suffix])).lower() - - await ctx.send_yield({topic: quote}) - - # ugh, clear ticks since we've consumed them + first_quote = {topic: quote} ticker.ticks = [] + + # yield first quote asap + await ctx.send_yield(first_quote) + + # real-time stream + async for ticker in stream: + quote = normalize( + ticker, + calc_price=calc_price + ) + # TODO: in theory you can send the IPC msg *before* + # writing to the sharedmem array to decrease latency, + # however, that will require `tractor.msg.pub` support + # here or at least some way to prevent task switching + # at the yield such that the array write isn't delayed + # while another consumer is serviced.. + + # if we are the lone tick writer start writing + # the buffer with appropriate trade data + if not writer_already_exists: + for tick in iterticks(quote, type='trade'): + last = tick['price'] + # print(f'broker last: {tick}') + + # update last entry + # benchmarked in the 4-5 us range + high, low = shm.array[-1][['high', 'low']] + shm.array[['high', 'low', 'close']][-1] = ( + max(high, last), + min(low, last), + last, + ) + + con = quote['contract'] + topic = '.'.join((con['symbol'], con[suffix])).lower() + + await ctx.send_yield({topic: quote}) + + # ugh, clear ticks since we've consumed them + ticker.ticks = [] diff --git a/piker/data/__init__.py b/piker/data/__init__.py index 153c1c8f..77bcba12 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -167,6 +167,7 @@ async def open_feed( # Attempt to allocate (or attach to) shm array for this broker/symbol shm, opened = maybe_open_shm_array( key=sym_to_shm_key(name, symbols[0]), + # use any broker defined ohlc dtype: dtype=getattr(mod, '_ohlc_dtype', base_ohlc_dtype), @@ -193,12 +194,13 @@ async def open_feed( # tests are in? shm_token, is_writer = await stream.receive() + if opened: + assert is_writer + log.info("Started shared mem bar writer") + shm_token['dtype_descr'] = list(shm_token['dtype_descr']) assert shm_token == shm.token # sanity - if is_writer: - log.info("Started shared mem bar writer") - yield Feed( name=name, stream=stream, From cb7266235009313b6efaf9d7c2ced840d568bb1f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 15 Oct 2020 15:07:56 -0400 Subject: [PATCH 144/206] Add warnings for shm cache misses --- piker/data/_sharedmem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index 96526206..c256da31 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -323,13 +323,13 @@ def maybe_open_shm_array( token = _known_tokens[key] return attach_shm_array(token=token, **kwargs), False except KeyError: - log.debug(f"Could not find {key} in shms cache") + log.warning(f"Could not find {key} in shms cache") if dtype: token = _make_token(key, dtype) try: return attach_shm_array(token=token, **kwargs), False except FileNotFoundError: - log.debug(f"Could not attach to shm with token {token}") + log.warning(f"Could not attach to shm with token {token}") # This actor does not know about memory # associated with the provided "key". From 6f429b11042e5c16313b4478f3ebc7b2ca54a486 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 15 Oct 2020 15:08:16 -0400 Subject: [PATCH 145/206] These seem to be "faster" crosshair settings? --- piker/ui/_graphics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 0297028f..79173530 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -17,7 +17,7 @@ from ._axes import YAxisLabel, XAxisLabel # - checkout pyqtgraph.PlotCurveItem.setCompositionMode _mouse_rate_limit = 60 -_debounce_delay = 20 +_debounce_delay = 1/2e3 _ch_label_opac = 1 From f4c38621d5dc15a126c68bafd47b02778a7a5a2b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 15 Oct 2020 15:08:56 -0400 Subject: [PATCH 146/206] Add a default "bars from right" style setting --- piker/ui/_chart.py | 15 ++++++++++++--- piker/ui/_style.py | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8ee6cf25..50220174 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -18,6 +18,7 @@ from ._axes import YSticky from ._style import ( _xaxis_at, _min_points_to_show, hcolor, CHART_MARGINS, + _bars_from_right_in_follow_mode, ) from ..data._source import Symbol from .. import brokers @@ -398,7 +399,10 @@ class ChartPlotWidget(pg.PlotWidget): xlast = data[-1]['index'] # show last 50 points on startup - self.plotItem.vb.setXRange(xlast - 50, xlast + 50) + self.plotItem.vb.setXRange( + xlast - 50, + xlast + _bars_from_right_in_follow_mode + ) self._add_sticky(name) @@ -457,7 +461,10 @@ class ChartPlotWidget(pg.PlotWidget): xlast = len(data) - 1 # show last 50 points on startup - self.plotItem.vb.setXRange(xlast - 50, xlast + 50) + self.plotItem.vb.setXRange( + xlast - 50, + xlast + _bars_from_right_in_follow_mode + ) # TODO: we should instead implement a diff based # "only update with new items" on the pg.PlotDataItem @@ -774,7 +781,9 @@ async def chart_from_fsp( dtype=fsp_dtype, readonly=True, ) - assert opened + # XXX: fsp may have been opened by a duplicate chart. Error for + # now until we figure out how to wrap fsps as "feeds". + assert opened, f"A chart for {key} likely already exists?" # start fsp sub-actor portal = await n.run_in_actor( diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 0378f079..d6f2adec 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -18,6 +18,7 @@ _xaxis_at = 'bottom' # charting config _min_points_to_show = 3 +_bars_from_right_in_follow_mode = 5 CHART_MARGINS = (0, 0, 2, 2) From 58d3234f744f1e9e0b9a51cbe2f0c139966eea67 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 16 Oct 2020 12:15:07 -0400 Subject: [PATCH 147/206] Drop lingering print --- piker/fsp/_momo.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 8c5f386c..934a5e70 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -72,7 +72,6 @@ def ema( else: s[0] = ylast - print(s) for i in range(1, n): s[i] = y[i] * alpha + s[i-1] * (1 - alpha) @@ -133,7 +132,6 @@ def wma( assert length == len(weights) - # return np.convolve(ohlcv.array['close'], weights, 'valid') return np.convolve(signal, weights, 'valid') From fc23b2180ddfd65e0c239533c63eed0a690d1e59 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 16 Oct 2020 12:15:33 -0400 Subject: [PATCH 148/206] Pass through fonts to axes --- piker/ui/_axes.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 1a8a65a3..45d325a3 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -19,16 +19,18 @@ class PriceAxis(pg.AxisItem): self, ) -> None: super().__init__(orientation='right') + self.setTickFont(_font) self.setStyle(**{ 'textFillLimits': [(0, 0.5)], - # 'tickTextWidth': 10, - # 'tickTextHeight': 25, + # 'tickTextWidth': 100, + # 'tickTextHeight': 20, + 'tickFont': _font, + # 'tickTextWidth': 40, # 'autoExpandTextSpace': True, # 'maxTickLength': -20, # 'stopAxisAtTick': (True, True), }) - self.setLabel(**{'font-size': '10pt'}) - self.setTickFont(_font) + # self.setLabel(**{'font-size': '10pt'}) self.setWidth(40) # XXX: drop for now since it just eats up h space @@ -60,13 +62,12 @@ class DynamicDateAxis(pg.AxisItem): self.setTickFont(_font) # default styling - self.setStyle( - tickTextOffset=4, - textFillLimits=[(0, 0.70)], - # TODO: doesn't seem to work -> bug in pyqtgraph? - # tickTextHeight=11, - ) - self.setHeight(10) + self.setStyle(**{ + # tickTextOffset=4, + 'textFillLimits': [(0, 0.70)], + 'tickFont': _font, + }) + self.setHeight(11) def _indexes_to_timestrs( self, From c7d5ea6e1552bebf8d5ab4a19328d4a6fd0b84d5 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 16 Oct 2020 12:18:14 -0400 Subject: [PATCH 149/206] Fix static yrange and last bar double draw issues --- piker/ui/_chart.py | 133 ++++++++++++++++++++++-------------------- piker/ui/_graphics.py | 5 +- piker/ui/_style.py | 4 +- 3 files changed, 76 insertions(+), 66 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 50220174..6f06b9cc 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -19,6 +19,7 @@ from ._style import ( _xaxis_at, _min_points_to_show, hcolor, CHART_MARGINS, _bars_from_right_in_follow_mode, + _bars_to_left_in_follow_mode, ) from ..data._source import Symbol from .. import brokers @@ -206,6 +207,7 @@ class LinkedSplitCharts(QtGui.QWidget): xaxis: DynamicDateAxis = None, ohlc: bool = False, _is_main: bool = False, + **cpw_kwargs, ) -> 'ChartPlotWidget': """Add (sub)plots to chart widget by name. @@ -226,6 +228,7 @@ class LinkedSplitCharts(QtGui.QWidget): parent=self.splitter, axisItems={'bottom': xaxis, 'right': PriceAxis()}, viewBox=cv, + **cpw_kwargs, ) # this name will be used to register the primary # graphics curve managed by the subchart @@ -283,7 +286,7 @@ class ChartPlotWidget(pg.PlotWidget): self, # the data view we generate graphics from array: np.ndarray, - yrange: Optional[Tuple[float, float]] = None, + static_yrange: Optional[Tuple[float, float]] = None, **kwargs, ): """Configure chart display settings. @@ -299,9 +302,8 @@ class ChartPlotWidget(pg.PlotWidget): self._overlays = {} # registry of overlay curves self._labels = {} # registry of underlying graphics self._ysticks = {} # registry of underlying graphics - self._yrange = yrange self._vb = self.plotItem.vb - self._static_yrange = None + self._static_yrange = static_yrange # for "known y-range style" # show only right side axes self.hideAxis('left') @@ -310,20 +312,21 @@ class ChartPlotWidget(pg.PlotWidget): # show background grid self.showGrid(x=True, y=True, alpha=0.4) - self.plotItem.vb.setXRange(0, 0) + # don't need right? + # self._vb.setXRange(0, 0) - # use cross-hair for cursor - self.setCursor(QtCore.Qt.CrossCursor) + # use cross-hair for cursor? + # self.setCursor(QtCore.Qt.CrossCursor) # Assign callback for rescaling y-axis automatically # based on data contents and ``ViewBox`` state. self.sigXRangeChanged.connect(self._set_yrange) - vb = self._vb # for mouse wheel which doesn't seem to emit XRangeChanged - vb.sigRangeChangedManually.connect(self._set_yrange) + self._vb.sigRangeChangedManually.connect(self._set_yrange) + # for when the splitter(s) are resized - vb.sigResized.connect(self._set_yrange) + self._vb.sigResized.connect(self._set_yrange) def _update_contents_label(self, index: int) -> None: if index >= 0 and index < len(self._array): @@ -369,6 +372,7 @@ class ChartPlotWidget(pg.PlotWidget): # adds all bar/candle graphics objects for each data point in # the np array buffer to be drawn on next render cycle self.addItem(graphics) + # draw after to allow self.scene() to work... graphics.draw_from_data(data) @@ -392,7 +396,7 @@ class ChartPlotWidget(pg.PlotWidget): ) self._labels[name] = (label, update) - self._update_contents_label(index=-1) + self._update_contents_label(len(data) - 1) label.show() # set xrange limits @@ -400,7 +404,7 @@ class ChartPlotWidget(pg.PlotWidget): # show last 50 points on startup self.plotItem.vb.setXRange( - xlast - 50, + xlast - _bars_to_left_in_follow_mode, xlast + _bars_from_right_in_follow_mode ) @@ -437,7 +441,7 @@ class ChartPlotWidget(pg.PlotWidget): # XXX: How to stack labels vertically? label = pg.LabelItem( justify='left', - size='4pt', + size='6px', ) label.setParentItem(self._vb) # label.setParentItem(self.getPlotItem()) @@ -455,14 +459,14 @@ class ChartPlotWidget(pg.PlotWidget): label.setText(f"{name} -> {data}") self._labels[name] = (label, update) - self._update_contents_label(index=-1) + self._update_contents_label(len(data) - 1) # set a "startup view" xlast = len(data) - 1 - # show last 50 points on startup + # configure "follow mode" view on startup self.plotItem.vb.setXRange( - xlast - 50, + xlast - _bars_to_left_in_follow_mode, xlast + _bars_from_right_in_follow_mode ) @@ -514,46 +518,47 @@ class ChartPlotWidget(pg.PlotWidget): that data always fits nicely inside the current view of the data set. """ - l, lbar, rbar, r = self.bars_range() - - # figure out x-range in view such that user can scroll "off" the data - # set up to the point where ``_min_points_to_show`` are left. - # if l < lbar or r > rbar: - view_len = r - l - # TODO: logic to check if end of bars in view - extra = view_len - _min_points_to_show - begin = 0 - extra - end = len(self._array) - 1 + extra - - # XXX: test code for only rendering lines for the bars in view. - # This turns out to be very very poor perf when scaling out to - # many bars (think > 1k) on screen. - # name = self.name - # bars = self._graphics[self.name] - # bars.draw_lines( - # istart=max(lbar, l), iend=min(rbar, r), just_history=True) - - # bars_len = rbar - lbar - # log.trace( - # f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" - # f"view_len: {view_len}, bars_len: {bars_len}\n" - # f"begin: {begin}, end: {end}, extra: {extra}" - # ) - self._set_xlimits(begin, end) # yrange - if self._static_yrange is not None: - yrange = self._static_yrange + # if self._static_yrange is not None: + # yrange = self._static_yrange - if yrange is not None: - ylow, yhigh = yrange - self._static_yrange = yrange - else: + if self._static_yrange is not None: + ylow, yhigh = self._static_yrange + + else: # determine max, min y values in viewable x-range + + l, lbar, rbar, r = self.bars_range() + + # figure out x-range in view such that user can scroll "off" + # the data set up to the point where ``_min_points_to_show`` + # are left. + view_len = r - l + # TODO: logic to check if end of bars in view + extra = view_len - _min_points_to_show + begin = 0 - extra + end = len(self._array) - 1 + extra + + # XXX: test code for only rendering lines for the bars in view. + # This turns out to be very very poor perf when scaling out to + # many bars (think > 1k) on screen. + # name = self.name + # bars = self._graphics[self.name] + # bars.draw_lines( + # istart=max(lbar, l), iend=min(rbar, r), just_history=True) + + # bars_len = rbar - lbar + # log.trace( + # f"\nl: {l}, lbar: {lbar}, rbar: {rbar}, r: {r}\n" + # f"view_len: {view_len}, bars_len: {bars_len}\n" + # f"begin: {begin}, end: {end}, extra: {extra}" + # ) + self._set_xlimits(begin, end) # TODO: this should be some kind of numpy view api bars = self._array[lbar:rbar] if not len(bars): - # likely no data loaded yet + # likely no data loaded yet or extreme scrolling? log.error(f"WTF bars_range = {lbar}:{rbar}") return @@ -562,19 +567,18 @@ class ChartPlotWidget(pg.PlotWidget): try: ylow = np.nanmin(bars['low']) yhigh = np.nanmax(bars['high']) - # std = np.std(bars['close']) except (IndexError, ValueError): # must be non-ohlc array? ylow = np.nanmin(bars) yhigh = np.nanmax(bars) - # std = np.std(bars) - # view margins: stay within 10% of the "true range" - diff = yhigh - ylow - ylow = ylow - (diff * 0.04) - yhigh = yhigh + (diff * 0.01) + # view margins: stay within a % of the "true range" + diff = yhigh - ylow + ylow = ylow - (diff * 0.08) + yhigh = yhigh + (diff * 0.01) # compute contents label "height" in view terms + # to avoid having data "contents" overlap with them if self._labels: label = self._labels[self.name][0] rect = label.itemRect() @@ -590,13 +594,14 @@ class ChartPlotWidget(pg.PlotWidget): else: label_h = 0 - chart = self - chart.setLimits( + if label_h > yhigh - ylow: + label_h = 0 + + self.setLimits( yMin=ylow, yMax=yhigh + label_h, - # minYRange=std ) - chart.setYRange(ylow, yhigh + label_h) + self.setYRange(ylow, yhigh + label_h) def enterEvent(self, ev): # noqa # pg.PlotWidget.enterEvent(self, ev) @@ -809,20 +814,23 @@ async def chart_from_fsp( chart = linked_charts.add_plot( name=func_name, - - # TODO: enforce type checking here? array=shm.array, + + # settings passed down to ``ChartPlotWidget`` + static_yrange=(0, 100), ) + # display contents labels asap + chart._update_contents_label(len(shm.array) - 1) + array = shm.array[func_name] value = array[-1] last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) chart.update_from_array(chart.name, array) - chart._set_yrange(yrange=(0, 100)) - chart._shm = shm + chart._set_yrange() # update chart graphics async for value in stream: @@ -830,7 +838,6 @@ async def chart_from_fsp( value = array[-1] last_val_sticky.update_from_data(-1, value) chart.update_from_array(chart.name, array) - # chart._set_yrange() async def check_for_new_bars(feed, ohlcv, linked_charts): diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 79173530..6a26d5fa 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -285,7 +285,10 @@ class BarItems(pg.GraphicsObject): index = len(lines) self.lines[:index] = lines self.index = index - self.draw_lines(just_history=True, iend=self.index) + + # up to last to avoid double draw of last bar + self.draw_lines(just_history=True, iend=self.index - 1) + self.draw_lines(iend=self.index) # @timeit def draw_lines( diff --git a/piker/ui/_style.py b/piker/ui/_style.py index d6f2adec..bb5b3fdf 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -17,10 +17,10 @@ _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) _xaxis_at = 'bottom' # charting config +CHART_MARGINS = (0, 0, 2, 2) _min_points_to_show = 3 _bars_from_right_in_follow_mode = 5 -CHART_MARGINS = (0, 0, 2, 2) - +_bars_to_left_in_follow_mode = 100 _tina_mode = False From d3dc8fb2191f59eb62c48e7432e66a184a93dfa7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 11:37:28 -0400 Subject: [PATCH 150/206] Differentiate array schema by close field --- piker/ui/_axes.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 45d325a3..b1310f1e 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -28,7 +28,7 @@ class PriceAxis(pg.AxisItem): # 'tickTextWidth': 40, # 'autoExpandTextSpace': True, # 'maxTickLength': -20, - # 'stopAxisAtTick': (True, True), + # 'stopAxisAtTick': (True, True), # doesn't work well on price }) # self.setLabel(**{'font-size': '10pt'}) self.setWidth(40) @@ -83,7 +83,7 @@ class DynamicDateAxis(pg.AxisItem): map(int, filter(lambda i: i < bars_len, indexes)) )] # TODO: **don't** have this hard coded shift to EST - dts = pd.to_datetime(epochs, unit='s') # - 4*pd.offsets.Hour() + dts = pd.to_datetime(epochs, unit='s') #- 4*pd.offsets.Hour() return dts.strftime(self.tick_tpl[delay]) def tickStrings(self, values: List[float], scale, spacing): @@ -244,9 +244,11 @@ class YSticky(YAxisLabel): def update_on_resize(self, vr, r): # TODO: add an `.index` to the array data-buffer layer # and make this way less shitty... - a = self._chart._array + chart = self._chart + name = chart.name + a = chart._array fields = a.dtype.fields - if fields and 'index' in fields: + if fields and 'close' in fields: index, last = a[-1][['index', 'close']] else: # non-ohlc case @@ -260,9 +262,9 @@ class YSticky(YAxisLabel): def update_from_data( self, index: int, - last: float, + value: float, ) -> None: self.update_label( - self._chart.mapFromView(QPointF(index, last)), - last + self._chart.mapFromView(QPointF(index, value)), + value ) From 32974a118c33d063c29ea55546d84512c5afefab Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 11:37:50 -0400 Subject: [PATCH 151/206] Add hidpi comments, 300 bars in view at startup --- piker/ui/_style.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index bb5b3fdf..24d32fea 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -8,9 +8,15 @@ from qdarkstyle.palette import DarkPalette # chart-wide font _font = QtGui.QFont("Hack") -# use pixel size to be cross-resolution compatible +# use pixel size to be cross-resolution compatible? _font.setPixelSize(6) +# TODO: use QScreen to determine the same physical font size +# on screen despite different displays? +# PyQt docs: https://doc.qt.io/qtforpython/PySide2/QtGui/QScreen.html +# - supposedly it's ``from QtGui import QScreen`` +# Qt forums: https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4 + _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config @@ -20,7 +26,7 @@ _xaxis_at = 'bottom' CHART_MARGINS = (0, 0, 2, 2) _min_points_to_show = 3 _bars_from_right_in_follow_mode = 5 -_bars_to_left_in_follow_mode = 100 +_bars_to_left_in_follow_mode = 300 _tina_mode = False From 1706b67e0014bec19ebaab2ec014735b90e76849 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 14:01:25 -0400 Subject: [PATCH 152/206] Note the issues with the shared fsp array index.. --- piker/fsp/__init__.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py index 0b73b071..66c606de 100644 --- a/piker/fsp/__init__.py +++ b/piker/fsp/__init__.py @@ -44,6 +44,10 @@ async def increment_signals( feed: Feed, dst_shm: 'SharedArray', # noqa ) -> None: + """Increment the underlying shared memory buffer on every "increment" + msg received from the underlying data feed. + + """ async for msg in await feed.index_stream(): array = dst_shm.array last = array[-1:].copy() @@ -64,6 +68,7 @@ async def cascade( ) -> AsyncIterator[dict]: """Chain streaming signal processors and deliver output to destination mem buf. + """ src = attach_shm_array(token=src_shm_token) dst = attach_shm_array(readonly=False, token=dst_shm_token) @@ -87,10 +92,30 @@ async def cascade( feed.shm, ) + # TODO: XXX: + # THERE'S A BIG BUG HERE WITH THE `index` field since we're + # prepending a copy of the first value a few times to make + # sub-curves align with the parent bar chart. + # + # This likely needs to be fixed either by, + # - manually assigning the index and historical data + # seperately to the shm array (i.e. not using .push()) + # - developing some system on top of the shared mem array that + # is `index` aware such that historical data can be indexed + # relative to the true first datum? Not sure if this is sane + # for derivatives. + # Conduct a single iteration of fsp with historical bars input # and get historical output - history = await out_stream.__anext__() + history_output = await out_stream.__anext__() + # build a struct array which includes an 'index' field to push + # as history + history = np.array( + np.arange(len(history_output)), + dtype=dst.array.dtype + ) + history[fsp_func_name] = history_output # TODO: talk to ``pyqtgraph`` core about proper way to solve this: # XXX: hack to get curves aligned with bars graphics: prepend From 851104dd311b4a6311c6c230c04f5d6f97ff98b1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 14:01:57 -0400 Subject: [PATCH 153/206] Add an inf horizontal line helper --- piker/ui/_graphics.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 6a26d5fa..e27c302a 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -404,6 +404,8 @@ class BarItems(pg.GraphicsObject): # @timeit def paint(self, p, opt, widget): + profiler = pg.debug.Profiler(disabled=False, delayed=False) + # TODO: use to avoid drawing artefacts? # self.prepareGeometryChange() @@ -423,6 +425,8 @@ class BarItems(pg.GraphicsObject): # self._pmi.setPixmap(self.picture) # print(self.scene()) + profiler('bars redraw:') + def boundingRect(self): # TODO: can we do rect caching to make this faster? @@ -475,3 +479,25 @@ class BarItems(pg.GraphicsObject): # p.setBrush(self.bear_brush) # p.drawRects(*rects[Quotes.close < Quotes.open]) + + +def h_line(level: float) -> pg.InfiniteLine: + + line = pg.InfiniteLine( + movable=True, + angle=0, + pos=level, + # label='{value:0.2f}', + # labelOpts={ + # 'position': 0.01, + # 'movable': True + # 'color': (200,200,100), + # 'fill': (200,200,200,50), + # } + ) + default_pen = pg.mkPen(hcolor('default')) + line.setPen(default_pen) + + # os_line.label.setColor(hcolor('default_light')) + # os_line.label.setFont(_font) + return line From c57f6782952a042223dbf889044a73e8b9554110 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 14:18:06 -0400 Subject: [PATCH 154/206] Fix contents labels issues Lookup overlay contents from the OHLC struct array (for now / to make things work) and fix anchoring logic with better offsets to keep contents labels super tight to the edge of the view box. Unfortunately, had to hack the label-height-calc thing for avoiding overlap of graphics with the label; haven't found a better solution yet and pyqtgraph seems to require more rabbit holing to figure out something better. Slap in some inf lines for over[sold/bought] rsi conditions thresholding. --- piker/ui/_chart.py | 111 +++++++++++++++++++++++++++++++++------------ 1 file changed, 81 insertions(+), 30 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 6f06b9cc..1ee13114 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -13,13 +13,14 @@ from ._axes import ( DynamicDateAxis, PriceAxis, ) -from ._graphics import CrossHair, BarItems +from ._graphics import CrossHair, BarItems, h_line from ._axes import YSticky from ._style import ( _xaxis_at, _min_points_to_show, hcolor, CHART_MARGINS, _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, + # _font, ) from ..data._source import Symbol from .. import brokers @@ -295,6 +296,7 @@ class ChartPlotWidget(pg.PlotWidget): background=hcolor('papas_special'), # parent=None, # plotItem=None, + # antialias=True, **kwargs ) self._array = array # readonly view of data @@ -382,10 +384,14 @@ class ChartPlotWidget(pg.PlotWidget): # Ogi says: "use ..." label = pg.LabelItem( justify='left', - size='4pt', + size='6px', ) + label.setParentItem(self._vb) self.scene().addItem(label) + # keep close to top + label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) + def update(index: int) -> None: label.setText( "{name}[{index}] -> O:{} H:{} L:{} C:{} V:{}".format( @@ -427,7 +433,7 @@ class ChartPlotWidget(pg.PlotWidget): curve = pg.PlotDataItem( data, - antialias=True, + # antialias=True, name=name, # TODO: see how this handles with custom ohlcv bars graphics clipToView=True, @@ -443,21 +449,33 @@ class ChartPlotWidget(pg.PlotWidget): justify='left', size='6px', ) + + # anchor to the viewbox label.setParentItem(self._vb) # label.setParentItem(self.getPlotItem()) if overlay: # position bottom left if an overlay - label.anchor(itemPos=(0, 1), parentPos=(0, 1), offset=(0, 14)) + label.anchor(itemPos=(1, 1), parentPos=(1, 1), offset=(0, 3)) self._overlays[name] = curve + def update(index: int) -> None: + data = self._array[index][name] + label.setText(f"{name}: {data:.2f}") + else: + label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) + + def update(index: int) -> None: + data = self._array[index] + label.setText(f"{name}: {data:.2f}") + + # def update(index: int) -> None: + # data = self._array[index] + # label.setText(f"{name} -> {data:.2f}") + label.show() self.scene().addItem(label) - def update(index: int) -> None: - data = self._array[index] - label.setText(f"{name} -> {data}") - self._labels[name] = (label, update) self._update_contents_label(len(data) - 1) @@ -572,28 +590,34 @@ class ChartPlotWidget(pg.PlotWidget): ylow = np.nanmin(bars) yhigh = np.nanmax(bars) - # view margins: stay within a % of the "true range" - diff = yhigh - ylow - ylow = ylow - (diff * 0.08) - yhigh = yhigh + (diff * 0.01) + # view margins: stay within a % of the "true range" + diff = yhigh - ylow + ylow = ylow - (diff * 0.04) + # yhigh = yhigh + (diff * 0.01) # compute contents label "height" in view terms # to avoid having data "contents" overlap with them if self._labels: label = self._labels[self.name][0] + rect = label.itemRect() tl, br = rect.topLeft(), rect.bottomRight() vb = self.plotItem.vb + try: # on startup labels might not yet be rendered top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y()) - label_h = top - bottom + + # XXX: hack, how do we compute exactly? + label_h = (top - bottom) * 0.42 + except np.linalg.LinAlgError: label_h = 0 - # print(f'label height {self.name}: {label_h}') else: label_h = 0 + # print(f'label height {self.name}: {label_h}') + if label_h > yhigh - ylow: label_h = 0 @@ -731,13 +755,16 @@ async def chart_from_quotes( for sym, quote in quotes.items(): for tick in iterticks(quote, type='trade'): array = ohlcv.array + + # update price sticky(s) + last = array[-1] + last_price_sticky.update_from_data(*last[['index', 'close']]) + + # update price bar chart.update_from_array( chart.name, array, ) - # update sticky(s) - last = array[-1] - last_price_sticky.update_from_data(*last[['index', 'close']]) # TODO: we need a streaming minmax algorithm here to brange, mx, mn = maxmin() @@ -762,7 +789,7 @@ async def chart_from_quotes( async def chart_from_fsp( linked_charts, - func_name, + fsp_func_name, sym, src_shm, brokermod, @@ -772,10 +799,13 @@ async def chart_from_fsp( Pass target entrypoint and historical data. """ - name = f'fsp.{func_name}' + name = f'fsp.{fsp_func_name}' + # TODO: load function here and introspect # return stream type(s) - fsp_dtype = np.dtype([('index', int), (func_name, float)]) + + # TODO: should `index` be a required internal field? + fsp_dtype = np.dtype([('index', int), (fsp_func_name, float)]) async with tractor.open_nursery() as n: key = f'{sym}.' + name @@ -786,13 +816,16 @@ async def chart_from_fsp( dtype=fsp_dtype, readonly=True, ) + # XXX: fsp may have been opened by a duplicate chart. Error for # now until we figure out how to wrap fsps as "feeds". assert opened, f"A chart for {key} likely already exists?" # start fsp sub-actor portal = await n.run_in_actor( - name, # name as title of sub-chart + + # name as title of sub-chart + name, # subactor entrypoint fsp.cascade, @@ -800,7 +833,7 @@ async def chart_from_fsp( src_shm_token=src_shm.token, dst_shm_token=shm.token, symbol=sym, - fsp_func_name=func_name, + fsp_func_name=fsp_func_name, # tractor config loglevel=loglevel, @@ -813,8 +846,8 @@ async def chart_from_fsp( _ = await stream.receive() chart = linked_charts.add_plot( - name=func_name, - array=shm.array, + name=fsp_func_name, + array=shm.array[fsp_func_name], # settings passed down to ``ChartPlotWidget`` static_yrange=(0, 100), @@ -823,21 +856,39 @@ async def chart_from_fsp( # display contents labels asap chart._update_contents_label(len(shm.array) - 1) - array = shm.array[func_name] - value = array[-1] + array = shm.array + value = array[fsp_func_name][-1] + last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, array) + + chart.update_from_array(chart.name, array[fsp_func_name]) + + # TODO: figure out if we can roll our own `FillToThreshold` to + # get brush filled polygons for OS/OB conditions. + # ``pg.FillBetweenItems`` seems to be one technique using + # generic fills between curve types while ``PlotCurveItem`` has + # logic inside ``.paint()`` for ``self.opts['fillLevel']`` which + # might be the best solution? + # graphics = chart.update_from_array(chart.name, array[fsp_func_name]) + # graphics.curve.setBrush(50, 50, 200, 100) + # graphics.curve.setFillLevel(50) + + # add moveable over-[sold/bought] lines + chart.plotItem.addItem(h_line(30)) + chart.plotItem.addItem(h_line(70)) chart._shm = shm chart._set_yrange() # update chart graphics async for value in stream: - array = shm.array[func_name] - value = array[-1] + # p = pg.debug.Profiler(disabled=False, delayed=False) + array = shm.array + value = array[-1][fsp_func_name] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, array) + chart.update_from_array(chart.name, array[fsp_func_name]) + # p('rendered rsi datum') async def check_for_new_bars(feed, ohlcv, linked_charts): From 19025077030d1c56085c9344f48bdb0670302597 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 19 Oct 2020 21:32:50 -0400 Subject: [PATCH 155/206] Change scroll "center" to rightmost bar on screen --- piker/ui/_interaction.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index bde6a4c6..6b0dcf1d 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -49,10 +49,10 @@ class ChartView(pg.ViewBox): vl = r - l if ev.delta() > 0 and vl <= _min_points_to_show: - log.trace("Max zoom bruh...") + log.debug("Max zoom bruh...") return if ev.delta() < 0 and vl >= len(self.linked_charts._array): - log.trace("Min zoom bruh...") + log.debug("Min zoom bruh...") return # actual scaling factor @@ -64,14 +64,21 @@ class ChartView(pg.ViewBox): # ) # XXX: scroll "around" the right most element in the view - furthest_right_coord = self.boundingRect().topRight() - center = pg.Point( - fn.invertQTransform( - self.childGroup.transform() - ).map(furthest_right_coord) - ) + # which stays "pinned" in place. + + # furthest_right_coord = self.boundingRect().topRight() + + # yaxis = pg.Point( + # fn.invertQTransform( + # self.childGroup.transform() + # ).map(furthest_right_coord) + # ) + + # This seems like the most "intuitive option, a hybrdid of + # tws and tv styles + last_bar = pg.Point(rbar) self._resetTarget() - self.scaleBy(s, center) + self.scaleBy(s, last_bar) ev.accept() self.sigRangeChangedManually.emit(mask) From 88583d999a791d322fa86e4234d580551c4e7a2a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 20 Oct 2020 08:43:51 -0400 Subject: [PATCH 156/206] Add "follow mode" Makes the chart act like tws where each new time step increment the chart shifts to the right so that the last bar stays in place. This gets things looking like a proper auto-trading UX. Added a couple methods to ``ChartPlotWidget`` to make this work: - ``.default_view()`` to set the preferred view based on user settings - ``.increment_view()`` to shift the view one time frame right Also, split up the `.update_from_array()` method to be curve/ohlc specific allowing for passing in a struct array with a named field containing curve data more straightforwardly. This also simplifies the contest label update functions. --- piker/ui/_axes.py | 2 +- piker/ui/_chart.py | 145 ++++++++++++++++++++++++++---------------- piker/ui/_graphics.py | 6 +- 3 files changed, 95 insertions(+), 58 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index b1310f1e..a2e6d5ea 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -253,7 +253,7 @@ class YSticky(YAxisLabel): else: # non-ohlc case index = len(a) - 1 - last = a[-1] + last = a[chart.name][-1] self.update_from_data( index, last, diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 1ee13114..8f17fcde 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -16,8 +16,10 @@ from ._axes import ( from ._graphics import CrossHair, BarItems, h_line from ._axes import YSticky from ._style import ( - _xaxis_at, _min_points_to_show, hcolor, + hcolor, CHART_MARGINS, + _xaxis_at, + _min_points_to_show, _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, # _font, @@ -155,7 +157,7 @@ class LinkedSplitCharts(QtGui.QWidget): def set_split_sizes( self, - prop: float = 0.25 # proportion allocated to consumer subcharts + prop: float = 0.28 # proportion allocated to consumer subcharts ) -> None: """Set the proportion of space allocated for linked subcharts. """ @@ -306,6 +308,7 @@ class ChartPlotWidget(pg.PlotWidget): self._ysticks = {} # registry of underlying graphics self._vb = self.plotItem.vb self._static_yrange = static_yrange # for "known y-range style" + self._view_mode: str = 'follow' # show only right side axes self.hideAxis('left') @@ -314,9 +317,6 @@ class ChartPlotWidget(pg.PlotWidget): # show background grid self.showGrid(x=True, y=True, alpha=0.4) - # don't need right? - # self._vb.setXRange(0, 0) - # use cross-hair for cursor? # self.setCursor(QtCore.Qt.CrossCursor) @@ -330,6 +330,9 @@ class ChartPlotWidget(pg.PlotWidget): # for when the splitter(s) are resized self._vb.sigResized.connect(self._set_yrange) + def last_bar_in_view(self) -> bool: + self._array[-1]['index'] + def _update_contents_label(self, index: int) -> None: if index >= 0 and index < len(self._array): for name, (label, update) in self._labels.items(): @@ -356,6 +359,8 @@ class ChartPlotWidget(pg.PlotWidget): def bars_range(self) -> Tuple[int, int, int, int]: """Return a range tuple for the bars present in view. """ + # vr = self.viewRect() + # l, r = int(vr.left()), int(vr.right()) l, r = self.view_range() lbar = max(l, 0) rbar = min(r, len(self._array)) @@ -405,19 +410,43 @@ class ChartPlotWidget(pg.PlotWidget): self._update_contents_label(len(data) - 1) label.show() - # set xrange limits - xlast = data[-1]['index'] - - # show last 50 points on startup - self.plotItem.vb.setXRange( - xlast - _bars_to_left_in_follow_mode, - xlast + _bars_from_right_in_follow_mode - ) - self._add_sticky(name) return graphics + def default_view( + self, + index: int = -1, + ) -> None: + """Set the view box to the "default" startup view of the scene. + + """ + xlast = self._array[index]['index'] + begin = xlast - _bars_to_left_in_follow_mode + end = xlast + _bars_from_right_in_follow_mode + + self.plotItem.vb.setXRange( + min=begin, + max=end, + padding=0, + ) + + def increment_view( + self, + ) -> None: + """Increment the data view one step to the right thus "following" + the current time slot/step/bar. + + """ + l, r = self.view_range() + self._vb.setXRange( + min=l + 1, + max=r + 1, + # holy shit, wtf dude... why tf would this not be 0 by + # default... speechless. + padding=0, + ) + def draw_curve( self, name: str, @@ -432,7 +461,7 @@ class ChartPlotWidget(pg.PlotWidget): pdi_kwargs.update(_pdi_defaults) curve = pg.PlotDataItem( - data, + data[name], # antialias=True, name=name, # TODO: see how this handles with custom ohlcv bars graphics @@ -459,19 +488,12 @@ class ChartPlotWidget(pg.PlotWidget): label.anchor(itemPos=(1, 1), parentPos=(1, 1), offset=(0, 3)) self._overlays[name] = curve - def update(index: int) -> None: - data = self._array[index][name] - label.setText(f"{name}: {data:.2f}") else: label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) - def update(index: int) -> None: - data = self._array[index] - label.setText(f"{name}: {data:.2f}") - - # def update(index: int) -> None: - # data = self._array[index] - # label.setText(f"{name} -> {data:.2f}") + def update(index: int) -> None: + data = self._array[index][name] + label.setText(f"{name}: {data:.2f}") label.show() self.scene().addItem(label) @@ -479,19 +501,6 @@ class ChartPlotWidget(pg.PlotWidget): self._labels[name] = (label, update) self._update_contents_label(len(data) - 1) - # set a "startup view" - xlast = len(data) - 1 - - # configure "follow mode" view on startup - self.plotItem.vb.setXRange( - xlast - _bars_to_left_in_follow_mode, - xlast + _bars_from_right_in_follow_mode - ) - - # TODO: we should instead implement a diff based - # "only update with new items" on the pg.PlotDataItem - curve.update_from_array = curve.setData - self._add_sticky(name) return curve @@ -511,20 +520,40 @@ class ChartPlotWidget(pg.PlotWidget): ) return last - def update_from_array( + def update_ohlc_from_array( self, name: str, array: np.ndarray, **kwargs, ) -> pg.GraphicsObject: + """Update the named internal graphics from ``array``. + + """ if name not in self._overlays: self._array = array graphics = self._graphics[name] graphics.update_from_array(array, **kwargs) - return graphics + def update_curve_from_array( + self, + name: str, + array: np.ndarray, + **kwargs, + ) -> pg.GraphicsObject: + """Update the named internal graphics from ``array``. + + """ + if name not in self._overlays: + self._array = array + + curve = self._graphics[name] + # TODO: we should instead implement a diff based + # "only update with new items" on the pg.PlotDataItem + curve.setData(array[name], **kwargs) + return curve + def _set_yrange( self, *, @@ -535,12 +564,8 @@ class ChartPlotWidget(pg.PlotWidget): This adds auto-scaling like zoom on the scroll wheel such that data always fits nicely inside the current view of the data set. + """ - - # yrange - # if self._static_yrange is not None: - # yrange = self._static_yrange - if self._static_yrange is not None: ylow, yhigh = self._static_yrange @@ -552,6 +577,7 @@ class ChartPlotWidget(pg.PlotWidget): # the data set up to the point where ``_min_points_to_show`` # are left. view_len = r - l + # TODO: logic to check if end of bars in view extra = view_len - _min_points_to_show begin = 0 - extra @@ -673,7 +699,7 @@ async def _async_main( vwap_in_history = True chart.draw_curve( name='vwap', - data=bars['vwap'], + data=bars, overlay=True, ) @@ -751,6 +777,8 @@ async def chart_from_quotes( last_bars_range, last_mx, last_mn = maxmin() + chart.default_view() + async for quotes in stream: for sym, quote in quotes.items(): for tick in iterticks(quote, type='trade'): @@ -761,7 +789,7 @@ async def chart_from_quotes( last_price_sticky.update_from_data(*last[['index', 'close']]) # update price bar - chart.update_from_array( + chart.update_ohlc_from_array( chart.name, array, ) @@ -778,7 +806,7 @@ async def chart_from_quotes( if vwap_in_history: # update vwap overlay line - chart.update_from_array('vwap', ohlcv.array['vwap']) + chart.update_curve_from_array('vwap', ohlcv.array) # TODO: # - eventually we'll want to update bid/ask labels and @@ -847,7 +875,10 @@ async def chart_from_fsp( chart = linked_charts.add_plot( name=fsp_func_name, - array=shm.array[fsp_func_name], + array=shm.array, + + # curve by default + ohlc=False, # settings passed down to ``ChartPlotWidget`` static_yrange=(0, 100), @@ -862,7 +893,8 @@ async def chart_from_fsp( last_val_sticky = chart._ysticks[chart.name] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, array[fsp_func_name]) + chart.update_curve_from_array(fsp_func_name, array) + chart.default_view() # TODO: figure out if we can roll our own `FillToThreshold` to # get brush filled polygons for OS/OB conditions. @@ -887,7 +919,7 @@ async def chart_from_fsp( array = shm.array value = array[-1][fsp_func_name] last_val_sticky.update_from_data(-1, value) - chart.update_from_array(chart.name, array[fsp_func_name]) + chart.update_curve_from_array(fsp_func_name, array) # p('rendered rsi datum') @@ -901,11 +933,12 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): # aware of the instrument's tradable hours? price_chart = linked_charts.chart + price_chart.default_view() async for index in await feed.index_stream(): # update chart historical bars graphics - price_chart.update_from_array( + price_chart.update_ohlc_from_array( price_chart.name, ohlcv.array, # When appending a new bar, in the time between the insert @@ -922,15 +955,17 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): # TODO: standard api for signal lookups per plot if name in price_chart._array.dtype.fields: # should have already been incremented above - price_chart.update_from_array( + price_chart.update_curve_from_array( name, - price_chart._array[name], + price_chart._array, ) for name, chart in linked_charts.subplots.items(): - chart.update_from_array(chart.name, chart._shm.array[chart.name]) + chart.update_curve_from_array(chart.name, chart._shm.array) # chart._set_yrange() + price_chart.increment_view() + def _main( sym: str, diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index e27c302a..d184996a 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -404,7 +404,7 @@ class BarItems(pg.GraphicsObject): # @timeit def paint(self, p, opt, widget): - profiler = pg.debug.Profiler(disabled=False, delayed=False) + # profiler = pg.debug.Profiler(disabled=False, delayed=False) # TODO: use to avoid drawing artefacts? # self.prepareGeometryChange() @@ -425,7 +425,7 @@ class BarItems(pg.GraphicsObject): # self._pmi.setPixmap(self.picture) # print(self.scene()) - profiler('bars redraw:') + # profiler('bars redraw:') def boundingRect(self): # TODO: can we do rect caching to make this faster? @@ -482,7 +482,9 @@ class BarItems(pg.GraphicsObject): def h_line(level: float) -> pg.InfiniteLine: + """Convenience routine to add a styled horizontal line to a plot. + """ line = pg.InfiniteLine( movable=True, angle=0, From 1f6b5da17ecda70b11a5d7faa6d7f6ca358f4156 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 21 Oct 2020 10:40:51 -0400 Subject: [PATCH 157/206] Add support for curve "cursors" using a filled dot Add a new graphic `LineDot` which is a `pg.CurvePoint` that draws a simple filled dot over a curve at the specified index. Add support for adding these cursor-dots to the crosshair/mouse through a new `CrossHair.add_curve_cursor()`. Discretized the vertical line updates on the crosshair such that it's only drawn in the middle of the current bar in the main chart. --- piker/ui/_graphics.py | 94 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index d184996a..80e09b73 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -16,11 +16,50 @@ from ._axes import YAxisLabel, XAxisLabel # TODO: # - checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 60 +_mouse_rate_limit = 60 # calc current screen refresh rate? _debounce_delay = 1/2e3 _ch_label_opac = 1 +class LineDot(pg.CurvePoint): + + def __init__( + self, + curve: pg.PlotCurveItem, + index: int, + pos=None, + size: int = 2, # in pxs + color: str = 'default_light', + ) -> None: + pg.CurvePoint.__init__( + self, + curve, + index=index, + pos=pos, + rotate=False, + ) + + # TODO: get pen from curve if not defined? + cdefault = hcolor(color) + pen = pg.mkPen(cdefault) + brush = pg.mkBrush(cdefault) + + # presuming this is fast since it's built in? + dot = self.dot = QtGui.QGraphicsEllipseItem( + QtCore.QRectF(-size/2, -size/2, size, size) + ) + # dot.translate(-size*0.5, -size*0.5) + dot.setPen(pen) + dot.setBrush(brush) + dot.setParentItem(self) + + # keep a static size + self.setFlag(self.ItemIgnoresTransformations) + + def paint(self, p, opt, widget): + p.drawPicture(0, 0, self._pic) + + class CrossHair(pg.GraphicsObject): def __init__( @@ -44,6 +83,7 @@ class CrossHair(pg.GraphicsObject): self.plots = [] self.active_plot = None self.digits = digits + self._lastx = None def add_plot( self, @@ -100,17 +140,32 @@ class CrossHair(pg.GraphicsObject): color=self.pen, ) + def add_curve_cursor( + self, + plot: 'ChartPlotWidget', # noqa + curve: 'PlotCurveItem', # noqa + ) -> LineDot: + # if this plot contains curves add line dot "cursors" to denote + # the current sample under the mouse + cursor = LineDot(curve, index=len(plot._array)) + plot.addItem(cursor) + self.graphics[plot].setdefault('cursors', []).append(cursor) + return cursor + def mouseAction(self, action, plot): # noqa if action == 'Enter': + self.active_plot = plot + # show horiz line and y-label self.graphics[plot]['hl'].show() self.graphics[plot]['yl'].show() - self.active_plot = plot + else: # Leave + self.active_plot = None + # hide horiz line and y-label self.graphics[plot]['hl'].hide() self.graphics[plot]['yl'].hide() - self.active_plot = None def mouseMoved( self, @@ -130,26 +185,37 @@ class CrossHair(pg.GraphicsObject): return x, y = mouse_point.x(), mouse_point.y() - plot = self.active_plot + # update y-range items self.graphics[plot]['hl'].setY(y) self.graphics[self.active_plot]['yl'].update_label( abs_pos=pos, data=y ) - for plot, opts in self.graphics.items(): - # move the vertical line to the current x - opts['vl'].setX(x) - # update the chart's "contents" label - plot._update_contents_label(int(x)) + # Update x if cursor changed after discretization calc + # (this saves draw cycles on small mouse moves) + lastx = self._lastx + newx = int(x) - # update the label on the bottom of the crosshair - self.xaxis_label.update_label( - abs_pos=pos, - data=x - ) + if newx != lastx: + for plot, opts in self.graphics.items(): + # move the vertical line to the current "center of bar" + opts['vl'].setX(newx + BarItems.w) + + # update the chart's "contents" label + plot._update_contents_label(newx + 1) + + # update the label on the bottom of the crosshair + self.xaxis_label.update_label( + abs_pos=pos, + data=x + ) + + # update all subscribed curve dots + for cursor in opts.get('cursors', ()): + cursor.setIndex(newx + 1) def boundingRect(self): try: From 875bc8be24b6e3d019cc58fbf4a12b71b370e1b2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 21 Oct 2020 10:46:56 -0400 Subject: [PATCH 158/206] Add line dots cursors to curves by default --- piker/ui/_chart.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8f17fcde..0ce0ed44 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -13,7 +13,11 @@ from ._axes import ( DynamicDateAxis, PriceAxis, ) -from ._graphics import CrossHair, BarItems, h_line +from ._graphics import ( + CrossHair, + BarItems, + h_line, +) from ._axes import YSticky from ._style import ( hcolor, @@ -231,6 +235,7 @@ class LinkedSplitCharts(QtGui.QWidget): parent=self.splitter, axisItems={'bottom': xaxis, 'right': PriceAxis()}, viewBox=cv, + cursor=self._ch, **cpw_kwargs, ) # this name will be used to register the primary @@ -245,15 +250,15 @@ class LinkedSplitCharts(QtGui.QWidget): # link chart x-axis to main quotes chart cpw.setXLink(self.chart) + # add to cross-hair's known plots + self._ch.add_plot(cpw) + # draw curve graphics if ohlc: cpw.draw_ohlc(name, array) else: cpw.draw_curve(name, array) - # add to cross-hair's known plots - self._ch.add_plot(cpw) - if not _is_main: # track by name self.subplots[name] = cpw @@ -290,6 +295,7 @@ class ChartPlotWidget(pg.PlotWidget): # the data view we generate graphics from array: np.ndarray, static_yrange: Optional[Tuple[float, float]] = None, + cursor: Optional[CrossHair] = None, **kwargs, ): """Configure chart display settings. @@ -309,6 +315,7 @@ class ChartPlotWidget(pg.PlotWidget): self._vb = self.plotItem.vb self._static_yrange = static_yrange # for "known y-range style" self._view_mode: str = 'follow' + self._cursor = cursor # placehold for mouse # show only right side axes self.hideAxis('left') @@ -359,8 +366,6 @@ class ChartPlotWidget(pg.PlotWidget): def bars_range(self) -> Tuple[int, int, int, int]: """Return a range tuple for the bars present in view. """ - # vr = self.viewRect() - # l, r = int(vr.left()), int(vr.right()) l, r = self.view_range() lbar = max(l, 0) rbar = min(r, len(self._array)) @@ -503,6 +508,9 @@ class ChartPlotWidget(pg.PlotWidget): self._add_sticky(name) + if self._cursor: + self._cursor.add_curve_cursor(self, curve) + return curve def _add_sticky( From cd828db9e9677b7f267422a002921a7118b1d740 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 21 Oct 2020 10:51:54 -0400 Subject: [PATCH 159/206] Show 24 bars to left on startup --- piker/ui/_style.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 24d32fea..db6eae5a 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -25,8 +25,8 @@ _xaxis_at = 'bottom' # charting config CHART_MARGINS = (0, 0, 2, 2) _min_points_to_show = 3 -_bars_from_right_in_follow_mode = 5 -_bars_to_left_in_follow_mode = 300 +_bars_from_right_in_follow_mode = 24 +_bars_to_left_in_follow_mode = 250 _tina_mode = False From f2c4a46c944c7144136eab5f195c745f0796db5b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 21 Oct 2020 11:19:41 -0400 Subject: [PATCH 160/206] Center bars around index, adjust curves back to match... --- piker/fsp/__init__.py | 2 +- piker/ui/_graphics.py | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py index 66c606de..c4b12242 100644 --- a/piker/fsp/__init__.py +++ b/piker/fsp/__init__.py @@ -120,7 +120,7 @@ async def cascade( # TODO: talk to ``pyqtgraph`` core about proper way to solve this: # XXX: hack to get curves aligned with bars graphics: prepend # a copy of the first datum.. - dst.push(history[:1]) + # dst.push(history[:1]) # check for data length mis-allignment and fill missing values diff = len(src.array) - len(history) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 80e09b73..bf42fd48 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -10,7 +10,7 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF # from .quantdom.utils import timeit -from ._style import _xaxis_at, hcolor +from ._style import _xaxis_at, hcolor, _font from ._axes import YAxisLabel, XAxisLabel # TODO: @@ -197,15 +197,15 @@ class CrossHair(pg.GraphicsObject): # Update x if cursor changed after discretization calc # (this saves draw cycles on small mouse moves) lastx = self._lastx - newx = int(x) + ix = round(x) # since bars are centered around index - if newx != lastx: + if ix != lastx: for plot, opts in self.graphics.items(): # move the vertical line to the current "center of bar" - opts['vl'].setX(newx + BarItems.w) + opts['vl'].setX(ix) # update the chart's "contents" label - plot._update_contents_label(newx + 1) + plot._update_contents_label(ix) # update the label on the bottom of the crosshair self.xaxis_label.update_label( @@ -215,7 +215,7 @@ class CrossHair(pg.GraphicsObject): # update all subscribed curve dots for cursor in opts.get('cursors', ()): - cursor.setIndex(newx + 1) + cursor.setIndex(ix) def boundingRect(self): try: @@ -250,7 +250,7 @@ def bars_from_ohlc( # place the x-coord start as "middle" of the drawing range such # that the open arm line-graphic is at the left-most-side of # the indexe's range according to the view mapping. - index_start = index + w + index_start = index # high - low line if low != high: @@ -265,7 +265,7 @@ def bars_from_ohlc( # open line o = QLineF(index_start - w, open, index_start, open) # close line - c = QLineF(index_start + w, close, index_start, close) + c = QLineF(index_start, close, index_start + w, close) # indexing here is as per the below comments lines[i] = (hl, o, c) @@ -439,7 +439,7 @@ class BarItems(pg.GraphicsObject): body, larm, rarm = self.lines[index-1] # XXX: is there a faster way to modify this? - # update right arm + # update close line / right arm rarm.setLine(rarm.x1(), last, rarm.x2(), last) # update body @@ -455,16 +455,13 @@ class BarItems(pg.GraphicsObject): # if the bar was flat it likely does not have # the index set correctly due to a rendering bug # see above - body.setLine(i + self.w, low, i + self.w, high) + body.setLine(i, low, i, high) body._flat = False else: body.setLine(body.x1(), low, body.x2(), high) self.draw_lines(just_history=False) - # be compat with ``pg.PlotCurveItem`` - setData = update_from_array - # XXX: From the customGraphicsItem.py example: # The only required methods are paint() and boundingRect() # @timeit @@ -566,6 +563,7 @@ def h_line(level: float) -> pg.InfiniteLine: default_pen = pg.mkPen(hcolor('default')) line.setPen(default_pen) - # os_line.label.setColor(hcolor('default_light')) - # os_line.label.setFont(_font) + if getattr(line, 'label', None): + line.label.setFont(_font) + return line From 18dc809acb2e3039a1d47e68265378c99b4a7ae1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Oct 2020 14:05:35 -0400 Subject: [PATCH 161/206] Add naive digits count routine --- piker/data/_source.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/piker/data/_source.py b/piker/data/_source.py index 74238bb1..1df55f53 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -1,7 +1,7 @@ """ Numpy data source machinery. """ -import math +import decimal from dataclasses import dataclass import numpy as np @@ -33,11 +33,18 @@ tf_in_1m = { } +def float_digits( + value: float, +) -> int: + return int(-decimal.Decimal(str(value)).as_tuple().exponent) + + def ohlc_zeros(length: int) -> np.ndarray: """Construct an OHLC field formatted structarray. For "why a structarray" see here: https://stackoverflow.com/a/52443038 Bottom line, they're faster then ``np.recarray``. + """ return np.zeros(length, dtype=base_ohlc_dtype) @@ -46,6 +53,7 @@ def ohlc_zeros(length: int) -> np.ndarray: class Symbol: """I guess this is some kinda container thing for dealing with all the different meta-data formats from brokers? + """ key: str = '' min_tick: float = 0.01 @@ -54,8 +62,9 @@ class Symbol: def digits(self) -> int: """Return the trailing number of digits specified by the min tick size for the instrument. + """ - return int(math.log(self.min_tick, 0.1)) + return float_digits(self.min_tick) def from_df( @@ -64,6 +73,7 @@ def from_df( default_tf=None ) -> np.recarray: """Convert OHLC formatted ``pandas.DataFrame`` to ``numpy.recarray``. + """ df.reset_index(inplace=True) @@ -103,6 +113,7 @@ def from_df( def _nan_to_closest_num(array: np.ndarray): """Return interpolated values instead of NaN. + """ for col in ['open', 'high', 'low', 'close']: mask = np.isnan(array[col]) From 7be624de39907c78ad1a013faa155ce18c107c59 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Oct 2020 20:22:21 -0400 Subject: [PATCH 162/206] Implement `tickStrings` for price axis; use float_digits() --- piker/ui/_axes.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index a2e6d5ea..34b73899 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -3,14 +3,13 @@ Chart axes graphics and behavior. """ from typing import List - -# import numpy as np import pandas as pd import pyqtgraph as pg from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QPointF from ._style import _font, hcolor +from ..data._source import float_digits class PriceAxis(pg.AxisItem): @@ -21,7 +20,7 @@ class PriceAxis(pg.AxisItem): super().__init__(orientation='right') self.setTickFont(_font) self.setStyle(**{ - 'textFillLimits': [(0, 0.5)], + 'textFillLimits': [(0, 0.666)], # 'tickTextWidth': 100, # 'tickTextHeight': 20, 'tickFont': _font, @@ -35,11 +34,15 @@ class PriceAxis(pg.AxisItem): # XXX: drop for now since it just eats up h space - # def tickStrings(self, vals, scale, spacing): - # digts = max(0, np.ceil(-np.log10(spacing * scale))) - # return [ - # ('{:<8,.%df}' % digts).format(v).replace(',', ' ') for v in vals - # ] + def tickStrings(self, vals, scale, spacing): + digits = float_digits(spacing * scale) + + # print(f'vals: {vals}\nscale: {scale}\nspacing: {spacing}') + # print(f'digits: {digits}') + + return [ + ('{:,.%df}' % digits).format(v).replace(',', ' ') for v in vals + ] class DynamicDateAxis(pg.AxisItem): @@ -83,7 +86,7 @@ class DynamicDateAxis(pg.AxisItem): map(int, filter(lambda i: i < bars_len, indexes)) )] # TODO: **don't** have this hard coded shift to EST - dts = pd.to_datetime(epochs, unit='s') #- 4*pd.offsets.Hour() + dts = pd.to_datetime(epochs, unit='s') # - 4*pd.offsets.Hour() return dts.strftime(self.tick_tpl[delay]) def tickStrings(self, values: List[float], scale, spacing): @@ -245,7 +248,6 @@ class YSticky(YAxisLabel): # TODO: add an `.index` to the array data-buffer layer # and make this way less shitty... chart = self._chart - name = chart.name a = chart._array fields = a.dtype.fields if fields and 'close' in fields: From 8c25892521c37d88bfad6a3a8a5542c0b0bd0e89 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Oct 2020 20:35:51 -0400 Subject: [PATCH 163/206] Fix (really sidestep) flat bar rendering issue(s) It seems a plethora of problems (including drawing performance) are due to trying to hack around the strange rendering bug in Qt with `QLineF` with y1 == y2. There was all sorts of weirdness that would show up with trying (a hack) to just set all 4 points to the same value including strange infinite diagonal ghost lines randomly on charts. Instead, just place hold these flat bar's 'body' line with a `None` and filter the null values out before calling `QPainter.drawLines()`. This results in simply no body lines drawn for these datums. We can probably `numba` the filtering too if it turns out to be a bottleneck. --- piker/ui/_graphics.py | 68 +++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index bf42fd48..501ecdd5 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -13,10 +13,8 @@ from PyQt5.QtCore import QLineF from ._style import _xaxis_at, hcolor, _font from ._axes import YAxisLabel, XAxisLabel -# TODO: -# - checkout pyqtgraph.PlotCurveItem.setCompositionMode -_mouse_rate_limit = 60 # calc current screen refresh rate? +_mouse_rate_limit = 40 # calc current screen refresh rate? _debounce_delay = 1/2e3 _ch_label_opac = 1 @@ -48,6 +46,7 @@ class LineDot(pg.CurvePoint): dot = self.dot = QtGui.QGraphicsEllipseItem( QtCore.QRectF(-size/2, -size/2, size, size) ) + # if we needed transformable dot? # dot.translate(-size*0.5, -size*0.5) dot.setPen(pen) dot.setBrush(brush) @@ -56,9 +55,6 @@ class LineDot(pg.CurvePoint): # keep a static size self.setFlag(self.ItemIgnoresTransformations) - def paint(self, p, opt, widget): - p.drawPicture(0, 0, self._pic) - class CrossHair(pg.GraphicsObject): @@ -234,12 +230,14 @@ def _mk_lines_array(data: List, size: int) -> np.ndarray: ) +# TODO: `numba` this? def bars_from_ohlc( data: np.ndarray, w: float, start: int = 0, ) -> np.ndarray: """Generate an array of lines objects from input ohlc data. + """ lines = _mk_lines_array(data, data.shape[0]) @@ -247,25 +245,22 @@ def bars_from_ohlc( open, high, low, close, index = q[ ['open', 'high', 'low', 'close', 'index']] - # place the x-coord start as "middle" of the drawing range such - # that the open arm line-graphic is at the left-most-side of - # the indexe's range according to the view mapping. - index_start = index - # high - low line if low != high: - # hl = QLineF(index, low, index, high) - hl = QLineF(index_start, low, index_start, high) + hl = QLineF(index, low, index, high) else: # XXX: if we don't do it renders a weird rectangle? # see below too for handling this later... - hl = QLineF(low, low, low, low) - hl._flat = True + hl = None + + # NOTE: place the x-coord start as "middle" of the drawing range such + # that the open arm line-graphic is at the left-most-side of + # the indexe's range according to the view mapping. # open line - o = QLineF(index_start - w, open, index_start, open) + o = QLineF(index - w, open, index, open) # close line - c = QLineF(index_start, close, index_start + w, close) + c = QLineF(index, close, index + w, close) # indexing here is as per the below comments lines[i] = (hl, o, c) @@ -274,6 +269,7 @@ def bars_from_ohlc( # array and avoiding the call to `np.ravel()` below? # lines[3*i:3*i+3] = (hl, o, c) + # XXX: legacy code from candles custom graphics: # if not _tina_mode: # else _tina_mode: # self.lines = lines = np.concatenate( @@ -379,7 +375,11 @@ class BarItems(pg.GraphicsObject): # use 2d array of lines objects, see conlusion on speed: # https://stackoverflow.com/a/60089929 - to_draw = np.ravel(self.lines[istart:iend]) + flat = np.ravel(self.lines[istart:iend]) + + # TODO: do this with numba for speed gain: + # https://stackoverflow.com/questions/58422690/filtering-a-numpy-array-what-is-the-best-approach + to_draw = flat[np.where(flat != None)] # noqa # pre-computing a QPicture object allows paint() to run much # more quickly, rather than re-drawing the shapes every time. @@ -435,35 +435,27 @@ class BarItems(pg.GraphicsObject): return # current bar update - i, open, last, = array[-1][['index', 'open', 'close']] - body, larm, rarm = self.lines[index-1] + i, high, low, last, = array[-1][['index', 'high', 'low', 'close']] + assert i == self.index-1 + body, larm, rarm = self.lines[i] # XXX: is there a faster way to modify this? # update close line / right arm rarm.setLine(rarm.x1(), last, rarm.x2(), last) - # update body - high = body.y2() - low = body.y1() - if last < low: - low = last - - if last > high: - high = last - - if getattr(body, '_flat', None) and low != high: - # if the bar was flat it likely does not have - # the index set correctly due to a rendering bug - # see above - body.setLine(i, low, i, high) - body._flat = False + if low != high: + if body is None: + body = self.lines[index-1][0] = QLineF(i, low, i, high) + else: + # update body + body.setLine(i, low, i, high) else: - body.setLine(body.x1(), low, body.x2(), high) + # XXX: high == low -> remove any HL line to avoid render bug + if body is not None: + body = self.lines[index-1][0] = None self.draw_lines(just_history=False) - # XXX: From the customGraphicsItem.py example: - # The only required methods are paint() and boundingRect() # @timeit def paint(self, p, opt, widget): From 8eb4344d86ce820e4346db8f2328f3c71e334136 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Oct 2020 20:42:46 -0400 Subject: [PATCH 164/206] More "thematic" default view values ;) --- piker/ui/_style.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index db6eae5a..1026ff5d 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -24,9 +24,9 @@ _xaxis_at = 'bottom' # charting config CHART_MARGINS = (0, 0, 2, 2) -_min_points_to_show = 3 -_bars_from_right_in_follow_mode = 24 -_bars_to_left_in_follow_mode = 250 +_min_points_to_show = 6 +_bars_from_right_in_follow_mode = int(6**2) +_bars_to_left_in_follow_mode = int(6**3) _tina_mode = False From 13f32acfdfa43e56dcd00aeabb740bfb4d0b29d8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Oct 2020 21:21:14 -0400 Subject: [PATCH 165/206] Only update history on bar increment With the improved update logic on `BarsItems` it doesn't seem to be necessary. Remove y sticky for overlays for now to avoid clutter that looks like double draws when the last overlay value is close to the last price. --- piker/ui/_chart.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 0ce0ed44..1da886e3 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -496,6 +496,10 @@ class ChartPlotWidget(pg.PlotWidget): else: label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) + # TODO: something instead of stickies for overlays + # (we need something that avoids clutter on x-axis). + self._add_sticky(name) + def update(index: int) -> None: data = self._array[index][name] label.setText(f"{name}: {data:.2f}") @@ -506,7 +510,6 @@ class ChartPlotWidget(pg.PlotWidget): self._labels[name] = (label, update) self._update_contents_label(len(data) - 1) - self._add_sticky(name) if self._cursor: self._cursor.add_curve_cursor(self, curve) @@ -642,7 +645,7 @@ class ChartPlotWidget(pg.PlotWidget): # on startup labels might not yet be rendered top, bottom = (vb.mapToView(tl).y(), vb.mapToView(br).y()) - # XXX: hack, how do we compute exactly? + # XXX: magic hack, how do we compute exactly? label_h = (top - bottom) * 0.42 except np.linalg.LinAlgError: @@ -946,27 +949,25 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): async for index in await feed.index_stream(): # update chart historical bars graphics + + # When appending a new bar, in the time between the insert + # here and the Qt render call the underlying price data may + # have already been updated, thus make sure to also update + # the last bar if necessary on this render cycle which is + # why we **don't** set: just_history=True price_chart.update_ohlc_from_array( - price_chart.name, - ohlcv.array, - # When appending a new bar, in the time between the insert - # here and the Qt render call the underlying price data may - # have already been updated, thus make sure to also update - # the last bar if necessary on this render cycle which is - # why we **don't** set: - # just_history=True - ) + price_chart.name, ohlcv.array, just_history=True) + # resize view # price_chart._set_yrange() for name, curve in price_chart._overlays.items(): + # TODO: standard api for signal lookups per plot if name in price_chart._array.dtype.fields: + # should have already been incremented above - price_chart.update_curve_from_array( - name, - price_chart._array, - ) + price_chart.update_curve_from_array(name, price_chart._array) for name, chart in linked_charts.subplots.items(): chart.update_curve_from_array(chart.name, chart._shm.array) From 94a8ee6270fb889aa33b614bbbb1348bc59cce67 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 24 Oct 2020 20:04:57 -0400 Subject: [PATCH 166/206] Optimize axis labels using `QPicture` This is likely a marginal improvement but is slightly less execution and adds a coolio black border around the label. Drop all the legacy code from quantdom which was quite a convoluted mess for "coloring". Had to tweak sticky offsets to get the crosshair to line up right; not sure what that's all about yet. --- piker/ui/_axes.py | 68 +++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 38 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 34b73899..ad25c66b 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -95,17 +95,13 @@ class DynamicDateAxis(pg.AxisItem): class AxisLabel(pg.GraphicsObject): - # bg_color = pg.mkColor('#a9a9a9') - bg_color = pg.mkColor(hcolor('pikers')) - fg_color = pg.mkColor(hcolor('black')) - def __init__( self, - parent=None, - digits=2, - color=None, - opacity=1, - **kwargs + parent: pg.GraphicsObject, + digits: int = 2, + bg_color: str = 'bracket', + fg_color: str = 'black', + opacity: int = 1, ): super().__init__(parent) self.parent = parent @@ -113,33 +109,31 @@ class AxisLabel(pg.GraphicsObject): self.label_str = '' self.digits = digits - # some weird color convertion logic? - if isinstance(color, QtGui.QPen): - self.bg_color = color.color() - self.fg_color = pg.mkColor(hcolor('black')) - elif isinstance(color, list): - self.bg_color = {'>0': color[0].color(), '<0': color[1].color()} - self.fg_color = pg.mkColor(hcolor('white')) + self.bg_color = pg.mkColor(hcolor(bg_color)) + self.fg_color = pg.mkColor(hcolor(fg_color)) + + self.pic = QtGui.QPicture() + p = QtGui.QPainter(self.pic) + + self.rect = QtCore.QRectF(0, 0, 40, 11) + + p.setPen(self.fg_color) + p.setOpacity(self.opacity) + p.fillRect(self.rect, self.bg_color) + + # this adds a nice black outline around the label for some odd + # reason; ok by us + p.drawRect(self.rect) self.setFlag(self.ItemIgnoresTransformations) def paint(self, p, option, widget): - p.setRenderHint(p.TextAntialiasing, True) - p.setPen(self.fg_color) - if self.label_str: - if not isinstance(self.bg_color, dict): - bg_color = self.bg_color - else: - if int(self.label_str.replace(' ', '')) > 0: - bg_color = self.bg_color['>0'] - else: - bg_color = self.bg_color['<0'] - p.setOpacity(self.opacity) - p.fillRect(option.rect, bg_color) - p.setOpacity(1) - p.setFont(_font) + p.drawPicture(0, 0, self.pic) - p.drawText(option.rect, self.text_flags, self.label_str) + if self.label_str: + p.setFont(_font) + p.setPen(self.fg_color) + p.drawText(self.rect, self.text_flags, self.label_str) # uggggghhhh @@ -170,22 +164,21 @@ class XAxisLabel(AxisLabel): QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter # | QtCore.Qt.AlignTop - | QtCore.Qt.AlignVCenter + # | QtCore.Qt.AlignVCenter # | QtCore.Qt.AlignHCenter ) - # text_flags = _common_text_flags def boundingRect(self): # noqa # TODO: we need to get the parent axe's dimensions transformed # to abs coords to be 100% correct here: # self.parent.boundingRect() - return QtCore.QRectF(0, 2, 40, 10) + return QtCore.QRectF(0, 0, 40, 11) def update_label( self, abs_pos: QPointF, # scene coords data: float, # data for text - offset: int = 0 # if have margins, k? + offset: int = 1 # if have margins, k? ) -> None: timestrs = self.parent._indexes_to_timestrs([int(data)]) if not timestrs.any(): @@ -198,7 +191,6 @@ class XAxisLabel(AxisLabel): class YAxisLabel(AxisLabel): - # text_flags = _common_text_flags text_flags = ( QtCore.Qt.AlignLeft | QtCore.Qt.TextDontClip @@ -210,13 +202,13 @@ class YAxisLabel(AxisLabel): return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 50, 11) + return QtCore.QRectF(0, 0, 50, 10) def update_label( self, abs_pos: QPointF, # scene coords data: float, # data for text - offset: int = 0 # if have margins, k? + offset: int = 1 # if have margins, k? ) -> None: self.label_str = self.tick_to_string(data) height = self.boundingRect().height() From ece57b2a1dcdf21b02c5dd0a85fb4421bf10a48b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 24 Oct 2020 20:18:03 -0400 Subject: [PATCH 167/206] Rename our main color --- piker/ui/_style.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 1026ff5d..210b4780 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -60,7 +60,8 @@ def hcolor(name: str) -> str: 'gunmetal': '#91A3B0', 'battleship': '#848482', 'davies': '#555555', - 'pikers': '#666666', # like the cult + 'bracket': '#666666', # like the logo + 'original': '#a9a9a9', # palette 'default': DarkPalette.COLOR_BACKGROUND_NORMAL, From f32763d9926773d0b318e413c7bb125c6db9ca42 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 24 Oct 2020 20:18:21 -0400 Subject: [PATCH 168/206] Only move x-axis sticky when we mouse-over a new index Avoid drawing a new new sticky position if the mouse hasn't moved to the next (rounded) index in terms of the scene's coordinates. This completes the "discrete-ization" of the mouse/cursor UX. Finalizing this feature helped discover and solve pyqtgraph/pyqtgraph#1418 which masssively improves interaction performance throughout the whole lib! Hide stickys on startup until cursor shows up on plot. --- piker/ui/_graphics.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 501ecdd5..19532693 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -7,14 +7,14 @@ from typing import List import numpy as np import pyqtgraph as pg from PyQt5 import QtCore, QtGui -from PyQt5.QtCore import QLineF +from PyQt5.QtCore import QLineF, QPointF # from .quantdom.utils import timeit from ._style import _xaxis_at, hcolor, _font from ._axes import YAxisLabel, XAxisLabel -_mouse_rate_limit = 40 # calc current screen refresh rate? +_mouse_rate_limit = 60 # calc current screen refresh rate? _debounce_delay = 1/2e3 _ch_label_opac = 1 @@ -89,13 +89,17 @@ class CrossHair(pg.GraphicsObject): # add ``pg.graphicsItems.InfiniteLine``s # vertical and horizonal lines and a y-axis label vl = plot.addLine(x=0, pen=self.lines_pen, movable=False) + hl = plot.addLine(y=0, pen=self.lines_pen, movable=False) + hl.hide() + yl = YAxisLabel( parent=plot.getAxis('right'), digits=digits or self.digits, opacity=_ch_label_opac, - color=self.pen, + bg_color='default', ) + yl.hide() # on startup if mouse is off screen # TODO: checkout what ``.sigDelayed`` can be used for # (emitted once a sufficient delay occurs in mouse movement) @@ -133,8 +137,10 @@ class CrossHair(pg.GraphicsObject): self.xaxis_label = XAxisLabel( parent=self.plots[plot_index].getAxis('bottom'), opacity=_ch_label_opac, - color=self.pen, + bg_color='default', ) + # place label off-screen during startup + self.xaxis_label.setPos(self.plots[0].mapFromView(QPointF(0, 0))) def add_curve_cursor( self, @@ -197,21 +203,30 @@ class CrossHair(pg.GraphicsObject): if ix != lastx: for plot, opts in self.graphics.items(): + # move the vertical line to the current "center of bar" opts['vl'].setX(ix) # update the chart's "contents" label plot._update_contents_label(ix) + # update all subscribed curve dots + for cursor in opts.get('cursors', ()): + cursor.setIndex(ix) + # update the label on the bottom of the crosshair self.xaxis_label.update_label( - abs_pos=pos, - data=x + + # XXX: requires: + # https://github.com/pyqtgraph/pyqtgraph/pull/1418 + # otherwise gobbles tons of CPU.. + + # map back to abs (label-local) coordinates + abs_pos=plot.mapFromView(QPointF(ix, y)), + data=x, ) - # update all subscribed curve dots - for cursor in opts.get('cursors', ()): - cursor.setIndex(ix) + self._lastx = ix def boundingRect(self): try: @@ -300,7 +315,7 @@ class BarItems(pg.GraphicsObject): # 0.5 is no overlap between arms, 1.0 is full overlap w: float = 0.43 - bars_pen = pg.mkPen(hcolor('pikers')) + bars_pen = pg.mkPen(hcolor('bracket')) # XXX: tina mode, see below # bull_brush = pg.mkPen('#00cc00') From bed6a631c0f5b5f114ccfbfd66776fd2cbedee23 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 25 Oct 2020 10:49:31 -0400 Subject: [PATCH 169/206] Don't pass color down to axis --- piker/ui/_chart.py | 3 +-- piker/ui/_interaction.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 1da886e3..c5329138 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -510,7 +510,6 @@ class ChartPlotWidget(pg.PlotWidget): self._labels[name] = (label, update) self._update_contents_label(len(data) - 1) - if self._cursor: self._cursor.add_curve_cursor(self, curve) @@ -525,9 +524,9 @@ class ChartPlotWidget(pg.PlotWidget): last = self._ysticks[name] = YSticky( chart=self, parent=self.getAxis('right'), + # TODO: pass this from symbol data # digits=0, opacity=1, - color=pg.mkPen(hcolor('pikers')) ) return last diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 6b0dcf1d..287a59c5 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -51,6 +51,7 @@ class ChartView(pg.ViewBox): if ev.delta() > 0 and vl <= _min_points_to_show: log.debug("Max zoom bruh...") return + if ev.delta() < 0 and vl >= len(self.linked_charts._array): log.debug("Min zoom bruh...") return From 7a268ea88ed67fb2c71aeeeb122f08d558a999c8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 25 Oct 2020 18:20:38 -0400 Subject: [PATCH 170/206] Draft screen / font / dpi info script --- snippets/qt_screen_info.py | 87 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 snippets/qt_screen_info.py diff --git a/snippets/qt_screen_info.py b/snippets/qt_screen_info.py new file mode 100644 index 00000000..6d775d6b --- /dev/null +++ b/snippets/qt_screen_info.py @@ -0,0 +1,87 @@ +from pyqtgraph import QtGui +from PyQt5 import QtCore +from PyQt5.QtCore import ( + Qt, QCoreApplication +) + +# Proper high DPI scaling is available in Qt >= 5.6.0. This attibute +# must be set before creating the application +if hasattr(Qt, 'AA_EnableHighDpiScaling'): + QCoreApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True) + +if hasattr(Qt, 'AA_UseHighDpiPixmaps'): + QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) + + +app = QtGui.QApplication([]) +window = QtGui.QMainWindow() +main_widget = QtGui.QWidget() +window.setCentralWidget(main_widget) +window.show() + +pxr = main_widget.devicePixelRatioF() + +# screen_num = app.desktop().screenNumber() +# screen = app.screens()[screen_num] + +screen = app.screenAt(main_widget.geometry().center()) + +name = screen.name() +size = screen.size() +geo = screen.availableGeometry() +phydpi = screen.physicalDotsPerInch() +logdpi = screen.logicalDotsPerInch() + +print( + # f'screen number: {screen_num}\n', + f'screen name: {name}\n' + f'screen size: {size}\n' + f'screen geometry: {geo}\n\n' + f'devicePixelRationF(): {pxr}\n' + f'physical dpi: {phydpi}\n' + f'logical dpi: {logdpi}\n' +) + +print('-'*50) + +screen = app.primaryScreen() + +name = screen.name() +size = screen.size() +geo = screen.availableGeometry() +phydpi = screen.physicalDotsPerInch() +logdpi = screen.logicalDotsPerInch() + +print( + # f'screen number: {screen_num}\n', + f'screen name: {name}\n' + f'screen size: {size}\n' + f'screen geometry: {geo}\n\n' + f'devicePixelRationF(): {pxr}\n' + f'physical dpi: {phydpi}\n' + f'logical dpi: {logdpi}\n' +) + + +# app-wide font +font = QtGui.QFont("Hack") +# use pixel size to be cross-resolution compatible? +font.setPixelSize(6) + + +fm = QtGui.QFontMetrics(font) +fontdpi = fm.fontDpi() +font_h = fm.height() + +string = '10000' +str_br = fm.boundingRect(string) +str_w = str_br.width() + + +print( + # f'screen number: {screen_num}\n', + f'font dpi: {fontdpi}\n' + f'font height: {font_h}\n' + f'string bounding rect: {str_br}\n' + f'string width : {str_w}\n' +) From 55f34dfed0ff7dbe2c4c66c60ed5ae88e756548f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 26 Oct 2020 10:54:46 -0400 Subject: [PATCH 171/206] Start a profiling mod --- piker/_profile.py | 20 ++++++++++ piker/ui/quantdom/utils.py | 81 -------------------------------------- 2 files changed, 20 insertions(+), 81 deletions(-) create mode 100644 piker/_profile.py delete mode 100644 piker/ui/quantdom/utils.py diff --git a/piker/_profile.py b/piker/_profile.py new file mode 100644 index 00000000..c14071d0 --- /dev/null +++ b/piker/_profile.py @@ -0,0 +1,20 @@ + +""" +Profiling wrappers for internal libs. +""" +import time +from functools import wraps + + +def timeit(fn): + @wraps(fn) + def wrapper(*args, **kwargs): + t = time.time() + res = fn(*args, **kwargs) + print( + '%s.%s: %.4f sec' + % (fn.__module__, fn.__qualname__, time.time() - t) + ) + return res + + return wrapper diff --git a/piker/ui/quantdom/utils.py b/piker/ui/quantdom/utils.py deleted file mode 100644 index af885cc4..00000000 --- a/piker/ui/quantdom/utils.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Utils.""" - -import importlib.util -import inspect -import logging -import os -import os.path -import sys -import time -from datetime import datetime -from functools import wraps - -from PyQt5 import QtCore - -__all__ = ( - 'BASE_DIR', - 'Settings', - 'timeit', - 'fromtimestamp', - 'get_data_path', - 'get_resource_path', - 'strategies_from_file', -) - - -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -def get_data_path(path=''): - data_path = QtCore.QStandardPaths.writableLocation( - QtCore.QStandardPaths.AppDataLocation - ) - data_path = os.path.join(data_path, path) - os.makedirs(data_path, mode=0o755, exist_ok=True) - return data_path - - -def get_resource_path(relative_path): - # PyInstaller creates a temp folder and stores path in _MEIPASS - base_path = getattr(sys, '_MEIPASS', BASE_DIR) - return os.path.join(base_path, relative_path) - - -config_path = os.path.join(get_data_path(), 'Quantdom', 'config.ini') -Settings = QtCore.QSettings(config_path, QtCore.QSettings.IniFormat) - - -def timeit(fn): - @wraps(fn) - def wrapper(*args, **kwargs): - t = time.time() - res = fn(*args, **kwargs) - print( - '%s.%s: %.4f sec' - % (fn.__module__, fn.__qualname__, time.time() - t) - ) - return res - - return wrapper - - -def fromtimestamp(timestamp): - if timestamp == 0: - # on Win zero timestamp cause error - return datetime(1970, 1, 1) - return datetime.fromtimestamp(timestamp) - - -def strategies_from_file(filepath): - from .strategy import AbstractStrategy - - spec = importlib.util.spec_from_file_location('Strategy', filepath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - is_strategy = lambda _class: ( # noqa:E731 - inspect.isclass(_class) - and issubclass(_class, AbstractStrategy) - and _class.__name__ != 'AbstractStrategy' - ) - return [_class for _, _class in inspect.getmembers(module, is_strategy)] From 89d48afb6ccc535d2020224f9fe0dd04baabc281 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 26 Oct 2020 23:34:48 -0400 Subject: [PATCH 172/206] Size axis labels based on text contents Compute the size in pixels the label based on the label's contents. Eventually we want to have an update system that can iterate through axes and labels to do this whenever needed (eg. after widget is moved to a new screen with a different DPI). --- piker/ui/_axes.py | 75 +++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index ad25c66b..e6233ad4 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -11,6 +11,8 @@ from PyQt5.QtCore import QPointF from ._style import _font, hcolor from ..data._source import float_digits +_axis_pen = pg.mkPen(hcolor('bracket')) + class PriceAxis(pg.AxisItem): @@ -21,15 +23,18 @@ class PriceAxis(pg.AxisItem): self.setTickFont(_font) self.setStyle(**{ 'textFillLimits': [(0, 0.666)], + 'tickFont': _font, # 'tickTextWidth': 100, # 'tickTextHeight': 20, - 'tickFont': _font, # 'tickTextWidth': 40, # 'autoExpandTextSpace': True, # 'maxTickLength': -20, # 'stopAxisAtTick': (True, True), # doesn't work well on price }) # self.setLabel(**{'font-size': '10pt'}) + self.setTickFont(_font) + self.setPen(_axis_pen) + self.setWidth(40) # XXX: drop for now since it just eats up h space @@ -63,11 +68,12 @@ class DynamicDateAxis(pg.AxisItem): super().__init__(*args, **kwargs) self.linked_charts = linked_charts self.setTickFont(_font) + self.setPen(_axis_pen) # default styling self.setStyle(**{ # tickTextOffset=4, - 'textFillLimits': [(0, 0.70)], + 'textFillLimits': [(0, 0.666)], 'tickFont': _font, }) self.setHeight(11) @@ -95,6 +101,10 @@ class DynamicDateAxis(pg.AxisItem): class AxisLabel(pg.GraphicsObject): + _font = _font + _w_margin = 0 + _h_margin = 3 + def __init__( self, parent: pg.GraphicsObject, @@ -108,6 +118,7 @@ class AxisLabel(pg.GraphicsObject): self.opacity = opacity self.label_str = '' self.digits = digits + self._txt_br: QtCore.QRect = None self.bg_color = pg.mkColor(hcolor(bg_color)) self.fg_color = pg.mkColor(hcolor(fg_color)) @@ -115,34 +126,52 @@ class AxisLabel(pg.GraphicsObject): self.pic = QtGui.QPicture() p = QtGui.QPainter(self.pic) - self.rect = QtCore.QRectF(0, 0, 40, 11) + self.rect = None p.setPen(self.fg_color) p.setOpacity(self.opacity) - p.fillRect(self.rect, self.bg_color) - - # this adds a nice black outline around the label for some odd - # reason; ok by us - p.drawRect(self.rect) self.setFlag(self.ItemIgnoresTransformations) + def _size_br_from_str(self, value: str) -> None: + """Do our best to render the bounding rect to a set margin + around provided string contents. + + """ + txt_br = self._font._fm.boundingRect(value) + h, w = txt_br.height(), txt_br.width() + self.rect = QtCore.QRectF( + 0, 0, + w + self._w_margin, + h + self._h_margin + ) + def paint(self, p, option, widget): p.drawPicture(0, 0, self.pic) if self.label_str: + + if not self.rect: + self._size_br_from_str(self.label_str) + p.setFont(_font) p.setPen(self.fg_color) - p.drawText(self.rect, self.text_flags, self.label_str) + p.fillRect(self.rect, self.bg_color) + + # this adds a nice black outline around the label for some odd + # reason; ok by us + p.drawRect(self.rect) + + p.drawText(option.rect, self.text_flags, self.label_str) + + def boundingRect(self): # noqa + return self.rect or QtCore.QRectF() # uggggghhhh def tick_to_string(self, tick_pos): raise NotImplementedError() - def boundingRect(self): # noqa - raise NotImplementedError() - def update_label(self, evt_post, point_view): raise NotImplementedError() @@ -160,6 +189,8 @@ class AxisLabel(pg.GraphicsObject): class XAxisLabel(AxisLabel): + _w_margin = 8 + text_flags = ( QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter @@ -168,12 +199,6 @@ class XAxisLabel(AxisLabel): # | QtCore.Qt.AlignHCenter ) - def boundingRect(self): # noqa - # TODO: we need to get the parent axe's dimensions transformed - # to abs coords to be 100% correct here: - # self.parent.boundingRect() - return QtCore.QRectF(0, 0, 40, 11) - def update_label( self, abs_pos: QPointF, # scene coords @@ -201,9 +226,6 @@ class YAxisLabel(AxisLabel): # WTF IS THIS FORMAT? return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') - def boundingRect(self): # noqa - return QtCore.QRectF(0, 0, 50, 10) - def update_label( self, abs_pos: QPointF, # scene coords @@ -225,15 +247,10 @@ class YSticky(YAxisLabel): *args, **kwargs ) -> None: - super().__init__(*args, **kwargs) - self._chart = chart - # XXX: not sure why this wouldn't work with a proxy? - # pg.SignalProxy( - # delay=0, - # rateLimit=60, - # slot=last.update_on_resize, - # ) + super().__init__(*args, **kwargs) + + self._chart = chart chart.sigRangeChanged.connect(self.update_on_resize) def update_on_resize(self, vr, r): From 23672fc22b26b60e14312382b53180694bf4cb08 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 27 Oct 2020 10:50:28 -0400 Subject: [PATCH 173/206] Rework axes types, sizing stuff Make our own ``Axis`` and have it call an impl specific ``.resize()`` such that different axes can size to their own spec. Allow passing in a "typical maximum value string" which will be used by default for sizing the axis' minor dimension; a common value should be passed to all axes in a linked split charts widget. Add size hinting for axes labels such that they can check their parent (axis) for desired dimensions if needed. --- piker/ui/_axes.py | 104 ++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 46 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index e6233ad4..6be1d154 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -1,7 +1,7 @@ """ Chart axes graphics and behavior. """ -from typing import List +from typing import List, Tuple import pandas as pd import pyqtgraph as pg @@ -14,12 +14,19 @@ from ..data._source import float_digits _axis_pen = pg.mkPen(hcolor('bracket')) -class PriceAxis(pg.AxisItem): +class Axis(pg.AxisItem): def __init__( self, + linked_charts, + typical_max_str: str = '100 000.00', + **kwargs ) -> None: - super().__init__(orientation='right') + + self.linked_charts = linked_charts + + super().__init__(**kwargs) + self.setTickFont(_font) self.setStyle(**{ 'textFillLimits': [(0, 0.666)], @@ -29,13 +36,31 @@ class PriceAxis(pg.AxisItem): # 'tickTextWidth': 40, # 'autoExpandTextSpace': True, # 'maxTickLength': -20, - # 'stopAxisAtTick': (True, True), # doesn't work well on price + + # doesn't work well on price? + # 'stopAxisAtTick': (True, True), }) # self.setLabel(**{'font-size': '10pt'}) + self.setTickFont(_font) self.setPen(_axis_pen) + self.typical_br = _font._fm.boundingRect(typical_max_str) - self.setWidth(40) + # size the pertinent axis dimension to a "typical value" + self.resize() + + +class PriceAxis(Axis): + + def __init__( + self, + *args, + **kwargs, + ) -> None: + super().__init__(*args, orientation='right', **kwargs) + + def resize(self) -> None: + self.setWidth(self.typical_br.width()) # XXX: drop for now since it just eats up h space @@ -50,7 +75,8 @@ class PriceAxis(pg.AxisItem): ] -class DynamicDateAxis(pg.AxisItem): +class DynamicDateAxis(Axis): + # time formats mapped by seconds between bars tick_tpl = { 60*60*24: '%Y-%b-%d', @@ -59,24 +85,8 @@ class DynamicDateAxis(pg.AxisItem): 5: '%H:%M:%S', } - def __init__( - self, - linked_charts, - *args, - **kwargs - ) -> None: - super().__init__(*args, **kwargs) - self.linked_charts = linked_charts - self.setTickFont(_font) - self.setPen(_axis_pen) - - # default styling - self.setStyle(**{ - # tickTextOffset=4, - 'textFillLimits': [(0, 0.666)], - 'tickFont': _font, - }) - self.setHeight(11) + def resize(self) -> None: + self.setHeight(self.typical_br.height() + 3) def _indexes_to_timestrs( self, @@ -107,7 +117,7 @@ class AxisLabel(pg.GraphicsObject): def __init__( self, - parent: pg.GraphicsObject, + parent: Axis, digits: int = 2, bg_color: str = 'bracket', fg_color: str = 'black', @@ -133,19 +143,6 @@ class AxisLabel(pg.GraphicsObject): self.setFlag(self.ItemIgnoresTransformations) - def _size_br_from_str(self, value: str) -> None: - """Do our best to render the bounding rect to a set margin - around provided string contents. - - """ - txt_br = self._font._fm.boundingRect(value) - h, w = txt_br.height(), txt_br.width() - self.rect = QtCore.QRectF( - 0, 0, - w + self._w_margin, - h + self._h_margin - ) - def paint(self, p, option, widget): p.drawPicture(0, 0, self.pic) @@ -167,15 +164,19 @@ class AxisLabel(pg.GraphicsObject): def boundingRect(self): # noqa return self.rect or QtCore.QRectF() - # uggggghhhh + def _size_br_from_str(self, value: str) -> None: + """Do our best to render the bounding rect to a set margin + around provided string contents. - def tick_to_string(self, tick_pos): - raise NotImplementedError() - - def update_label(self, evt_post, point_view): - raise NotImplementedError() - - # end uggggghhhh + """ + txt_br = self._font._fm.boundingRect(value) + txt_h, txt_w = txt_br.height(), txt_br.width() + h, w = self.size_hint() + self.rect = QtCore.QRectF( + 0, 0, + (w or txt_w) + self._w_margin, + (h or txt_h) + self._h_margin, + ) # _common_text_flags = ( @@ -190,6 +191,7 @@ class AxisLabel(pg.GraphicsObject): class XAxisLabel(AxisLabel): _w_margin = 8 + _h_margin = 0 text_flags = ( QtCore.Qt.TextDontClip @@ -199,6 +201,10 @@ class XAxisLabel(AxisLabel): # | QtCore.Qt.AlignHCenter ) + def size_hint(self) -> Tuple[float, float]: + # size to parent axis height + return self.parent.height(), None + def update_label( self, abs_pos: QPointF, # scene coords @@ -206,8 +212,10 @@ class XAxisLabel(AxisLabel): offset: int = 1 # if have margins, k? ) -> None: timestrs = self.parent._indexes_to_timestrs([int(data)]) + if not timestrs.any(): return + self.label_str = timestrs[0] width = self.boundingRect().width() new_pos = QPointF(abs_pos.x() - width / 2 - offset, 0) @@ -222,6 +230,10 @@ class YAxisLabel(AxisLabel): | QtCore.Qt.AlignVCenter ) + def size_hint(self) -> Tuple[float, float]: + # size to parent axis width + return None, self.parent.width() + def tick_to_string(self, tick_pos): # WTF IS THIS FORMAT? return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') From 751cca35e12fb7e278eccf9b726832e26883a6b6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 27 Oct 2020 15:15:31 -0400 Subject: [PATCH 174/206] Attempt to calculate font size by DPI --- piker/ui/_chart.py | 22 ++++++++++++++++++-- piker/ui/_exec.py | 25 ++++++++++++++++++++++- piker/ui/_style.py | 51 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index c5329138..57c9a292 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -36,7 +36,7 @@ from ..data import ( maybe_open_shm_array, ) from ..log import get_logger -from ._exec import run_qtractor +from ._exec import run_qtractor, current_screen from ._interaction import ChartView from .. import fsp @@ -233,7 +233,10 @@ class LinkedSplitCharts(QtGui.QWidget): cpw = ChartPlotWidget( array=array, parent=self.splitter, - axisItems={'bottom': xaxis, 'right': PriceAxis()}, + axisItems={ + 'bottom': xaxis, + 'right': PriceAxis(linked_charts=self) + }, viewBox=cv, cursor=self._ch, **cpw_kwargs, @@ -688,6 +691,21 @@ async def _async_main( """ chart_app = widgets['main'] + screen = current_screen() + + from ._style import configure_font_to_dpi + print( + f'screen: {screen.name()} {screen.size()}') + + configure_font_to_dpi(screen) + + # from ._exec import get_screen + # screen = get_screen(chart_app.geometry().bottomRight()) + + # XXX: bug zone if you try to ctl-c after this we get hangs again? + # wtf... + # await tractor.breakpoint() + # historical data fetch brokermod = brokers.get_brokermod(brokername) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 1270d8ac..d9ac918e 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -16,12 +16,22 @@ from PyQt5.QtCore import ( pyqtRemoveInputHook, Qt, QCoreApplication ) import qdarkstyle - import trio import tractor from outcome import Error +# singleton app per actor +_qt_app: QtGui.QApplication = None +_qt_win: QtGui.QMainWindow = None + + +def current_screen() -> QtGui.QScreen: + + global _qt_win, _qt_app + return _qt_app.screenAt(_qt_win.centralWidget().geometry().center()) + + # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute # must be set before creating the application if hasattr(Qt, 'AA_EnableHighDpiScaling'): @@ -62,6 +72,10 @@ def run_qtractor( # currently seem tricky.. app.setQuitOnLastWindowClosed(False) + # set global app singleton + global _qt_app + _qt_app = app + # This code is from Nathaniel, and I quote: # "This is substantially faster than using a signal... for some # reason Qt signal dispatch is really slow (and relies on events @@ -84,10 +98,13 @@ def run_qtractor( app.postEvent(reenter, event) def done_callback(outcome): + print(f"Outcome: {outcome}") + if isinstance(outcome, Error): exc = outcome.error traceback.print_exception(type(exc), exc, exc.__traceback__) + app.quit() # load dark theme @@ -125,6 +142,12 @@ def run_qtractor( window.main_widget = main_widget window.setCentralWidget(instance) + + # store global ref + # set global app singleton + global _qt_win + _qt_win = window + # actually render to screen window.show() app.exec_() diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 210b4780..87e94c4d 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -5,19 +5,54 @@ import pyqtgraph as pg from PyQt5 import QtGui from qdarkstyle.palette import DarkPalette +from ..log import get_logger + +log = get_logger(__name__) # chart-wide font -_font = QtGui.QFont("Hack") +# font size 6px / 53 dpi (3x scaled down on 4k hidpi) +_font_inches_we_like = 6 / 53 + # use pixel size to be cross-resolution compatible? -_font.setPixelSize(6) +_font = QtGui.QFont("Hack") +_font.setPixelSize(6) # default -# TODO: use QScreen to determine the same physical font size -# on screen despite different displays? -# PyQt docs: https://doc.qt.io/qtforpython/PySide2/QtGui/QScreen.html -# - supposedly it's ``from QtGui import QScreen`` -# Qt forums: https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4 +# _physical_font_height_in = 1/6 # inches +_font._fm = QtGui.QFontMetrics(_font) -_i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) +# TODO: re-compute font size when main widget switches screens? +# https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 + + +def configure_font_to_dpi(screen: QtGui.QScreen): + """Set an appropriately sized font size depending on the screen DPI. + + If we end up needing to generalize this more here are some resources: + + - https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms + - https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628 + - https://doc.qt.io/qt-5/highdpi.html + - https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere + - https://stackoverflow.com/a/20465247 + - https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width + - https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 + - https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4 + + Also, see the script in ``snippets/qt_screen_info.py``. + + """ + dpi = screen.physicalDotsPerInch() + font_size = round(_font_inches_we_like * dpi) + log.info( + f"\nscreen:{screen.name()} with DPI: {dpi}" + f"\nbest font size is {font_size}\n" + ) + global _font + _font.setPixelSize(font_size) + return _font + + +# _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config _xaxis_at = 'bottom' From 20a4aed6721f740e0b61f2434497cf09dfb32d4e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 28 Oct 2020 08:05:15 -0400 Subject: [PATCH 175/206] Update font metrics after dpi calc; facepalm. --- piker/ui/_style.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 87e94c4d..24a4eac1 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -17,9 +17,6 @@ _font_inches_we_like = 6 / 53 _font = QtGui.QFont("Hack") _font.setPixelSize(6) # default -# _physical_font_height_in = 1/6 # inches -_font._fm = QtGui.QFontMetrics(_font) - # TODO: re-compute font size when main widget switches screens? # https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 @@ -27,18 +24,8 @@ _font._fm = QtGui.QFontMetrics(_font) def configure_font_to_dpi(screen: QtGui.QScreen): """Set an appropriately sized font size depending on the screen DPI. - If we end up needing to generalize this more here are some resources: - - - https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms - - https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628 - - https://doc.qt.io/qt-5/highdpi.html - - https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere - - https://stackoverflow.com/a/20465247 - - https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width - - https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 - - https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4 - - Also, see the script in ``snippets/qt_screen_info.py``. + If we end up needing to generalize this more here there are resources + listed in the script in ``snippets/qt_screen_info.py``. """ dpi = screen.physicalDotsPerInch() @@ -47,8 +34,11 @@ def configure_font_to_dpi(screen: QtGui.QScreen): f"\nscreen:{screen.name()} with DPI: {dpi}" f"\nbest font size is {font_size}\n" ) + global _font _font.setPixelSize(font_size) + _font._fm = QtGui.QFontMetrics(_font) + return _font From 031eab28c7757ec983ff07c20be7b1c59ca81ce8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 28 Oct 2020 08:24:13 -0400 Subject: [PATCH 176/206] Adjust contents label font for DPI --- piker/ui/_chart.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 57c9a292..1718c4ee 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -20,13 +20,14 @@ from ._graphics import ( ) from ._axes import YSticky from ._style import ( + configure_font_to_dpi, hcolor, CHART_MARGINS, _xaxis_at, _min_points_to_show, _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, - # _font, + _font, ) from ..data._source import Symbol from .. import brokers @@ -397,7 +398,7 @@ class ChartPlotWidget(pg.PlotWidget): # Ogi says: "use ..." label = pg.LabelItem( justify='left', - size='6px', + size=f'{_font.pixelSize()}px', ) label.setParentItem(self._vb) self.scene().addItem(label) @@ -484,7 +485,7 @@ class ChartPlotWidget(pg.PlotWidget): # XXX: How to stack labels vertically? label = pg.LabelItem( justify='left', - size='6px', + size=f'{_font.pixelSize()}px', ) # anchor to the viewbox @@ -691,13 +692,8 @@ async def _async_main( """ chart_app = widgets['main'] - screen = current_screen() - - from ._style import configure_font_to_dpi - print( - f'screen: {screen.name()} {screen.size()}') - - configure_font_to_dpi(screen) + # attempt to configure DPI aware font size + configure_font_to_dpi(current_screen()) # from ._exec import get_screen # screen = get_screen(chart_app.geometry().bottomRight()) From 6fd310473c30d2613b4d0f991c0bd4945bc62d44 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 28 Oct 2020 09:27:44 -0400 Subject: [PATCH 177/206] Add resource links to DPI snippet --- snippets/qt_screen_info.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/snippets/qt_screen_info.py b/snippets/qt_screen_info.py index 6d775d6b..238367c0 100644 --- a/snippets/qt_screen_info.py +++ b/snippets/qt_screen_info.py @@ -1,5 +1,20 @@ +""" +Resource list for mucking with DPIs on multiple screens: + +- https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms +- https://stackoverflow.com/questions/25761556/qt5-font-rendering-different-on-various-platforms/25929628#25929628 +- https://doc.qt.io/qt-5/highdpi.html +- https://stackoverflow.com/questions/20464814/changing-dpi-scaling-size-of-display-make-qt-applications-font-size-get-rendere +- https://stackoverflow.com/a/20465247 +- https://doc.qt.io/archives/qt-4.8/qfontmetrics.html#width +- https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 +- https://forum.qt.io/topic/43625/point-sizes-are-they-reliable/4 +- https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt +- https://doc.qt.io/qt-5/qguiapplication.html#screenAt + +""" + from pyqtgraph import QtGui -from PyQt5 import QtCore from PyQt5.QtCore import ( Qt, QCoreApplication ) From 68304b79bc1cd19fb032e41ed8d071d69f50a962 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 28 Oct 2020 09:28:37 -0400 Subject: [PATCH 178/206] More thematic max datums on screen --- piker/ui/_interaction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 287a59c5..444dfeda 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -52,7 +52,7 @@ class ChartView(pg.ViewBox): log.debug("Max zoom bruh...") return - if ev.delta() < 0 and vl >= len(self.linked_charts._array): + if ev.delta() < 0 and vl >= len(self.linked_charts._array) + 666: log.debug("Min zoom bruh...") return From aade0e5ea1995bc29120fd2c51624c0e06eb5bd0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 28 Oct 2020 15:03:51 -0400 Subject: [PATCH 179/206] Pin to our pyqtgraph branch; fixes transform invert performance --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 18ec5994..71fbd571 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # no pypi package for tractor (yet) # we require the asyncio-via-guest-mode dev branch -e git+git://github.com/goodboy/tractor.git@infect_asyncio#egg=tractor +-e git+git://github.com/pikers/pyqtgraph.git@use_qt_inverted#egg=pyqtgraph From 416f027c5f734b7b40b4ea4056a36d97c3f1f994 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 29 Oct 2020 17:08:03 -0400 Subject: [PATCH 180/206] Flip contents label stuff into a type --- piker/ui/_chart.py | 115 +++++++++++++++--------------------------- piker/ui/_graphics.py | 96 ++++++++++++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 76 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 1718c4ee..80c9b369 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -2,6 +2,7 @@ High level Qt chart widgets. """ from typing import Tuple, Dict, Any, Optional +from functools import partial from PyQt5 import QtCore, QtGui import numpy as np @@ -15,6 +16,7 @@ from ._axes import ( ) from ._graphics import ( CrossHair, + ContentsLabel, BarItems, h_line, ) @@ -27,7 +29,6 @@ from ._style import ( _min_points_to_show, _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, - _font, ) from ..data._source import Symbol from .. import brokers @@ -326,7 +327,7 @@ class ChartPlotWidget(pg.PlotWidget): self.showAxis('right') # show background grid - self.showGrid(x=True, y=True, alpha=0.4) + self.showGrid(x=True, y=True, alpha=0.5) # use cross-hair for cursor? # self.setCursor(QtCore.Qt.CrossCursor) @@ -344,10 +345,12 @@ class ChartPlotWidget(pg.PlotWidget): def last_bar_in_view(self) -> bool: self._array[-1]['index'] - def _update_contents_label(self, index: int) -> None: + def update_contents_labels(self, index: int) -> None: if index >= 0 and index < len(self._array): + array = self._array + for name, (label, update) in self._labels.items(): - update(index) + update(index, array) def _set_xlimits( self, @@ -375,54 +378,6 @@ class ChartPlotWidget(pg.PlotWidget): rbar = min(r, len(self._array)) return l, lbar, rbar, r - def draw_ohlc( - self, - name: str, - data: np.ndarray, - # XXX: pretty sure this is dumb and we don't need an Enum - style: pg.GraphicsObject = BarItems, - ) -> pg.GraphicsObject: - """Draw OHLC datums to chart. - """ - graphics = style(self.plotItem) - # adds all bar/candle graphics objects for each data point in - # the np array buffer to be drawn on next render cycle - self.addItem(graphics) - - # draw after to allow self.scene() to work... - graphics.draw_from_data(data) - - self._graphics[name] = graphics - - # XXX: How to stack labels vertically? - # Ogi says: "use ..." - label = pg.LabelItem( - justify='left', - size=f'{_font.pixelSize()}px', - ) - label.setParentItem(self._vb) - self.scene().addItem(label) - - # keep close to top - label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) - - def update(index: int) -> None: - label.setText( - "{name}[{index}] -> O:{} H:{} L:{} C:{} V:{}".format( - *self._array[index].item()[2:8], - name=name, - index=index, - ) - ) - - self._labels[name] = (label, update) - self._update_contents_label(len(data) - 1) - label.show() - - self._add_sticky(name) - - return graphics - def default_view( self, index: int = -1, @@ -456,6 +411,34 @@ class ChartPlotWidget(pg.PlotWidget): padding=0, ) + def draw_ohlc( + self, + name: str, + data: np.ndarray, + # XXX: pretty sure this is dumb and we don't need an Enum + style: pg.GraphicsObject = BarItems, + ) -> pg.GraphicsObject: + """Draw OHLC datums to chart. + """ + graphics = style(self.plotItem) + # adds all bar/candle graphics objects for each data point in + # the np array buffer to be drawn on next render cycle + self.addItem(graphics) + + # draw after to allow self.scene() to work... + graphics.draw_from_data(data) + + self._graphics[name] = graphics + + label = ContentsLabel(chart=self, anchor_at=('top', 'left')) + self._labels[name] = (label, partial(label.update_from_ohlc, name)) + label.show() + self.update_contents_labels(len(data) - 1) + + self._add_sticky(name) + + return graphics + def draw_curve( self, name: str, @@ -482,37 +465,21 @@ class ChartPlotWidget(pg.PlotWidget): # register overlay curve with name self._graphics[name] = curve - # XXX: How to stack labels vertically? - label = pg.LabelItem( - justify='left', - size=f'{_font.pixelSize()}px', - ) - - # anchor to the viewbox - label.setParentItem(self._vb) - # label.setParentItem(self.getPlotItem()) - if overlay: - # position bottom left if an overlay - label.anchor(itemPos=(1, 1), parentPos=(1, 1), offset=(0, 3)) + anchor_at = ('bottom', 'right') self._overlays[name] = curve else: - label.anchor(itemPos=(0, 0), parentPos=(0, 0), offset=(0, -4)) + anchor_at = ('top', 'right') # TODO: something instead of stickies for overlays # (we need something that avoids clutter on x-axis). self._add_sticky(name) - def update(index: int) -> None: - data = self._array[index][name] - label.setText(f"{name}: {data:.2f}") - + label = ContentsLabel(chart=self, anchor_at=anchor_at) + self._labels[name] = (label, partial(label.update_from_value, name)) label.show() - self.scene().addItem(label) - - self._labels[name] = (label, update) - self._update_contents_label(len(data) - 1) + self.update_contents_labels(len(data) - 1) if self._cursor: self._cursor.add_curve_cursor(self, curve) @@ -909,7 +876,7 @@ async def chart_from_fsp( ) # display contents labels asap - chart._update_contents_label(len(shm.array) - 1) + chart.update_contents_labels(len(shm.array) - 1) array = shm.array value = array[fsp_func_name][-1] diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 19532693..88089812 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -6,10 +6,11 @@ from typing import List import numpy as np import pyqtgraph as pg +# from numba import jit, float64, optional, int64 from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF, QPointF -# from .quantdom.utils import timeit +# from .._profile import timeit from ._style import _xaxis_at, hcolor, _font from ._axes import YAxisLabel, XAxisLabel @@ -56,6 +57,78 @@ class LineDot(pg.CurvePoint): self.setFlag(self.ItemIgnoresTransformations) +_corner_anchors = { + 'top': 0, + 'left': 0, + 'bottom': 1, + 'right': 1, +} +# XXX: fyi naming here is confusing / opposite to coords +_corner_margins = { + ('top', 'left'): (-4, -5), + ('top', 'right'): (4, -5), + ('bottom', 'left'): (-4, 5), + ('bottom', 'right'): (4, 5), +} + + +class ContentsLabel(pg.LabelItem): + """Label anchored to a ``ViewBox`` typically for displaying + datum-wise points from the "viewed" contents. + + """ + def __init__( + self, + chart: 'ChartPlotWidget', # noqa + anchor_at: str = ('top', 'right'), + justify_text: str = 'left', + size: str = f'{_font.pixelSize()}px', + ) -> None: + + super().__init__(justify=justify_text, size=size) + + # anchor to viewbox + self.setParentItem(chart._vb) + chart.scene().addItem(self) + self.chart = chart + + v, h = anchor_at + index = (_corner_anchors[h], _corner_anchors[v]) + margins = _corner_margins[(v, h)] + + self.anchor(itemPos=index, parentPos=index, offset=margins) + + def update_from_ohlc( + self, + name: str, + index: int, + array: np.ndarray, + ) -> None: + # this being "html" is the dumbest shit :eyeroll: + self.setText( + "i:{index}
" + "O:{}
" + "H:{}
" + "L:{}
" + "C:{}
" + "V:{}".format( + # *self._array[index].item()[2:8], + *array[index].item()[2:8], + name=name, + index=index, + ) + ) + + def update_from_value( + self, + name: str, + index: int, + array: np.ndarray, + ) -> None: + data = array[index][name] + self.setText(f"{name}: {data:.2f}") + + class CrossHair(pg.GraphicsObject): def __init__( @@ -208,7 +281,7 @@ class CrossHair(pg.GraphicsObject): opts['vl'].setX(ix) # update the chart's "contents" label - plot._update_contents_label(ix) + plot.update_contents_labels(ix) # update all subscribed curve dots for cursor in opts.get('cursors', ()): @@ -235,6 +308,15 @@ class CrossHair(pg.GraphicsObject): return self.plots[0].boundingRect() +# @jit( +# # float64[:]( +# # float64[:], +# # optional(float64), +# # optional(int16) +# # ), +# nopython=True, +# nogil=True +# ) def _mk_lines_array(data: List, size: int) -> np.ndarray: """Create an ndarray to hold lines graphics objects. """ @@ -246,6 +328,16 @@ def _mk_lines_array(data: List, size: int) -> np.ndarray: # TODO: `numba` this? + +# @jit( +# # float64[:]( +# # float64[:], +# # optional(float64), +# # optional(int16) +# # ), +# nopython=True, +# nogil=True +# ) def bars_from_ohlc( data: np.ndarray, w: float, From da0789e184b00805e2c35c8444ad4e389cbbab40 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 29 Oct 2020 17:08:25 -0400 Subject: [PATCH 181/206] Add symbol field to kraken quotes --- piker/brokers/kraken.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index a79a6399..3d1d8e7a 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -216,7 +216,7 @@ def normalize( quote = asdict(ohlc) quote['broker_ts'] = quote['time'] quote['brokerd_ts'] = time.time() - quote['pair'] = quote['pair'].replace('/', '') + quote['symbol'] = quote['pair'] = quote['pair'].replace('/', '') # seriously eh? what's with this non-symmetry everywhere # in subscription systems... From 307c50176390f730428b59b456ef21e785a11d21 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 29 Oct 2020 17:21:41 -0400 Subject: [PATCH 182/206] Add symbol field to ib quotes --- piker/brokers/ib.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 9f40a7d9..c2067665 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -33,6 +33,7 @@ from ..data import ( subscribe_ohlc_for_increment, ) from ..data._source import from_df +from ._util import SymbolNotFound log = get_logger(__name__) @@ -88,8 +89,7 @@ class NonShittyWrapper(Wrapper): class NonShittyIB(ibis.IB): - """The beginning of overriding quite a few quetionable decisions - in this lib. + """The beginning of overriding quite a few decisions in this lib. - Don't use datetimes - Don't use named tuples @@ -117,12 +117,14 @@ class Client: """IB wrapped for our broker backend API. Note: this client requires running inside an ``asyncio`` loop. + """ def __init__( self, ib: ibis.IB, ) -> None: self.ib = ib + self.ib.RaiseRequestErrors = True async def bars( self, @@ -563,6 +565,9 @@ async def stream_quotes( symbol=sym, ) + if bars is None: + raise SymbolNotFound(sym) + # write historical data to buffer shm.push(bars) shm_token = shm.token @@ -576,7 +581,9 @@ async def stream_quotes( # first quote can be ignored as a 2nd with newer data is sent? first_ticker = await stream.__anext__() + quote = normalize(first_ticker) + # ugh, clear ticks since we've consumed them # (ahem, ib_insync is stateful trash) first_ticker.ticks = [] @@ -608,9 +615,11 @@ async def stream_quotes( calc_price = True ticker = first_ticker - con = quote['contract'] quote = normalize(ticker, calc_price=calc_price) + con = quote['contract'] topic = '.'.join((con['symbol'], con[suffix])).lower() + quote['symbol'] = topic + first_quote = {topic: quote} ticker.ticks = [] @@ -623,6 +632,7 @@ async def stream_quotes( ticker, calc_price=calc_price ) + quote['symbol'] = topic # TODO: in theory you can send the IPC msg *before* # writing to the sharedmem array to decrease latency, # however, that will require `tractor.msg.pub` support @@ -648,6 +658,7 @@ async def stream_quotes( con = quote['contract'] topic = '.'.join((con['symbol'], con[suffix])).lower() + quote['symbol'] = topic await ctx.send_yield({topic: quote}) From 9e7aa3f9bfbc2f102cf29a44604e7dca367222be Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 Nov 2020 12:02:05 -0500 Subject: [PATCH 183/206] Add a dpi aware font wrapper type --- piker/ui/_axes.py | 143 +++++++++++++++++++++++++++++---------------- piker/ui/_style.py | 139 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 207 insertions(+), 75 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 6be1d154..b4b910ce 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -1,21 +1,23 @@ """ Chart axes graphics and behavior. """ -from typing import List, Tuple +from typing import List, Tuple, Optional import pandas as pd import pyqtgraph as pg from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QPointF -from ._style import _font, hcolor +from ._style import DpiAwareFont, hcolor, _font from ..data._source import float_digits _axis_pen = pg.mkPen(hcolor('bracket')) class Axis(pg.AxisItem): + """A better axis that sizes to typical tick contents considering font size. + """ def __init__( self, linked_charts, @@ -27,10 +29,10 @@ class Axis(pg.AxisItem): super().__init__(**kwargs) - self.setTickFont(_font) + self.setTickFont(_font.font) self.setStyle(**{ 'textFillLimits': [(0, 0.666)], - 'tickFont': _font, + 'tickFont': _font.font, # 'tickTextWidth': 100, # 'tickTextHeight': 20, # 'tickTextWidth': 40, @@ -42,9 +44,9 @@ class Axis(pg.AxisItem): }) # self.setLabel(**{'font-size': '10pt'}) - self.setTickFont(_font) + self.setTickFont(_font.font) self.setPen(_axis_pen) - self.typical_br = _font._fm.boundingRect(typical_max_str) + self.typical_br = _font._qfm.boundingRect(typical_max_str) # size the pertinent axis dimension to a "typical value" self.resize() @@ -92,17 +94,18 @@ class DynamicDateAxis(Axis): self, indexes: List[int], ) -> List[str]: + bars = self.linked_charts.chart._array - times = bars['time'] bars_len = len(bars) - # delay = times[-1] - times[times != times[-1]][-1] - delay = times[-1] - times[-2] + times = bars['time'] epochs = times[list( map(int, filter(lambda i: i < bars_len, indexes)) )] # TODO: **don't** have this hard coded shift to EST dts = pd.to_datetime(epochs, unit='s') # - 4*pd.offsets.Hour() + + delay = times[-1] - times[-2] return dts.strftime(self.tick_tpl[delay]) def tickStrings(self, values: List[float], scale, spacing): @@ -111,9 +114,8 @@ class DynamicDateAxis(Axis): class AxisLabel(pg.GraphicsObject): - _font = _font _w_margin = 0 - _h_margin = 3 + _h_margin = 0 def __init__( self, @@ -122,46 +124,62 @@ class AxisLabel(pg.GraphicsObject): bg_color: str = 'bracket', fg_color: str = 'black', opacity: int = 1, + font_size: Optional[int] = None, ): super().__init__(parent) self.parent = parent self.opacity = opacity self.label_str = '' self.digits = digits + self._txt_br: QtCore.QRect = None + self._dpifont = DpiAwareFont() + self._dpifont.configure_to_dpi(_font._screen) + if font_size is not None: + self._dpifont._set_qfont_px_size(font_size) + + # self._font._fm = QtGui.QFontMetrics(self._font) + self.bg_color = pg.mkColor(hcolor(bg_color)) self.fg_color = pg.mkColor(hcolor(fg_color)) - self.pic = QtGui.QPicture() - p = QtGui.QPainter(self.pic) + # self.pic = QtGui.QPicture() + # p = QtGui.QPainter(self.pic) self.rect = None - p.setPen(self.fg_color) - p.setOpacity(self.opacity) + # p.setPen(self.fg_color) self.setFlag(self.ItemIgnoresTransformations) def paint(self, p, option, widget): - p.drawPicture(0, 0, self.pic) + # p.drawPicture(0, 0, self.pic) + p.setCompositionMode(QtGui.QPainter.CompositionMode_SourceOver) if self.label_str: if not self.rect: self._size_br_from_str(self.label_str) - p.setFont(_font) + p.setFont(self._dpifont.font) p.setPen(self.fg_color) + p.setOpacity(self.opacity) p.fillRect(self.rect, self.bg_color) # this adds a nice black outline around the label for some odd # reason; ok by us p.drawRect(self.rect) - p.drawText(option.rect, self.text_flags, self.label_str) + p.drawText(self.rect, self.text_flags, self.label_str) def boundingRect(self): # noqa + # if self.label_str: + # self._size_br_from_str(self.label_str) + # return self.rect + + # return QtCore.QRectF() + return self.rect or QtCore.QRectF() def _size_br_from_str(self, value: str) -> None: @@ -169,9 +187,21 @@ class AxisLabel(pg.GraphicsObject): around provided string contents. """ - txt_br = self._font._fm.boundingRect(value) - txt_h, txt_w = txt_br.height(), txt_br.width() + # size the filled rect to text and/or parent axis + br = self._txt_br = self._dpifont.boundingRect(value) + + # px_per_char = self._font._fm.averageCharWidth() + # br = br * 1.88 + txt_h, txt_w = br.height(), br.width() + print(f'orig: {txt_h}') + # txt_h = (br.topLeft() - br.bottomRight()).y() + # txt_w = len(value) * px_per_char + # txt_w *= 1.88 + # txt_h *= 1.88 + # print(f'calced: {txt_h}') + h, w = self.size_hint() + self.rect = QtCore.QRectF( 0, 0, (w or txt_w) + self._w_margin, @@ -190,15 +220,12 @@ class AxisLabel(pg.GraphicsObject): class XAxisLabel(AxisLabel): - _w_margin = 8 + _w_margin = 0 _h_margin = 0 text_flags = ( QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter - # | QtCore.Qt.AlignTop - # | QtCore.Qt.AlignVCenter - # | QtCore.Qt.AlignHCenter ) def size_hint(self) -> Tuple[float, float]: @@ -211,43 +238,50 @@ class XAxisLabel(AxisLabel): data: float, # data for text offset: int = 1 # if have margins, k? ) -> None: + timestrs = self.parent._indexes_to_timestrs([int(data)]) if not timestrs.any(): return self.label_str = timestrs[0] + width = self.boundingRect().width() - new_pos = QPointF(abs_pos.x() - width / 2 - offset, 0) - self.setPos(new_pos) + self.setPos(QPointF( + abs_pos.x() - width / 2, # - offset, + 0 + )) class YAxisLabel(AxisLabel): text_flags = ( - QtCore.Qt.AlignLeft + # QtCore.Qt.AlignLeft + QtCore.Qt.AlignVCenter | QtCore.Qt.TextDontClip - | QtCore.Qt.AlignVCenter ) def size_hint(self) -> Tuple[float, float]: # size to parent axis width return None, self.parent.width() - def tick_to_string(self, tick_pos): - # WTF IS THIS FORMAT? - return ('{: ,.%df}' % self.digits).format(tick_pos).replace(',', ' ') - def update_label( self, abs_pos: QPointF, # scene coords data: float, # data for text offset: int = 1 # if have margins, k? ) -> None: - self.label_str = self.tick_to_string(data) - height = self.boundingRect().height() - new_pos = QPointF(0, abs_pos.y() - height / 2 - offset) - self.setPos(new_pos) + + # this is read inside ``.paint()`` + self.label_str = '{data: ,.{digits}f}'.format( + digits=self.digits, data=data).replace(',', ' ') + + br = self.boundingRect() + h = br.height() + self.setPos(QPointF( + 0, + abs_pos.y() - h / 2 #- offset + )) class YSticky(YAxisLabel): @@ -264,29 +298,40 @@ class YSticky(YAxisLabel): self._chart = chart chart.sigRangeChanged.connect(self.update_on_resize) + self._last_datum = (None, None) def update_on_resize(self, vr, r): # TODO: add an `.index` to the array data-buffer layer # and make this way less shitty... - chart = self._chart - a = chart._array - fields = a.dtype.fields - if fields and 'close' in fields: - index, last = a[-1][['index', 'close']] - else: - # non-ohlc case - index = len(a) - 1 - last = a[chart.name][-1] - self.update_from_data( - index, - last, - ) + index, last = self._last_datum + if index is not None: + self.update_from_data( + index, + last, + ) + + # chart = self._chart + # a = chart._array + # fields = a.dtype.fields + + # if fields and 'close' in fields: + # index, last = a[-1][['index', 'close']] + + # else: # non-ohlc case + # index = len(a) - 1 + # last = a[chart.name][-1] + + # self.update_from_data( + # index, + # last, + # ) def update_from_data( self, index: int, value: float, ) -> None: + self._last_datum = (index, value) self.update_label( self._chart.mapFromView(QPointF(index, value)), value diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 24a4eac1..8f1e9425 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -2,46 +2,129 @@ Qt UI styling. """ import pyqtgraph as pg -from PyQt5 import QtGui +from PyQt5 import QtCore, QtGui from qdarkstyle.palette import DarkPalette from ..log import get_logger log = get_logger(__name__) + +# def configure_font_to_dpi(screen: QtGui.QScreen): +# """Set an appropriately sized font size depending on the screen DPI. + +# If we end up needing to generalize this more here there are resources +# listed in the script in ``snippets/qt_screen_info.py``. + +# """ +# dpi = screen.physicalDotsPerInch() +# font_size = round(_font_inches_we_like * dpi) +# log.info( +# f"\nscreen:{screen.name()} with DPI: {dpi}" +# f"\nbest font size is {font_size}\n" +# ) + +# global _font +# _font.setPixelSize(font_size) +# _font._fm = QtGui.QFontMetrics(_font) + +# return _font + + # chart-wide font # font size 6px / 53 dpi (3x scaled down on 4k hidpi) -_font_inches_we_like = 6 / 53 +_font_inches_we_like = 6 / 53 # px / (px / inch) = inch + + +class DpiAwareFont: + def __init__( + self, + name: str = 'Hack', + ) -> None: + self.name = name + self._qfont = QtGui.QFont(name) + self._iwl = _font_inches_we_like + self._qfm = QtGui.QFontMetrics(self._qfont) + self._font_size = None + self._physical_dpi = None + self._screen = None + + def _set_qfont_px_size(self, px_size: int) -> None: + # self._qfont = QtGui.Qfont(self.name) + self._qfont.setPixelSize(px_size) + self._qfm = QtGui.QFontMetrics(self._qfont) + + @property + def font(self): + return self._qfont + + @property + def px_size(self): + return self._qfont.pixelSize() + + # def set_px_size(self, size: int) -> None: + # pass + + def configure_to_dpi(self, screen: QtGui.QScreen): + """Set an appropriately sized font size depending on the screen DPI. + + If we end up needing to generalize this more here there are resources + listed in the script in ``snippets/qt_screen_info.py``. + + """ + dpi = screen.physicalDotsPerInch() + font_size = round(self._iwl * dpi) + log.info( + f"\nscreen:{screen.name()} with DPI: {dpi}" + f"\nbest font size is {font_size}\n" + ) + self._set_qfont_px_size(font_size) + self._font_size = font_size + self._physical_dpi = dpi + self._screen = screen + + def boundingRect(self, value: str) -> QtCore.QRectF: + screen = self._screen + if screen is None: + raise RuntimeError("You must call .configure_to_dpi() first!") + + unscaled_br = self._qfm.boundingRect(value) + + # XXX: for wtv absolutely fucked reason, the scaling only applies + # to everything when the current font size **is not** the size + # needed to get the original desired text height... :mindblow: + if self._font_size != 6: + # scalar = self._qfm.fontDpi() / self._physical_dpi + scalar = screen.logicalDotsPerInch() / screen.physicalDotsPerInch() + # scalar = 100 / screen.physicalDotsPerInch() + # assert 0 + print(f'SCALAR {scalar}') + + + return QtCore.QRectF( + # unscaled_br.x(), + # unscaled_br.y(), + 0, + 0, + unscaled_br.width() * scalar, + unscaled_br.height() * scalar, + ) + else: + return QtCore.QRectF( + # unscaled_br.x(), + # unscaled_br.y(), + 0, + 0, + unscaled_br.width(), + unscaled_br.height(), + ) # use pixel size to be cross-resolution compatible? -_font = QtGui.QFont("Hack") -_font.setPixelSize(6) # default +_font = DpiAwareFont() # TODO: re-compute font size when main widget switches screens? # https://forum.qt.io/topic/54136/how-do-i-get-the-qscreen-my-widget-is-on-qapplication-desktop-screen-returns-a-qwidget-and-qobject_cast-qscreen-returns-null/3 - -def configure_font_to_dpi(screen: QtGui.QScreen): - """Set an appropriately sized font size depending on the screen DPI. - - If we end up needing to generalize this more here there are resources - listed in the script in ``snippets/qt_screen_info.py``. - - """ - dpi = screen.physicalDotsPerInch() - font_size = round(_font_inches_we_like * dpi) - log.info( - f"\nscreen:{screen.name()} with DPI: {dpi}" - f"\nbest font size is {font_size}\n" - ) - - global _font - _font.setPixelSize(font_size) - _font._fm = QtGui.QFontMetrics(_font) - - return _font - - # _i3_rgba = QtGui.QColor.fromRgbF(*[0.14]*3 + [1]) # splitter widget config @@ -69,6 +152,7 @@ def hcolor(name: str) -> str: """Hex color codes by hipster speak. """ return { + # lives matter 'black': '#000000', 'erie_black': '#1B1B1B', @@ -99,6 +183,9 @@ def hcolor(name: str) -> str: 'vwap_blue': '#0582fb', 'dodger_blue': '#1e90ff', # like the team? 'panasonic_blue': '#0040be', # from japan + # 'bid_blue': '#0077ea', # like the L1 + 'bid_blue': '#3094d9', # like the L1 + 'aquaman': '#39abd0', # traditional 'tina_green': '#00cc00', From 96f700a76258425d1922c038f80e9056b4a26e3d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 Nov 2020 13:33:19 -0500 Subject: [PATCH 184/206] Add level line type with custom label --- piker/ui/_graphics.py | 95 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 88089812..5dafc81b 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -2,7 +2,7 @@ Chart graphics for displaying a slew of different data types. """ # import time -from typing import List +from typing import List, Optional, Tuple import numpy as np import pyqtgraph as pg @@ -12,7 +12,7 @@ from PyQt5.QtCore import QLineF, QPointF # from .._profile import timeit from ._style import _xaxis_at, hcolor, _font -from ._axes import YAxisLabel, XAxisLabel +from ._axes import YAxisLabel, XAxisLabel, YSticky _mouse_rate_limit = 60 # calc current screen refresh rate? @@ -82,10 +82,10 @@ class ContentsLabel(pg.LabelItem): chart: 'ChartPlotWidget', # noqa anchor_at: str = ('top', 'right'), justify_text: str = 'left', - size: str = f'{_font.pixelSize()}px', + font_size: Optional[int] = None, ) -> None: - - super().__init__(justify=justify_text, size=size) + font_size = font_size or _font.font.pixelSize() + super().__init__(justify=justify_text, size=f'{str(font_size)}px') # anchor to viewbox self.setParentItem(chart._vb) @@ -642,27 +642,84 @@ class BarItems(pg.GraphicsObject): # p.setBrush(self.bear_brush) # p.drawRects(*rects[Quotes.close < Quotes.open]) +class LevelLabel(YSticky): -def h_line(level: float) -> pg.InfiniteLine: + def update_label( + self, + abs_pos: QPointF, # scene coords + data: float, # data for text + offset: int = 1 # if have margins, k? + ) -> None: + + # this is read inside ``.paint()`` + self.label_str = '{data: ,.{digits}f}'.format( + digits=self.digits, data=data).replace(',', ' ') + + self._size_br_from_str(self.label_str) + + br = self.boundingRect() + w, h = br.height(), br.width() + + self.setPos(QPointF( + # *2 why? wat..? + 0 - w*2, + abs_pos.y(), # - h / 2 - offset + )) + + def size_hint(self) -> Tuple[None, None]: + return None, None + + +class LevelLine(pg.InfiniteLine): + def __init__( + self, + label: LevelLabel, + **kwargs, + ) -> None: + self.label = label + super().__init__(**kwargs) + self.sigPositionChanged.connect(self.set_value) + + def set_value(self, value: float) -> None: + self.label.update_from_data(0, self.value()) + + # def valueChanged(self) -> None: + # print('yooo') + # self.label.update_from_data(0, self.value()) + + +def level_line( + chart: 'ChartPlogWidget', # noqa + level: float, + digits: int = 3, + # label_precision: int = 0, +) -> LevelLine: """Convenience routine to add a styled horizontal line to a plot. """ - line = pg.InfiniteLine( + label = LevelLabel( + chart=chart, + parent=chart.getAxis('right'), + # TODO: pass this from symbol data + digits=digits, + opacity=1, + font_size=4, + bg_color='default', + ) + label.update_from_data(0, level) + # label._size_br_from_str( + + line = LevelLine( + label, movable=True, angle=0, - pos=level, - # label='{value:0.2f}', - # labelOpts={ - # 'position': 0.01, - # 'movable': True - # 'color': (200,200,100), - # 'fill': (200,200,200,50), - # } ) - default_pen = pg.mkPen(hcolor('default')) - line.setPen(default_pen) + line.setValue(level) + line.setPen(pg.mkPen(hcolor('default'))) + # activate/draw label + line.setValue(level) - if getattr(line, 'label', None): - line.label.setFont(_font) + # line.show() + chart.plotItem.addItem(line) return line From 119196f2ff9834b2960c61882e2f6a9c690f4a39 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 Nov 2020 15:27:48 -0500 Subject: [PATCH 185/206] Scale level label correctly to small(er) fonts Not sure what fixed it exactly, and I guess we didn't need any relative DPI scaling factor after all. Using the 3px margin on the level label seems to make it look nice for any font size (i think) as well. Gonna need some cleanup after this one. --- piker/ui/_axes.py | 44 ++++++++---------------- piker/ui/_chart.py | 10 +++--- piker/ui/_graphics.py | 26 +++++++------- piker/ui/_style.py | 79 ++++++++++++++++--------------------------- 4 files changed, 60 insertions(+), 99 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index b4b910ce..4e4bcb16 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -136,7 +136,9 @@ class AxisLabel(pg.GraphicsObject): self._dpifont = DpiAwareFont() self._dpifont.configure_to_dpi(_font._screen) + if font_size is not None: + # print(f"SETTING FONT TO: {font_size}") self._dpifont._set_qfont_px_size(font_size) # self._font._fm = QtGui.QFontMetrics(self._font) @@ -155,7 +157,7 @@ class AxisLabel(pg.GraphicsObject): def paint(self, p, option, widget): # p.drawPicture(0, 0, self.pic) - p.setCompositionMode(QtGui.QPainter.CompositionMode_SourceOver) + # p.setCompositionMode(QtGui.QPainter.CompositionMode_SourceOver) if self.label_str: @@ -174,13 +176,13 @@ class AxisLabel(pg.GraphicsObject): p.drawText(self.rect, self.text_flags, self.label_str) def boundingRect(self): # noqa - # if self.label_str: - # self._size_br_from_str(self.label_str) - # return self.rect + if self.label_str: + self._size_br_from_str(self.label_str) + return self.rect - # return QtCore.QRectF() + return QtCore.QRectF() - return self.rect or QtCore.QRectF() + # return self.rect or QtCore.QRectF() def _size_br_from_str(self, value: str) -> None: """Do our best to render the bounding rect to a set margin @@ -193,7 +195,7 @@ class AxisLabel(pg.GraphicsObject): # px_per_char = self._font._fm.averageCharWidth() # br = br * 1.88 txt_h, txt_w = br.height(), br.width() - print(f'orig: {txt_h}') + # print(f'orig: {txt_h}') # txt_h = (br.topLeft() - br.bottomRight()).y() # txt_w = len(value) * px_per_char # txt_w *= 1.88 @@ -220,9 +222,6 @@ class AxisLabel(pg.GraphicsObject): class XAxisLabel(AxisLabel): - _w_margin = 0 - _h_margin = 0 - text_flags = ( QtCore.Qt.TextDontClip | QtCore.Qt.AlignCenter @@ -246,14 +245,15 @@ class XAxisLabel(AxisLabel): self.label_str = timestrs[0] - width = self.boundingRect().width() + w = self.boundingRect().width() self.setPos(QPointF( - abs_pos.x() - width / 2, # - offset, - 0 + abs_pos.x() - w / 2 - offset, + 0, )) class YAxisLabel(AxisLabel): + _h_margin = 3 text_flags = ( # QtCore.Qt.AlignLeft @@ -280,7 +280,7 @@ class YAxisLabel(AxisLabel): h = br.height() self.setPos(QPointF( 0, - abs_pos.y() - h / 2 #- offset + abs_pos.y() - h / 2 - offset )) @@ -310,22 +310,6 @@ class YSticky(YAxisLabel): last, ) - # chart = self._chart - # a = chart._array - # fields = a.dtype.fields - - # if fields and 'close' in fields: - # index, last = a[-1][['index', 'close']] - - # else: # non-ohlc case - # index = len(a) - 1 - # last = a[chart.name][-1] - - # self.update_from_data( - # index, - # last, - # ) - def update_from_data( self, index: int, diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 80c9b369..268c8332 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -18,11 +18,11 @@ from ._graphics import ( CrossHair, ContentsLabel, BarItems, - h_line, + level_line, ) from ._axes import YSticky from ._style import ( - configure_font_to_dpi, + _font, hcolor, CHART_MARGINS, _xaxis_at, @@ -660,7 +660,7 @@ async def _async_main( chart_app = widgets['main'] # attempt to configure DPI aware font size - configure_font_to_dpi(current_screen()) + _font.configure_to_dpi(current_screen()) # from ._exec import get_screen # screen = get_screen(chart_app.geometry().bottomRight()) @@ -898,8 +898,8 @@ async def chart_from_fsp( # graphics.curve.setFillLevel(50) # add moveable over-[sold/bought] lines - chart.plotItem.addItem(h_line(30)) - chart.plotItem.addItem(h_line(70)) + level_line(chart, 30) + level_line(chart, 70) chart._shm = shm chart._set_yrange() diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 5dafc81b..5281b1a6 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -644,6 +644,9 @@ class BarItems(pg.GraphicsObject): class LevelLabel(YSticky): + _w_margin = 3 + _h_margin = 3 + def update_label( self, abs_pos: QPointF, # scene coords @@ -653,17 +656,18 @@ class LevelLabel(YSticky): # this is read inside ``.paint()`` self.label_str = '{data: ,.{digits}f}'.format( - digits=self.digits, data=data).replace(',', ' ') + digits=self.digits, + data=data + ).replace(',', ' ') self._size_br_from_str(self.label_str) br = self.boundingRect() - w, h = br.height(), br.width() + h, w = br.height(), br.width() self.setPos(QPointF( - # *2 why? wat..? - 0 - w*2, - abs_pos.y(), # - h / 2 - offset + -w, + abs_pos.y() - offset )) def size_hint(self) -> Tuple[None, None]: @@ -683,16 +687,12 @@ class LevelLine(pg.InfiniteLine): def set_value(self, value: float) -> None: self.label.update_from_data(0, self.value()) - # def valueChanged(self) -> None: - # print('yooo') - # self.label.update_from_data(0, self.value()) - def level_line( chart: 'ChartPlogWidget', # noqa level: float, - digits: int = 3, - # label_precision: int = 0, + digits: int = 1, + font_size: int = 4, ) -> LevelLine: """Convenience routine to add a styled horizontal line to a plot. @@ -703,11 +703,10 @@ def level_line( # TODO: pass this from symbol data digits=digits, opacity=1, - font_size=4, + font_size=font_size, bg_color='default', ) label.update_from_data(0, level) - # label._size_br_from_str( line = LevelLine( label, @@ -719,7 +718,6 @@ def level_line( # activate/draw label line.setValue(level) - # line.show() chart.plotItem.addItem(line) return line diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 8f1e9425..dba601d0 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -9,28 +9,6 @@ from ..log import get_logger log = get_logger(__name__) - -# def configure_font_to_dpi(screen: QtGui.QScreen): -# """Set an appropriately sized font size depending on the screen DPI. - -# If we end up needing to generalize this more here there are resources -# listed in the script in ``snippets/qt_screen_info.py``. - -# """ -# dpi = screen.physicalDotsPerInch() -# font_size = round(_font_inches_we_like * dpi) -# log.info( -# f"\nscreen:{screen.name()} with DPI: {dpi}" -# f"\nbest font size is {font_size}\n" -# ) - -# global _font -# _font.setPixelSize(font_size) -# _font._fm = QtGui.QFontMetrics(_font) - -# return _font - - # chart-wide font # font size 6px / 53 dpi (3x scaled down on 4k hidpi) _font_inches_we_like = 6 / 53 # px / (px / inch) = inch @@ -45,9 +23,9 @@ class DpiAwareFont: self._qfont = QtGui.QFont(name) self._iwl = _font_inches_we_like self._qfm = QtGui.QFontMetrics(self._qfont) - self._font_size = None self._physical_dpi = None self._screen = None + self._dpi_scalar = 1. def _set_qfont_px_size(self, px_size: int) -> None: # self._qfont = QtGui.Qfont(self.name) @@ -62,9 +40,6 @@ class DpiAwareFont: def px_size(self): return self._qfont.pixelSize() - # def set_px_size(self, size: int) -> None: - # pass - def configure_to_dpi(self, screen: QtGui.QScreen): """Set an appropriately sized font size depending on the screen DPI. @@ -79,11 +54,12 @@ class DpiAwareFont: f"\nbest font size is {font_size}\n" ) self._set_qfont_px_size(font_size) - self._font_size = font_size self._physical_dpi = dpi self._screen = screen def boundingRect(self, value: str) -> QtCore.QRectF: + # print(f'boundingRect STRING: {value}') + screen = self._screen if screen is None: raise RuntimeError("You must call .configure_to_dpi() first!") @@ -93,31 +69,34 @@ class DpiAwareFont: # XXX: for wtv absolutely fucked reason, the scaling only applies # to everything when the current font size **is not** the size # needed to get the original desired text height... :mindblow: - if self._font_size != 6: - # scalar = self._qfm.fontDpi() / self._physical_dpi - scalar = screen.logicalDotsPerInch() / screen.physicalDotsPerInch() - # scalar = 100 / screen.physicalDotsPerInch() - # assert 0 - print(f'SCALAR {scalar}') + + # if self.px_size != 6: + # # scalar = self._qfm.fontDpi() / self._physical_dpi + # # self._dpi_scalar = scalar = screen.logicalDotsPerInch() / screen.physicalDotsPerInch() + # # self._dpi_scalar = scalar = 96 / screen.physicalDotsPerInch() + # # # assert 0 + # # print(f'SCALAR {scalar}') + # # w = min(self._qfm.averageCharWidth() * len(value), unscaled_br.width()) - return QtCore.QRectF( - # unscaled_br.x(), - # unscaled_br.y(), - 0, - 0, - unscaled_br.width() * scalar, - unscaled_br.height() * scalar, - ) - else: - return QtCore.QRectF( - # unscaled_br.x(), - # unscaled_br.y(), - 0, - 0, - unscaled_br.width(), - unscaled_br.height(), - ) + # return QtCore.QRectF( + # # unscaled_br.x(), + # # unscaled_br.y(), + # 0, + # 0, + # # w * scalar, + # unscaled_br.width(), # * scalar, + # unscaled_br.height(),# * scalar, + # ) + # else: + return QtCore.QRectF( + # unscaled_br.x(), + # unscaled_br.y(), + 0, + 0, + unscaled_br.width(), + unscaled_br.height(), + ) # use pixel size to be cross-resolution compatible? _font = DpiAwareFont() From b23e459027973f3cfaf201a28798402313824158 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 2 Nov 2020 15:43:19 -0500 Subject: [PATCH 186/206] Cleanup unneeded commented stuff --- piker/ui/_axes.py | 46 +++++++++------------------------------------- piker/ui/_style.py | 27 --------------------------- 2 files changed, 9 insertions(+), 64 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 4e4bcb16..ac527fa8 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -25,24 +25,14 @@ class Axis(pg.AxisItem): **kwargs ) -> None: - self.linked_charts = linked_charts - super().__init__(**kwargs) + self.linked_charts = linked_charts self.setTickFont(_font.font) self.setStyle(**{ 'textFillLimits': [(0, 0.666)], 'tickFont': _font.font, - # 'tickTextWidth': 100, - # 'tickTextHeight': 20, - # 'tickTextWidth': 40, - # 'autoExpandTextSpace': True, - # 'maxTickLength': -20, - - # doesn't work well on price? - # 'stopAxisAtTick': (True, True), }) - # self.setLabel(**{'font-size': '10pt'}) self.setTickFont(_font.font) self.setPen(_axis_pen) @@ -127,6 +117,8 @@ class AxisLabel(pg.GraphicsObject): font_size: Optional[int] = None, ): super().__init__(parent) + self.setFlag(self.ItemIgnoresTransformations) + self.parent = parent self.opacity = opacity self.label_str = '' @@ -138,25 +130,14 @@ class AxisLabel(pg.GraphicsObject): self._dpifont.configure_to_dpi(_font._screen) if font_size is not None: - # print(f"SETTING FONT TO: {font_size}") self._dpifont._set_qfont_px_size(font_size) - # self._font._fm = QtGui.QFontMetrics(self._font) - self.bg_color = pg.mkColor(hcolor(bg_color)) self.fg_color = pg.mkColor(hcolor(fg_color)) - # self.pic = QtGui.QPicture() - # p = QtGui.QPainter(self.pic) - self.rect = None - # p.setPen(self.fg_color) - - self.setFlag(self.ItemIgnoresTransformations) - def paint(self, p, option, widget): - # p.drawPicture(0, 0, self.pic) # p.setCompositionMode(QtGui.QPainter.CompositionMode_SourceOver) if self.label_str: @@ -176,13 +157,13 @@ class AxisLabel(pg.GraphicsObject): p.drawText(self.rect, self.text_flags, self.label_str) def boundingRect(self): # noqa - if self.label_str: - self._size_br_from_str(self.label_str) - return self.rect + # if self.label_str: + # self._size_br_from_str(self.label_str) + # return self.rect - return QtCore.QRectF() + # return QtCore.QRectF() - # return self.rect or QtCore.QRectF() + return self.rect or QtCore.QRectF() def _size_br_from_str(self, value: str) -> None: """Do our best to render the bounding rect to a set margin @@ -192,16 +173,7 @@ class AxisLabel(pg.GraphicsObject): # size the filled rect to text and/or parent axis br = self._txt_br = self._dpifont.boundingRect(value) - # px_per_char = self._font._fm.averageCharWidth() - # br = br * 1.88 txt_h, txt_w = br.height(), br.width() - # print(f'orig: {txt_h}') - # txt_h = (br.topLeft() - br.bottomRight()).y() - # txt_w = len(value) * px_per_char - # txt_w *= 1.88 - # txt_h *= 1.88 - # print(f'calced: {txt_h}') - h, w = self.size_hint() self.rect = QtCore.QRectF( @@ -269,7 +241,7 @@ class YAxisLabel(AxisLabel): self, abs_pos: QPointF, # scene coords data: float, # data for text - offset: int = 1 # if have margins, k? + offset: int = 1 # on odd dimension and/or adds nice black line ) -> None: # this is read inside ``.paint()`` diff --git a/piker/ui/_style.py b/piker/ui/_style.py index dba601d0..773937eb 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -28,7 +28,6 @@ class DpiAwareFont: self._dpi_scalar = 1. def _set_qfont_px_size(self, px_size: int) -> None: - # self._qfont = QtGui.Qfont(self.name) self._qfont.setPixelSize(px_size) self._qfm = QtGui.QFontMetrics(self._qfont) @@ -58,7 +57,6 @@ class DpiAwareFont: self._screen = screen def boundingRect(self, value: str) -> QtCore.QRectF: - # print(f'boundingRect STRING: {value}') screen = self._screen if screen is None: @@ -66,32 +64,7 @@ class DpiAwareFont: unscaled_br = self._qfm.boundingRect(value) - # XXX: for wtv absolutely fucked reason, the scaling only applies - # to everything when the current font size **is not** the size - # needed to get the original desired text height... :mindblow: - - # if self.px_size != 6: - # # scalar = self._qfm.fontDpi() / self._physical_dpi - # # self._dpi_scalar = scalar = screen.logicalDotsPerInch() / screen.physicalDotsPerInch() - # # self._dpi_scalar = scalar = 96 / screen.physicalDotsPerInch() - # # # assert 0 - # # print(f'SCALAR {scalar}') - # # w = min(self._qfm.averageCharWidth() * len(value), unscaled_br.width()) - - - # return QtCore.QRectF( - # # unscaled_br.x(), - # # unscaled_br.y(), - # 0, - # 0, - # # w * scalar, - # unscaled_br.width(), # * scalar, - # unscaled_br.height(),# * scalar, - # ) - # else: return QtCore.QRectF( - # unscaled_br.x(), - # unscaled_br.y(), 0, 0, unscaled_br.width(), From 1640906b097b19be359b1b3bd27c6da4f6b37c84 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 08:14:37 -0500 Subject: [PATCH 187/206] Write shm volume (facepalm), update open=close on first volume --- piker/brokers/ib.py | 17 ++++++++++++++--- piker/brokers/kraken.py | 23 +++++++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index c2067665..94adb39e 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -645,15 +645,26 @@ async def stream_quotes( if not writer_already_exists: for tick in iterticks(quote, type='trade'): last = tick['price'] - # print(f'broker last: {tick}') # update last entry # benchmarked in the 4-5 us range - high, low = shm.array[-1][['high', 'low']] - shm.array[['high', 'low', 'close']][-1] = ( + o, high, low, v = shm.array[-1][ + ['open', 'high', 'low', 'volume'] + ] + + new_v = tick['size'] + + if v == 0 and new_v: + # no trades for this bar yet so the open + # is also the close/last trade price + o = last + + shm.array[['open', 'high', 'low', 'close', 'volume']][-1] = ( + o, max(high, last), min(low, last), last, + v + new_v, ) con = quote['contract'] diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 3d1d8e7a..21627c69 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -342,12 +342,31 @@ async def stream_quotes( if not writer_exists: # update last entry # benchmarked in the 4-5 us range - high, low = shm.array[-1][['high', 'low']] - shm.array[['high', 'low', 'close', 'vwap']][-1] = ( + o, high, low, v = shm.array[-1][ + ['open', 'high', 'low', 'volume'] + ] + new_v = tick_volume + + if v == 0 and new_v: + # no trades for this bar yet so the open + # is also the close/last trade price + o = last + + # write shm + shm.array[ + ['open', + 'high', + 'low', + 'close', + 'vwap', + 'volume'] + ][-1] = ( + o, max(high, last), min(low, last), last, ohlc.vwap, + volume, ) # XXX: format required by ``tractor.msg.pub`` From 22f1b56b369abc1e3dba48e3885320c188c8657f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 08:15:00 -0500 Subject: [PATCH 188/206] Always update left (open) arm --- piker/ui/_graphics.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 5281b1a6..280224f7 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -542,22 +542,25 @@ class BarItems(pg.GraphicsObject): return # current bar update - i, high, low, last, = array[-1][['index', 'high', 'low', 'close']] - assert i == self.index-1 + i, o, h, l, last, v = array[-1][ + ['index', 'open', 'high', 'low', 'close', 'volume'] + ] + assert i == self.index - 1 body, larm, rarm = self.lines[i] # XXX: is there a faster way to modify this? - # update close line / right arm rarm.setLine(rarm.x1(), last, rarm.x2(), last) + # writer is responsible for changing open on "first" volume of bar + larm.setLine(larm.x1(), o, larm.x2(), o) - if low != high: + if l != h: if body is None: - body = self.lines[index-1][0] = QLineF(i, low, i, high) + body = self.lines[index-1][0] = QLineF(i, l, i, h) else: # update body - body.setLine(i, low, i, high) + body.setLine(i, l, i, h) else: - # XXX: high == low -> remove any HL line to avoid render bug + # XXX: h == l -> remove any HL line to avoid render bug if body is not None: body = self.lines[index-1][0] = None From d92e02db862dcd8c0333abe59bf81338f1d901d1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 12:22:57 -0500 Subject: [PATCH 189/206] Add back min window size i guess --- piker/ui/_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index d9ac918e..55a703b3 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -48,7 +48,7 @@ class MainWindow(QtGui.QMainWindow): def __init__(self, parent=None): super().__init__(parent) - # self.setMinimumSize(*self.size) + self.setMinimumSize(*self.size) self.setWindowTitle(self.title) From e65f5116484147eeb5119a16a608e858a9d21b59 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 12:25:08 -0500 Subject: [PATCH 190/206] Draw flat line on every new time step Until we get a better datum "cursor" figured out just draw the flat bar despite the extra overhead. The reason to do this in 2 separate calls is detailed in the comment but basic gist is that there's a race between writer and reader of the last shm index. Oh, and toss in some draft symbol search label code. --- piker/ui/_chart.py | 81 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 268c8332..b324325b 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -53,18 +53,37 @@ class ChartSpace(QtGui.QWidget): """ def __init__(self, parent=None): super().__init__(parent) + self.v_layout = QtGui.QVBoxLayout(self) self.v_layout.setContentsMargins(0, 0, 0, 0) + self.v_layout.setSpacing(0) + self.toolbar_layout = QtGui.QHBoxLayout() - self.toolbar_layout.setContentsMargins(5, 5, 10, 0) + self.toolbar_layout.setContentsMargins(0, 0, 0, 0) + self.h_layout = QtGui.QHBoxLayout() + self.h_layout.setContentsMargins(0, 0, 0, 0) # self.init_timeframes_ui() # self.init_strategy_ui() - self.v_layout.addLayout(self.toolbar_layout) self.v_layout.addLayout(self.h_layout) self._chart_cache = {} + self.symbol_label: Optional[QtGui.QLabel] = None + + def init_search(self): + self.symbol_label = label = QtGui.QLabel() + label.setTextFormat(3) # markdown + label.setFont(_font.font) + label.setMargin(0) + # title = f'sym: {self.symbol}' + # label.setText(title) + + label.setAlignment( + QtCore.Qt.AlignVCenter + | QtCore.Qt.AlignLeft + ) + self.v_layout.addWidget(label) def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() @@ -95,6 +114,15 @@ class ChartSpace(QtGui.QWidget): """ # XXX: let's see if this causes mem problems self.window.setWindowTitle(f'piker chart {symbol}') + + # TODO: symbol search + # # of course this doesn't work :eyeroll: + # h = _font.boundingRect('Ag').height() + # print(f'HEIGHT {h}') + # self.symbol_label.setFixedHeight(h + 4) + # self.v_layout.update() + # self.symbol_label.setText(f'/`{symbol}`') + linkedcharts = self._chart_cache.setdefault( symbol, LinkedSplitCharts() @@ -102,11 +130,12 @@ class ChartSpace(QtGui.QWidget): s = Symbol(key=symbol) # remove any existing plots - if not self.h_layout.isEmpty(): - self.h_layout.removeWidget(linkedcharts) + if not self.v_layout.isEmpty(): + self.v_layout.removeWidget(linkedcharts) main_chart = linkedcharts.plot_main(s, data) - self.h_layout.addWidget(linkedcharts) + self.v_layout.addWidget(linkedcharts) + return linkedcharts, main_chart # TODO: add signalling painter system @@ -154,7 +183,7 @@ class LinkedSplitCharts(QtGui.QWidget): # self.xaxis.hide() self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical) - self.splitter.setMidLineWidth(3) + self.splitter.setMidLineWidth(2) self.splitter.setHandleWidth(0) self.layout = QtGui.QVBoxLayout(self) @@ -247,10 +276,10 @@ class LinkedSplitCharts(QtGui.QWidget): # graphics curve managed by the subchart cpw.name = name cpw.plotItem.vb.linked_charts = self - cpw.setFrameStyle(QtGui.QFrame.StyledPanel) # | QtGui.QFrame.Plain) - cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) cpw.hideButtons() + # XXX: gives us outline on backside of y-axis + cpw.getPlotItem().setContentsMargins(*CHART_MARGINS) # link chart x-axis to main quotes chart cpw.setXLink(self.chart) @@ -312,6 +341,7 @@ class ChartPlotWidget(pg.PlotWidget): # antialias=True, **kwargs ) + # self.setViewportMargins(0, 0, 0, 0) self._array = array # readonly view of data self._graphics = {} # registry of underlying graphics self._overlays = {} # registry of overlay curves @@ -662,6 +692,8 @@ async def _async_main( # attempt to configure DPI aware font size _font.configure_to_dpi(current_screen()) + # chart_app.init_search() + # from ._exec import get_screen # screen = get_screen(chart_app.geometry().bottomRight()) @@ -928,15 +960,36 @@ async def check_for_new_bars(feed, ohlcv, linked_charts): async for index in await feed.index_stream(): - # update chart historical bars graphics + # update chart historical bars graphics by incrementing + # a time step and drawing the history and new bar # When appending a new bar, in the time between the insert - # here and the Qt render call the underlying price data may - # have already been updated, thus make sure to also update - # the last bar if necessary on this render cycle which is - # why we **don't** set: just_history=True + # from the writing process and the Qt render call, here, + # the index of the shm buffer may be incremented and the + # (render) call here might read the new flat bar appended + # to the buffer (since -1 index read). In that case H==L and the + # body will be set as None (not drawn) on what this render call + # *thinks* is the curent bar (even though it's reading data from + # the newly inserted flat bar. + # + # HACK: We need to therefore write only the history (not the + # current bar) and then either write the current bar manually + # or place a cursor for visual cue of the current time step. + price_chart.update_ohlc_from_array( - price_chart.name, ohlcv.array, just_history=True) + price_chart.name, + ohlcv.array, + just_history=True, + ) + + # XXX: this puts a flat bar on the current time step + # TODO: if we eventually have an x-axis time-step "cursor" + # we can get rid of this since it is extra overhead. + price_chart.update_ohlc_from_array( + price_chart.name, + ohlcv.array, + just_history=False, + ) # resize view # price_chart._set_yrange() From a5d5208cfa50c2628a6a82ed516d72e01d2f325b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 16:21:21 -0500 Subject: [PATCH 191/206] Add sticky "orientation", allow overriding label draw instructions. --- piker/ui/_axes.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index ac527fa8..081254a8 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -150,12 +150,20 @@ class AxisLabel(pg.GraphicsObject): p.setOpacity(self.opacity) p.fillRect(self.rect, self.bg_color) - # this adds a nice black outline around the label for some odd - # reason; ok by us - p.drawRect(self.rect) + # can be overrided in subtype + self.draw(p, self.rect) p.drawText(self.rect, self.text_flags, self.label_str) + def draw( + self, + p: QtGui.QPainter, + rect: QtCore.QRectF + ) -> None: + # this adds a nice black outline around the label for some odd + # reason; ok by us + p.drawRect(self.rect) + def boundingRect(self): # noqa # if self.label_str: # self._size_br_from_str(self.label_str) @@ -263,14 +271,21 @@ class YSticky(YAxisLabel): self, chart, *args, + orient_v: str = 'bottom', + orient_h: str = 'left', **kwargs ) -> None: super().__init__(*args, **kwargs) + self._orient_v = orient_v + self._orient_h = orient_h + self._chart = chart chart.sigRangeChanged.connect(self.update_on_resize) self._last_datum = (None, None) + self._v_shift = {'top': 1., 'bottom': 0, 'middle': 1/2.}[orient_v] + self._h_shift = {'left': -1., 'right': 0}[orient_h] def update_on_resize(self, vr, r): # TODO: add an `.index` to the array data-buffer layer From 987c13c5841c363778ef028313e60d2ec67090ea Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 16:22:04 -0500 Subject: [PATCH 192/206] Classify L1 tick types --- piker/brokers/ib.py | 24 +++++++++++++++++++++--- piker/data/_normalize.py | 8 ++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 94adb39e..8d3fa183 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -285,7 +285,7 @@ class Client: self, symbol: str, to_trio, - opts: Tuple[int] = ('375',), # '233', ), + opts: Tuple[int] = ('375', '233',), # opts: Tuple[int] = ('459',), ) -> None: """Stream a ticker using the std L1 api. @@ -469,9 +469,27 @@ def normalize( for tick in ticker.ticks: td = tick._asdict() - if td['tickType'] in (48, 77): + if td['tickType'] in (77,): td['type'] = 'trade' + if td['tickType'] in (48,): + td['type'] = 'utrade' + + elif td['tickType'] in (0,): + td['type'] = 'bsize' + + elif td['tickType'] in (1,): + td['type'] = 'bid' + + elif td['tickType'] in (2,): + td['type'] = 'ask' + + elif td['tickType'] in (3,): + td['type'] = 'asize' + + elif td['tickType'] in (5,): + td['type'] = 'size' + new_ticks.append(td) ticker.ticks = new_ticks @@ -643,7 +661,7 @@ async def stream_quotes( # if we are the lone tick writer start writing # the buffer with appropriate trade data if not writer_already_exists: - for tick in iterticks(quote, type='trade'): + for tick in iterticks(quote, types=('trade', 'utrade',)): last = tick['price'] # update last entry diff --git a/piker/data/_normalize.py b/piker/data/_normalize.py index 9f73858d..e1120278 100644 --- a/piker/data/_normalize.py +++ b/piker/data/_normalize.py @@ -1,14 +1,14 @@ """ Stream format enforcement. """ -from typing import AsyncIterator, Optional +from typing import AsyncIterator, Optional, Tuple import numpy as np def iterticks( quote: dict, - type: str = 'trade', + types: Tuple[str] = ('trade', 'utrade'), ) -> AsyncIterator: """Iterate through ticks delivered per quote cycle. """ @@ -16,6 +16,6 @@ def iterticks( ticks = quote.get('ticks', ()) if ticks: for tick in ticks: - # print(tick) - if tick.get('type') == type: + print(f"{quote['symbol']}: {tick}") + if tick.get('type') in types: yield tick From 73e54a225952c83b6cbf499d8a11813110991651 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 19:48:27 -0500 Subject: [PATCH 193/206] Add L1 labels wrapper type Start a simple API for L1 bid/ask labels. Make `LevelLabel` draw a line above/below it's text (instead of the rect fill we had before) since it looks much simpler/slicker. Generalize the label text orientation through bounding rect geometry positioning. --- piker/ui/_graphics.py | 64 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 280224f7..5913bbbf 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -352,17 +352,17 @@ def bars_from_ohlc( open, high, low, close, index = q[ ['open', 'high', 'low', 'close', 'index']] - # high - low line + # high -> low vertical (body) line if low != high: hl = QLineF(index, low, index, high) else: # XXX: if we don't do it renders a weird rectangle? - # see below too for handling this later... + # see below for filtering this later... hl = None # NOTE: place the x-coord start as "middle" of the drawing range such # that the open arm line-graphic is at the left-most-side of - # the indexe's range according to the view mapping. + # the index's range according to the view mapping. # open line o = QLineF(index - w, open, index, open) @@ -647,7 +647,7 @@ class BarItems(pg.GraphicsObject): class LevelLabel(YSticky): - _w_margin = 3 + _w_margin = 4 _h_margin = 3 def update_label( @@ -669,13 +669,60 @@ class LevelLabel(YSticky): h, w = br.height(), br.width() self.setPos(QPointF( - -w, - abs_pos.y() - offset + self._h_shift * w - offset, + abs_pos.y() - (self._v_shift * h) - offset )) def size_hint(self) -> Tuple[None, None]: return None, None + def draw( + self, + p: QtGui.QPainter, + rect: QtCore.QRectF + ) -> None: + if self._orient_v == 'bottom': + p.drawLine(rect.topLeft(), rect.topRight()) + elif self._orient_v == 'top': + p.drawLine(rect.bottomLeft(), rect.bottomRight()) + + +class L1Labels: + """Level 1 bid ask labels for dynamic update on price-axis. + + """ + def __init__( + self, + chart: 'ChartPlotWidget', # noqa + digits: int = 2, + font_size: int = 4, + ) -> None: + self.chart = chart + + self.bid_label = LevelLabel( + chart=chart, + parent=chart.getAxis('right'), + # TODO: pass this from symbol data + digits=digits, + opacity=1, + font_size=font_size, + bg_color='papas_special', + fg_color='bracket', + orient_v='bottom', + ) + + self.ask_label = LevelLabel( + chart=chart, + parent=chart.getAxis('right'), + # TODO: pass this from symbol data + digits=digits, + opacity=1, + font_size=font_size, + bg_color='papas_special', + fg_color='bracket', + orient_v='top', + ) + class LevelLine(pg.InfiniteLine): def __init__( @@ -696,6 +743,7 @@ def level_line( level: float, digits: int = 1, font_size: int = 4, + **linelabelkwargs ) -> LevelLine: """Convenience routine to add a styled horizontal line to a plot. @@ -707,7 +755,9 @@ def level_line( digits=digits, opacity=1, font_size=font_size, - bg_color='default', + bg_color='papas_special', + fg_color='default', + **linelabelkwargs ) label.update_from_data(0, level) From f438139ad7213d949a946cc9cc519214024c9e66 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 3 Nov 2020 22:03:49 -0500 Subject: [PATCH 194/206] Update L1 labels in price loop --- piker/ui/_chart.py | 87 +++++++++++++++++++++++++++++++--------------- 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b324325b..2e5f98c5 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -19,6 +19,7 @@ from ._graphics import ( ContentsLabel, BarItems, level_line, + L1Labels, ) from ._axes import YSticky from ._style import ( @@ -504,7 +505,7 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: something instead of stickies for overlays # (we need something that avoids clutter on x-axis). - self._add_sticky(name) + self._add_sticky(name, bg_color='default_light') label = ContentsLabel(chart=self, anchor_at=anchor_at) self._labels[name] = (label, partial(label.update_from_value, name)) @@ -519,6 +520,7 @@ class ChartPlotWidget(pg.PlotWidget): def _add_sticky( self, name: str, + bg_color='bracket', # retreive: Callable[None, np.ndarray], ) -> YSticky: # add y-axis "last" value label @@ -528,6 +530,7 @@ class ChartPlotWidget(pg.PlotWidget): # TODO: pass this from symbol data # digits=0, opacity=1, + bg_color=bg_color, ) return last @@ -801,41 +804,69 @@ async def chart_from_quotes( last_bars_range, last_mx, last_mn = maxmin() chart.default_view() + l1 = L1Labels(chart) + + last_bid = last_ask = ohlcv.array[-1]['close'] async for quotes in stream: for sym, quote in quotes.items(): - for tick in iterticks(quote, type='trade'): - array = ohlcv.array + for tick in quote.get('ticks', ()): + # for tick in iterticks(quote, type='trade'): - # update price sticky(s) - last = array[-1] - last_price_sticky.update_from_data(*last[['index', 'close']]) + ticktype = tick.get('type') + price = tick.get('price') - # update price bar - chart.update_ohlc_from_array( - chart.name, - array, - ) + if ticktype in ('trade', 'utrade'): + array = ohlcv.array - # TODO: we need a streaming minmax algorithm here to - brange, mx, mn = maxmin() - if brange != last_bars_range: - if mx > last_mx or mn < last_mn: - # avoid running this every cycle. - chart._set_yrange() + # update price sticky(s) + last = array[-1] + last_price_sticky.update_from_data(*last[['index', 'close']]) - # save for next cycle - last_mx, last_mn = mx, mn + # update price bar + chart.update_ohlc_from_array( + chart.name, + array, + ) - if vwap_in_history: - # update vwap overlay line - chart.update_curve_from_array('vwap', ohlcv.array) + # TODO: we need a streaming minmax algorithm here to + brange, mx, mn = maxmin() + if brange != last_bars_range: + if mx > last_mx or mn < last_mn: + # avoid running this every cycle. + chart._set_yrange() - # TODO: - # - eventually we'll want to update bid/ask labels and - # other data as subscribed by underlying UI consumers. - # - in theory we should be able to read buffer data - # faster then msgs arrive.. needs some tinkering and testing + # save for next cycle + last_mx, last_mn = mx, mn + + if vwap_in_history: + # update vwap overlay line + chart.update_curve_from_array('vwap', ohlcv.array) + + # TODO: + # - eventually we'll want to update bid/ask labels and + # other data as subscribed by underlying UI consumers. + # - in theory we should be able to read buffer data + # faster then msgs arrive.. needs some tinkering and testing + + # if trade volume jumps above / below prior L1 price + # levels adjust bid / ask lines to match + if price > last_ask: + l1.ask_label.update_from_data(0, price) + last_ask = price + + elif price < last_bid: + l1.bid_label.update_from_data(0, price) + last_bid = price + + + elif ticktype == 'ask': + l1.ask_label.update_from_data(0, price) + last_ask = price + + elif ticktype == 'bid': + l1.bid_label.update_from_data(0, price) + last_bid = price async def chart_from_fsp( @@ -931,7 +962,7 @@ async def chart_from_fsp( # add moveable over-[sold/bought] lines level_line(chart, 30) - level_line(chart, 70) + level_line(chart, 70, orient_v='top') chart._shm = shm chart._set_yrange() From 9c3850874d8b5cffa650453ece32c1101dc9541f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Nov 2020 08:18:55 -0500 Subject: [PATCH 195/206] Add all L1 tick types for ib --- piker/brokers/ib.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 8d3fa183..2a1680ad 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -460,6 +460,20 @@ async def get_client( yield get_method_proxy(portal, Client) +# https://interactivebrokers.github.io/tws-api/tick_types.html +tick_types = { + 77: 'trade', + 48: 'utrade', + 0: 'bsize', + 1: 'bid', + 2: 'ask', + 3: 'asize', + 4: 'last', + 5: 'size', + 8: 'volume', +} + + def normalize( ticker: Ticker, calc_price: bool = False @@ -468,27 +482,7 @@ def normalize( new_ticks = [] for tick in ticker.ticks: td = tick._asdict() - - if td['tickType'] in (77,): - td['type'] = 'trade' - - if td['tickType'] in (48,): - td['type'] = 'utrade' - - elif td['tickType'] in (0,): - td['type'] = 'bsize' - - elif td['tickType'] in (1,): - td['type'] = 'bid' - - elif td['tickType'] in (2,): - td['type'] = 'ask' - - elif td['tickType'] in (3,): - td['type'] = 'asize' - - elif td['tickType'] in (5,): - td['type'] = 'size' + td['type'] = tick_types.get(td['tickType'], 'n/a') new_ticks.append(td) @@ -507,7 +501,7 @@ def normalize( # add time stamps for downstream latency measurements data['brokerd_ts'] = time.time() - # stupid stupid shit...don't even care any more + # stupid stupid shit...don't even care any more.. # leave it until we do a proper latency study # if ticker.rtTime is not None: # data['broker_ts'] = data['rtTime_s'] = float( From db075b81ac6875b2ae0272c42ed551c5f8e1a427 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Nov 2020 12:08:02 -0500 Subject: [PATCH 196/206] Specialize `LevelLabel` for orientation-around-axis gymnastics --- piker/ui/_axes.py | 31 ++++++------- piker/ui/_graphics.py | 102 ++++++++++++++++++++++++++++++++++-------- 2 files changed, 96 insertions(+), 37 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index 081254a8..b508cb58 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -71,7 +71,7 @@ class DynamicDateAxis(Axis): # time formats mapped by seconds between bars tick_tpl = { - 60*60*24: '%Y-%b-%d', + 60 * 60 * 24: '%Y-%b-%d', 60: '%H:%M', 30: '%H:%M:%S', 5: '%H:%M:%S', @@ -113,7 +113,7 @@ class AxisLabel(pg.GraphicsObject): digits: int = 2, bg_color: str = 'bracket', fg_color: str = 'black', - opacity: int = 1, + opacity: int = 0, font_size: Optional[int] = None, ): super().__init__(parent) @@ -162,6 +162,7 @@ class AxisLabel(pg.GraphicsObject): ) -> None: # this adds a nice black outline around the label for some odd # reason; ok by us + p.setOpacity(self.opacity) p.drawRect(self.rect) def boundingRect(self): # noqa @@ -214,11 +215,11 @@ class XAxisLabel(AxisLabel): def update_label( self, abs_pos: QPointF, # scene coords - data: float, # data for text + value: float, # data for text offset: int = 1 # if have margins, k? ) -> None: - timestrs = self.parent._indexes_to_timestrs([int(data)]) + timestrs = self.parent._indexes_to_timestrs([int(value)]) if not timestrs.any(): return @@ -230,6 +231,7 @@ class XAxisLabel(AxisLabel): abs_pos.x() - w / 2 - offset, 0, )) + self.update() class YAxisLabel(AxisLabel): @@ -248,13 +250,13 @@ class YAxisLabel(AxisLabel): def update_label( self, abs_pos: QPointF, # scene coords - data: float, # data for text + value: float, # data for text offset: int = 1 # on odd dimension and/or adds nice black line ) -> None: # this is read inside ``.paint()`` - self.label_str = '{data: ,.{digits}f}'.format( - digits=self.digits, data=data).replace(',', ' ') + self.label_str = '{value: ,.{digits}f}'.format( + digits=self.digits, value=value).replace(',', ' ') br = self.boundingRect() h = br.height() @@ -262,6 +264,7 @@ class YAxisLabel(AxisLabel): 0, abs_pos.y() - h / 2 - offset )) + self.update() class YSticky(YAxisLabel): @@ -271,31 +274,23 @@ class YSticky(YAxisLabel): self, chart, *args, - orient_v: str = 'bottom', - orient_h: str = 'left', **kwargs ) -> None: super().__init__(*args, **kwargs) - self._orient_v = orient_v - self._orient_h = orient_h - self._chart = chart chart.sigRangeChanged.connect(self.update_on_resize) self._last_datum = (None, None) - self._v_shift = {'top': 1., 'bottom': 0, 'middle': 1/2.}[orient_v] - self._h_shift = {'left': -1., 'right': 0}[orient_h] def update_on_resize(self, vr, r): # TODO: add an `.index` to the array data-buffer layer # and make this way less shitty... + + # pretty sure we did that ^ ? index, last = self._last_datum if index is not None: - self.update_from_data( - index, - last, - ) + self.update_from_data(index, last) def update_from_data( self, diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index 5913bbbf..f2172c6d 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -15,8 +15,11 @@ from ._style import _xaxis_at, hcolor, _font from ._axes import YAxisLabel, XAxisLabel, YSticky +# XXX: these settings seem to result in really decent mouse scroll +# latency (in terms of perceived lag in cross hair) so really be sure +# there's an improvement if you want to change it. _mouse_rate_limit = 60 # calc current screen refresh rate? -_debounce_delay = 1/2e3 +_debounce_delay = 1 / 2e3 _ch_label_opac = 1 @@ -45,7 +48,7 @@ class LineDot(pg.CurvePoint): # presuming this is fast since it's built in? dot = self.dot = QtGui.QGraphicsEllipseItem( - QtCore.QRectF(-size/2, -size/2, size, size) + QtCore.QRectF(-size / 2, -size / 2, size, size) ) # if we needed transformable dot? # dot.translate(-size*0.5, -size*0.5) @@ -266,7 +269,7 @@ class CrossHair(pg.GraphicsObject): self.graphics[plot]['hl'].setY(y) self.graphics[self.active_plot]['yl'].update_label( - abs_pos=pos, data=y + abs_pos=pos, value=y ) # Update x if cursor changed after discretization calc @@ -296,7 +299,7 @@ class CrossHair(pg.GraphicsObject): # map back to abs (label-local) coordinates abs_pos=plot.mapFromView(QPointF(ix, y)), - data=x, + value=x, ) self._lastx = ix @@ -553,16 +556,16 @@ class BarItems(pg.GraphicsObject): # writer is responsible for changing open on "first" volume of bar larm.setLine(larm.x1(), o, larm.x2(), o) - if l != h: + if l != h: # noqa if body is None: - body = self.lines[index-1][0] = QLineF(i, l, i, h) + body = self.lines[index - 1][0] = QLineF(i, l, i, h) else: # update body body.setLine(i, l, i, h) else: # XXX: h == l -> remove any HL line to avoid render bug if body is not None: - body = self.lines[index-1][0] = None + body = self.lines[index - 1][0] = None self.draw_lines(just_history=False) @@ -649,29 +652,61 @@ class LevelLabel(YSticky): _w_margin = 4 _h_margin = 3 + level: float = 0 + + def __init__( + self, + chart, + *args, + orient_v: str = 'bottom', + orient_h: str = 'left', + **kwargs + ) -> None: + super().__init__(chart, *args, **kwargs) + + # orientation around axis options + self._orient_v = orient_v + self._orient_h = orient_h + self._v_shift = { + 'top': 1., + 'bottom': 0, + 'middle': 1 / 2. + }[orient_v] + + self._h_shift = { + 'left': -1., 'right': 0 + }[orient_h] def update_label( self, abs_pos: QPointF, # scene coords - data: float, # data for text + level: float, # data for text offset: int = 1 # if have margins, k? ) -> None: - # this is read inside ``.paint()`` - self.label_str = '{data: ,.{digits}f}'.format( - digits=self.digits, - data=data - ).replace(',', ' ') - - self._size_br_from_str(self.label_str) + # write contents, type specific + self.set_label_str(level) br = self.boundingRect() h, w = br.height(), br.width() + # this triggers ``.pain()`` implicitly? self.setPos(QPointF( self._h_shift * w - offset, abs_pos.y() - (self._v_shift * h) - offset )) + self.update() + + self.level = level + + def set_label_str(self, level: float): + # this is read inside ``.paint()`` + # self.label_str = '{size} x {level:.{digits}f}'.format( + self.label_str = '{level:.{digits}f}'.format( + # size=self._size, + digits=self.digits, + level=level + ).replace(',', ' ') def size_hint(self) -> Tuple[None, None]: return None, None @@ -687,19 +722,43 @@ class LevelLabel(YSticky): p.drawLine(rect.bottomLeft(), rect.bottomRight()) +class L1Label(LevelLabel): + + size: float = 0 + + text_flags = ( + QtCore.Qt.TextDontClip + | QtCore.Qt.AlignLeft + ) + + def set_label_str(self, level: float) -> None: + """Reimplement the label string write to include the level's order-queue's + size in the text, eg. 100 x 323.3. + + """ + self.label_str = '{size} x {level:,.{digits}f}'.format( + size=self.size or '?', + digits=self.digits, + level=level + ).replace(',', ' ') + + class L1Labels: """Level 1 bid ask labels for dynamic update on price-axis. """ + max_value: float = '100 x 100 000' + def __init__( self, chart: 'ChartPlotWidget', # noqa + # level: float, digits: int = 2, font_size: int = 4, ) -> None: self.chart = chart - self.bid_label = LevelLabel( + self.bid_label = L1Label( chart=chart, parent=chart.getAxis('right'), # TODO: pass this from symbol data @@ -710,8 +769,9 @@ class L1Labels: fg_color='bracket', orient_v='bottom', ) + self.bid_label._size_br_from_str(self.max_value) - self.ask_label = LevelLabel( + self.ask_label = L1Label( chart=chart, parent=chart.getAxis('right'), # TODO: pass this from symbol data @@ -722,6 +782,7 @@ class L1Labels: fg_color='bracket', orient_v='top', ) + self.ask_label._size_br_from_str(self.max_value) class LevelLine(pg.InfiniteLine): @@ -732,9 +793,9 @@ class LevelLine(pg.InfiniteLine): ) -> None: self.label = label super().__init__(**kwargs) - self.sigPositionChanged.connect(self.set_value) + self.sigPositionChanged.connect(self.set_level) - def set_value(self, value: float) -> None: + def set_level(self, value: float) -> None: self.label.update_from_data(0, self.value()) @@ -755,11 +816,14 @@ def level_line( digits=digits, opacity=1, font_size=font_size, + # TODO: make this take the view's bg pen bg_color='papas_special', fg_color='default', **linelabelkwargs ) label.update_from_data(0, level) + # TODO: can we somehow figure out a max value from the parent axis? + label._size_br_from_str(label.label_str) line = LevelLine( label, From cc88300ac50c9c2a6fe387ec969553cc4a6a0b34 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Nov 2020 12:08:29 -0500 Subject: [PATCH 197/206] Fix L1 updates to be like TWS I think this gets us to the same output as TWS both on booktrader and the quote details pane. In theory there might be logic needed to decreases an L1 queue size on trades but can't seem to get it without getting -ves displayed occasionally - thus leaving it for now. Also, fix the max-min streaming logic to actually do its job, lel. --- piker/ui/_chart.py | 104 +++++++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 37 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 2e5f98c5..b0859aac 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -34,10 +34,7 @@ from ._style import ( from ..data._source import Symbol from .. import brokers from .. import data -from ..data import ( - iterticks, - maybe_open_shm_array, -) +from ..data import maybe_open_shm_array from ..log import get_logger from ._exec import run_qtractor, current_screen from ._interaction import ChartView @@ -583,7 +580,12 @@ class ChartPlotWidget(pg.PlotWidget): if self._static_yrange is not None: ylow, yhigh = self._static_yrange - else: # determine max, min y values in viewable x-range + elif yrange is not None: + ylow, yhigh = yrange + + else: + # Determine max, min y values in viewable x-range from data. + # Make sure min bars/datums on screen is adhered. l, lbar, rbar, r = self.bars_range() @@ -794,6 +796,10 @@ async def chart_from_quotes( last_price_sticky = chart._ysticks[chart.name] def maxmin(): + # TODO: implement this + # https://arxiv.org/abs/cs/0610046 + # https://github.com/lemire/pythonmaxmin + array = chart._array last_bars_range = chart.bars_range() l, lbar, rbar, r = last_bars_range @@ -804,24 +810,27 @@ async def chart_from_quotes( last_bars_range, last_mx, last_mn = maxmin() chart.default_view() - l1 = L1Labels(chart) - last_bid = last_ask = ohlcv.array[-1]['close'] + # last_bid = last_ask = ohlcv.array[-1]['close'] + l1 = L1Labels(chart) async for quotes in stream: for sym, quote in quotes.items(): for tick in quote.get('ticks', ()): - # for tick in iterticks(quote, type='trade'): + # print(f"CHART: {quote['symbol']}: {tick}") ticktype = tick.get('type') price = tick.get('price') + size = tick.get('size') if ticktype in ('trade', 'utrade'): array = ohlcv.array # update price sticky(s) last = array[-1] - last_price_sticky.update_from_data(*last[['index', 'close']]) + last_price_sticky.update_from_data( + *last[['index', 'close']] + ) # update price bar chart.update_ohlc_from_array( @@ -829,44 +838,65 @@ async def chart_from_quotes( array, ) - # TODO: we need a streaming minmax algorithm here to - brange, mx, mn = maxmin() - if brange != last_bars_range: - if mx > last_mx or mn < last_mn: - # avoid running this every cycle. - chart._set_yrange() - - # save for next cycle - last_mx, last_mn = mx, mn - if vwap_in_history: # update vwap overlay line chart.update_curve_from_array('vwap', ohlcv.array) # TODO: - # - eventually we'll want to update bid/ask labels and - # other data as subscribed by underlying UI consumers. - # - in theory we should be able to read buffer data - # faster then msgs arrive.. needs some tinkering and testing + # - eventually we'll want to update bid/ask labels + # and other data as subscribed by underlying UI + # consumers. + # - in theory we should be able to read buffer data faster + # then msgs arrive.. needs some tinkering and testing # if trade volume jumps above / below prior L1 price # levels adjust bid / ask lines to match - if price > last_ask: - l1.ask_label.update_from_data(0, price) - last_ask = price - elif price < last_bid: - l1.bid_label.update_from_data(0, price) - last_bid = price + # compute max and min trade values to display in view + # TODO: we need a streaming minmax algorithm here, see + # def above. + brange, mx_in_view, mn_in_view = maxmin() + # XXX: prettty sure this is correct? + # if ticktype in ('trade', 'last'): + if ticktype in ('last',): # 'size'): - elif ticktype == 'ask': + label = { + l1.ask_label.level: l1.ask_label, + l1.bid_label.level: l1.bid_label, + }.get(price) + + if label is not None: + label.size = size + label.update_from_data(0, price) + + # on trades should we be knocking down + # the relevant L1 queue? + # label.size -= size + + elif ticktype in ('ask', 'asize'): + l1.ask_label.size = size l1.ask_label.update_from_data(0, price) - last_ask = price - elif ticktype == 'bid': + # update max price in view to keep ask on screen + mx_in_view = max(price, mx_in_view) + + elif ticktype in ('bid', 'bsize'): + l1.bid_label.size = size l1.bid_label.update_from_data(0, price) - last_bid = price + + # update min price in view to keep bid on screen + mn_in_view = max(price, mn_in_view) + + if mx_in_view > last_mx or mn_in_view < last_mn: + chart._set_yrange(yrange=(mn_in_view, mx_in_view)) + last_mx, last_mn = mx_in_view, mn_in_view + + if brange != last_bars_range: + # we **must always** update the last values due to + # the x-range change + last_mx, last_mn = mx_in_view, mn_in_view + last_bars_range = brange async def chart_from_fsp( @@ -1049,8 +1079,8 @@ def _main( """ # Qt entry point run_qtractor( - func=_async_main, - args=(sym, brokername), - main_widget=ChartSpace, - tractor_kwargs=tractor_kwargs, + func=_async_main, + args=(sym, brokername), + main_widget=ChartSpace, + tractor_kwargs=tractor_kwargs, ) From 1e491fb1bbf2ac76e678165929822997b312dd6d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Nov 2020 13:23:29 -0500 Subject: [PATCH 198/206] Use pyqtgraph default pen for level lines --- piker/ui/_graphics.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index f2172c6d..f881e136 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -650,6 +650,8 @@ class BarItems(pg.GraphicsObject): class LevelLabel(YSticky): + line_pen = pg.mkPen(hcolor('bracket')) + _w_margin = 4 _h_margin = 3 level: float = 0 @@ -716,10 +718,15 @@ class LevelLabel(YSticky): p: QtGui.QPainter, rect: QtCore.QRectF ) -> None: + p.setPen(self.line_pen) + if self._orient_v == 'bottom': - p.drawLine(rect.topLeft(), rect.topRight()) + lp, rp = rect.topLeft(), rect.topRight() + # p.drawLine(rect.topLeft(), rect.topRight()) elif self._orient_v == 'top': - p.drawLine(rect.bottomLeft(), rect.bottomRight()) + lp, rp = rect.bottomLeft(), rect.bottomRight() + + p.drawLine(lp.x(), lp.y(), rp.x(), rp.y()) class L1Label(LevelLabel): From 205bedce85b33e013fec7e510ca2f8da9fda8590 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 5 Nov 2020 20:32:35 -0500 Subject: [PATCH 199/206] Spec dpi aware font size in inches --- piker/ui/_axes.py | 7 ++----- piker/ui/_graphics.py | 21 ++++++++++++++------- piker/ui/_style.py | 11 ++++++++--- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index b508cb58..f567c877 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -114,7 +114,7 @@ class AxisLabel(pg.GraphicsObject): bg_color: str = 'bracket', fg_color: str = 'black', opacity: int = 0, - font_size: Optional[int] = None, + font_size_inches: Optional[float] = None, ): super().__init__(parent) self.setFlag(self.ItemIgnoresTransformations) @@ -126,12 +126,9 @@ class AxisLabel(pg.GraphicsObject): self._txt_br: QtCore.QRect = None - self._dpifont = DpiAwareFont() + self._dpifont = DpiAwareFont(size_in_inches=font_size_inches) self._dpifont.configure_to_dpi(_font._screen) - if font_size is not None: - self._dpifont._set_qfont_px_size(font_size) - self.bg_color = pg.mkColor(hcolor(bg_color)) self.fg_color = pg.mkColor(hcolor(fg_color)) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index f881e136..c8dd1632 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -11,7 +11,11 @@ from PyQt5 import QtCore, QtGui from PyQt5.QtCore import QLineF, QPointF # from .._profile import timeit -from ._style import _xaxis_at, hcolor, _font +from ._style import ( + _xaxis_at, + hcolor, + _font, +) from ._axes import YAxisLabel, XAxisLabel, YSticky @@ -759,10 +763,10 @@ class L1Labels: def __init__( self, chart: 'ChartPlotWidget', # noqa - # level: float, digits: int = 2, - font_size: int = 4, + font_size_inches: float = 4 / 53., ) -> None: + self.chart = chart self.bid_label = L1Label( @@ -771,7 +775,7 @@ class L1Labels: # TODO: pass this from symbol data digits=digits, opacity=1, - font_size=font_size, + font_size_inches=font_size_inches, bg_color='papas_special', fg_color='bracket', orient_v='bottom', @@ -784,7 +788,7 @@ class L1Labels: # TODO: pass this from symbol data digits=digits, opacity=1, - font_size=font_size, + font_size_inches=font_size_inches, bg_color='papas_special', fg_color='bracket', orient_v='top', @@ -810,7 +814,10 @@ def level_line( chart: 'ChartPlogWidget', # noqa level: float, digits: int = 1, - font_size: int = 4, + + # size 4 font on 4k screen scaled down, so small-ish. + font_size_inches: float = 4 / 53., + **linelabelkwargs ) -> LevelLine: """Convenience routine to add a styled horizontal line to a plot. @@ -822,7 +829,7 @@ def level_line( # TODO: pass this from symbol data digits=digits, opacity=1, - font_size=font_size, + font_size_inches=font_size_inches, # TODO: make this take the view's bg pen bg_color='papas_special', fg_color='default', diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 773937eb..98db4bb2 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -1,6 +1,8 @@ """ Qt UI styling. """ +from typing import Optional + import pyqtgraph as pg from PyQt5 import QtCore, QtGui from qdarkstyle.palette import DarkPalette @@ -11,17 +13,19 @@ log = get_logger(__name__) # chart-wide font # font size 6px / 53 dpi (3x scaled down on 4k hidpi) -_font_inches_we_like = 6 / 53 # px / (px / inch) = inch +_default_font_inches_we_like = 6 / 53 # px / (px / inch) = inch +_down_2_font_inches_we_like = 4 / 53 class DpiAwareFont: def __init__( self, name: str = 'Hack', + size_in_inches: Optional[float] = None, ) -> None: self.name = name self._qfont = QtGui.QFont(name) - self._iwl = _font_inches_we_like + self._iwl = size_in_inches or _default_font_inches_we_like self._qfm = QtGui.QFontMetrics(self._qfont) self._physical_dpi = None self._screen = None @@ -71,7 +75,8 @@ class DpiAwareFont: unscaled_br.height(), ) -# use pixel size to be cross-resolution compatible? + +# use inches size to be cross-resolution compatible? _font = DpiAwareFont() # TODO: re-compute font size when main widget switches screens? From e27fece4e6ce72d704040b0ef94fd10bdadd5369 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 11:34:32 -0500 Subject: [PATCH 200/206] Add L1 queue size precision controls --- piker/ui/_graphics.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index c8dd1632..b8dff5d4 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -652,6 +652,7 @@ class BarItems(pg.GraphicsObject): # p.setBrush(self.bear_brush) # p.drawRects(*rects[Quotes.close < Quotes.open]) + class LevelLabel(YSticky): line_pen = pg.mkPen(hcolor('bracket')) @@ -736,6 +737,7 @@ class LevelLabel(YSticky): class L1Label(LevelLabel): size: float = 0 + size_digits: float = 3 text_flags = ( QtCore.Qt.TextDontClip @@ -747,7 +749,8 @@ class L1Label(LevelLabel): size in the text, eg. 100 x 323.3. """ - self.label_str = '{size} x {level:,.{digits}f}'.format( + self.label_str = '{size:.{size_digits}f} x {level:,.{digits}f}'.format( + size_digits=self.size_digits, size=self.size or '?', digits=self.digits, level=level @@ -758,12 +761,13 @@ class L1Labels: """Level 1 bid ask labels for dynamic update on price-axis. """ - max_value: float = '100 x 100 000' + max_value: float = '100.0 x 100 000.00' def __init__( self, chart: 'ChartPlotWidget', # noqa digits: int = 2, + size_digits: int = 0, font_size_inches: float = 4 / 53., ) -> None: @@ -780,6 +784,7 @@ class L1Labels: fg_color='bracket', orient_v='bottom', ) + self.bid_label.size_digits = size_digits self.bid_label._size_br_from_str(self.max_value) self.ask_label = L1Label( @@ -793,6 +798,7 @@ class L1Labels: fg_color='bracket', orient_v='top', ) + self.ask_label.size_digits = size_digits self.ask_label._size_br_from_str(self.max_value) From 043bc985df6de9dbb8ba966920fb14349973f4b2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 11:35:10 -0500 Subject: [PATCH 201/206] Configure L1 queue size precisions from history --- piker/ui/_chart.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index b0859aac..004e6b8f 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -31,7 +31,7 @@ from ._style import ( _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, ) -from ..data._source import Symbol +from ..data._source import Symbol, float_digits from .. import brokers from .. import data from ..data import maybe_open_shm_array @@ -811,11 +811,19 @@ async def chart_from_quotes( chart.default_view() - # last_bid = last_ask = ohlcv.array[-1]['close'] - l1 = L1Labels(chart) + last, volume = ohlcv.array[-1][['close', 'volume']] + + l1 = L1Labels( + chart, + # determine precision/decimal lengths + digits=max(float_digits(last), 2), + size_digits=min(float_digits(volume), 3) + ) async for quotes in stream: for sym, quote in quotes.items(): + # print(f'CHART: {quote}') + for tick in quote.get('ticks', ()): # print(f"CHART: {quote['symbol']}: {tick}") From be4a3df7ba5db6c034b2f6569c11bb36befae551 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 11:35:40 -0500 Subject: [PATCH 202/206] Add L1 spread streaming to kraken --- piker/brokers/kraken.py | 196 ++++++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 67 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 21627c69..90bd6476 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -176,8 +176,9 @@ class OHLC: setattr(self, f, val.type(getattr(self, f))) -async def recv_ohlc(recv): +async def recv_msg(recv): too_slow_count = last_hb = 0 + while True: with trio.move_on_after(1.5) as cs: msg = await recv() @@ -194,20 +195,50 @@ async def recv_ohlc(recv): if isinstance(msg, dict): if msg.get('event') == 'heartbeat': + now = time.time() delay = now - last_hb last_hb = now log.trace(f"Heartbeat after {delay}") + # TODO: hmm i guess we should use this # for determining when to do connection # resets eh? continue + err = msg.get('errorMessage') if err: raise BrokerError(err) else: - chan_id, ohlc_array, chan_name, pair = msg - yield OHLC(chan_id, chan_name, pair, *ohlc_array) + chan_id, *payload_array, chan_name, pair = msg + + if 'ohlc' in chan_name: + + yield 'ohlc', OHLC(chan_id, chan_name, pair, *payload_array[0]) + + elif 'spread' in chan_name: + + bid, ask, ts, bsize, asize = map(float, payload_array[0]) + + # TODO: really makes you think IB has a horrible API... + quote = { + 'symbol': pair.replace('/', ''), + 'ticks': [ + {'type': 'bid', 'price': bid, 'size': bsize}, + {'type': 'bsize', 'price': bid, 'size': bsize}, + + {'type': 'ask', 'price': ask, 'size': asize}, + {'type': 'asize', 'price': ask, 'size': asize}, + ], + } + yield 'l1', quote + + # elif 'book' in msg[-2]: + # chan_id, *payload_array, chan_name, pair = msg + # print(msg) + + else: + print(f'UNHANDLED MSG: {msg}') def normalize( @@ -226,6 +257,21 @@ def normalize( return topic, quote +def make_sub(pairs: List[str], data: Dict[str, Any]) -> Dict[str, str]: + """Create a request subscription packet dict. + + https://docs.kraken.com/websockets/#message-subscribe + + """ + # eg. specific logic for this in kraken's sync client: + # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188 + return { + 'pair': pairs, + 'event': 'subscribe', + 'subscription': data, + } + + # @tractor.msg.pub async def stream_quotes( # get_topics: Callable, @@ -247,6 +293,7 @@ async def stream_quotes( ws_pairs = {} async with get_client() as client: + # keep client cached for real-time section for sym in symbols: ws_pairs[sym] = (await client.symbol_info(sym))['wsname'] @@ -280,31 +327,36 @@ async def stream_quotes( async with trio_websocket.open_websocket_url( 'wss://ws.kraken.com', ) as ws: - # setup subs + + # XXX: setup subs # https://docs.kraken.com/websockets/#message-subscribe - subs = { - 'pair': list(ws_pairs.values()), - 'event': 'subscribe', - 'subscription': { - 'name': sub_type, - 'interval': 1, # 1 min - # 'name': 'ticker', - # 'name': 'openOrders', - # 'depth': '25', - }, - } + # specific logic for this in kraken's shitty sync client: + # https://github.com/krakenfx/kraken-wsclient-py/blob/master/kraken_wsclient_py/kraken_wsclient_py.py#L188 + ohlc_sub = make_sub( + list(ws_pairs.values()), + {'name': 'ohlc', 'interval': 1} + ) + # TODO: we want to eventually allow unsubs which should # be completely fine to request from a separate task # since internally the ws methods appear to be FIFO # locked. - await ws.send_message(json.dumps(subs)) + await ws.send_message(json.dumps(ohlc_sub)) + + # trade data (aka L1) + l1_sub = make_sub( + list(ws_pairs.values()), + {'name': 'spread'} # 'depth': 10} + + ) + await ws.send_message(json.dumps(l1_sub)) async def recv(): return json.loads(await ws.get_message()) # pull a first quote and deliver - ohlc_gen = recv_ohlc(recv) - ohlc_last = await ohlc_gen.__anext__() + msg_gen = recv_msg(recv) + typ, ohlc_last = await msg_gen.__anext__() topic, quote = normalize(ohlc_last) @@ -315,65 +367,75 @@ async def stream_quotes( last_interval_start = ohlc_last.etime # start streaming - async for ohlc in ohlc_gen: + async for typ, ohlc in msg_gen: - # generate tick values to match time & sales pane: - # https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m - volume = ohlc.volume - if ohlc.etime > last_interval_start: # new interval - last_interval_start = ohlc.etime - tick_volume = volume - else: - # this is the tick volume *within the interval* - tick_volume = volume - ohlc_last.volume + if typ == 'ohlc': - last = ohlc.close - if tick_volume: - ohlc.ticks.append({ - 'type': 'trade', - 'price': last, - 'size': tick_volume, - }) + # TODO: can get rid of all this by using + # ``trades`` subscription... - topic, quote = normalize(ohlc) + # generate tick values to match time & sales pane: + # https://trade.kraken.com/charts/KRAKEN:BTC-USD?period=1m + volume = ohlc.volume - # if we are the lone tick writer start writing - # the buffer with appropriate trade data - if not writer_exists: - # update last entry - # benchmarked in the 4-5 us range - o, high, low, v = shm.array[-1][ - ['open', 'high', 'low', 'volume'] - ] - new_v = tick_volume + # new interval + if ohlc.etime > last_interval_start: + last_interval_start = ohlc.etime + tick_volume = volume + else: + # this is the tick volume *within the interval* + tick_volume = volume - ohlc_last.volume - if v == 0 and new_v: - # no trades for this bar yet so the open - # is also the close/last trade price - o = last + last = ohlc.close + if tick_volume: + ohlc.ticks.append({ + 'type': 'trade', + 'price': last, + 'size': tick_volume, + }) - # write shm - shm.array[ - ['open', - 'high', - 'low', - 'close', - 'vwap', - 'volume'] - ][-1] = ( - o, - max(high, last), - min(low, last), - last, - ohlc.vwap, - volume, - ) + topic, quote = normalize(ohlc) + + # if we are the lone tick writer start writing + # the buffer with appropriate trade data + if not writer_exists: + # update last entry + # benchmarked in the 4-5 us range + o, high, low, v = shm.array[-1][ + ['open', 'high', 'low', 'volume'] + ] + new_v = tick_volume + + if v == 0 and new_v: + # no trades for this bar yet so the open + # is also the close/last trade price + o = last + + # write shm + shm.array[ + ['open', + 'high', + 'low', + 'close', + 'vwap', + 'volume'] + ][-1] = ( + o, + max(high, last), + min(low, last), + last, + ohlc.vwap, + volume, + ) + ohlc_last = ohlc + + elif typ == 'l1': + quote = ohlc + topic = quote['symbol'] # XXX: format required by ``tractor.msg.pub`` # requires a ``Dict[topic: str, quote: dict]`` yield {topic: quote} - ohlc_last = ohlc - except (ConnectionClosed, DisconnectionTimeout): log.exception("Good job kraken...reconnecting") From c1109ee3fb848962226c8570c7033f16934dd306 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 12:23:14 -0500 Subject: [PATCH 203/206] Add license headers to pertinent files --- piker/_async_utils.py | 16 ++++++++++++++++ piker/_profile.py | 15 +++++++++++++++ piker/brokers/__init__.py | 16 ++++++++++++++++ piker/brokers/_util.py | 16 ++++++++++++++++ piker/brokers/api.py | 16 ++++++++++++++++ piker/brokers/cli.py | 16 ++++++++++++++++ piker/brokers/config.py | 16 ++++++++++++++++ piker/brokers/core.py | 16 ++++++++++++++++ piker/brokers/data.py | 16 ++++++++++++++++ piker/brokers/ib.py | 18 +++++++++++++++++- piker/brokers/kraken.py | 16 ++++++++++++++++ piker/brokers/questrade.py | 16 ++++++++++++++++ piker/brokers/robinhood.py | 16 ++++++++++++++++ piker/calc.py | 16 ++++++++++++++++ piker/data/__init__.py | 19 +++++++++++++++++++ piker/data/_buffer.py | 16 ++++++++++++++++ piker/data/_normalize.py | 17 +++++++++++++++++ piker/data/_sharedmem.py | 16 ++++++++++++++++ piker/data/_source.py | 16 ++++++++++++++++ piker/data/cli.py | 16 ++++++++++++++++ piker/data/marketstore.py | 16 ++++++++++++++++ piker/fsp/__init__.py | 17 +++++++++++++++++ piker/fsp/_momo.py | 18 +++++++++++++++++- piker/log.py | 16 ++++++++++++++++ piker/ui/__init__.py | 16 ++++++++++++++++ piker/ui/_axes.py | 17 +++++++++++++++++ piker/ui/_chart.py | 16 ++++++++++++++++ piker/ui/_exec.py | 16 ++++++++++++++++ piker/ui/_graphics.py | 17 +++++++++++++++++ piker/ui/_interaction.py | 16 ++++++++++++++++ piker/ui/_signalling.py | 16 ++++++++++++++++ piker/ui/_style.py | 16 ++++++++++++++++ piker/ui/cli.py | 16 ++++++++++++++++ piker/watchlists/__init__.py | 16 ++++++++++++++++ piker/watchlists/cli.py | 15 +++++++++++++++ setup.py | 2 +- 36 files changed, 568 insertions(+), 3 deletions(-) diff --git a/piker/_async_utils.py b/piker/_async_utils.py index 7069d597..b358e2f0 100644 --- a/piker/_async_utils.py +++ b/piker/_async_utils.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Async utils no one seems to have built into a core lib (yet). """ diff --git a/piker/_profile.py b/piker/_profile.py index c14071d0..a6f171c1 100644 --- a/piker/_profile.py +++ b/piker/_profile.py @@ -1,3 +1,18 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . """ Profiling wrappers for internal libs. diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index 852b3db2..06328d4f 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Broker clients, daemons and general back end machinery. """ diff --git a/piker/brokers/_util.py b/piker/brokers/_util.py index b8b6fccc..64f0ad3a 100644 --- a/piker/brokers/_util.py +++ b/piker/brokers/_util.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Handy utils. """ diff --git a/piker/brokers/api.py b/piker/brokers/api.py index 29fe5577..ba54a565 100644 --- a/piker/brokers/api.py +++ b/piker/brokers/api.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Actor-aware broker agnostic interface. """ diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index b1047b18..1c340fc3 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Console interface to broker client/daemons. """ diff --git a/piker/brokers/config.py b/piker/brokers/config.py index bbf4d407..6718e794 100644 --- a/piker/brokers/config.py +++ b/piker/brokers/config.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Broker configuration mgmt. """ diff --git a/piker/brokers/core.py b/piker/brokers/core.py index 67255a41..5189df85 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Broker high level cross-process API layer. diff --git a/piker/brokers/data.py b/piker/brokers/data.py index ea2076bb..cdf056b4 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Real-time data feed machinery """ diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 2a1680ad..cf509bfb 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Interactive Brokers API backend. @@ -150,7 +166,7 @@ class Client: # durationStr='1 D', # time length calcs - durationStr='{count} S'.format(count=3000 * 5), + durationStr='{count} S'.format(count=5000 * 5), barSizeSetting='5 secs', # always use extended hours diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 90bd6476..4329a981 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Kraken backend. """ diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 6063b9f6..e54c75a2 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Questrade API backend. """ diff --git a/piker/brokers/robinhood.py b/piker/brokers/robinhood.py index 34be0627..71b21055 100644 --- a/piker/brokers/robinhood.py +++ b/piker/brokers/robinhood.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Robinhood API backend. diff --git a/piker/calc.py b/piker/calc.py index 679e2782..2e64c684 100644 --- a/piker/calc.py +++ b/piker/calc.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Handy financial calculations. """ diff --git a/piker/data/__init__.py b/piker/data/__init__.py index 77bcba12..cae1347c 100644 --- a/piker/data/__init__.py +++ b/piker/data/__init__.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Data feed apis and infra. @@ -80,6 +96,9 @@ async def maybe_spawn_brokerd( if loglevel: get_console_log(loglevel) + # disable debugger in brokerd? + # tractor._state._runtime_vars['_debug_mode'] = False + tractor_kwargs['loglevel'] = loglevel brokermod = get_brokermod(brokername) diff --git a/piker/data/_buffer.py b/piker/data/_buffer.py index 5e1c3588..64460476 100644 --- a/piker/data/_buffer.py +++ b/piker/data/_buffer.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Data buffers for fast shared humpy. """ diff --git a/piker/data/_normalize.py b/piker/data/_normalize.py index e1120278..cbda6062 100644 --- a/piker/data/_normalize.py +++ b/piker/data/_normalize.py @@ -1,6 +1,23 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Stream format enforcement. """ + from typing import AsyncIterator, Optional, Tuple import numpy as np diff --git a/piker/data/_sharedmem.py b/piker/data/_sharedmem.py index c256da31..7f90d1ae 100644 --- a/piker/data/_sharedmem.py +++ b/piker/data/_sharedmem.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ NumPy compatible shared memory buffers for real-time FSP. """ diff --git a/piker/data/_source.py b/piker/data/_source.py index 1df55f53..3ad6d3e8 100644 --- a/piker/data/_source.py +++ b/piker/data/_source.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Numpy data source machinery. """ diff --git a/piker/data/cli.py b/piker/data/cli.py index 4088f392..7a774fb5 100644 --- a/piker/data/cli.py +++ b/piker/data/cli.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ marketstore cli. """ diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index 0c68adf7..d8cb3930 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ ``marketstore`` integration. diff --git a/piker/fsp/__init__.py b/piker/fsp/__init__.py index c4b12242..6e56c667 100644 --- a/piker/fsp/__init__.py +++ b/piker/fsp/__init__.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Financial signal processing for the peeps. """ @@ -125,6 +141,7 @@ async def cascade( # check for data length mis-allignment and fill missing values diff = len(src.array) - len(history) if diff >= 0: + print(f"WTF DIFFZZZ {diff}") for _ in range(diff): dst.push(history[:1]) diff --git a/piker/fsp/_momo.py b/piker/fsp/_momo.py index 934a5e70..13bad728 100644 --- a/piker/fsp/_momo.py +++ b/piker/fsp/_momo.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Momentum bby. """ @@ -136,7 +152,7 @@ def wma( # @piker.fsp( -# aggregates=['30s', '1m', '5m', '1H', '4H', '1D'], + # aggregates=[60, 60*5, 60*60, '4H', '1D'], # ) async def _rsi( source: 'QuoteStream[Dict[str, Any]]', # noqa diff --git a/piker/log.py b/piker/log.py index 218a9fdb..7c8bb798 100644 --- a/piker/log.py +++ b/piker/log.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Log like a forester! """ diff --git a/piker/ui/__init__.py b/piker/ui/__init__.py index a7fb1052..88771b2d 100644 --- a/piker/ui/__init__.py +++ b/piker/ui/__init__.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Stuff for your eyes, aka super hawt Qt UI components. diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index f567c877..dfce7559 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -1,6 +1,23 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Chart axes graphics and behavior. """ + from typing import List, Tuple, Optional import pandas as pd diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 004e6b8f..0caf0d17 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ High level Qt chart widgets. """ diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 55a703b3..b0424c32 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Trio - Qt integration diff --git a/piker/ui/_graphics.py b/piker/ui/_graphics.py index b8dff5d4..88193a72 100644 --- a/piker/ui/_graphics.py +++ b/piker/ui/_graphics.py @@ -1,6 +1,23 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Chart graphics for displaying a slew of different data types. """ + # import time from typing import List, Optional, Tuple diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 444dfeda..af5bcec3 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ UX interaction customs. """ diff --git a/piker/ui/_signalling.py b/piker/ui/_signalling.py index 66a5bf1c..dbb4f467 100644 --- a/piker/ui/_signalling.py +++ b/piker/ui/_signalling.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Signalling graphics and APIs. diff --git a/piker/ui/_style.py b/piker/ui/_style.py index 98db4bb2..23a3ac09 100644 --- a/piker/ui/_style.py +++ b/piker/ui/_style.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Qt UI styling. """ diff --git a/piker/ui/cli.py b/piker/ui/cli.py index 74a3cae3..0b2422da 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + """ Console interface to UI components. """ diff --git a/piker/watchlists/__init__.py b/piker/watchlists/__init__.py index 707c602f..7448a5c2 100644 --- a/piker/watchlists/__init__.py +++ b/piker/watchlists/__init__.py @@ -1,3 +1,19 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + import os import json from collections import defaultdict diff --git a/piker/watchlists/cli.py b/piker/watchlists/cli.py index 7ed089d1..3d657348 100644 --- a/piker/watchlists/cli.py +++ b/piker/watchlists/cli.py @@ -1,3 +1,18 @@ +# piker: trading gear for hackers +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . """ Watchlist management commands. diff --git a/setup.py b/setup.py index db505cfd..4f8818f5 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # piker: trading gear for hackers -# Copyright 2018 Tyler Goodlet +# Copyright (C) 2018-present Tyler Goodlet (in stewardship of piker0) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by From bdcee2b2104b99aba57c22d2cace91cce418b4c7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 12:46:04 -0500 Subject: [PATCH 204/206] Readme bump --- README.rst | 87 +++++++++++++++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/README.rst b/README.rst index ac8dfb02..67ca4eba 100644 --- a/README.rst +++ b/README.rst @@ -2,15 +2,17 @@ piker ----- trading gear for hackers. -|travis| +|gh_actions| -``piker`` is an attempt at a pro-grade, broker agnostic, next-gen FOSS -toolset for real-time trading and financial analysis targetted at -hardcore Linux users. +.. |gh_actions| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Fpikers%2Fpiker%2Fbadge&style=popout-square + :target: https://actions-badge.atrox.dev/piker/pikers/goto -it tries to use as much bleeding edge tech as possible including (but not limited to): +``piker`` is a broker agnostic, next-gen FOSS toolset for real-time +trading targeted at hardcore Linux users. -- latest Python for glue_ and business logic +we use as much bleeding edge tech as possible including (but not limited to): + +- latest python for glue_ - trio_ for `structured concurrency`_ - tractor_ for distributed, multi-core, real-time streaming - marketstore_ for historical and real-time tick data persistence and sharing @@ -32,23 +34,19 @@ it tries to use as much bleeding edge tech as possible including (but not limite .. _fast numerics: https://zerowithdot.com/python-numpy-and-pandas-performance/ -Focus and Features: +focus and features: ******************* -- zero web -- zero pump -- zero "backtesting" (aka yabf) -- zero "cloud" -- 100% federated: your code, your hardware, your broker's data feeds +- zero web, cloud or "backtesting frameworks" (aka yabf) +- zero self promotion (aka pump); expected throughout the community +- 100% federated: your code, your hardware, your data feeds, your broker fills +- broker/exchange/asset-class agnostic - privacy -- broker/exchange agnostic -- built on a structured concurrent actor model -- production grade, highly attractive native UIs -- expected to be used from tiling wms -- sophisticated rt charting -- emphasis on collaboration through UI and data sharing -- zero interest in adoption by suits +- real-time financial signal processing from the ground up +- high quality, attractive, native UX with expected use in tiling wms +- sophisticated rt charting and data sharing facilities +- geared for collaboration within trader communities +- zero interest in adoption by suits; no corporate friendly license, ever. - not built for *sale*; built for *people* -- no corporate friendly license, ever. fitting with these tenets, we're always open to new framework suggestions and ideas. @@ -57,49 +55,58 @@ building the best looking, most reliable, keyboard friendly trading platform is the dream. feel free to pipe in with your ideas and quiffs. -Install +install ******* -``piker`` is currently under heavy pre-alpha development and as such should -be cloned from this repo and hacked on directly. +``piker`` is currently under heavy pre-alpha development and as such +should be cloned from this repo and hacked on directly. -A couple bleeding edge components are being used atm pertaining to +a couple bleeding edge components are being used atm pertaining to new components within `trio`_. -For a development install:: +for a development install:: git clone git@github.com:pikers/piker.git cd piker pip install -e . -Broker Support +broker Support ************** -For live data feeds the in-progress set of supported brokers is: +for live data feeds the in-progress set of supported brokers is: -- Questrade_ which comes with effectively free L1 - IB_ via ``ib_insync`` -- Webull_ via the reverse engineered public API -- Kraken_ for crypto over their public websocket API +- questrade_ which comes with effectively free L1 +- kraken_ for crypto over their public websocket API -If you want your broker supported and they have an API let us know. +coming soon... + +- webull_ via the reverse engineered public API +- yahoo via yliveticker_ +- coinbase_ through websocket feed + +if you want your broker supported and they have an API let us know. -.. _Questrade: https://www.questrade.com/api/documentation .. _IB: https://interactivebrokers.github.io/tws-api/index.html -.. _Webull: https://www.kraken.com/features/api#public-market-data -.. _Kraken: https://www.kraken.com/features/api#public-market-data +.. _questrade: https://www.questrade.com/api/documentation +.. _kraken: https://www.kraken.com/features/api#public-market-data +.. _webull: https://github.com/tedchou12/webull +.. _yliveticker: https://github.com/yahoofinancelive/yliveticker +.. _coinbase: https://docs.pro.coinbase.com/#websocket-feed - -Check out our charts +check out our charts ******************** bet you weren't expecting this from the foss bby:: piker -b kraken chart XBTUSD -If anyone asks you what this project is about +if anyone asks you what this project is about ********************************************* -tell them *it's a broken crypto trading platform that doesn't scale*. +you don't talk about it. -How do i get involved? +how do i get involved? ********************** -coming soon. +enter the matrix. + +learning the code is to your benefit and acts as a filter for desired +users; many alpha nuggets within. From 0f458f826317ec0fc9c5fa7fdd2a0ef81152fe14 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 15:24:01 -0500 Subject: [PATCH 205/206] Add min tick setting to axis type --- piker/ui/_axes.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/piker/ui/_axes.py b/piker/ui/_axes.py index dfce7559..e0be7178 100644 --- a/piker/ui/_axes.py +++ b/piker/ui/_axes.py @@ -39,11 +39,13 @@ class Axis(pg.AxisItem): self, linked_charts, typical_max_str: str = '100 000.00', + min_tick: int = 2, **kwargs ) -> None: super().__init__(**kwargs) self.linked_charts = linked_charts + self._min_tick = min_tick self.setTickFont(_font.font) self.setStyle(**{ @@ -58,6 +60,9 @@ class Axis(pg.AxisItem): # size the pertinent axis dimension to a "typical value" self.resize() + def set_min_tick(self, size: int) -> None: + self._min_tick = size + class PriceAxis(Axis): @@ -74,13 +79,20 @@ class PriceAxis(Axis): # XXX: drop for now since it just eats up h space def tickStrings(self, vals, scale, spacing): - digits = float_digits(spacing * scale) + + # TODO: figure out how to enforce min tick spacing by passing + # it into the parent type + digits = max(float_digits(spacing * scale), self._min_tick) # print(f'vals: {vals}\nscale: {scale}\nspacing: {spacing}') # print(f'digits: {digits}') return [ - ('{:,.%df}' % digits).format(v).replace(',', ' ') for v in vals + ('{value:,.{digits}f}') + .format( + digits=digits, + value=v, + ).replace(',', ' ') for v in vals ] @@ -250,10 +262,12 @@ class XAxisLabel(AxisLabel): class YAxisLabel(AxisLabel): _h_margin = 3 + # _w_margin = 1 text_flags = ( # QtCore.Qt.AlignLeft - QtCore.Qt.AlignVCenter + QtCore.Qt.AlignHCenter + | QtCore.Qt.AlignVCenter | QtCore.Qt.TextDontClip ) @@ -269,7 +283,7 @@ class YAxisLabel(AxisLabel): ) -> None: # this is read inside ``.paint()`` - self.label_str = '{value: ,.{digits}f}'.format( + self.label_str = '{value:,.{digits}f}'.format( digits=self.digits, value=value).replace(',', ' ') br = self.boundingRect() From 1972740d0cf20dce1d2511e45f4cea3788b02a1f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 6 Nov 2020 15:35:10 -0500 Subject: [PATCH 206/206] Make salotz stop grumbling like an old man --- piker/ui/_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index b0424c32..732db3e2 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -60,7 +60,7 @@ if hasattr(Qt, 'AA_UseHighDpiPixmaps'): class MainWindow(QtGui.QMainWindow): size = (800, 500) - title = 'piker chart (bby)' + title = 'piker chart (ur symbol is loading bby)' def __init__(self, parent=None): super().__init__(parent)