From 5fdec8012dc8d7493655f7b18f37e0ecaed1d5cf Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 12:42:37 -0500 Subject: [PATCH 01/81] Add cryptofeeds data feed module, Add Kucoin backend client wip --- piker/brokers/kucoin.py | 137 +++++++++++++++++ piker/data/cryptofeeds.py | 313 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 piker/brokers/kucoin.py create mode 100644 piker/data/cryptofeeds.py diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py new file mode 100644 index 00000000..677e5456 --- /dev/null +++ b/piker/brokers/kucoin.py @@ -0,0 +1,137 @@ +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +""" + +""" + +from typing import Any, Optional, Literal +from contextlib import asynccontextmanager as acm + +import asks +import tractor +import trio +from trio_typing import TaskStatus +from fuzzywuzzy import process as fuzzy +from cryptofeed.defines import ( + KUCOIN, + TRADES, + L2_BOOK +) +from piker.data.cryptofeeds import mk_stream_quotes +from piker._cacheables import open_cached_client +from piker.log import get_logger +from ._util import SymbolNotFound + +_spawn_kwargs = { + "infect_asyncio": True, +} + +log = get_logger(__name__) + +class Client: + def __init__(self) -> None: + self._pairs: dict[str, Any] = None + # TODO" Shouldn't have to write kucoin twice here + + # config = get_config('kucoin').get('kucoin', {}) + # + # if ('key_id' in config) and ('key_secret' in config): + # self._key_id = config['key_id'] + # self._key_secret = config['key_secret'] + # + # else: + # self._key_id = None + # self._key_secret = None + + async def symbol_info( + self, + sym: str = None, + ) -> dict[str, Any]: + + if self._pairs: + return self._pairs + + entries = await self.request("GET", "/symbols") + if not entries: + raise SymbolNotFound(f'{sym} not found') + + syms = {item['name']: item for item in entries} + return syms + + + async def request(self, action: Literal["POST", "GET", "PUT", "DELETE"], route: str): + api_url = f"https://api.kucoin.com/api/v2{route}" + res = await asks.request(action, api_url) + return res.json()['data'] + + async def cache_symbols( + self, + ) -> dict: + if not self._pairs: + self._pairs = await self.symbol_info() + + return self._pairs + + async def search_symbols( + self, + pattern: str, + limit: int = 30, + ) -> dict[str, Any]: + data = await self.symbol_info() + + matches = fuzzy.extractBests(pattern, data, score_cutoff=35, limit=limit) + # repack in dict form + return {item[0]["instrument_name"].lower(): item[0] for item in matches} + + +@acm +async def get_client(): + client = Client() + # Do we need to open a nursery here? + await client.cache_symbols() + yield client + + +@tractor.context +async def open_symbol_search( + ctx: tractor.Context, +): + async with open_cached_client("kucoin") as client: + # load all symbols locally for fast search + cache = await client.cache_symbols() + await ctx.started() + + # async with ctx.open_stream() as stream: + # async for pattern in stream: + # # repack in dict form + # await stream.send(await client.search_symbols(pattern)) + + +async def stream_quotes( + send_chan: trio.abc.SendChannel, + symbols: list[str], + feed_is_live: trio.Event, + loglevel: str = None, + # startup sync + task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, +): + return await mk_stream_quotes( + KUCOIN, + [L2_BOOK], + send_chan, + symbols, + feed_is_live, + loglevel, + task_status, + ) diff --git a/piker/data/cryptofeeds.py b/piker/data/cryptofeeds.py new file mode 100644 index 00000000..931207c6 --- /dev/null +++ b/piker/data/cryptofeeds.py @@ -0,0 +1,313 @@ +# piker: trading gear for hackers +# Copyright (C) Guillermo Rodriguez (in stewardship for piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +""" +""" +from os import walk +from contextlib import asynccontextmanager as acm +from datetime import datetime +from types import ModuleType +from typing import Any, Literal, Optional, Callable +import time +from functools import partial + +import trio +from trio_typing import TaskStatus +from tractor.trionics import broadcast_receiver, maybe_open_context +import pendulum +from fuzzywuzzy import process as fuzzy +import numpy as np +import tractor +from tractor import to_asyncio +from cryptofeed import FeedHandler +from cryptofeed.defines import TRADES, L2_BOOK +from cryptofeed.symbols import Symbol +import asyncio + +from piker._cacheables import open_cached_client +from piker.log import get_logger, get_console_log +from piker.data import ShmArray +from piker.brokers._util import ( + BrokerError, + DataUnavailable, +) +from piker.pp import config + +_spawn_kwargs = { + "infect_asyncio": True, +} + +log = get_logger(__name__) + + +def deribit_timestamp(when): + return int((when.timestamp() * 1000) + (when.microsecond / 1000)) + + +# def str_to_cb_sym(name: str) -> Symbol: +# base, strike_price, expiry_date, option_type = name.split("-") +# +# quote = base +# +# if option_type == "put": +# option_type = PUT +# elif option_type == "call": +# option_type = CALL +# else: +# raise Exception("Couldn't parse option type") +# +# return Symbol( +# base, +# quote, +# type=OPTION, +# strike_price=strike_price, +# option_type=option_type, +# expiry_date=expiry_date, +# expiry_normalize=False, +# ) +# + + +def piker_sym_to_cb_sym(symbol) -> Symbol: + return Symbol( + base=symbol['baseCurrency'], + quote=symbol['quoteCurrency'] + ) + + +def cb_sym_to_deribit_inst(sym: Symbol): + # cryptofeed normalized + cb_norm = ["F", "G", "H", "J", "K", "M", "N", "Q", "U", "V", "X", "Z"] + + # deribit specific + months = [ + "JAN", + "FEB", + "MAR", + "APR", + "MAY", + "JUN", + "JUL", + "AUG", + "SEP", + "OCT", + "NOV", + "DEC", + ] + + exp = sym.expiry_date + + # YYMDD + # 01234 + year, month, day = (exp[:2], months[cb_norm.index(exp[2:3])], exp[3:]) + + otype = "C" if sym.option_type == CALL else "P" + + return f"{sym.base}-{day}{month}{year}-{sym.strike_price}-{otype}" + + +def get_config(exchange: str) -> dict[str, Any]: + conf, path = config.load() + + section = conf.get(exchange.lower()) + breakpoint() + + # TODO: document why we send this, basically because logging params for cryptofeed + conf["log"] = {} + conf["log"]["disabled"] = True + + if section is None: + log.warning(f"No config section found for deribit in {path}") + + return conf + + +async def mk_stream_quotes( + exchange: str, + channels: list[str], + send_chan: trio.abc.SendChannel, + symbols: list[str], + feed_is_live: trio.Event, + loglevel: str = None, + # startup sync + task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, +) -> None: + + # XXX: required to propagate ``tractor`` loglevel to piker logging + get_console_log(loglevel or tractor.current_actor().loglevel) + + sym = symbols[0] + + async with (open_cached_client(exchange.lower()) as client, send_chan as send_chan): + # create init message here + + cache = await client.cache_symbols() + + cf_syms = {} + for key, value in cache.items(): + cf_sym = key.lower().replace('-', '') + cf_syms[cf_sym] = value + + cf_sym = cf_syms[sym] + + async with maybe_open_price_feed(cf_sym, exchange, channels) as stream: + + init_msgs = { + # pass back token, and bool, signalling if we're the writer + # and that history has been written + sym: { + 'symbol_info': { + 'asset_type': 'crypto', + 'price_tick_size': 0.0005 + }, + 'shm_write_opts': {'sum_tick_vml': False}, + 'fqsn': sym, + }, + } + + # broker schemas to validate symbol data + quote_msg = {"symbol": cf_sym["name"], "last": 0, "ticks": []} + + task_status.started((init_msgs, quote_msg)) + + feed_is_live.set() + + async for typ, quote in stream: + topic = quote["symbol"] + await send_chan.send({topic: quote}) + +@acm +async def maybe_open_price_feed( + symbol, exchange, channels +) -> trio.abc.ReceiveStream: + # TODO: add a predicate to maybe_open_context + # TODO: ensure we can dynamically pass down args here + async with maybe_open_context( + acm_func=open_price_feed, + kwargs={ + "symbol": symbol, + "exchange": exchange, + "channels": channels, + }, + key=symbol['name'], + ) as (cache_hit, feed): + if cache_hit: + yield broadcast_receiver(feed, 10) + else: + yield feed + + +@acm +async def open_price_feed(symbol: str, exchange, channels) -> trio.abc.ReceiveStream: + async with maybe_open_feed_handler(exchange) as fh: + async with to_asyncio.open_channel_from( + partial(aio_price_feed_relay, exchange, channels, fh, symbol) + ) as (first, chan): + yield chan + + +@acm +async def maybe_open_feed_handler(exchange: str) -> trio.abc.ReceiveStream: + async with maybe_open_context( + acm_func=open_feed_handler, + kwargs={ + 'exchange': exchange, + }, + key="feedhandler", + ) as (cache_hit, fh): + yield fh + + +@acm +async def open_feed_handler(exchange: str): + fh = FeedHandler(config=get_config(exchange)) + yield fh + await to_asyncio.run_task(fh.stop_async) + + +async def aio_price_feed_relay( + exchange: str, + channels: list[str], + fh: FeedHandler, + symbol: Symbol, + from_trio: asyncio.Queue, + to_trio: trio.abc.SendChannel, +) -> None: + async def _trade(data: dict, receipt_timestamp): + breakpoint() + # to_trio.send_nowait( + # ( + # "trade", + # { + # "symbol": cb_sym_to_deribit_inst( + # str_to_cb_sym(data.symbol) + # ).lower(), + # "last": data, + # "broker_ts": time.time(), + # "data": data.to_dict(), + # "receipt": receipt_timestamp, + # }, + # ) + # ) + + async def _l1(data: dict, receipt_timestamp): + breakpoint() + # to_trio.send_nowait( + # ( + # "l1", + # { + # "symbol": cb_sym_to_deribit_inst( + # str_to_cb_sym(data.symbol) + # ).lower(), + # "ticks": [ + # { + # "type": "bid", + # "price": float(data.bid_price), + # "size": float(data.bid_size), + # }, + # { + # "type": "bsize", + # "price": float(data.bid_price), + # "size": float(data.bid_size), + # }, + # { + # "type": "ask", + # "price": float(data.ask_price), + # "size": float(data.ask_size), + # }, + # { + # "type": "asize", + # "price": float(data.ask_price), + # "size": float(data.ask_size), + # }, + # ], + # }, + # ) + # ) + fh.add_feed( + exchange, + channels=channels, + symbols=[piker_sym_to_cb_sym(symbol)], + callbacks={TRADES: _trade, L2_BOOK: _l1}, + ) + + if not fh.running: + fh.run(start_loop=False, install_signal_handlers=False) + + # sync with trio + to_trio.send_nowait(None) + + await asyncio.sleep(float("inf")) From c96d4387c525604a137237b247cabd921aac047d Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 12:56:12 -0500 Subject: [PATCH 02/81] Start adding history client --- piker/brokers/kucoin.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 677e5456..75d72037 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -53,6 +53,10 @@ class Client: # else: # self._key_id = None # self._key_secret = None + async def request(self, action: Literal["POST", "GET", "PUT", "DELETE"], route: str, api_v: str = 'v2'): + api_url = f"https://api.kucoin.com/api/{api_v}{route}" + res = await asks.request(action, api_url) + return res.json()['data'] async def symbol_info( self, @@ -68,12 +72,6 @@ class Client: syms = {item['name']: item for item in entries} return syms - - - async def request(self, action: Literal["POST", "GET", "PUT", "DELETE"], route: str): - api_url = f"https://api.kucoin.com/api/v2{route}" - res = await asks.request(action, api_url) - return res.json()['data'] async def cache_symbols( self, @@ -135,3 +133,6 @@ async def stream_quotes( loglevel, task_status, ) + +async def open_history_client(): + From ad9d6457823737fb4670f8257e512d5c7bfdab0e Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 2 Mar 2023 23:01:17 -0500 Subject: [PATCH 03/81] WIP - setup basic history and streaming client --- piker/brokers/__init__.py | 3 +- piker/brokers/kucoin.py | 226 +++++++++++++++++++++++++++++++------- piker/data/cryptofeeds.py | 226 +++++++++++++++----------------------- 3 files changed, 273 insertions(+), 182 deletions(-) diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index a35e4aea..c67f4003 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -24,7 +24,7 @@ __brokers__ = [ 'binance', 'ib', 'kraken', - + 'kucoin' # broken but used to work # 'questrade', # 'robinhood', @@ -35,7 +35,6 @@ __brokers__ = [ # iex # deribit - # kucoin # bitso ] diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 75d72037..edf5d508 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -15,89 +15,196 @@ """ +from dataclasses import field from typing import Any, Optional, Literal from contextlib import asynccontextmanager as acm +from datetime import datetime +import time +import math +from os import path, walk import asks import tractor import trio from trio_typing import TaskStatus from fuzzywuzzy import process as fuzzy -from cryptofeed.defines import ( - KUCOIN, - TRADES, - L2_BOOK +from cryptofeed.defines import KUCOIN, TRADES, L2_BOOK +from cryptofeed.symbols import Symbol +import pendulum +import numpy as np +from piker.data.cryptofeeds import ( + fqsn_to_cf_sym, + mk_stream_quotes, + get_config, ) -from piker.data.cryptofeeds import mk_stream_quotes from piker._cacheables import open_cached_client from piker.log import get_logger -from ._util import SymbolNotFound +from piker.pp import config +from ._util import DataUnavailable _spawn_kwargs = { "infect_asyncio": True, } log = get_logger(__name__) +_ohlc_dtype = [ + ('index', int), + ('time', int), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', float), + ('bar_wap', float), # will be zeroed by sampler if not filled +] + class Client: def __init__(self) -> None: - self._pairs: dict[str, Any] = None + self._pairs: dict[str, Symbol] = {} + self._bars: list[list] = [] # TODO" Shouldn't have to write kucoin twice here - # config = get_config('kucoin').get('kucoin', {}) + config = get_config("kucoin").get("kucoin", {}) # - # if ('key_id' in config) and ('key_secret' in config): - # self._key_id = config['key_id'] - # self._key_secret = config['key_secret'] - # - # else: - # self._key_id = None - # self._key_secret = None - async def request(self, action: Literal["POST", "GET", "PUT", "DELETE"], route: str, api_v: str = 'v2'): + if ("key_id" in config) and ("key_secret" in config): + self._key_id = config["key_id"] + self._key_secret = config["key_secret"] + + else: + self._key_id = None + self._key_secret = None + + async def _request( + self, + action: Literal["POST", "GET", "PUT", "DELETE"], + route: str, + api_v: str = "v2", + ) -> Any: api_url = f"https://api.kucoin.com/api/{api_v}{route}" res = await asks.request(action, api_url) - return res.json()['data'] + #breakpoint() + try: + return res.json()["data"] + except KeyError as e: + print(f'KUCOIN ERROR: {res.json()["msg"]}') + breakpoint() - async def symbol_info( + async def get_pairs( self, - sym: str = None, ) -> dict[str, Any]: - if self._pairs: return self._pairs - entries = await self.request("GET", "/symbols") - if not entries: - raise SymbolNotFound(f'{sym} not found') - - syms = {item['name']: item for item in entries} + entries = await self._request("GET", "/symbols") + syms = {item["name"]: item for item in entries} return syms - async def cache_symbols( + async def cache_pairs( self, - ) -> dict: + normalize: bool = True, + ) -> dict[str, Symbol]: if not self._pairs: - self._pairs = await self.symbol_info() - + self._pairs = await self.get_pairs() + if normalize: + self._pairs = self.normalize_pairs(self._pairs) return self._pairs + def normalize_pairs(self, pairs: dict[str, Symbol]) -> dict[str, Symbol]: + """ + Map crypfeeds symbols to fqsn strings + + """ + norm_pairs = {} + + for key, value in pairs.items(): + fqsn = key.lower().replace("-", "") + norm_pairs[fqsn] = value + + return norm_pairs + async def search_symbols( self, pattern: str, limit: int = 30, ) -> dict[str, Any]: - data = await self.symbol_info() + data = await self.get_pairs() matches = fuzzy.extractBests(pattern, data, score_cutoff=35, limit=limit) # repack in dict form - return {item[0]["instrument_name"].lower(): item[0] for item in matches} + return {item[0]["name"].lower(): item[0] for item in matches} + + async def get_bars( + self, + fqsn: str, + start_dt: Optional[datetime] = None, + end_dt: Optional[datetime] = None, + limit: int = 1000, + as_np: bool = True, + type: str = "1min", + ): + if len(self._bars): + return self._bars + + if end_dt is None: + end_dt = pendulum.now("UTC").add(minutes=1) + + if start_dt is None: + start_dt = end_dt.start_of("minute").subtract(minutes=limit) + + # Format datetime to unix + start_dt = math.trunc(time.mktime(start_dt.timetuple())) + end_dt = math.trunc(time.mktime(end_dt.timetuple())) + kucoin_sym = fqsn_to_cf_sym(fqsn, self._pairs) + url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" + + bars = await self._request( + "GET", + url, + api_v="v1", + ) + + new_bars = [] + for i, bar in enumerate(bars[::-1]): + # TODO: implement struct/typecasting/validation here + + data = { + 'index': i, + 'time': bar[0], + 'open': bar[1], + 'close': bar[2], + 'high': bar[3], + 'low': bar[4], + 'volume': bar[5], + 'amount': bar [6], + 'bar_wap': 0.0, + } + + row = [] + for j, (field_name, field_type) in enumerate(_ohlc_dtype): + + value = data[field_name] + match field_name: + case 'index' | 'time': + row.append(int(value)) + # case 'time': + # dt_from_unix_ts = datetime.utcfromtimestamp(int(value)) + # # convert unix time to epoch seconds + # row.append(int(dt_from_unix_ts.timestamp())) + case _: + row.append(float(value)) + + new_bars.append(tuple(row)) + + self._bars = array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + return array @acm async def get_client(): client = Client() # Do we need to open a nursery here? - await client.cache_symbols() + await client.cache_pairs() yield client @@ -107,13 +214,13 @@ async def open_symbol_search( ): async with open_cached_client("kucoin") as client: # load all symbols locally for fast search - cache = await client.cache_symbols() + cache = await client.cache_pairs() await ctx.started() - # async with ctx.open_stream() as stream: - # async for pattern in stream: - # # repack in dict form - # await stream.send(await client.search_symbols(pattern)) + async with ctx.open_stream() as stream: + async for pattern in stream: + # repack in dict form + await stream.send(await client.search_symbols(pattern)) async def stream_quotes( @@ -126,7 +233,7 @@ async def stream_quotes( ): return await mk_stream_quotes( KUCOIN, - [L2_BOOK], + [L2_BOOK, TRADES], send_chan, symbols, feed_is_live, @@ -134,5 +241,44 @@ async def stream_quotes( task_status, ) -async def open_history_client(): - + +@acm +async def open_history_client( + symbol: str, + type: str = "1m", +): + async with open_cached_client("kucoin") as client: + # call bars on kucoin + async def get_ohlc_history( + timeframe: float, + end_dt: datetime | None = None, + start_dt: datetime | None = None, + ) -> tuple[ + np.ndarray, + datetime | None, # start + datetime | None, # end + ]: + if timeframe != 60: + raise DataUnavailable('Only 1m bars are supported') + + array = await client.get_bars( + symbol, + start_dt=start_dt, + end_dt=end_dt, + ) + + times = array['time'] + + if ( + end_dt is None + ): + inow = round(time.time()) + print(f'difference in time between load and processing {inow - times[-1]}') + if (inow - times[-1]) > 60: + await tractor.breakpoint() + + start_dt = pendulum.from_timestamp(times[0]) + end_dt = pendulum.from_timestamp(times[-1]) + return array, start_dt, end_dt + + yield get_ohlc_history, {'erlangs': 3, 'rate': 3} diff --git a/piker/data/cryptofeeds.py b/piker/data/cryptofeeds.py index 931207c6..727c3a3c 100644 --- a/piker/data/cryptofeeds.py +++ b/piker/data/cryptofeeds.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) Guillermo Rodriguez (in stewardship for piker0) +# Copyright (C) Jared Goldman (in stewardship for piker0) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -35,6 +35,7 @@ from tractor import to_asyncio from cryptofeed import FeedHandler from cryptofeed.defines import TRADES, L2_BOOK from cryptofeed.symbols import Symbol +from cryptofeed.types import OrderBook import asyncio from piker._cacheables import open_cached_client @@ -53,84 +54,34 @@ _spawn_kwargs = { log = get_logger(__name__) -def deribit_timestamp(when): - return int((when.timestamp() * 1000) + (when.microsecond / 1000)) +def fqsn_to_cb_sym(pair_data: Symbol) -> Symbol: + return Symbol(base=pair_data["baseCurrency"], quote=pair_data["quoteCurrency"]) -# def str_to_cb_sym(name: str) -> Symbol: -# base, strike_price, expiry_date, option_type = name.split("-") -# -# quote = base -# -# if option_type == "put": -# option_type = PUT -# elif option_type == "call": -# option_type = CALL -# else: -# raise Exception("Couldn't parse option type") -# -# return Symbol( -# base, -# quote, -# type=OPTION, -# strike_price=strike_price, -# option_type=option_type, -# expiry_date=expiry_date, -# expiry_normalize=False, -# ) -# +def fqsn_to_cf_sym(fqsn: str, pairs: dict[str, Symbol]) -> str: + pair_data = pairs[fqsn] + return pair_data["baseCurrency"] + "-" + pair_data["quoteCurrency"] -def piker_sym_to_cb_sym(symbol) -> Symbol: - return Symbol( - base=symbol['baseCurrency'], - quote=symbol['quoteCurrency'] - ) +def pair_data_to_cf_sym(sym_data: Symbol): + return sym_data["baseCurrency"] + "-" + sym_data["quoteCurrency"] -def cb_sym_to_deribit_inst(sym: Symbol): - # cryptofeed normalized - cb_norm = ["F", "G", "H", "J", "K", "M", "N", "Q", "U", "V", "X", "Z"] - - # deribit specific - months = [ - "JAN", - "FEB", - "MAR", - "APR", - "MAY", - "JUN", - "JUL", - "AUG", - "SEP", - "OCT", - "NOV", - "DEC", - ] - - exp = sym.expiry_date - - # YYMDD - # 01234 - year, month, day = (exp[:2], months[cb_norm.index(exp[2:3])], exp[3:]) - - otype = "C" if sym.option_type == CALL else "P" - - return f"{sym.base}-{day}{month}{year}-{sym.strike_price}-{otype}" +def cf_sym_to_fqsn(sym: str) -> str: + return sym.lower().replace("-", "") def get_config(exchange: str) -> dict[str, Any]: conf, path = config.load() section = conf.get(exchange.lower()) - breakpoint() # TODO: document why we send this, basically because logging params for cryptofeed conf["log"] = {} conf["log"]["disabled"] = True if section is None: - log.warning(f"No config section found for deribit in {path}") + log.warning(f"No config section found for deribit in {exchange}") return conf @@ -145,64 +96,53 @@ async def mk_stream_quotes( # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: - # XXX: required to propagate ``tractor`` loglevel to piker logging get_console_log(loglevel or tractor.current_actor().loglevel) sym = symbols[0] async with (open_cached_client(exchange.lower()) as client, send_chan as send_chan): - # create init message here + pairs = await client.cache_pairs() - cache = await client.cache_symbols() - - cf_syms = {} - for key, value in cache.items(): - cf_sym = key.lower().replace('-', '') - cf_syms[cf_sym] = value - - cf_sym = cf_syms[sym] - - async with maybe_open_price_feed(cf_sym, exchange, channels) as stream: + pair_data = pairs[sym] + async with maybe_open_price_feed(pair_data, exchange, channels) as stream: init_msgs = { # pass back token, and bool, signalling if we're the writer # and that history has been written sym: { - 'symbol_info': { - 'asset_type': 'crypto', - 'price_tick_size': 0.0005 - }, - 'shm_write_opts': {'sum_tick_vml': False}, - 'fqsn': sym, + "symbol_info": {"asset_type": "crypto", "price_tick_size": 0.0005}, + "shm_write_opts": {"sum_tick_vml": False}, + "fqsn": sym, }, } # broker schemas to validate symbol data - quote_msg = {"symbol": cf_sym["name"], "last": 0, "ticks": []} - + quote_msg = {"symbol": pair_data["name"], "last": 0, "ticks": []} task_status.started((init_msgs, quote_msg)) feed_is_live.set() async for typ, quote in stream: + print(f'streaming {typ} quote: {quote}') topic = quote["symbol"] await send_chan.send({topic: quote}) + @acm async def maybe_open_price_feed( - symbol, exchange, channels + pair_data: Symbol, exchange: str, channels ) -> trio.abc.ReceiveStream: # TODO: add a predicate to maybe_open_context # TODO: ensure we can dynamically pass down args here async with maybe_open_context( acm_func=open_price_feed, kwargs={ - "symbol": symbol, + "pair_data": pair_data, "exchange": exchange, "channels": channels, }, - key=symbol['name'], + key=pair_data["name"], ) as (cache_hit, feed): if cache_hit: yield broadcast_receiver(feed, 10) @@ -211,10 +151,12 @@ async def maybe_open_price_feed( @acm -async def open_price_feed(symbol: str, exchange, channels) -> trio.abc.ReceiveStream: +async def open_price_feed( + pair_data: Symbol, exchange, channels +) -> trio.abc.ReceiveStream: async with maybe_open_feed_handler(exchange) as fh: async with to_asyncio.open_channel_from( - partial(aio_price_feed_relay, exchange, channels, fh, symbol) + partial(aio_price_feed_relay, pair_data, exchange, channels, fh) ) as (first, chan): yield chan @@ -224,7 +166,7 @@ async def maybe_open_feed_handler(exchange: str) -> trio.abc.ReceiveStream: async with maybe_open_context( acm_func=open_feed_handler, kwargs={ - 'exchange': exchange, + "exchange": exchange, }, key="feedhandler", ) as (cache_hit, fh): @@ -239,74 +181,78 @@ async def open_feed_handler(exchange: str): async def aio_price_feed_relay( + pair_data: Symbol, exchange: str, channels: list[str], fh: FeedHandler, - symbol: Symbol, from_trio: asyncio.Queue, to_trio: trio.abc.SendChannel, ) -> None: async def _trade(data: dict, receipt_timestamp): - breakpoint() - # to_trio.send_nowait( - # ( - # "trade", - # { - # "symbol": cb_sym_to_deribit_inst( - # str_to_cb_sym(data.symbol) - # ).lower(), - # "last": data, - # "broker_ts": time.time(), - # "data": data.to_dict(), - # "receipt": receipt_timestamp, - # }, - # ) - # ) + print(f' trade data: {data}') + to_trio.send_nowait( + ( + "trade", + { + "symbol": cf_sym_to_fqsn(data.symbol), + "last": float(data.to_dict()['price']), + "broker_ts": time.time(), + "data": data.to_dict(), + "receipt": receipt_timestamp, + }, + ) + ) async def _l1(data: dict, receipt_timestamp): - breakpoint() - # to_trio.send_nowait( - # ( - # "l1", - # { - # "symbol": cb_sym_to_deribit_inst( - # str_to_cb_sym(data.symbol) - # ).lower(), - # "ticks": [ - # { - # "type": "bid", - # "price": float(data.bid_price), - # "size": float(data.bid_size), - # }, - # { - # "type": "bsize", - # "price": float(data.bid_price), - # "size": float(data.bid_size), - # }, - # { - # "type": "ask", - # "price": float(data.ask_price), - # "size": float(data.ask_size), - # }, - # { - # "type": "asize", - # "price": float(data.ask_price), - # "size": float(data.ask_size), - # }, - # ], - # }, - # ) - # ) + print(f'l2 data: {data}') + bid = data.book.to_dict()['bid'] + ask = data.book.to_dict()['ask'] + l1_ask_price, l1_ask_size = next(iter(ask.items())) + l1_bid_price, l1_bid_size = next(iter(bid.items())) + + to_trio.send_nowait( + ( + "l1", + { + "symbol": cf_sym_to_fqsn(data.symbol), + "ticks": [ + { + "type": "bid", + "price": float(l1_bid_price), + "size": float(l1_bid_size), + }, + { + "type": "bsize", + "price": float(l1_bid_price), + "size": float(l1_bid_size), + }, + { + "type": "ask", + "price": float(l1_ask_price), + "size": float(l1_ask_size), + }, + { + "type": "asize", + "price": float(l1_ask_price), + "size": float(l1_ask_size), + }, + ] + } + ) + ) + fh.add_feed( exchange, channels=channels, - symbols=[piker_sym_to_cb_sym(symbol)], - callbacks={TRADES: _trade, L2_BOOK: _l1}, + symbols=[pair_data_to_cf_sym(pair_data)], + callbacks={TRADES: _trade, L2_BOOK: _l1} ) if not fh.running: - fh.run(start_loop=False, install_signal_handlers=False) - + try: + fh.run(start_loop=False, install_signal_handlers=False) + except BaseExceptionGroup as e: + breakpoint() # sync with trio to_trio.send_nowait(None) From c751c36a8bf9df6d6bc3538a4bc6ecc01ae1289a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 3 Mar 2023 15:24:32 -0500 Subject: [PATCH 04/81] Update trade message format --- piker/data/cryptofeeds.py | 80 +++++++++++++++++++++++---------------- 1 file changed, 48 insertions(+), 32 deletions(-) diff --git a/piker/data/cryptofeeds.py b/piker/data/cryptofeeds.py index 727c3a3c..7e6e9de2 100644 --- a/piker/data/cryptofeeds.py +++ b/piker/data/cryptofeeds.py @@ -101,33 +101,46 @@ async def mk_stream_quotes( sym = symbols[0] - async with (open_cached_client(exchange.lower()) as client, send_chan as send_chan): + async with ( + open_cached_client(exchange.lower()) as client, + send_chan as send_chan + ): pairs = await client.cache_pairs() pair_data = pairs[sym] async with maybe_open_price_feed(pair_data, exchange, channels) as stream: + init_msgs = { - # pass back token, and bool, signalling if we're the writer - # and that history has been written sym: { "symbol_info": {"asset_type": "crypto", "price_tick_size": 0.0005}, "shm_write_opts": {"sum_tick_vml": False}, "fqsn": sym, }, } - - # broker schemas to validate symbol data quote_msg = {"symbol": pair_data["name"], "last": 0, "ticks": []} + task_status.started((init_msgs, quote_msg)) feed_is_live.set() - - async for typ, quote in stream: - print(f'streaming {typ} quote: {quote}') - topic = quote["symbol"] - await send_chan.send({topic: quote}) - + # try: + # async for typ, quote in stream: + # print(f'streaming {typ} quote: {quote}') + # topic = quote["symbobl"] + # await send_chan.send({topic: quote}) + # finally: + # breakpoint() + + while True: + with trio.move_on_after(4) as cancel_scope: + log.warning(f'WAITING FOR MESSAGE') + msg = await stream.receive() + log.warning(f'RECEIVED MSG: {msg}') + topic = msg["symbol"] + await send_chan.send({topic: msg}) + log.warning(f'SENT TO CHAN') + if cancel_scope.cancelled_caught: + await tractor.breakpoint() @acm async def maybe_open_price_feed( @@ -144,10 +157,7 @@ async def maybe_open_price_feed( }, key=pair_data["name"], ) as (cache_hit, feed): - if cache_hit: - yield broadcast_receiver(feed, 10) - else: - yield feed + yield feed @acm @@ -189,32 +199,37 @@ async def aio_price_feed_relay( to_trio: trio.abc.SendChannel, ) -> None: async def _trade(data: dict, receipt_timestamp): - print(f' trade data: {data}') - to_trio.send_nowait( - ( + data = data.to_dict() + message = ( "trade", { - "symbol": cf_sym_to_fqsn(data.symbol), - "last": float(data.to_dict()['price']), + "symbol": cf_sym_to_fqsn(data['symbol']), + "last": float(data['price']), "broker_ts": time.time(), - "data": data.to_dict(), - "receipt": receipt_timestamp, + "ticks": [{ + 'type': 'trade', + 'price': float(data['price']), + 'size': float(data['amount']), + 'broker_ts': receipt_timestamp + }], }, ) - ) + print(f'trade message: {message}') + # try: + to_trio.send_nowait(message) + # except trio.WouldBlock as e: + #breakpoint() async def _l1(data: dict, receipt_timestamp): - print(f'l2 data: {data}') bid = data.book.to_dict()['bid'] ask = data.book.to_dict()['ask'] l1_ask_price, l1_ask_size = next(iter(ask.items())) l1_bid_price, l1_bid_size = next(iter(bid.items())) - - to_trio.send_nowait( - ( + message = ( "l1", { "symbol": cf_sym_to_fqsn(data.symbol), + "broker_ts": time.time(), "ticks": [ { "type": "bid", @@ -239,7 +254,10 @@ async def aio_price_feed_relay( ] } ) - ) + try: + to_trio.send_nowait(message) + except trio.WouldBlock as e: + print(e) fh.add_feed( exchange, @@ -249,10 +267,8 @@ async def aio_price_feed_relay( ) if not fh.running: - try: - fh.run(start_loop=False, install_signal_handlers=False) - except BaseExceptionGroup as e: - breakpoint() + fh.run(start_loop=False, install_signal_handlers=False) + # sync with trio to_trio.send_nowait(None) From 8e91e215b3d142d31308cad4d4f2011b42864114 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 3 Mar 2023 16:24:44 -0500 Subject: [PATCH 05/81] WIP - ensure `asyncio` pumps the event loop each send --- piker/brokers/kucoin.py | 3 ++ piker/data/cryptofeeds.py | 109 ++++++++++++++++++++++++++------------ 2 files changed, 78 insertions(+), 34 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index edf5d508..14929d53 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,3 +1,6 @@ +# piker: trading gear for hackers +# Copyright (C) Jared Goldman (in stewardship for pikers) + # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/piker/data/cryptofeeds.py b/piker/data/cryptofeeds.py index 7e6e9de2..5605993d 100644 --- a/piker/data/cryptofeeds.py +++ b/piker/data/cryptofeeds.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) Jared Goldman (in stewardship for piker0) +# Copyright (C) Jared Goldman (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -102,14 +102,17 @@ async def mk_stream_quotes( sym = symbols[0] async with ( - open_cached_client(exchange.lower()) as client, - send_chan as send_chan + open_cached_client(exchange.lower()) as client, + # send_chan as send_chan, ): pairs = await client.cache_pairs() - pair_data = pairs[sym] - async with maybe_open_price_feed(pair_data, exchange, channels) as stream: + async with maybe_open_price_feed( + pair_data, + exchange, + channels, + ) as stream: init_msgs = { sym: { @@ -121,30 +124,41 @@ async def mk_stream_quotes( quote_msg = {"symbol": pair_data["name"], "last": 0, "ticks": []} task_status.started((init_msgs, quote_msg)) - feed_is_live.set() + + async for typ, quote in stream: + topic = quote["symbol"] + await send_chan.send({topic: quote}) + log.info( + f'sending {typ} quote:\n' + f'{quote}' + ) # try: - # async for typ, quote in stream: - # print(f'streaming {typ} quote: {quote}') - # topic = quote["symbobl"] - # await send_chan.send({topic: quote}) # finally: # breakpoint() - - while True: - with trio.move_on_after(4) as cancel_scope: - log.warning(f'WAITING FOR MESSAGE') - msg = await stream.receive() - log.warning(f'RECEIVED MSG: {msg}') - topic = msg["symbol"] - await send_chan.send({topic: msg}) - log.warning(f'SENT TO CHAN') - if cancel_scope.cancelled_caught: - await tractor.breakpoint() + + # while True: + # with trio.move_on_after(16) as cancel_scope: + + # log.warning(f'WAITING FOR MESSAGE') + # typ, quote = await stream.receive() + + # log.warning(f'RECEIVED MSG: {quote}') + + # topic = quote["symbol"] + # await send_chan.send({topic: quote}) + + # log.warning(f'SENT TO CHAN') + + # if cancel_scope.cancelled_caught: + # await tractor.breakpoint() @acm async def maybe_open_price_feed( - pair_data: Symbol, exchange: str, channels + pair_data: Symbol, + exchange: str, + channels, + ) -> trio.abc.ReceiveStream: # TODO: add a predicate to maybe_open_context # TODO: ensure we can dynamically pass down args here @@ -166,7 +180,13 @@ async def open_price_feed( ) -> trio.abc.ReceiveStream: async with maybe_open_feed_handler(exchange) as fh: async with to_asyncio.open_channel_from( - partial(aio_price_feed_relay, pair_data, exchange, channels, fh) + partial( + aio_price_feed_relay, + pair_data, + exchange, + channels, + fh, + ) ) as (first, chan): yield chan @@ -195,9 +215,15 @@ async def aio_price_feed_relay( exchange: str, channels: list[str], fh: FeedHandler, + from_trio: asyncio.Queue, to_trio: trio.abc.SendChannel, + ) -> None: + + # sync with trio + to_trio.send_nowait(None) + async def _trade(data: dict, receipt_timestamp): data = data.to_dict() message = ( @@ -214,18 +240,28 @@ async def aio_price_feed_relay( }], }, ) - print(f'trade message: {message}') - # try: - to_trio.send_nowait(message) - # except trio.WouldBlock as e: - #breakpoint() + try: + to_trio.send_nowait(message) + await asyncio.sleep(0.001) + except trio.WouldBlock as e: + log.exception( + 'l1: OVERRUN ASYNCIO -> TRIO\n' + f'TO_TRIO.stats -> {to_trio.statistics()}' + + ) + await asyncio.sleep(0) + + async def _l1( + data: dict, + receipt_timestamp: str | None, + ) -> None: + log.info(f'RECV L1 {receipt_timestamp}') - async def _l1(data: dict, receipt_timestamp): bid = data.book.to_dict()['bid'] ask = data.book.to_dict()['ask'] l1_ask_price, l1_ask_size = next(iter(ask.items())) l1_bid_price, l1_bid_size = next(iter(bid.items())) - message = ( + message = ( "l1", { "symbol": cf_sym_to_fqsn(data.symbol), @@ -256,8 +292,16 @@ async def aio_price_feed_relay( ) try: to_trio.send_nowait(message) + await asyncio.sleep(0.001) except trio.WouldBlock as e: - print(e) + log.exception( + 'l1: OVERRUN ASYNCIO -> TRIO\n' + f'TO_TRIO.stats -> {to_trio.statistics()}' + + ) + await asyncio.sleep(0) + # breakpoint() + # raise fh.add_feed( exchange, @@ -268,8 +312,5 @@ async def aio_price_feed_relay( if not fh.running: fh.run(start_loop=False, install_signal_handlers=False) - - # sync with trio - to_trio.send_nowait(None) await asyncio.sleep(float("inf")) From 7074ca771303f34b83b38bfa769edb00c5e22220 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 8 Mar 2023 23:31:28 -0500 Subject: [PATCH 06/81] Implement Kucoin auth and last trades call --- piker/brokers/kucoin.py | 211 +++++++++++++++++++++++++--------------- 1 file changed, 135 insertions(+), 76 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 14929d53..e1e9ebd9 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -18,32 +18,28 @@ """ -from dataclasses import field +from logging import warning from typing import Any, Optional, Literal from contextlib import asynccontextmanager as acm from datetime import datetime import time import math -from os import path, walk +import base64 +import hmac +import hashlib import asks import tractor import trio from trio_typing import TaskStatus from fuzzywuzzy import process as fuzzy -from cryptofeed.defines import KUCOIN, TRADES, L2_BOOK -from cryptofeed.symbols import Symbol import pendulum import numpy as np -from piker.data.cryptofeeds import ( - fqsn_to_cf_sym, - mk_stream_quotes, - get_config, -) + from piker._cacheables import open_cached_client from piker.log import get_logger -from piker.pp import config from ._util import DataUnavailable +from piker.pp import config _spawn_kwargs = { "infect_asyncio": True, @@ -51,45 +47,80 @@ _spawn_kwargs = { log = get_logger(__name__) _ohlc_dtype = [ - ('index', int), - ('time', int), - ('open', float), - ('high', float), - ('low', float), - ('close', float), - ('volume', float), - ('bar_wap', float), # will be zeroed by sampler if not filled + ("index", int), + ("time", int), + ("open", float), + ("high", float), + ("low", float), + ("close", float), + ("volume", float), + ("bar_wap", float), # will be zeroed by sampler if not filled ] class Client: def __init__(self) -> None: - self._pairs: dict[str, Symbol] = {} + self._pairs: dict[str, any] = {} self._bars: list[list] = [] - # TODO" Shouldn't have to write kucoin twice here + self._key_id: str + self._key_secret: str + self._key_passphrase: str + self._authenticated: bool = False - config = get_config("kucoin").get("kucoin", {}) - # - if ("key_id" in config) and ("key_secret" in config): + config = get_config() + breakpoint() + if ("key_id" in config) and \ + ("key_secret" in config) and \ + ("key_passphrase" in config): + self._authenticated = True self._key_id = config["key_id"] self._key_secret = config["key_secret"] - - else: - self._key_id = None - self._key_secret = None + self._key_passphrase = config["key_passphrase"] async def _request( self, action: Literal["POST", "GET", "PUT", "DELETE"], - route: str, + endpoint: str, api_v: str = "v2", ) -> Any: - api_url = f"https://api.kucoin.com/api/{api_v}{route}" - res = await asks.request(action, api_url) - #breakpoint() - try: + + now = int(time.time() * 1000) + path = f'/api/{api_v}{endpoint}' + str_to_sign = str(now) + action + path + headers = {} + + # Add headers to request if authenticated + if self._authenticated: + signature = base64.b64encode( + hmac.new( + self._key_secret.encode('utf-8'), + str_to_sign.encode('utf-8'), + hashlib.sha256 + ).digest() + ) + + passphrase = base64.b64encode( + hmac.new( + self._key_secret.encode('utf-8'), + self._key_passphrase.encode('utf-8'), + hashlib.sha256 + ).digest() + ) + + headers = { + "KC-API-SIGN": signature, + "KC-API-TIMESTAMP": str(now), + "KC-API-KEY": self._key_id, + "KC-API-PASSPHRASE": passphrase, + "KC-API-KEY-VERSION": "2" + } + + api_url = f"https://api.kucoin.com{path}" + res = await asks.request(action, api_url, headers=headers) + # breakpoint() + if "data" in res.json(): return res.json()["data"] - except KeyError as e: + else: print(f'KUCOIN ERROR: {res.json()["msg"]}') breakpoint() @@ -106,14 +137,14 @@ class Client: async def cache_pairs( self, normalize: bool = True, - ) -> dict[str, Symbol]: + ) -> dict[str, any]: if not self._pairs: self._pairs = await self.get_pairs() if normalize: self._pairs = self.normalize_pairs(self._pairs) return self._pairs - def normalize_pairs(self, pairs: dict[str, Symbol]) -> dict[str, Symbol]: + def normalize_pairs(self, pairs: dict[str, any]) -> dict[str, any]: """ Map crypfeeds symbols to fqsn strings @@ -137,6 +168,10 @@ class Client: # repack in dict form return {item[0]["name"].lower(): item[0] for item in matches} + async def last_trades(self, sym: str): + trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") + return trades.items + async def get_bars( self, fqsn: str, @@ -158,7 +193,7 @@ class Client: # Format datetime to unix start_dt = math.trunc(time.mktime(start_dt.timetuple())) end_dt = math.trunc(time.mktime(end_dt.timetuple())) - kucoin_sym = fqsn_to_cf_sym(fqsn, self._pairs) + kucoin_sym = fqsn_to_cf_sym(fqsn, self._pairs) url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" bars = await self._request( @@ -170,39 +205,58 @@ class Client: new_bars = [] for i, bar in enumerate(bars[::-1]): # TODO: implement struct/typecasting/validation here - + data = { - 'index': i, - 'time': bar[0], - 'open': bar[1], - 'close': bar[2], - 'high': bar[3], - 'low': bar[4], - 'volume': bar[5], - 'amount': bar [6], - 'bar_wap': 0.0, + "index": i, + "time": bar[0], + "open": bar[1], + "close": bar[2], + "high": bar[3], + "low": bar[4], + "volume": bar[5], + "amount": bar[6], + "bar_wap": 0.0, } row = [] for j, (field_name, field_type) in enumerate(_ohlc_dtype): - - value = data[field_name] + value = data[field_name] match field_name: - case 'index' | 'time': + case "index" | "time": row.append(int(value)) # case 'time': - # dt_from_unix_ts = datetime.utcfromtimestamp(int(value)) - # # convert unix time to epoch seconds - # row.append(int(dt_from_unix_ts.timestamp())) - case _: + # dt_from_unix_ts = datetime.utcfromtimestamp(int(value)) + # # convert unix time to epoch seconds + # row.append(int(dt_from_unix_ts.timestamp())) + case _: row.append(float(value)) new_bars.append(tuple(row)) - + self._bars = array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars return array +def fqsn_to_cf_sym(fqsn: str, pairs: dict[str, any]) -> str: + pair_data = pairs[fqsn] + return pair_data["baseCurrency"] + "-" + pair_data["quoteCurrency"] + + +def get_config() -> dict[str, Any]: + conf, path = config.load() + + section = conf.get('kucoin') + + # TODO: document why we send this, basically because logging params for cryptofeed + conf["log"] = {} + conf["log"]["disabled"] = True + + if section is None: + log.warning("No config section found for deribit in kucoin") + + return section + + @acm async def get_client(): client = Client() @@ -217,7 +271,7 @@ async def open_symbol_search( ): async with open_cached_client("kucoin") as client: # load all symbols locally for fast search - cache = await client.cache_pairs() + await client.cache_pairs() await ctx.started() async with ctx.open_stream() as stream: @@ -234,15 +288,24 @@ async def stream_quotes( # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ): - return await mk_stream_quotes( - KUCOIN, - [L2_BOOK, TRADES], - send_chan, - symbols, - feed_is_live, - loglevel, - task_status, - ) + sym = symbols[0] + + async with open_cached_client("kucoin") as client: + init_msgs = { + # pass back token, and bool, signalling if we're the writer + # and that history has been written + sym: { + "symbol_info": { + "asset_type": "option", + "price_tick_size": 0.0005, + "lot_tick_size": 0.1, + }, + "shm_write_opts": {"sum_tick_vml": False}, + "fqsn": sym, + }, + } + + last_trades = await client.last_trades(sym) @acm @@ -256,13 +319,9 @@ async def open_history_client( timeframe: float, end_dt: datetime | None = None, start_dt: datetime | None = None, - ) -> tuple[ - np.ndarray, - datetime | None, # start - datetime | None, # end - ]: + ) -> tuple[np.ndarray, datetime | None, datetime | None,]: # start # end if timeframe != 60: - raise DataUnavailable('Only 1m bars are supported') + raise DataUnavailable("Only 1m bars are supported") array = await client.get_bars( symbol, @@ -270,13 +329,13 @@ async def open_history_client( end_dt=end_dt, ) - times = array['time'] + times = array["time"] - if ( - end_dt is None - ): + if end_dt is None: inow = round(time.time()) - print(f'difference in time between load and processing {inow - times[-1]}') + print( + f"difference in time between load and processing {inow - times[-1]}" + ) if (inow - times[-1]) > 60: await tractor.breakpoint() @@ -284,4 +343,4 @@ async def open_history_client( end_dt = pendulum.from_timestamp(times[-1]) return array, start_dt, end_dt - yield get_ohlc_history, {'erlangs': 3, 'rate': 3} + yield get_ohlc_history, {"erlangs": 3, "rate": 3} From cda045f123bcddab5b601a5f98a44d563460138b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 8 Mar 2023 23:45:51 -0500 Subject: [PATCH 07/81] Abstract header gen to seperate function --- piker/brokers/kucoin.py | 68 +++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index e1e9ebd9..1a7861cb 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -77,47 +77,55 @@ class Client: self._key_secret = config["key_secret"] self._key_passphrase = config["key_passphrase"] + def _gen_auth_req_headers( + self, + action: Literal["POST", "GET", "PUT", "DELETE"], + endpoint: str, + api_v: str = "v2", + ): + now = int(time.time() * 1000) + path = f'/api/{api_v}{endpoint}' + str_to_sign = str(now) + action + path + + # Add headers to request if authenticated + signature = base64.b64encode( + hmac.new( + self._key_secret.encode('utf-8'), + str_to_sign.encode('utf-8'), + hashlib.sha256 + ).digest() + ) + + passphrase = base64.b64encode( + hmac.new( + self._key_secret.encode('utf-8'), + self._key_passphrase.encode('utf-8'), + hashlib.sha256 + ).digest() + ) + + return { + "KC-API-SIGN": signature, + "KC-API-TIMESTAMP": str(now), + "KC-API-KEY": self._key_id, + "KC-API-PASSPHRASE": passphrase, + "KC-API-KEY-VERSION": "2" + } + async def _request( self, action: Literal["POST", "GET", "PUT", "DELETE"], endpoint: str, api_v: str = "v2", ) -> Any: - - now = int(time.time() * 1000) - path = f'/api/{api_v}{endpoint}' - str_to_sign = str(now) + action + path headers = {} - # Add headers to request if authenticated if self._authenticated: - signature = base64.b64encode( - hmac.new( - self._key_secret.encode('utf-8'), - str_to_sign.encode('utf-8'), - hashlib.sha256 - ).digest() - ) + headers = self._gen_auth_req_headers(action, endpoint, api_v) - passphrase = base64.b64encode( - hmac.new( - self._key_secret.encode('utf-8'), - self._key_passphrase.encode('utf-8'), - hashlib.sha256 - ).digest() - ) - - headers = { - "KC-API-SIGN": signature, - "KC-API-TIMESTAMP": str(now), - "KC-API-KEY": self._key_id, - "KC-API-PASSPHRASE": passphrase, - "KC-API-KEY-VERSION": "2" - } - - api_url = f"https://api.kucoin.com{path}" + api_url = f"https://api.kucoin.com/api/{api_v}{endpoint}" res = await asks.request(action, api_url, headers=headers) - # breakpoint() + if "data" in res.json(): return res.json()["data"] else: From 1a655b7e396b96898616a4898c046e72ddc04778 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 11 Mar 2023 16:21:42 -0500 Subject: [PATCH 08/81] Ensure we're passing the correct api version to the header builder, make headers a default arg --- piker/brokers/kucoin.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 1a7861cb..f4ef0b97 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -15,10 +15,9 @@ # along with this program. If not, see . """ - +Kucoin broker backend """ -from logging import warning from typing import Any, Optional, Literal from contextlib import asynccontextmanager as acm from datetime import datetime @@ -41,10 +40,6 @@ from piker.log import get_logger from ._util import DataUnavailable from piker.pp import config -_spawn_kwargs = { - "infect_asyncio": True, -} - log = get_logger(__name__) _ohlc_dtype = [ ("index", int), @@ -68,7 +63,7 @@ class Client: self._authenticated: bool = False config = get_config() - breakpoint() + if ("key_id" in config) and \ ("key_secret" in config) and \ ("key_passphrase" in config): @@ -83,11 +78,13 @@ class Client: endpoint: str, api_v: str = "v2", ): + ''' + https://docs.kucoin.com/#authentication + ''' now = int(time.time() * 1000) path = f'/api/{api_v}{endpoint}' str_to_sign = str(now) + action + path - # Add headers to request if authenticated signature = base64.b64encode( hmac.new( self._key_secret.encode('utf-8'), @@ -109,7 +106,7 @@ class Client: "KC-API-TIMESTAMP": str(now), "KC-API-KEY": self._key_id, "KC-API-PASSPHRASE": passphrase, - "KC-API-KEY-VERSION": "2" + "KC-API-KEY-VERSION": api_v[1] } async def _request( @@ -117,8 +114,8 @@ class Client: action: Literal["POST", "GET", "PUT", "DELETE"], endpoint: str, api_v: str = "v2", + headers: dict = {} ) -> Any: - headers = {} if self._authenticated: headers = self._gen_auth_req_headers(action, endpoint, api_v) From 109e7d7b43237ac579a207271ae47d73cc1dd4f1 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 11 Mar 2023 18:55:40 -0500 Subject: [PATCH 09/81] Add back static API version in headers --- piker/brokers/kucoin.py | 95 +++++++++++++++++++++++++++-------------- 1 file changed, 62 insertions(+), 33 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index f4ef0b97..64152b18 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -18,7 +18,7 @@ Kucoin broker backend """ -from typing import Any, Optional, Literal +from typing import Any, Optional, Literal, AsyncGenerator from contextlib import asynccontextmanager as acm from datetime import datetime import time @@ -26,6 +26,7 @@ import math import base64 import hmac import hashlib +import wsproto import asks import tractor @@ -39,6 +40,10 @@ from piker._cacheables import open_cached_client from piker.log import get_logger from ._util import DataUnavailable from piker.pp import config +from ..data._web_bs import ( + open_autorecon_ws, + NoBsWs, +) log = get_logger(__name__) _ohlc_dtype = [ @@ -53,6 +58,21 @@ _ohlc_dtype = [ ] +def get_config() -> dict[str, Any]: + conf, path = config.load() + + section = conf.get("kucoin") + + # TODO: document why we send this, basically because logging params for cryptofeed + conf["log"] = {} + conf["log"]["disabled"] = True + breakpoint() + if section is None: + log.warning("No config section found for kucoin in config") + + return section + + class Client: def __init__(self) -> None: self._pairs: dict[str, any] = {} @@ -64,9 +84,11 @@ class Client: config = get_config() - if ("key_id" in config) and \ - ("key_secret" in config) and \ - ("key_passphrase" in config): + if ( + ("key_id" in config) + and ("key_secret" in config) + and ("key_passphrase" in config) + ): self._authenticated = True self._key_id = config["key_id"] self._key_secret = config["key_secret"] @@ -74,30 +96,30 @@ class Client: def _gen_auth_req_headers( self, - action: Literal["POST", "GET", "PUT", "DELETE"], + action: Literal["POST", "GET"], endpoint: str, api_v: str = "v2", ): - ''' + """ https://docs.kucoin.com/#authentication - ''' + """ now = int(time.time() * 1000) - path = f'/api/{api_v}{endpoint}' + path = f"/api/{api_v}{endpoint}" str_to_sign = str(now) + action + path signature = base64.b64encode( hmac.new( - self._key_secret.encode('utf-8'), - str_to_sign.encode('utf-8'), - hashlib.sha256 + self._key_secret.encode("utf-8"), + str_to_sign.encode("utf-8"), + hashlib.sha256, ).digest() ) passphrase = base64.b64encode( hmac.new( - self._key_secret.encode('utf-8'), - self._key_passphrase.encode('utf-8'), - hashlib.sha256 + self._key_secret.encode("utf-8"), + self._key_passphrase.encode("utf-8"), + hashlib.sha256, ).digest() ) @@ -106,17 +128,16 @@ class Client: "KC-API-TIMESTAMP": str(now), "KC-API-KEY": self._key_id, "KC-API-PASSPHRASE": passphrase, - "KC-API-KEY-VERSION": api_v[1] + "KC-API-KEY-VERSION": "2", } async def _request( self, - action: Literal["POST", "GET", "PUT", "DELETE"], + action: Literal["POST", "GET"], endpoint: str, api_v: str = "v2", - headers: dict = {} + headers: dict = {}, ) -> Any: - if self._authenticated: headers = self._gen_auth_req_headers(action, endpoint, api_v) @@ -129,6 +150,11 @@ class Client: print(f'KUCOIN ERROR: {res.json()["msg"]}') breakpoint() + async def _get_ws_token(self, private: bool) -> str: + token_type = "private" if private else "public" + token = await self._request("POST", f"/bullet-{token_type}", "v1") + return token + async def get_pairs( self, ) -> dict[str, Any]: @@ -175,6 +201,7 @@ class Client: async def last_trades(self, sym: str): trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") + breakpoint() return trades.items async def get_bars( @@ -247,21 +274,6 @@ def fqsn_to_cf_sym(fqsn: str, pairs: dict[str, any]) -> str: return pair_data["baseCurrency"] + "-" + pair_data["quoteCurrency"] -def get_config() -> dict[str, Any]: - conf, path = config.load() - - section = conf.get('kucoin') - - # TODO: document why we send this, basically because logging params for cryptofeed - conf["log"] = {} - conf["log"]["disabled"] = True - - if section is None: - log.warning("No config section found for deribit in kucoin") - - return section - - @acm async def get_client(): client = Client() @@ -312,6 +324,23 @@ async def stream_quotes( last_trades = await client.last_trades(sym) + # @acm + # async def subscribe(ws: wsproto.WSConnection): + + token = await client._get_ws_token(True) + async with open_autorecon_ws( + f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={12345}]" + ) as ws: + msg_gen = stream_messageS(ws) + + +async def stream_messageS(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]: + timeouts = 0 + while True: + with trio.move_on_after(3) as cs: + msg = await ws.recv_msg() + print(f"msg: {msg}") + @acm async def open_history_client( From ade2c32adbaff130af407055650ca6430f3dec6a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 14 Mar 2023 14:15:10 -0400 Subject: [PATCH 10/81] Succesfully connect to kucoin ws --- piker/brokers/kucoin.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 64152b18..ef70bfc0 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -26,7 +26,8 @@ import math import base64 import hmac import hashlib -import wsproto +# import wsproto +from uuid import uuid4 import asks import tractor @@ -66,7 +67,6 @@ def get_config() -> dict[str, Any]: # TODO: document why we send this, basically because logging params for cryptofeed conf["log"] = {} conf["log"]["disabled"] = True - breakpoint() if section is None: log.warning("No config section found for kucoin in config") @@ -152,8 +152,12 @@ class Client: async def _get_ws_token(self, private: bool) -> str: token_type = "private" if private else "public" - token = await self._request("POST", f"/bullet-{token_type}", "v1") - return token + data = await self._request("POST", f"/bullet-{token_type}", "v1") + if "token" in data: + return data["token"] + else: + print(f'KUCOIN ERROR: {data.json()["msg"]}') + breakpoint() async def get_pairs( self, @@ -201,7 +205,6 @@ class Client: async def last_trades(self, sym: str): trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") - breakpoint() return trades.items async def get_bars( @@ -328,13 +331,15 @@ async def stream_quotes( # async def subscribe(ws: wsproto.WSConnection): token = await client._get_ws_token(True) + connect_id = str(uuid4()) async with open_autorecon_ws( - f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={12345}]" + f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={connect_id}]" ) as ws: - msg_gen = stream_messageS(ws) + breakpoint() + msg_gen = stream_messages(ws) -async def stream_messageS(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]: +async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 while True: with trio.move_on_after(3) as cs: From ac34ca7cad20c115dc3e73bc1940de368688e2a0 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 14 Mar 2023 15:05:04 -0400 Subject: [PATCH 11/81] Add sub method to flow Stash for checkout of master --- piker/brokers/kucoin.py | 79 +++++++++++++++++++++++++++++++++-------- piker/data/_web_bs.py | 3 +- 2 files changed, 67 insertions(+), 15 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index ef70bfc0..4472bee7 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,5 +1,3 @@ -# piker: trading gear for hackers -# Copyright (C) Jared Goldman (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -26,7 +24,8 @@ import math import base64 import hmac import hashlib -# import wsproto + +import wsproto from uuid import uuid4 import asks @@ -150,9 +149,10 @@ class Client: print(f'KUCOIN ERROR: {res.json()["msg"]}') breakpoint() - async def _get_ws_token(self, private: bool) -> str: + async def _get_ws_token(self, private: bool = False) -> str | None: token_type = "private" if private else "public" data = await self._request("POST", f"/bullet-{token_type}", "v1") + breakpoint() if "token" in data: return data["token"] else: @@ -308,15 +308,21 @@ async def stream_quotes( # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ): + + # TODO: Add multi-symbol functionality here sym = symbols[0] + connect_id = 0 async with open_cached_client("kucoin") as client: + + pairs = await client.cache_pairs() + kucoin_sym = pairs[sym]['symbol'] init_msgs = { # pass back token, and bool, signalling if we're the writer # and that history has been written sym: { "symbol_info": { - "asset_type": "option", + "asset_type": "crypto", "price_tick_size": 0.0005, "lot_tick_size": 0.1, }, @@ -327,25 +333,70 @@ async def stream_quotes( last_trades = await client.last_trades(sym) - # @acm - # async def subscribe(ws: wsproto.WSConnection): - - token = await client._get_ws_token(True) - connect_id = str(uuid4()) - async with open_autorecon_ws( - f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={connect_id}]" - ) as ws: + @acm + async def subscribe(ws: wsproto.WSConnection): + await ws.send_msg({ + "id": connect_id, + "type": "ping" + }) + res = await ws.recv_msg() breakpoint() + yield + # l1_sub = make_sub(kucoin_sym, sub_id) + # await ws.send_msg(l1_sub) + # res = await ws.recv_msg() + # breakpoint() + # assert res['id'] == connect_id + # + # yield + # + # # unsub + # ws.send_msg({ + # "id": sub_id, + # "type": "unsubscribe", + # "topic": f"/market/ticker:{sym}", + # "privateChannel": False, + # "response": True, + # }) + + token = await client._get_ws_token() + breakpoint() + async with open_autorecon_ws( + f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={connect_id}]", + fixture=subscribe, + ) as ws: msg_gen = stream_messages(ws) +def make_sub(sym, connect_id): + breakpoint() + return { + "id": connect_id, + "type": "subscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]: + timeouts = 0 + while True: with trio.move_on_after(3) as cs: msg = await ws.recv_msg() - print(f"msg: {msg}") + if cs.cancelled_caught: + + timeouts += 1 + + if timeouts > 2: + log.error("kucoin feed is sh**ing the bed... rebooting...") + await ws._connect() + + continue + + breakpoint() @acm async def open_history_client( diff --git a/piker/data/_web_bs.py b/piker/data/_web_bs.py index 2dd7f4af..3a397f7e 100644 --- a/piker/data/_web_bs.py +++ b/piker/data/_web_bs.py @@ -100,6 +100,7 @@ class NoBsWs: last_err = None for i in range(tries): try: + breakpoint() self._ws = await self._stack.enter_async_context( trio_websocket.open_websocket_url(self.url) ) @@ -166,7 +167,7 @@ async def open_autorecon_ws( # TODO: proper type cannot smh fixture: Optional[Callable] = None, -) -> AsyncGenerator[tuple[...], NoBsWs]: +) -> AsyncGenerator[tuple[...], NoBsWs]: """Apparently we can QoS for all sorts of reasons..so catch em. """ From a3c7bec5768c4345e436a0efb90dccc96991dd3b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 15 Mar 2023 20:03:16 -0400 Subject: [PATCH 12/81] Implement working message streaming --- piker/brokers/kucoin.py | 90 ++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 4472bee7..50a4ba5e 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,4 +1,3 @@ - # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -152,7 +151,6 @@ class Client: async def _get_ws_token(self, private: bool = False) -> str | None: token_type = "private" if private else "public" data = await self._request("POST", f"/bullet-{token_type}", "v1") - breakpoint() if "token" in data: return data["token"] else: @@ -308,15 +306,13 @@ async def stream_quotes( # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ): - # TODO: Add multi-symbol functionality here sym = symbols[0] - connect_id = 0 + connect_id = str(uuid4()) async with open_cached_client("kucoin") as client: - pairs = await client.cache_pairs() - kucoin_sym = pairs[sym]['symbol'] + kucoin_sym = pairs[sym]["symbol"] init_msgs = { # pass back token, and bool, signalling if we're the writer # and that history has been written @@ -335,40 +331,47 @@ async def stream_quotes( @acm async def subscribe(ws: wsproto.WSConnection): - await ws.send_msg({ - "id": connect_id, - "type": "ping" - }) - res = await ws.recv_msg() - breakpoint() - yield - # l1_sub = make_sub(kucoin_sym, sub_id) - # await ws.send_msg(l1_sub) + # await ws.send_msg({"id": connect_id, "type": "ping"}) # res = await ws.recv_msg() + l1_sub = make_sub(kucoin_sym, connect_id) + await ws.send_msg(l1_sub) + res = await ws.recv_msg() # breakpoint() - # assert res['id'] == connect_id - # - # yield - # - # # unsub - # ws.send_msg({ - # "id": sub_id, - # "type": "unsubscribe", - # "topic": f"/market/ticker:{sym}", - # "privateChannel": False, - # "response": True, - # }) + # assert res["id"] == connect_id + + yield + + # unsub + await ws.send_msg( + { + "id": connect_id, + "type": "unsubscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + ) token = await client._get_ws_token() - breakpoint() async with open_autorecon_ws( - f"wss://ws-api-spot.kucoin.com/?token=={token}&[connectId={connect_id}]", + f"wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]", fixture=subscribe, ) as ws: - msg_gen = stream_messages(ws) + msg_gen = stream_messages(ws, sym) + typ, quote = await msg_gen.__anext__() + # + while typ != "trade": + # TODO: use ``anext()`` when it lands in 3.10! + typ, quote = await msg_gen.__anext__() + + task_status.started((init_msgs, quote)) + feed_is_live.set() + + async for typ, msg in msg_gen: + await send_chan.send({sym: msg}) + def make_sub(sym, connect_id): - breakpoint() return { "id": connect_id, "type": "subscribe", @@ -378,25 +381,38 @@ def make_sub(sym, connect_id): } -async def stream_messages(ws: NoBsWs) -> AsyncGenerator[NoBsWs, dict]: - +async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 while True: with trio.move_on_after(3) as cs: msg = await ws.recv_msg() - if cs.cancelled_caught: - timeouts += 1 - if timeouts > 2: log.error("kucoin feed is sh**ing the bed... rebooting...") await ws._connect() continue + if "subject" in msg and msg["subject"] == "trade.ticker": + # TODO: cast msg into class + trade_data = msg["data"] + yield "trade", { + "symbol": sym, + "last": trade_data["price"], + "brokerd_ts": trade_data["time"], + "ticks": [ + { + "type": "trade", + "price": float(trade_data["price"]), + "size": float(trade_data["size"]), + "broker_ts": trade_data["time"], + } + ], + } + else: + continue - breakpoint() @acm async def open_history_client( From b14b3230685fe026a99edad4efc60dff9a7bd56a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 15 Mar 2023 20:26:51 -0400 Subject: [PATCH 13/81] Remove breakpoint in web_bs, ensure we only unsub if ws is connected --- piker/brokers/kucoin.py | 19 ++++++++++--------- piker/data/_web_bs.py | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 50a4ba5e..2d0efd7d 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -342,15 +342,16 @@ async def stream_quotes( yield # unsub - await ws.send_msg( - { - "id": connect_id, - "type": "unsubscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, - } - ) + if ws.connected(): + await ws.send_msg( + { + "id": connect_id, + "type": "unsubscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + ) token = await client._get_ws_token() async with open_autorecon_ws( diff --git a/piker/data/_web_bs.py b/piker/data/_web_bs.py index 3a397f7e..21b06d68 100644 --- a/piker/data/_web_bs.py +++ b/piker/data/_web_bs.py @@ -100,7 +100,6 @@ class NoBsWs: last_err = None for i in range(tries): try: - breakpoint() self._ws = await self._stack.enter_async_context( trio_websocket.open_websocket_url(self.url) ) From 199a70880c5cebb03967d6aa60d22dac54548dca Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 18 Mar 2023 11:21:23 -0400 Subject: [PATCH 14/81] Spawn background ping task --- piker/brokers/kucoin.py | 66 +++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 2d0efd7d..fc497eb0 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -29,6 +29,7 @@ from uuid import uuid4 import asks import tractor +from tractor.trionics import maybe_open_context import trio from trio_typing import TaskStatus from fuzzywuzzy import process as fuzzy @@ -148,11 +149,13 @@ class Client: print(f'KUCOIN ERROR: {res.json()["msg"]}') breakpoint() - async def _get_ws_token(self, private: bool = False) -> str | None: + async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: token_type = "private" if private else "public" data = await self._request("POST", f"/bullet-{token_type}", "v1") if "token" in data: - return data["token"] + # return token and ping interval + ping_interval = data["instanceServers"][0]["pingInterval"] + return data["token"], ping_interval else: print(f'KUCOIN ERROR: {data.json()["msg"]}') breakpoint() @@ -311,6 +314,7 @@ async def stream_quotes( connect_id = str(uuid4()) async with open_cached_client("kucoin") as client: + token, ping_interval = await client._get_ws_token() pairs = await client.cache_pairs() kucoin_sym = pairs[sym]["symbol"] init_msgs = { @@ -331,29 +335,45 @@ async def stream_quotes( @acm async def subscribe(ws: wsproto.WSConnection): - # await ws.send_msg({"id": connect_id, "type": "ping"}) - # res = await ws.recv_msg() - l1_sub = make_sub(kucoin_sym, connect_id) - await ws.send_msg(l1_sub) - res = await ws.recv_msg() - # breakpoint() - # assert res["id"] == connect_id - yield + @acm + async def open_ping_task(ws: wsproto.WSConnection): + async with trio.open_nursery() as n: - # unsub - if ws.connected(): - await ws.send_msg( - { - "id": connect_id, - "type": "unsubscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, - } - ) + async def ping_server(): + while True: + await trio.sleep((ping_interval - 1000) / 1000) + print("PINGING") + await ws.send_msg({"id": connect_id, "type": "ping"}) + + n.start_soon(ping_server) + + yield ws + + n.cancel_scope.cancel() + + # Spawn the ping task here + async with open_ping_task(ws) as _ws: + + # subscribe to market feedz here + l1_sub = make_sub(kucoin_sym, connect_id) + await _ws.send_msg(l1_sub) + res = await _ws.recv_msg() + + yield + + # unsub + if _ws.connected(): + await _ws.send_msg( + { + "id": connect_id, + "type": "unsubscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + ) - token = await client._get_ws_token() async with open_autorecon_ws( f"wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]", fixture=subscribe, @@ -395,6 +415,7 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: await ws._connect() continue + if "subject" in msg and msg["subject"] == "trade.ticker": # TODO: cast msg into class trade_data = msg["data"] @@ -411,6 +432,7 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: } ], } + else: continue From 1c4c19b35153a4e68455a57810f48d6ce7ef3aff Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 13:11:33 -0400 Subject: [PATCH 15/81] Clean up broker code, Add typecasting for messages/rt-data and historcal user trades ensure we're fetching all history add multi-symbol support ' --- piker/brokers/kucoin.py | 324 ++++++++++++++++++++++++++-------------- 1 file changed, 213 insertions(+), 111 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index fc497eb0..35d8f690 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,3 +1,6 @@ +# piker: trading gear for hackers +# Copyright (C) Jared Goldman (in stewardship for piker0) + # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -13,9 +16,10 @@ """ Kucoin broker backend + """ -from typing import Any, Optional, Literal, AsyncGenerator +from typing import Any, Callable, Optional, Literal, AsyncGenerator from contextlib import asynccontextmanager as acm from datetime import datetime import time @@ -23,13 +27,11 @@ import math import base64 import hmac import hashlib - import wsproto from uuid import uuid4 import asks import tractor -from tractor.trionics import maybe_open_context import trio from trio_typing import TaskStatus from fuzzywuzzy import process as fuzzy @@ -40,12 +42,14 @@ from piker._cacheables import open_cached_client from piker.log import get_logger from ._util import DataUnavailable from piker.pp import config +from ..data.types import Struct from ..data._web_bs import ( open_autorecon_ws, NoBsWs, ) log = get_logger(__name__) + _ohlc_dtype = [ ("index", int), ("time", int), @@ -58,24 +62,92 @@ _ohlc_dtype = [ ] -def get_config() -> dict[str, Any]: +def get_config() -> dict[str, dict]: conf, path = config.load() section = conf.get("kucoin") - # TODO: document why we send this, basically because logging params for cryptofeed - conf["log"] = {} - conf["log"]["disabled"] = True if section is None: log.warning("No config section found for kucoin in config") return section +class KucoinMktPair(Struct, frozen=True): + ''' + Kucoin's pair format + + ''' + baseCurrency: str + baseIncrement: float + baseMaxSize: float + baseMinSize: float + enableTrading: bool + feeCurrency: str + isMarginEnabled: bool + market: str + minFunds: float + name: str + priceIncrement: float + priceLimitRate: float + quoteCurrency: str + quoteIncrement: float + quoteMaxSize: float + quoteMinSize: float + symbol: str + + +class AccountTrade(Struct, frozen=True): + ''' + Historical trade format + + ''' + id: str + currency: str + amount: float + fee: float + balance: float + accountType: str + bizType: str + direction: Literal["in", "out"] + createdAt: float + context: list[str] + + +class AccountResponse(Struct, frozen=True): + currentPage: int + pageSize: int + totalNum: int + totalPage: int + items: list[AccountTrade] + + +class KucoinTrade(Struct, frozen=True): + ''' + Real-time trade format + + ''' + bestAsk: float + bestAskSize: float + bestBid: float + bestBidSize: float + price: float + sequence: float + size: float + time: float + + +class KucoinTradeMsg(Struct, frozen=True): + type: str + topic: str + subject: str + data: list[KucoinTrade] + + class Client: def __init__(self) -> None: - self._pairs: dict[str, any] = {} - self._bars: list[list] = [] + self._pairs: dict[str, KucoinMktPair] = {} + self._bars: list[list[float]] = [] self._key_id: str self._key_secret: str self._key_passphrase: str @@ -84,7 +156,7 @@ class Client: config = get_config() if ( - ("key_id" in config) + float("key_id" in config) and ("key_secret" in config) and ("key_passphrase" in config) ): @@ -98,10 +170,12 @@ class Client: action: Literal["POST", "GET"], endpoint: str, api_v: str = "v2", - ): - """ + ) -> dict[str, str]: + ''' + Generate authenticated request headers https://docs.kucoin.com/#authentication - """ + + ''' now = int(time.time() * 1000) path = f"/api/{api_v}{endpoint}" str_to_sign = str(now) + action + path @@ -127,6 +201,7 @@ class Client: "KC-API-TIMESTAMP": str(now), "KC-API-KEY": self._key_id, "KC-API-PASSPHRASE": passphrase, + # XXX: Even if using the v1 api - this stays the same "KC-API-KEY-VERSION": "2", } @@ -136,7 +211,11 @@ class Client: endpoint: str, api_v: str = "v2", headers: dict = {}, - ) -> Any: + ) -> dict[str, Any]: + ''' + Generic request wrapper for Kucoin API + + ''' if self._authenticated: headers = self._gen_auth_req_headers(action, endpoint, api_v) @@ -146,43 +225,56 @@ class Client: if "data" in res.json(): return res.json()["data"] else: - print(f'KUCOIN ERROR: {res.json()["msg"]}') - breakpoint() + log.error(f'Error making request to {api_url} -> {res.json()["msg"]}') - async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: + async def _get_ws_token( + self, + private: bool = False + ) -> tuple[str, int] | None: + ''' + Fetch ws token needed for sub access + + ''' token_type = "private" if private else "public" data = await self._request("POST", f"/bullet-{token_type}", "v1") + if "token" in data: - # return token and ping interval ping_interval = data["instanceServers"][0]["pingInterval"] return data["token"], ping_interval else: - print(f'KUCOIN ERROR: {data.json()["msg"]}') - breakpoint() + log.error( + f'Error making request for Kucoin ws token -> {res.json()["msg"]}' + ) async def get_pairs( self, - ) -> dict[str, Any]: + ) -> dict[str, KucoinMktPair]: if self._pairs: return self._pairs entries = await self._request("GET", "/symbols") - syms = {item["name"]: item for item in entries} + syms = {item["name"]: KucoinMktPair(**item) for item in entries} return syms async def cache_pairs( self, normalize: bool = True, - ) -> dict[str, any]: + ) -> dict[str, KucoinMktPair]: + ''' + Get cached pairs and convert keyed symbols into fqsns if ya want + + ''' if not self._pairs: self._pairs = await self.get_pairs() if normalize: self._pairs = self.normalize_pairs(self._pairs) return self._pairs - def normalize_pairs(self, pairs: dict[str, any]) -> dict[str, any]: + def normalize_pairs( + self, pairs: dict[str, KucoinMktPair] + ) -> dict[str, KucoinMktPair]: """ - Map crypfeeds symbols to fqsn strings + Map kucoin pairs to fqsn strings """ norm_pairs = {} @@ -197,14 +289,14 @@ class Client: self, pattern: str, limit: int = 30, - ) -> dict[str, Any]: + ) -> dict[str, KucoinMktPair]: data = await self.get_pairs() matches = fuzzy.extractBests(pattern, data, score_cutoff=35, limit=limit) # repack in dict form - return {item[0]["name"].lower(): item[0] for item in matches} + return {kucoin_sym_to_fqsn(item[0].name): item[0] for item in matches} - async def last_trades(self, sym: str): + async def last_trades(self, sym: str) -> AccountResponse: trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") return trades.items @@ -216,20 +308,23 @@ class Client: limit: int = 1000, as_np: bool = True, type: str = "1min", - ): - if len(self._bars): - return self._bars + ) -> np.ndarray: + ''' + Get OHLC data and convert to numpy array for perffff + ''' + # Generate generic end and start time if values not passed if end_dt is None: end_dt = pendulum.now("UTC").add(minutes=1) if start_dt is None: start_dt = end_dt.start_of("minute").subtract(minutes=limit) - # Format datetime to unix + # Format datetime to unix timestamp start_dt = math.trunc(time.mktime(start_dt.timetuple())) end_dt = math.trunc(time.mktime(end_dt.timetuple())) - kucoin_sym = fqsn_to_cf_sym(fqsn, self._pairs) + kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) + url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" bars = await self._request( @@ -238,9 +333,9 @@ class Client: api_v="v1", ) + # Map to OHLC values to dict then to np array new_bars = [] for i, bar in enumerate(bars[::-1]): - # TODO: implement struct/typecasting/validation here data = { "index": i, @@ -256,14 +351,12 @@ class Client: row = [] for j, (field_name, field_type) in enumerate(_ohlc_dtype): + value = data[field_name] + match field_name: case "index" | "time": row.append(int(value)) - # case 'time': - # dt_from_unix_ts = datetime.utcfromtimestamp(int(value)) - # # convert unix time to epoch seconds - # row.append(int(dt_from_unix_ts.timestamp())) case _: row.append(float(value)) @@ -273,23 +366,31 @@ class Client: return array -def fqsn_to_cf_sym(fqsn: str, pairs: dict[str, any]) -> str: +def fqsn_to_kucoin_sym( + fqsn: str, + pairs: dict[str, KucoinMktPair] +) -> str: pair_data = pairs[fqsn] - return pair_data["baseCurrency"] + "-" + pair_data["quoteCurrency"] + return pair_data.baseCurrency + "-" + pair_data.quoteCurrency + + +def kucoin_sym_to_fqsn(sym: str) -> str: + return sym.lower().replace("-", "") @acm -async def get_client(): +async def get_client() -> AsyncGenerator[Client, None]: + client = Client() - # Do we need to open a nursery here? await client.cache_pairs() + yield client @tractor.context async def open_symbol_search( ctx: tractor.Context, -): +) -> None: async with open_cached_client("kucoin") as client: # load all symbols locally for fast search await client.cache_pairs() @@ -297,7 +398,6 @@ async def open_symbol_search( async with ctx.open_stream() as stream: async for pattern in stream: - # repack in dict form await stream.send(await client.search_symbols(pattern)) @@ -308,91 +408,93 @@ async def stream_quotes( loglevel: str = None, # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, -): - # TODO: Add multi-symbol functionality here - sym = symbols[0] +) -> None: + ''' + Required piker api to stream real-time data. + Where the rubber hits the road baby + + ''' connect_id = str(uuid4()) async with open_cached_client("kucoin") as client: - token, ping_interval = await client._get_ws_token() - pairs = await client.cache_pairs() - kucoin_sym = pairs[sym]["symbol"] - init_msgs = { - # pass back token, and bool, signalling if we're the writer - # and that history has been written - sym: { - "symbol_info": { - "asset_type": "crypto", - "price_tick_size": 0.0005, - "lot_tick_size": 0.1, + + # map through symbols and sub to feedz + for sym in symbols: + + token, ping_interval = await client._get_ws_token() + pairs = await client.cache_pairs() + kucoin_sym = pairs[sym].symbol + + init_msgs = { + # pass back token, and bool, signalling if we're the writer + # and that history has been written + sym: { + "symbol_info": { + "asset_type": "crypto", + "price_tick_size": 0.0005, + "lot_tick_size": 0.1, + }, + "shm_write_opts": {"sum_tick_vml": False}, + "fqsn": sym, }, - "shm_write_opts": {"sum_tick_vml": False}, - "fqsn": sym, - }, - } - - last_trades = await client.last_trades(sym) - - @acm - async def subscribe(ws: wsproto.WSConnection): + } @acm - async def open_ping_task(ws: wsproto.WSConnection): - async with trio.open_nursery() as n: + async def subscribe(ws: wsproto.WSConnection): + @acm + async def open_ping_task(ws: wsproto.WSConnection): + async with trio.open_nursery() as n: - async def ping_server(): - while True: - await trio.sleep((ping_interval - 1000) / 1000) - print("PINGING") - await ws.send_msg({"id": connect_id, "type": "ping"}) + async def ping_server(): + while True: + await trio.sleep((ping_interval - 1000) / 1000) + await ws.send_msg({"id": connect_id, "type": "ping"}) - n.start_soon(ping_server) + n.start_soon(ping_server) - yield ws + yield ws - n.cancel_scope.cancel() + n.cancel_scope.cancel() - # Spawn the ping task here - async with open_ping_task(ws) as _ws: + # Spawn the ping task here + async with open_ping_task(ws) as ws: + # subscribe to market feedz here + l1_sub = make_sub(kucoin_sym, connect_id) + await ws.send_msg(l1_sub) - # subscribe to market feedz here - l1_sub = make_sub(kucoin_sym, connect_id) - await _ws.send_msg(l1_sub) - res = await _ws.recv_msg() + yield - yield + # unsub + if ws.connected(): + await ws.send_msg( + { + "id": connect_id, + "type": "unsubscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + ) - # unsub - if _ws.connected(): - await _ws.send_msg( - { - "id": connect_id, - "type": "unsubscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, - } - ) - - async with open_autorecon_ws( - f"wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]", - fixture=subscribe, - ) as ws: - msg_gen = stream_messages(ws, sym) - typ, quote = await msg_gen.__anext__() - # - while typ != "trade": - # TODO: use ``anext()`` when it lands in 3.10! + async with open_autorecon_ws( + f"wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]", + fixture=subscribe, + ) as ws: + msg_gen = stream_messages(ws, sym) typ, quote = await msg_gen.__anext__() + # + while typ != "trade": + # TODO: use ``anext()`` when it lands in 3.10! + typ, quote = await msg_gen.__anext__() - task_status.started((init_msgs, quote)) - feed_is_live.set() + task_status.started((init_msgs, quote)) + feed_is_live.set() - async for typ, msg in msg_gen: - await send_chan.send({sym: msg}) + async for typ, msg in msg_gen: + await send_chan.send({sym: msg}) -def make_sub(sym, connect_id): +def make_sub(sym, connect_id) -> dict[str, str | bool]: return { "id": connect_id, "type": "subscribe", @@ -441,7 +543,7 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: async def open_history_client( symbol: str, type: str = "1m", -): +) -> AsyncGenerator[Callable, None]: async with open_cached_client("kucoin") as client: # call bars on kucoin async def get_ohlc_history( From 50e1070004472443f6986c653b7f50a14d40b9c4 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 13:27:31 -0400 Subject: [PATCH 16/81] More cleanup, add comments re sub func --- piker/brokers/kucoin.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 35d8f690..1f87e701 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -441,8 +441,14 @@ async def stream_quotes( @acm async def subscribe(ws: wsproto.WSConnection): + @acm async def open_ping_task(ws: wsproto.WSConnection): + ''' + Ping ws server every ping_interval + so Kucoin doesn't drop our connection + + ''' async with trio.open_nursery() as n: async def ping_server(): @@ -504,7 +510,11 @@ def make_sub(sym, connect_id) -> dict[str, str | bool]: } -async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: +async def stream_messages( + ws: NoBsWs, + sym: str +) -> AsyncGenerator[NoBsWs, dict]: + timeouts = 0 while True: From 9bf6f557ed15add6ca87157f8fe1a9f08081a8c5 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 14:14:33 -0400 Subject: [PATCH 17/81] Label private methods accordingly, remove cryptofeeds module --- piker/brokers/kucoin.py | 14 +- piker/data/cryptofeeds.py | 316 -------------------------------------- 2 files changed, 7 insertions(+), 323 deletions(-) delete mode 100644 piker/data/cryptofeeds.py diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 1f87e701..bfa4fc8f 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -246,7 +246,7 @@ class Client: f'Error making request for Kucoin ws token -> {res.json()["msg"]}' ) - async def get_pairs( + async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: if self._pairs: @@ -265,12 +265,12 @@ class Client: ''' if not self._pairs: - self._pairs = await self.get_pairs() + self._pairs = await self._get_pairs() if normalize: - self._pairs = self.normalize_pairs(self._pairs) + self._pairs = self._normalize_pairs(self._pairs) return self._pairs - def normalize_pairs( + def _normalize_pairs( self, pairs: dict[str, KucoinMktPair] ) -> dict[str, KucoinMktPair]: """ @@ -290,7 +290,7 @@ class Client: pattern: str, limit: int = 30, ) -> dict[str, KucoinMktPair]: - data = await self.get_pairs() + data = await self._get_pairs() matches = fuzzy.extractBests(pattern, data, score_cutoff=35, limit=limit) # repack in dict form @@ -300,7 +300,7 @@ class Client: trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") return trades.items - async def get_bars( + async def _get_bars( self, fqsn: str, start_dt: Optional[datetime] = None, @@ -564,7 +564,7 @@ async def open_history_client( if timeframe != 60: raise DataUnavailable("Only 1m bars are supported") - array = await client.get_bars( + array = await client._get_bars( symbol, start_dt=start_dt, end_dt=end_dt, diff --git a/piker/data/cryptofeeds.py b/piker/data/cryptofeeds.py deleted file mode 100644 index 5605993d..00000000 --- a/piker/data/cryptofeeds.py +++ /dev/null @@ -1,316 +0,0 @@ -# piker: trading gear for hackers -# Copyright (C) Jared Goldman (in stewardship for pikers) - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. - -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -""" -""" -from os import walk -from contextlib import asynccontextmanager as acm -from datetime import datetime -from types import ModuleType -from typing import Any, Literal, Optional, Callable -import time -from functools import partial - -import trio -from trio_typing import TaskStatus -from tractor.trionics import broadcast_receiver, maybe_open_context -import pendulum -from fuzzywuzzy import process as fuzzy -import numpy as np -import tractor -from tractor import to_asyncio -from cryptofeed import FeedHandler -from cryptofeed.defines import TRADES, L2_BOOK -from cryptofeed.symbols import Symbol -from cryptofeed.types import OrderBook -import asyncio - -from piker._cacheables import open_cached_client -from piker.log import get_logger, get_console_log -from piker.data import ShmArray -from piker.brokers._util import ( - BrokerError, - DataUnavailable, -) -from piker.pp import config - -_spawn_kwargs = { - "infect_asyncio": True, -} - -log = get_logger(__name__) - - -def fqsn_to_cb_sym(pair_data: Symbol) -> Symbol: - return Symbol(base=pair_data["baseCurrency"], quote=pair_data["quoteCurrency"]) - - -def fqsn_to_cf_sym(fqsn: str, pairs: dict[str, Symbol]) -> str: - pair_data = pairs[fqsn] - return pair_data["baseCurrency"] + "-" + pair_data["quoteCurrency"] - - -def pair_data_to_cf_sym(sym_data: Symbol): - return sym_data["baseCurrency"] + "-" + sym_data["quoteCurrency"] - - -def cf_sym_to_fqsn(sym: str) -> str: - return sym.lower().replace("-", "") - - -def get_config(exchange: str) -> dict[str, Any]: - conf, path = config.load() - - section = conf.get(exchange.lower()) - - # TODO: document why we send this, basically because logging params for cryptofeed - conf["log"] = {} - conf["log"]["disabled"] = True - - if section is None: - log.warning(f"No config section found for deribit in {exchange}") - - return conf - - -async def mk_stream_quotes( - exchange: str, - channels: list[str], - send_chan: trio.abc.SendChannel, - symbols: list[str], - feed_is_live: trio.Event, - loglevel: str = None, - # startup sync - task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, -) -> None: - # XXX: required to propagate ``tractor`` loglevel to piker logging - get_console_log(loglevel or tractor.current_actor().loglevel) - - sym = symbols[0] - - async with ( - open_cached_client(exchange.lower()) as client, - # send_chan as send_chan, - ): - pairs = await client.cache_pairs() - pair_data = pairs[sym] - - async with maybe_open_price_feed( - pair_data, - exchange, - channels, - ) as stream: - - init_msgs = { - sym: { - "symbol_info": {"asset_type": "crypto", "price_tick_size": 0.0005}, - "shm_write_opts": {"sum_tick_vml": False}, - "fqsn": sym, - }, - } - quote_msg = {"symbol": pair_data["name"], "last": 0, "ticks": []} - - task_status.started((init_msgs, quote_msg)) - feed_is_live.set() - - async for typ, quote in stream: - topic = quote["symbol"] - await send_chan.send({topic: quote}) - log.info( - f'sending {typ} quote:\n' - f'{quote}' - ) - # try: - # finally: - # breakpoint() - - # while True: - # with trio.move_on_after(16) as cancel_scope: - - # log.warning(f'WAITING FOR MESSAGE') - # typ, quote = await stream.receive() - - # log.warning(f'RECEIVED MSG: {quote}') - - # topic = quote["symbol"] - # await send_chan.send({topic: quote}) - - # log.warning(f'SENT TO CHAN') - - # if cancel_scope.cancelled_caught: - # await tractor.breakpoint() - -@acm -async def maybe_open_price_feed( - pair_data: Symbol, - exchange: str, - channels, - -) -> trio.abc.ReceiveStream: - # TODO: add a predicate to maybe_open_context - # TODO: ensure we can dynamically pass down args here - async with maybe_open_context( - acm_func=open_price_feed, - kwargs={ - "pair_data": pair_data, - "exchange": exchange, - "channels": channels, - }, - key=pair_data["name"], - ) as (cache_hit, feed): - yield feed - - -@acm -async def open_price_feed( - pair_data: Symbol, exchange, channels -) -> trio.abc.ReceiveStream: - async with maybe_open_feed_handler(exchange) as fh: - async with to_asyncio.open_channel_from( - partial( - aio_price_feed_relay, - pair_data, - exchange, - channels, - fh, - ) - ) as (first, chan): - yield chan - - -@acm -async def maybe_open_feed_handler(exchange: str) -> trio.abc.ReceiveStream: - async with maybe_open_context( - acm_func=open_feed_handler, - kwargs={ - "exchange": exchange, - }, - key="feedhandler", - ) as (cache_hit, fh): - yield fh - - -@acm -async def open_feed_handler(exchange: str): - fh = FeedHandler(config=get_config(exchange)) - yield fh - await to_asyncio.run_task(fh.stop_async) - - -async def aio_price_feed_relay( - pair_data: Symbol, - exchange: str, - channels: list[str], - fh: FeedHandler, - - from_trio: asyncio.Queue, - to_trio: trio.abc.SendChannel, - -) -> None: - - # sync with trio - to_trio.send_nowait(None) - - async def _trade(data: dict, receipt_timestamp): - data = data.to_dict() - message = ( - "trade", - { - "symbol": cf_sym_to_fqsn(data['symbol']), - "last": float(data['price']), - "broker_ts": time.time(), - "ticks": [{ - 'type': 'trade', - 'price': float(data['price']), - 'size': float(data['amount']), - 'broker_ts': receipt_timestamp - }], - }, - ) - try: - to_trio.send_nowait(message) - await asyncio.sleep(0.001) - except trio.WouldBlock as e: - log.exception( - 'l1: OVERRUN ASYNCIO -> TRIO\n' - f'TO_TRIO.stats -> {to_trio.statistics()}' - - ) - await asyncio.sleep(0) - - async def _l1( - data: dict, - receipt_timestamp: str | None, - ) -> None: - log.info(f'RECV L1 {receipt_timestamp}') - - bid = data.book.to_dict()['bid'] - ask = data.book.to_dict()['ask'] - l1_ask_price, l1_ask_size = next(iter(ask.items())) - l1_bid_price, l1_bid_size = next(iter(bid.items())) - message = ( - "l1", - { - "symbol": cf_sym_to_fqsn(data.symbol), - "broker_ts": time.time(), - "ticks": [ - { - "type": "bid", - "price": float(l1_bid_price), - "size": float(l1_bid_size), - }, - { - "type": "bsize", - "price": float(l1_bid_price), - "size": float(l1_bid_size), - }, - { - "type": "ask", - "price": float(l1_ask_price), - "size": float(l1_ask_size), - }, - { - "type": "asize", - "price": float(l1_ask_price), - "size": float(l1_ask_size), - }, - ] - } - ) - try: - to_trio.send_nowait(message) - await asyncio.sleep(0.001) - except trio.WouldBlock as e: - log.exception( - 'l1: OVERRUN ASYNCIO -> TRIO\n' - f'TO_TRIO.stats -> {to_trio.statistics()}' - - ) - await asyncio.sleep(0) - # breakpoint() - # raise - - fh.add_feed( - exchange, - channels=channels, - symbols=[pair_data_to_cf_sym(pair_data)], - callbacks={TRADES: _trade, L2_BOOK: _l1} - ) - - if not fh.running: - fh.run(start_loop=False, install_signal_handlers=False) - - await asyncio.sleep(float("inf")) From 6ad1e3da38fa3078c15018fa12dd1806254436d3 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 14:15:14 -0400 Subject: [PATCH 18/81] Correct typo in license --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index bfa4fc8f..d0fca3be 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,5 +1,5 @@ # piker: trading gear for hackers -# Copyright (C) Jared Goldman (in stewardship for piker0) +# Copyright (C) Jared Goldman (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by From 5ff0cc7905f4176c22abfd25f5f9ac4121e979d2 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 14:22:56 -0400 Subject: [PATCH 19/81] Cast/validate streamed messages Update comments Minor formatting Minor formatting --- piker/brokers/kucoin.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index d0fca3be..63502099 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -418,7 +418,7 @@ async def stream_quotes( async with open_cached_client("kucoin") as client: - # map through symbols and sub to feedz + # loop through symbols and sub to feedz for sym in symbols: token, ping_interval = await client._get_ws_token() @@ -445,8 +445,9 @@ async def stream_quotes( @acm async def open_ping_task(ws: wsproto.WSConnection): ''' - Ping ws server every ping_interval - so Kucoin doesn't drop our connection + Spawn a non-blocking task that pings the ws + server every ping_interval so Kucoin doesn't drop + our connection ''' async with trio.open_nursery() as n: @@ -468,7 +469,7 @@ async def stream_quotes( l1_sub = make_sub(kucoin_sym, connect_id) await ws.send_msg(l1_sub) - yield + yieldhttps: // github.com/pikers/piker/pull/494 # unsub if ws.connected(): @@ -529,18 +530,20 @@ async def stream_messages( continue if "subject" in msg and msg["subject"] == "trade.ticker": - # TODO: cast msg into class - trade_data = msg["data"] + + trade_msg = KucoinTradeMsg(**msg) + trade_data = KucoinTrade(**trade_msg.data) + yield "trade", { "symbol": sym, - "last": trade_data["price"], - "brokerd_ts": trade_data["time"], + "last": trade_data.price, + "brokerd_ts": trade_data.time, "ticks": [ { "type": "trade", - "price": float(trade_data["price"]), - "size": float(trade_data["size"]), - "broker_ts": trade_data["time"], + "price": float(trade_data.price), + "size": float(trade_data.size), + "broker_ts": trade_data.time, } ], } @@ -555,12 +558,15 @@ async def open_history_client( type: str = "1m", ) -> AsyncGenerator[Callable, None]: async with open_cached_client("kucoin") as client: - # call bars on kucoin + async def get_ohlc_history( + timeframe: float, end_dt: datetime | None = None, start_dt: datetime | None = None, - ) -> tuple[np.ndarray, datetime | None, datetime | None,]: # start # end + + ) -> tuple[np.ndarray, datetime | None, datetime | None]: # start # end + if timeframe != 60: raise DataUnavailable("Only 1m bars are supported") @@ -573,10 +579,13 @@ async def open_history_client( times = array["time"] if end_dt is None: + inow = round(time.time()) + print( f"difference in time between load and processing {inow - times[-1]}" ) + if (inow - times[-1]) > 60: await tractor.breakpoint() From 52070c00f95cfe93e27a754a9e9268458a72ba13 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 14:43:51 -0400 Subject: [PATCH 20/81] Remove typo --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 63502099..b8d9c503 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -469,7 +469,7 @@ async def stream_quotes( l1_sub = make_sub(kucoin_sym, connect_id) await ws.send_msg(l1_sub) - yieldhttps: // github.com/pikers/piker/pull/494 + yield # unsub if ws.connected(): From ac31bca181d570e5f58743346ef8dd392a9a89cc Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 19 Mar 2023 14:48:47 -0400 Subject: [PATCH 21/81] Make broker creds/auth optional --- piker/brokers/kucoin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index b8d9c503..947d0a3b 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -156,7 +156,8 @@ class Client: config = get_config() if ( - float("key_id" in config) + config + and float("key_id" in config) and ("key_secret" in config) and ("key_passphrase" in config) ): From 7bdebd47d18e323e53c1945ddc45eab8054bc6c7 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 20 Mar 2023 21:14:58 -0400 Subject: [PATCH 22/81] Add exponential retry case for history client --- piker/brokers/kucoin.py | 84 ++++++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 31 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 947d0a3b..65f47a40 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -19,6 +19,7 @@ Kucoin broker backend """ +from random import randint from typing import Any, Callable, Optional, Literal, AsyncGenerator from contextlib import asynccontextmanager as acm from datetime import datetime @@ -62,17 +63,6 @@ _ohlc_dtype = [ ] -def get_config() -> dict[str, dict]: - conf, path = config.load() - - section = conf.get("kucoin") - - if section is None: - log.warning("No config section found for kucoin in config") - - return section - - class KucoinMktPair(Struct, frozen=True): ''' Kucoin's pair format @@ -137,6 +127,12 @@ class KucoinTrade(Struct, frozen=True): time: float +class BrokerConfig(Struct, frozen=True): + key_id: str + key_secret: str + key_passphrase: str + + class KucoinTradeMsg(Struct, frozen=True): type: str topic: str @@ -144,6 +140,18 @@ class KucoinTradeMsg(Struct, frozen=True): data: list[KucoinTrade] +def get_config() -> BrokerConfig | None: + conf, path = config.load() + + section = conf.get("kucoin") + + if section is None: + log.warning("No config section found for kucoin in config") + return None + + return BrokerConfig(**section) + + class Client: def __init__(self) -> None: self._pairs: dict[str, KucoinMktPair] = {} @@ -153,25 +161,25 @@ class Client: self._key_passphrase: str self._authenticated: bool = False - config = get_config() + config: BrokerConfig | None = get_config() if ( config - and float("key_id" in config) - and ("key_secret" in config) - and ("key_passphrase" in config) + and float(config.key_id) + and config.key_secret + and config.key_passphrase ): self._authenticated = True - self._key_id = config["key_id"] - self._key_secret = config["key_secret"] - self._key_passphrase = config["key_passphrase"] + self._key_id = config.key_id + self._key_secret = config.key_secret + self._key_passphrase = config.key_passphrase def _gen_auth_req_headers( self, action: Literal["POST", "GET"], endpoint: str, api_v: str = "v2", - ) -> dict[str, str]: + ) -> dict[str, str | bytes]: ''' Generate authenticated request headers https://docs.kucoin.com/#authentication @@ -212,7 +220,7 @@ class Client: endpoint: str, api_v: str = "v2", headers: dict = {}, - ) -> dict[str, Any]: + ) -> Any: ''' Generic request wrapper for Kucoin API @@ -221,12 +229,14 @@ class Client: headers = self._gen_auth_req_headers(action, endpoint, api_v) api_url = f"https://api.kucoin.com/api/{api_v}{endpoint}" + res = await asks.request(action, api_url, headers=headers) if "data" in res.json(): return res.json()["data"] else: log.error(f'Error making request to {api_url} -> {res.json()["msg"]}') + return res.json()["msg"] async def _get_ws_token( self, @@ -237,14 +247,18 @@ class Client: ''' token_type = "private" if private else "public" - data = await self._request("POST", f"/bullet-{token_type}", "v1") + data: dict[str, Any] | None = await self._request( + "POST", + f"/bullet-{token_type}", + "v1" + ) - if "token" in data: - ping_interval = data["instanceServers"][0]["pingInterval"] + if data and "token" in data: + ping_interval: int = data["instanceServers"][0]["pingInterval"] return data["token"], ping_interval - else: + elif data: log.error( - f'Error making request for Kucoin ws token -> {res.json()["msg"]}' + f'Error making request for Kucoin ws token -> {data.json()["msg"]}' ) async def _get_pairs( @@ -297,8 +311,9 @@ class Client: # repack in dict form return {kucoin_sym_to_fqsn(item[0].name): item[0] for item in matches} - async def last_trades(self, sym: str) -> AccountResponse: + async def last_trades(self, sym: str) -> list[AccountTrade]: trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") + trades = AccountResponse(**trades) return trades.items async def _get_bars( @@ -327,12 +342,19 @@ class Client: kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" + bars = [] + for i in range(10): - bars = await self._request( - "GET", - url, - api_v="v1", - ) + res = await self._request( + "GET", + url, + api_v="v1", + ) + if not isinstance(res, list): + await trio.sleep(i + (randint(0, 1000) / 1000)) + else: + bars = res + break # Map to OHLC values to dict then to np array new_bars = [] From 32107d0ac3b97a9bd2d398b51c26136e7c9050e3 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 20 Mar 2023 21:24:23 -0400 Subject: [PATCH 23/81] Strengthen retry case and add comments --- piker/brokers/kucoin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 65f47a40..ee7c04f7 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -343,6 +343,7 @@ class Client: url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" bars = [] + for i in range(10): res = await self._request( @@ -350,7 +351,9 @@ class Client: url, api_v="v1", ) - if not isinstance(res, list): + + if not isinstance(res, list) or not len(bars): + # Do a gradual backoff if Kucoin is rate limiting us await trio.sleep(i + (randint(0, 1000) / 1000)) else: bars = res From dcbb7fa64fbc44b62652cef63797e0237278bbb3 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 18:50:21 -0400 Subject: [PATCH 24/81] Remove float conversion for config key id --- piker/brokers/kucoin.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index ee7c04f7..50222fa5 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -164,10 +164,7 @@ class Client: config: BrokerConfig | None = get_config() if ( - config - and float(config.key_id) - and config.key_secret - and config.key_passphrase + config and config.key_id and config.key_secret and config.key_passphrase ): self._authenticated = True self._key_id = config.key_id From e2e5191ded68e3cff1663405eecbebffdf5f626a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 19:56:38 -0400 Subject: [PATCH 25/81] Remove breaking useless condition for determining if res is list of ohlc values --- piker/brokers/kucoin.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 50222fa5..a8eae81b 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -164,7 +164,7 @@ class Client: config: BrokerConfig | None = get_config() if ( - config and config.key_id and config.key_secret and config.key_passphrase + config and float(config.key_id) and config.key_secret and config.key_passphrase ): self._authenticated = True self._key_id = config.key_id @@ -343,17 +343,19 @@ class Client: for i in range(10): - res = await self._request( + data = await self._request( "GET", url, api_v="v1", ) - if not isinstance(res, list) or not len(bars): + if not isinstance(data, list): # Do a gradual backoff if Kucoin is rate limiting us - await trio.sleep(i + (randint(0, 1000) / 1000)) + backoff_interval = i + (randint(0, 1000) / 1000) + log.warn(f'History call failed, backing off for {backoff_interval}s') + await trio.sleep(backoff_interval) else: - bars = res + bars = data break # Map to OHLC values to dict then to np array @@ -392,6 +394,8 @@ class Client: def fqsn_to_kucoin_sym( fqsn: str, pairs: dict[str, KucoinMktPair] + + ) -> str: pair_data = pairs[fqsn] return pair_data.baseCurrency + "-" + pair_data.quoteCurrency @@ -401,7 +405,7 @@ def kucoin_sym_to_fqsn(sym: str) -> str: return sym.lower().replace("-", "") -@acm +@ acm async def get_client() -> AsyncGenerator[Client, None]: client = Client() @@ -410,7 +414,7 @@ async def get_client() -> AsyncGenerator[Client, None]: yield client -@tractor.context +@ tractor.context async def open_symbol_search( ctx: tractor.Context, ) -> None: @@ -581,6 +585,7 @@ async def open_history_client( type: str = "1m", ) -> AsyncGenerator[Callable, None]: async with open_cached_client("kucoin") as client: + log.info("Attempting to open kucoin history client") async def get_ohlc_history( From ae170f2645b3c70c3a9e4a593ca33c8737f742d8 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 20:02:08 -0400 Subject: [PATCH 26/81] Add more informative logs on startup --- piker/brokers/kucoin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index a8eae81b..8cbc17d0 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -426,6 +426,7 @@ async def open_symbol_search( async with ctx.open_stream() as stream: async for pattern in stream: await stream.send(await client.search_symbols(pattern)) + log.info("Kucoin symbol search opened") async def stream_quotes( @@ -444,7 +445,7 @@ async def stream_quotes( connect_id = str(uuid4()) async with open_cached_client("kucoin") as client: - + log.info("Starting up quote stream") # loop through symbols and sub to feedz for sym in symbols: @@ -493,6 +494,7 @@ async def stream_quotes( # Spawn the ping task here async with open_ping_task(ws) as ws: # subscribe to market feedz here + log.info(f'Subscribing to {kucoin_sym} feed') l1_sub = make_sub(kucoin_sym, connect_id) await ws.send_msg(l1_sub) @@ -500,6 +502,7 @@ async def stream_quotes( # unsub if ws.connected(): + log.info(f'Unsubscribing to {kucoin_sym} feed') await ws.send_msg( { "id": connect_id, @@ -619,6 +622,7 @@ async def open_history_client( start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) + log.info('History succesfully fetched baby') return array, start_dt, end_dt yield get_ohlc_history, {"erlangs": 3, "rate": 3} From 81890a39d93f96eb990876e8d81503c2063e49e0 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 20:11:59 -0400 Subject: [PATCH 27/81] Leave datetimes alone! --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 8cbc17d0..70e2ed46 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -380,7 +380,7 @@ class Client: value = data[field_name] match field_name: - case "index" | "time": + case "index": row.append(int(value)) case _: row.append(float(value)) From 788e158d9f762b5dbd94c5feceecbb1e42ad2b4d Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 20:16:18 -0400 Subject: [PATCH 28/81] Stop still converting datetime to float --- piker/brokers/kucoin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 70e2ed46..2748a778 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -382,6 +382,8 @@ class Client: match field_name: case "index": row.append(int(value)) + case "time": + row.append(value) case _: row.append(float(value)) From dfd030a6aa074419f417b3e75bd87038cf96712f Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Mar 2023 20:23:33 -0400 Subject: [PATCH 29/81] Remove float conversion of key_id again --- piker/brokers/kucoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 2748a778..2fbfcbc5 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -164,7 +164,7 @@ class Client: config: BrokerConfig | None = get_config() if ( - config and float(config.key_id) and config.key_secret and config.key_passphrase + config and config.key_id and config.key_secret and config.key_passphrase ): self._authenticated = True self._key_id = config.key_id @@ -397,7 +397,6 @@ def fqsn_to_kucoin_sym( fqsn: str, pairs: dict[str, KucoinMktPair] - ) -> str: pair_data = pairs[fqsn] return pair_data.baseCurrency + "-" + pair_data.quoteCurrency From 52aadb374bf50548764bad64cb9a70e385118f09 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 27 Mar 2023 21:28:11 -0400 Subject: [PATCH 30/81] Add L1 data feed and correct history issue --- piker/brokers/kucoin.py | 138 ++++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 46 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 2fbfcbc5..901b24c9 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -1,4 +1,3 @@ -# piker: trading gear for hackers # Copyright (C) Jared Goldman (in stewardship for pikers) # This program is free software: you can redistribute it and/or modify @@ -127,19 +126,29 @@ class KucoinTrade(Struct, frozen=True): time: float +class KucoinL2(Struct, frozen=True): + ''' + Real-time L2 order book format + + ''' + asks: list[list[float]] + bids: list[list[float]] + timestamp: float + + +class KucoinMsg(Struct, frozen=True): + type: str + topic: str + subject: str + data: list[KucoinTrade | KucoinL2] + + class BrokerConfig(Struct, frozen=True): key_id: str key_secret: str key_passphrase: str -class KucoinTradeMsg(Struct, frozen=True): - type: str - topic: str - subject: str - data: list[KucoinTrade] - - def get_config() -> BrokerConfig | None: conf, path = config.load() @@ -182,6 +191,7 @@ class Client: https://docs.kucoin.com/#authentication ''' + breakpoint() now = int(time.time() * 1000) path = f"/api/{api_v}{endpoint}" str_to_sign = str(now) + action + path @@ -327,22 +337,22 @@ class Client: ''' # Generate generic end and start time if values not passed + # Currently gives us 12hrs of data if end_dt is None: end_dt = pendulum.now("UTC").add(minutes=1) if start_dt is None: start_dt = end_dt.start_of("minute").subtract(minutes=limit) - # Format datetime to unix timestamp - start_dt = math.trunc(time.mktime(start_dt.timetuple())) - end_dt = math.trunc(time.mktime(end_dt.timetuple())) + start_dt = int(start_dt.timestamp()) + end_dt = int(end_dt.timestamp()) + kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" bars = [] for i in range(10): - data = await self._request( "GET", url, @@ -375,7 +385,7 @@ class Client: } row = [] - for j, (field_name, field_type) in enumerate(_ohlc_dtype): + for _, (field_name, field_type) in enumerate(_ohlc_dtype): value = data[field_name] @@ -383,16 +393,21 @@ class Client: case "index": row.append(int(value)) case "time": + # row.append(int(value) + (3600 * 4)) row.append(value) case _: row.append(float(value)) new_bars.append(tuple(row)) - self._bars = array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars return array +def kucoin_timestamp(dt: datetime): + return math.trunc(time.mktime(dt.timetuple())) + + def fqsn_to_kucoin_sym( fqsn: str, pairs: dict[str, KucoinMktPair] @@ -474,8 +489,8 @@ async def stream_quotes( @acm async def open_ping_task(ws: wsproto.WSConnection): ''' - Spawn a non-blocking task that pings the ws - server every ping_interval so Kucoin doesn't drop + Spawn a non-blocking task that pings the ws + server every ping_interval so Kucoin doesn't drop our connection ''' @@ -496,7 +511,9 @@ async def stream_quotes( async with open_ping_task(ws) as ws: # subscribe to market feedz here log.info(f'Subscribing to {kucoin_sym} feed') - l1_sub = make_sub(kucoin_sym, connect_id) + trade_sub = make_sub(kucoin_sym, connect_id, level='l3') + l1_sub = make_sub(kucoin_sym, connect_id, level='l1') + await ws.send_msg(trade_sub) await ws.send_msg(l1_sub) yield @@ -520,7 +537,7 @@ async def stream_quotes( ) as ws: msg_gen = stream_messages(ws, sym) typ, quote = await msg_gen.__anext__() - # + while typ != "trade": # TODO: use ``anext()`` when it lands in 3.10! typ, quote = await msg_gen.__anext__() @@ -532,14 +549,26 @@ async def stream_quotes( await send_chan.send({sym: msg}) -def make_sub(sym, connect_id) -> dict[str, str | bool]: - return { - "id": connect_id, - "type": "subscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, - } +def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: + match level: + case 'l1': + return { + "id": connect_id, + "type": "subscribe", + "topic": f"/spotMarket/level2Depth5:{sym}", + "privateChannel": False, + "response": True, + } + case 'l3': + return { + "id": connect_id, + "type": "subscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + case _: + return {} async def stream_messages( @@ -560,28 +589,43 @@ async def stream_messages( continue - if "subject" in msg and msg["subject"] == "trade.ticker": + if msg.get("subject") != None: - trade_msg = KucoinTradeMsg(**msg) - trade_data = KucoinTrade(**trade_msg.data) + msg = KucoinMsg(**msg) - yield "trade", { - "symbol": sym, - "last": trade_data.price, - "brokerd_ts": trade_data.time, - "ticks": [ - { - "type": "trade", - "price": float(trade_data.price), - "size": float(trade_data.size), - "broker_ts": trade_data.time, + match msg.subject: + case "trade.ticker": + + trade_data = KucoinTrade(**msg.data) + yield "trade", { + "symbol": sym, + "last": trade_data.price, + "brokerd_ts": trade_data.time, + "ticks": [ + { + "type": "trade", + "price": float(trade_data.price), + "size": float(trade_data.size), + "broker_ts": trade_data.time, + } + ], } - ], - } - else: - continue + case "level2": + l2_data = KucoinL2(**msg.data) + + ticks = [] + for trade in l2_data.bids: + tick = {'type': 'bid', 'price': float(trade[0]), 'size': float(trade[1])} + ticks.append(tick) + for trade in l2_data.asks: + tick = {'type': 'ask', 'price': float(trade[0]), 'size': float(trade[1])} + ticks.append(tick) + yield 'l1', { + 'symbol': sym, + 'ticks': ticks, + } @acm async def open_history_client( @@ -609,7 +653,6 @@ async def open_history_client( ) times = array["time"] - if end_dt is None: inow = round(time.time()) @@ -620,10 +663,13 @@ async def open_history_client( if (inow - times[-1]) > 60: await tractor.breakpoint() - start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) log.info('History succesfully fetched baby') + # breakpoint() + # print(f'OUTPUTTED END TIME: {time.ctime(kucoin_timestamp(end_dt))}') + # print(f'OUTPUTTED START TIME: {time.ctime(kucoin_timestamp(start_dt))}') + # print(f'DIFFERENCE IN MINUTES {(end_dt - start_dt).in_minutes()}') return array, start_dt, end_dt - yield get_ohlc_history, {"erlangs": 3, "rate": 3} + yield get_ohlc_history, {} From 68a06093e97bef50714914e48e2c0946ff733e10 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 27 Mar 2023 21:51:54 -0400 Subject: [PATCH 31/81] Format and ensure we're only grabbing the most closest bid and ask --- piker/brokers/kucoin.py | 293 ++++++++++++++++++++-------------------- 1 file changed, 143 insertions(+), 150 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 901b24c9..e42ed37d 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -13,10 +13,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" +''' Kucoin broker backend -""" +''' from random import randint from typing import Any, Callable, Optional, Literal, AsyncGenerator @@ -51,14 +51,14 @@ from ..data._web_bs import ( log = get_logger(__name__) _ohlc_dtype = [ - ("index", int), - ("time", int), - ("open", float), - ("high", float), - ("low", float), - ("close", float), - ("volume", float), - ("bar_wap", float), # will be zeroed by sampler if not filled + ('index', int), + ('time', int), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', float), + ('bar_wap', float), # will be zeroed by sampler if not filled ] @@ -67,6 +67,7 @@ class KucoinMktPair(Struct, frozen=True): Kucoin's pair format ''' + baseCurrency: str baseIncrement: float baseMaxSize: float @@ -91,6 +92,7 @@ class AccountTrade(Struct, frozen=True): Historical trade format ''' + id: str currency: str amount: float @@ -98,7 +100,7 @@ class AccountTrade(Struct, frozen=True): balance: float accountType: str bizType: str - direction: Literal["in", "out"] + direction: Literal['in', 'out'] createdAt: float context: list[str] @@ -116,6 +118,7 @@ class KucoinTrade(Struct, frozen=True): Real-time trade format ''' + bestAsk: float bestAskSize: float bestBid: float @@ -131,6 +134,7 @@ class KucoinL2(Struct, frozen=True): Real-time L2 order book format ''' + asks: list[list[float]] bids: list[list[float]] timestamp: float @@ -152,10 +156,10 @@ class BrokerConfig(Struct, frozen=True): def get_config() -> BrokerConfig | None: conf, path = config.load() - section = conf.get("kucoin") + section = conf.get('kucoin') if section is None: - log.warning("No config section found for kucoin in config") + log.warning('No config section found for kucoin in config') return None return BrokerConfig(**section) @@ -172,9 +176,7 @@ class Client: config: BrokerConfig | None = get_config() - if ( - config and config.key_id and config.key_secret and config.key_passphrase - ): + if config and config.key_id and config.key_secret and config.key_passphrase: self._authenticated = True self._key_id = config.key_id self._key_secret = config.key_secret @@ -182,9 +184,9 @@ class Client: def _gen_auth_req_headers( self, - action: Literal["POST", "GET"], + action: Literal['POST', 'GET'], endpoint: str, - api_v: str = "v2", + api_v: str = 'v2', ) -> dict[str, str | bytes]: ''' Generate authenticated request headers @@ -193,39 +195,39 @@ class Client: ''' breakpoint() now = int(time.time() * 1000) - path = f"/api/{api_v}{endpoint}" + path = f'/api/{api_v}{endpoint}' str_to_sign = str(now) + action + path signature = base64.b64encode( hmac.new( - self._key_secret.encode("utf-8"), - str_to_sign.encode("utf-8"), + self._key_secret.encode('utf-8'), + str_to_sign.encode('utf-8'), hashlib.sha256, ).digest() ) passphrase = base64.b64encode( hmac.new( - self._key_secret.encode("utf-8"), - self._key_passphrase.encode("utf-8"), + self._key_secret.encode('utf-8'), + self._key_passphrase.encode('utf-8'), hashlib.sha256, ).digest() ) return { - "KC-API-SIGN": signature, - "KC-API-TIMESTAMP": str(now), - "KC-API-KEY": self._key_id, - "KC-API-PASSPHRASE": passphrase, + 'KC-API-SIGN': signature, + 'KC-API-TIMESTAMP': str(now), + 'KC-API-KEY': self._key_id, + 'KC-API-PASSPHRASE': passphrase, # XXX: Even if using the v1 api - this stays the same - "KC-API-KEY-VERSION": "2", + 'KC-API-KEY-VERSION': '2', } async def _request( self, - action: Literal["POST", "GET"], + action: Literal['POST', 'GET'], endpoint: str, - api_v: str = "v2", + api_v: str = 'v2', headers: dict = {}, ) -> Any: ''' @@ -235,34 +237,29 @@ class Client: if self._authenticated: headers = self._gen_auth_req_headers(action, endpoint, api_v) - api_url = f"https://api.kucoin.com/api/{api_v}{endpoint}" + api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' res = await asks.request(action, api_url, headers=headers) - if "data" in res.json(): - return res.json()["data"] + if 'data' in res.json(): + return res.json()['data'] else: log.error(f'Error making request to {api_url} -> {res.json()["msg"]}') - return res.json()["msg"] + return res.json()['msg'] - async def _get_ws_token( - self, - private: bool = False - ) -> tuple[str, int] | None: + async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: ''' Fetch ws token needed for sub access ''' - token_type = "private" if private else "public" + token_type = 'private' if private else 'public' data: dict[str, Any] | None = await self._request( - "POST", - f"/bullet-{token_type}", - "v1" + 'POST', f'/bullet-{token_type}', 'v1' ) - if data and "token" in data: - ping_interval: int = data["instanceServers"][0]["pingInterval"] - return data["token"], ping_interval + if data and 'token' in data: + ping_interval: int = data['instanceServers'][0]['pingInterval'] + return data['token'], ping_interval elif data: log.error( f'Error making request for Kucoin ws token -> {data.json()["msg"]}' @@ -274,8 +271,8 @@ class Client: if self._pairs: return self._pairs - entries = await self._request("GET", "/symbols") - syms = {item["name"]: KucoinMktPair(**item) for item in entries} + entries = await self._request('GET', '/symbols') + syms = {item['name']: KucoinMktPair(**item) for item in entries} return syms async def cache_pairs( @@ -295,14 +292,14 @@ class Client: def _normalize_pairs( self, pairs: dict[str, KucoinMktPair] ) -> dict[str, KucoinMktPair]: - """ + ''' Map kucoin pairs to fqsn strings - """ + ''' norm_pairs = {} for key, value in pairs.items(): - fqsn = key.lower().replace("-", "") + fqsn = key.lower().replace('-', '') norm_pairs[fqsn] = value return norm_pairs @@ -319,7 +316,7 @@ class Client: return {kucoin_sym_to_fqsn(item[0].name): item[0] for item in matches} async def last_trades(self, sym: str) -> list[AccountTrade]: - trades = await self._request("GET", f"/accounts/ledgers?currency={sym}", "v1") + trades = await self._request('GET', f'/accounts/ledgers?currency={sym}', 'v1') trades = AccountResponse(**trades) return trades.items @@ -330,7 +327,7 @@ class Client: end_dt: Optional[datetime] = None, limit: int = 1000, as_np: bool = True, - type: str = "1min", + type: str = '1min', ) -> np.ndarray: ''' Get OHLC data and convert to numpy array for perffff @@ -339,24 +336,24 @@ class Client: # Generate generic end and start time if values not passed # Currently gives us 12hrs of data if end_dt is None: - end_dt = pendulum.now("UTC").add(minutes=1) + end_dt = pendulum.now('UTC').add(minutes=1) if start_dt is None: - start_dt = end_dt.start_of("minute").subtract(minutes=limit) + start_dt = end_dt.start_of('minute').subtract(minutes=limit) start_dt = int(start_dt.timestamp()) end_dt = int(end_dt.timestamp()) kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) - url = f"/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}" + url = f'/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}' bars = [] for i in range(10): data = await self._request( - "GET", + 'GET', url, - api_v="v1", + api_v='v1', ) if not isinstance(data, list): @@ -371,28 +368,26 @@ class Client: # Map to OHLC values to dict then to np array new_bars = [] for i, bar in enumerate(bars[::-1]): - data = { - "index": i, - "time": bar[0], - "open": bar[1], - "close": bar[2], - "high": bar[3], - "low": bar[4], - "volume": bar[5], - "amount": bar[6], - "bar_wap": 0.0, + 'index': i, + 'time': bar[0], + 'open': bar[1], + 'close': bar[2], + 'high': bar[3], + 'low': bar[4], + 'volume': bar[5], + 'amount': bar[6], + 'bar_wap': 0.0, } row = [] for _, (field_name, field_type) in enumerate(_ohlc_dtype): - value = data[field_name] match field_name: - case "index": + case 'index': row.append(int(value)) - case "time": + case 'time': # row.append(int(value) + (3600 * 4)) row.append(value) case _: @@ -408,33 +403,28 @@ def kucoin_timestamp(dt: datetime): return math.trunc(time.mktime(dt.timetuple())) -def fqsn_to_kucoin_sym( - fqsn: str, - pairs: dict[str, KucoinMktPair] - -) -> str: +def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: pair_data = pairs[fqsn] - return pair_data.baseCurrency + "-" + pair_data.quoteCurrency + return pair_data.baseCurrency + '-' + pair_data.quoteCurrency def kucoin_sym_to_fqsn(sym: str) -> str: - return sym.lower().replace("-", "") + return sym.lower().replace('-', '') -@ acm +@acm async def get_client() -> AsyncGenerator[Client, None]: - client = Client() await client.cache_pairs() yield client -@ tractor.context +@tractor.context async def open_symbol_search( ctx: tractor.Context, ) -> None: - async with open_cached_client("kucoin") as client: + async with open_cached_client('kucoin') as client: # load all symbols locally for fast search await client.cache_pairs() await ctx.started() @@ -442,7 +432,7 @@ async def open_symbol_search( async with ctx.open_stream() as stream: async for pattern in stream: await stream.send(await client.search_symbols(pattern)) - log.info("Kucoin symbol search opened") + log.info('Kucoin symbol search opened') async def stream_quotes( @@ -460,11 +450,10 @@ async def stream_quotes( ''' connect_id = str(uuid4()) - async with open_cached_client("kucoin") as client: - log.info("Starting up quote stream") + async with open_cached_client('kucoin') as client: + log.info('Starting up quote stream') # loop through symbols and sub to feedz for sym in symbols: - token, ping_interval = await client._get_ws_token() pairs = await client.cache_pairs() kucoin_sym = pairs[sym].symbol @@ -473,19 +462,18 @@ async def stream_quotes( # pass back token, and bool, signalling if we're the writer # and that history has been written sym: { - "symbol_info": { - "asset_type": "crypto", - "price_tick_size": 0.0005, - "lot_tick_size": 0.1, + 'symbol_info': { + 'asset_type': 'crypto', + 'price_tick_size': 0.0005, + 'lot_tick_size': 0.1, }, - "shm_write_opts": {"sum_tick_vml": False}, - "fqsn": sym, + 'shm_write_opts': {'sum_tick_vml': False}, + 'fqsn': sym, }, } @acm async def subscribe(ws: wsproto.WSConnection): - @acm async def open_ping_task(ws: wsproto.WSConnection): ''' @@ -499,7 +487,7 @@ async def stream_quotes( async def ping_server(): while True: await trio.sleep((ping_interval - 1000) / 1000) - await ws.send_msg({"id": connect_id, "type": "ping"}) + await ws.send_msg({'id': connect_id, 'type': 'ping'}) n.start_soon(ping_server) @@ -523,22 +511,22 @@ async def stream_quotes( log.info(f'Unsubscribing to {kucoin_sym} feed') await ws.send_msg( { - "id": connect_id, - "type": "unsubscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'unsubscribe', + 'topic': f'/market/ticker:{sym}', + 'privateChannel': False, + 'response': True, } ) async with open_autorecon_ws( - f"wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]", + f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]', fixture=subscribe, ) as ws: msg_gen = stream_messages(ws, sym) typ, quote = await msg_gen.__anext__() - while typ != "trade": + while typ != 'trade': # TODO: use ``anext()`` when it lands in 3.10! typ, quote = await msg_gen.__anext__() @@ -553,29 +541,25 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: match level: case 'l1': return { - "id": connect_id, - "type": "subscribe", - "topic": f"/spotMarket/level2Depth5:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/spotMarket/level2Depth5:{sym}', + 'privateChannel': False, + 'response': True, } case 'l3': return { - "id": connect_id, - "type": "subscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/market/ticker:{sym}', + 'privateChannel': False, + 'response': True, } case _: return {} -async def stream_messages( - ws: NoBsWs, - sym: str -) -> AsyncGenerator[NoBsWs, dict]: - +async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 while True: @@ -584,67 +568,77 @@ async def stream_messages( if cs.cancelled_caught: timeouts += 1 if timeouts > 2: - log.error("kucoin feed is sh**ing the bed... rebooting...") + log.error('kucoin feed is sh**ing the bed... rebooting...') await ws._connect() continue - if msg.get("subject") != None: - + if msg.get('subject') != None: msg = KucoinMsg(**msg) match msg.subject: - case "trade.ticker": - + case 'trade.ticker': trade_data = KucoinTrade(**msg.data) - yield "trade", { - "symbol": sym, - "last": trade_data.price, - "brokerd_ts": trade_data.time, - "ticks": [ + yield 'trade', { + 'symbol': sym, + 'last': trade_data.price, + 'brokerd_ts': trade_data.time, + 'ticks': [ { - "type": "trade", - "price": float(trade_data.price), - "size": float(trade_data.size), - "broker_ts": trade_data.time, + 'type': 'trade', + 'price': float(trade_data.price), + 'size': float(trade_data.size), + 'broker_ts': trade_data.time, } ], } - case "level2": - + case 'level2': l2_data = KucoinL2(**msg.data) - - ticks = [] - for trade in l2_data.bids: - tick = {'type': 'bid', 'price': float(trade[0]), 'size': float(trade[1])} - ticks.append(tick) - for trade in l2_data.asks: - tick = {'type': 'ask', 'price': float(trade[0]), 'size': float(trade[1])} - ticks.append(tick) + first_ask = l2_data.asks[0] + first_bid = l2_data.bids[0] yield 'l1', { 'symbol': sym, - 'ticks': ticks, + 'ticks': [ + { + 'type': 'bid', + 'price': float(first_bid[0]), + 'size': float(first_bid[1]), + }, + { + 'type': 'bsize', + 'price': float(first_bid[0]), + 'size': float(first_bid[1]), + }, + { + 'type': 'ask', + 'price': float(first_ask[0]), + 'size': float(first_ask[1]), + }, + { + 'type': 'asize', + 'price': float(first_ask[0]), + 'size': float(first_ask[1]), + }, + ], } + @acm async def open_history_client( symbol: str, - type: str = "1m", + type: str = '1m', ) -> AsyncGenerator[Callable, None]: - async with open_cached_client("kucoin") as client: - log.info("Attempting to open kucoin history client") + async with open_cached_client('kucoin') as client: + log.info('Attempting to open kucoin history client') async def get_ohlc_history( - timeframe: float, end_dt: datetime | None = None, start_dt: datetime | None = None, - ) -> tuple[np.ndarray, datetime | None, datetime | None]: # start # end - if timeframe != 60: - raise DataUnavailable("Only 1m bars are supported") + raise DataUnavailable('Only 1m bars are supported') array = await client._get_bars( symbol, @@ -652,13 +646,12 @@ async def open_history_client( end_dt=end_dt, ) - times = array["time"] + times = array['time'] if end_dt is None: - inow = round(time.time()) print( - f"difference in time between load and processing {inow - times[-1]}" + f'difference in time between load and processing {inow - times[-1]}' ) if (inow - times[-1]) > 60: From 68d0327d4128565b9b4992304345485918fd4c13 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 27 Mar 2023 22:01:44 -0400 Subject: [PATCH 32/81] Remove breakpoints, simplify backoff logic --- piker/brokers/kucoin.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index e42ed37d..c851ed6d 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -181,6 +181,7 @@ class Client: self._key_id = config.key_id self._key_secret = config.key_secret self._key_passphrase = config.key_passphrase + log.info('User credentials added') def _gen_auth_req_headers( self, @@ -193,7 +194,6 @@ class Client: https://docs.kucoin.com/#authentication ''' - breakpoint() now = int(time.time() * 1000) path = f'/api/{api_v}{endpoint}' str_to_sign = str(now) + action + path @@ -273,6 +273,8 @@ class Client: entries = await self._request('GET', '/symbols') syms = {item['name']: KucoinMktPair(**item) for item in entries} + + log.info('Kucoin market pairs fetches') return syms async def cache_pairs( @@ -358,7 +360,7 @@ class Client: if not isinstance(data, list): # Do a gradual backoff if Kucoin is rate limiting us - backoff_interval = i + (randint(0, 1000) / 1000) + backoff_interval = i log.warn(f'History call failed, backing off for {backoff_interval}s') await trio.sleep(backoff_interval) else: @@ -388,7 +390,6 @@ class Client: case 'index': row.append(int(value)) case 'time': - # row.append(int(value) + (3600 * 4)) row.append(value) case _: row.append(float(value)) @@ -399,10 +400,6 @@ class Client: return array -def kucoin_timestamp(dt: datetime): - return math.trunc(time.mktime(dt.timetuple())) - - def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: pair_data = pairs[fqsn] return pair_data.baseCurrency + '-' + pair_data.quoteCurrency @@ -483,12 +480,13 @@ async def stream_quotes( ''' async with trio.open_nursery() as n: - + # TODO: cache this task so it's only called once async def ping_server(): while True: await trio.sleep((ping_interval - 1000) / 1000) await ws.send_msg({'id': connect_id, 'type': 'ping'}) + log.info(f'Starting ping task for {sym}') n.start_soon(ping_server) yield ws @@ -497,12 +495,13 @@ async def stream_quotes( # Spawn the ping task here async with open_ping_task(ws) as ws: - # subscribe to market feedz here - log.info(f'Subscribing to {kucoin_sym} feed') - trade_sub = make_sub(kucoin_sym, connect_id, level='l3') - l1_sub = make_sub(kucoin_sym, connect_id, level='l1') - await ws.send_msg(trade_sub) - await ws.send_msg(l1_sub) + tasks = [] + tasks.append(make_sub(kucoin_sym, connect_id, level='l3')) + tasks.append(make_sub(kucoin_sym, connect_id, level='l1')) + + for task in tasks: + log.info(f'Subscribing to {task.level} feed for {sym}') + await ws.send_msg(task) yield @@ -547,6 +546,7 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: 'privateChannel': False, 'response': True, } + case 'l3': return { 'id': connect_id, @@ -555,8 +555,6 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: 'privateChannel': False, 'response': True, } - case _: - return {} async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: @@ -630,6 +628,7 @@ async def open_history_client( type: str = '1m', ) -> AsyncGenerator[Callable, None]: async with open_cached_client('kucoin') as client: + log.info('Attempting to open kucoin history client') async def get_ohlc_history( @@ -637,6 +636,7 @@ async def open_history_client( end_dt: datetime | None = None, start_dt: datetime | None = None, ) -> tuple[np.ndarray, datetime | None, datetime | None]: # start # end + if timeframe != 60: raise DataUnavailable('Only 1m bars are supported') @@ -647,6 +647,7 @@ async def open_history_client( ) times = array['time'] + if end_dt is None: inow = round(time.time()) @@ -656,13 +657,12 @@ async def open_history_client( if (inow - times[-1]) > 60: await tractor.breakpoint() + start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) + log.info('History succesfully fetched baby') - # breakpoint() - # print(f'OUTPUTTED END TIME: {time.ctime(kucoin_timestamp(end_dt))}') - # print(f'OUTPUTTED START TIME: {time.ctime(kucoin_timestamp(start_dt))}') - # print(f'DIFFERENCE IN MINUTES {(end_dt - start_dt).in_minutes()}') + return array, start_dt, end_dt yield get_ohlc_history, {} From 54cf648d74e5b3144e7f384bd61cc85a3a88236b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 27 Mar 2023 22:08:55 -0400 Subject: [PATCH 33/81] Ensure sub logging dict attritbutes will be there --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index c851ed6d..0c3d1c7e 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -500,7 +500,7 @@ async def stream_quotes( tasks.append(make_sub(kucoin_sym, connect_id, level='l1')) for task in tasks: - log.info(f'Subscribing to {task.level} feed for {sym}') + log.info(f'Subscribing to {task["topic"]} feed for {sym}') await ws.send_msg(task) yield From b71f6b6c67ba2669274ec9a9bfb0a85047142403 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 27 Mar 2023 22:53:08 -0400 Subject: [PATCH 34/81] Strip uneccesary data from ticks in l1 data feed --- piker/brokers/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 4bd16f22..37377136 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -551,7 +551,7 @@ async def stream_quotes( while typ != 'trade': typ, quote = await anext(msg_gen) - task_status.started((init_msgs, quote)) + task_status.started((init_msgs, quote)) # signal to caller feed is ready for consumption feed_is_live.set() From 48c3b333b24c8e01a493152b83ed24729bd5c9f2 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 19:42:09 -0400 Subject: [PATCH 35/81] Format imports with parenthesis --- piker/brokers/kucoin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 0c3d1c7e..4fa69943 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -19,7 +19,13 @@ Kucoin broker backend ''' from random import randint -from typing import Any, Callable, Optional, Literal, AsyncGenerator +from typing import ( + Any, + Callable, + Optional, + Literal, + AsyncGenerator +) from contextlib import asynccontextmanager as acm from datetime import datetime import time From c68fcf7e1c18c83759a2c7e5a818951bba7238bd Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 19:45:24 -0400 Subject: [PATCH 36/81] Remove extra line from docstrings --- piker/brokers/kucoin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 4fa69943..de049999 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -18,7 +18,6 @@ Kucoin broker backend ''' -from random import randint from typing import ( Any, Callable, @@ -73,7 +72,6 @@ class KucoinMktPair(Struct, frozen=True): Kucoin's pair format ''' - baseCurrency: str baseIncrement: float baseMaxSize: float @@ -98,7 +96,6 @@ class AccountTrade(Struct, frozen=True): Historical trade format ''' - id: str currency: str amount: float @@ -140,7 +137,6 @@ class KucoinL2(Struct, frozen=True): Real-time L2 order book format ''' - asks: list[list[float]] bids: list[list[float]] timestamp: float From ca937dff5e3c4f2018ea95cc59e9fc5a933c4d39 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:00:42 -0400 Subject: [PATCH 37/81] Add api doc links in structs --- piker/brokers/kucoin.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index de049999..cb225387 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -69,7 +69,8 @@ _ohlc_dtype = [ class KucoinMktPair(Struct, frozen=True): ''' - Kucoin's pair format + Kucoin's pair format: + https://docs.kucoin.com/#get-symbols-list ''' baseCurrency: str @@ -93,7 +94,8 @@ class KucoinMktPair(Struct, frozen=True): class AccountTrade(Struct, frozen=True): ''' - Historical trade format + Historical trade format: + https://docs.kucoin.com/#get-account-ledgers ''' id: str @@ -109,6 +111,10 @@ class AccountTrade(Struct, frozen=True): class AccountResponse(Struct, frozen=True): + ''' + https://docs.kucoin.com/#get-account-ledgers + + ''' currentPage: int pageSize: int totalNum: int @@ -118,8 +124,8 @@ class AccountResponse(Struct, frozen=True): class KucoinTrade(Struct, frozen=True): ''' - Real-time trade format - + Real-time trade format: + https://docs.kucoin.com/#symbol-ticker ''' bestAsk: float @@ -134,7 +140,8 @@ class KucoinTrade(Struct, frozen=True): class KucoinL2(Struct, frozen=True): ''' - Real-time L2 order book format + Real-time L2 order book format: + https://docs.kucoin.com/#level2-5-best-ask-bid-orders ''' asks: list[list[float]] @@ -143,6 +150,10 @@ class KucoinL2(Struct, frozen=True): class KucoinMsg(Struct, frozen=True): + ''' + Generic outer-wrapper for any Kucoin ws msg + + ''' type: str topic: str subject: str From 208a8e5d7ae5042e37e795324bf985db728a6f7d Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:12:25 -0400 Subject: [PATCH 38/81] Remove unecessary config vars --- piker/brokers/kucoin.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index cb225387..a205c31e 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -182,19 +182,7 @@ class Client: def __init__(self) -> None: self._pairs: dict[str, KucoinMktPair] = {} self._bars: list[list[float]] = [] - self._key_id: str - self._key_secret: str - self._key_passphrase: str - self._authenticated: bool = False - - config: BrokerConfig | None = get_config() - - if config and config.key_id and config.key_secret and config.key_passphrase: - self._authenticated = True - self._key_id = config.key_id - self._key_secret = config.key_secret - self._key_passphrase = config.key_passphrase - log.info('User credentials added') + self._config: BrokerConfig | None = get_config() def _gen_auth_req_headers( self, @@ -247,7 +235,7 @@ class Client: Generic request wrapper for Kucoin API ''' - if self._authenticated: + if self._config: headers = self._gen_auth_req_headers(action, endpoint, api_v) api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' From 13df3e70d5284cae5ab14cd680802d57a71aed8c Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:14:25 -0400 Subject: [PATCH 39/81] Refactor sign gen into one line --- piker/brokers/kucoin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index a205c31e..f3aee52f 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -195,9 +195,7 @@ class Client: https://docs.kucoin.com/#authentication ''' - now = int(time.time() * 1000) - path = f'/api/{api_v}{endpoint}' - str_to_sign = str(now) + action + path + str_to_sign = str(int(time.time() * 1000)) + action + f'/api/{api_v}{endpoint}' signature = base64.b64encode( hmac.new( From 5a0d29c774def0faa2033f8220ee285dfa27afd0 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:23:29 -0400 Subject: [PATCH 40/81] Add ws token api doc link --- piker/brokers/kucoin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index f3aee52f..13d58974 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -248,7 +248,8 @@ class Client: async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: ''' - Fetch ws token needed for sub access + Fetch ws token needed for sub access: + https://docs.kucoin.com/#apply-connect-token ''' token_type = 'private' if private else 'public' From ea21656624bfb28b3094808878b57fbc2e46a713 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:28:52 -0400 Subject: [PATCH 41/81] Don't cache pairs in _get_pairs call --- piker/brokers/kucoin.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 13d58974..97c332b7 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -268,9 +268,6 @@ class Client: async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: - if self._pairs: - return self._pairs - entries = await self._request('GET', '/symbols') syms = {item['name']: KucoinMktPair(**item) for item in entries} From 9db84e80299dcbd70fd344f212560bd73dffa504 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:48:32 -0400 Subject: [PATCH 42/81] Remove norm_pairs method and do all normalization in initial _get_pairs call --- piker/brokers/kucoin.py | 53 ++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 97c332b7..e66384d6 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -19,11 +19,11 @@ Kucoin broker backend ''' from typing import ( - Any, - Callable, - Optional, - Literal, - AsyncGenerator + Any, + Callable, + Optional, + Literal, + AsyncGenerator ) from contextlib import asynccontextmanager as acm from datetime import datetime @@ -126,8 +126,8 @@ class KucoinTrade(Struct, frozen=True): ''' Real-time trade format: https://docs.kucoin.com/#symbol-ticker - ''' + ''' bestAsk: float bestAskSize: float bestBid: float @@ -195,7 +195,8 @@ class Client: https://docs.kucoin.com/#authentication ''' - str_to_sign = str(int(time.time() * 1000)) + action + f'/api/{api_v}{endpoint}' + str_to_sign = str(int(time.time() * 1000)) + \ + action + f'/api/{api_v}{endpoint}' signature = base64.b64encode( hmac.new( @@ -243,7 +244,8 @@ class Client: if 'data' in res.json(): return res.json()['data'] else: - log.error(f'Error making request to {api_url} -> {res.json()["msg"]}') + log.error( + f'Error making request to {api_url} -> {res.json()["msg"]}') return res.json()['msg'] async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: @@ -269,14 +271,14 @@ class Client: self, ) -> dict[str, KucoinMktPair]: entries = await self._request('GET', '/symbols') - syms = {item['name']: KucoinMktPair(**item) for item in entries} + syms = {kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item) for item in entries} - log.info('Kucoin market pairs fetches') + log.info('Kucoin market pairs fetched') return syms async def cache_pairs( self, - normalize: bool = True, + # normalize: bool = True, ) -> dict[str, KucoinMktPair]: ''' Get cached pairs and convert keyed symbols into fqsns if ya want @@ -284,25 +286,9 @@ class Client: ''' if not self._pairs: self._pairs = await self._get_pairs() - if normalize: - self._pairs = self._normalize_pairs(self._pairs) + return self._pairs - def _normalize_pairs( - self, pairs: dict[str, KucoinMktPair] - ) -> dict[str, KucoinMktPair]: - ''' - Map kucoin pairs to fqsn strings - - ''' - norm_pairs = {} - - for key, value in pairs.items(): - fqsn = key.lower().replace('-', '') - norm_pairs[fqsn] = value - - return norm_pairs - async def search_symbols( self, pattern: str, @@ -310,9 +296,10 @@ class Client: ) -> dict[str, KucoinMktPair]: data = await self._get_pairs() - matches = fuzzy.extractBests(pattern, data, score_cutoff=35, limit=limit) + matches = fuzzy.extractBests( + pattern, data, score_cutoff=35, limit=limit) # repack in dict form - return {kucoin_sym_to_fqsn(item[0].name): item[0] for item in matches} + return {item[0].name: item[0] for item in matches} async def last_trades(self, sym: str) -> list[AccountTrade]: trades = await self._request('GET', f'/accounts/ledgers?currency={sym}', 'v1') @@ -358,7 +345,8 @@ class Client: if not isinstance(data, list): # Do a gradual backoff if Kucoin is rate limiting us backoff_interval = i - log.warn(f'History call failed, backing off for {backoff_interval}s') + log.warn( + f'History call failed, backing off for {backoff_interval}s') await trio.sleep(backoff_interval) else: bars = data @@ -497,7 +485,8 @@ async def stream_quotes( tasks.append(make_sub(kucoin_sym, connect_id, level='l1')) for task in tasks: - log.info(f'Subscribing to {task["topic"]} feed for {sym}') + log.info( + f'Subscribing to {task["topic"]} feed for {sym}') await ws.send_msg(task) yield From 93e7d54c5ee71a3e8698a0d7ef34d26370f4052c Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 3 Apr 2023 20:50:38 -0400 Subject: [PATCH 43/81] Add api doc links to _get_bars def --- piker/brokers/kucoin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index e66384d6..8e9d398c 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -316,7 +316,8 @@ class Client: type: str = '1min', ) -> np.ndarray: ''' - Get OHLC data and convert to numpy array for perffff + Get OHLC data and convert to numpy array for perffff: + https://docs.kucoin.com/#get-klines ''' # Generate generic end and start time if values not passed From 3bed3a64c3038505709645d494774e603151127c Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 10 Apr 2023 19:59:50 -0400 Subject: [PATCH 44/81] Implement duplicate filtering at message level --- piker/brokers/kucoin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 8e9d398c..a8d04337 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -546,6 +546,7 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 + last_trade_data: KucoinTrade | dict = {} while True: with trio.move_on_after(3) as cs: @@ -558,12 +559,19 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: continue + if msg.get('subject') != None: msg = KucoinMsg(**msg) - match msg.subject: case 'trade.ticker': trade_data = KucoinTrade(**msg.data) + + # Filter for duplicate messages + if last_trade_data and trade_data.time == last_trade_data.time: + continue + + last_trade_data = trade_data + yield 'trade', { 'symbol': sym, 'last': trade_data.price, @@ -580,6 +588,7 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: case 'level2': l2_data = KucoinL2(**msg.data) + breakpoint() first_ask = l2_data.asks[0] first_bid = l2_data.bids[0] yield 'l1', { From d1b0608c88fa427219a4e277f56767ae82795684 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Mon, 10 Apr 2023 20:01:08 -0400 Subject: [PATCH 45/81] Remove breakpoint --- piker/brokers/kucoin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index a8d04337..e05d191a 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -588,7 +588,6 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: case 'level2': l2_data = KucoinL2(**msg.data) - breakpoint() first_ask = l2_data.asks[0] first_bid = l2_data.bids[0] yield 'l1', { From 6e55f6706f692af4c3d6fa2226a8f2b42c24fed3 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 11 Apr 2023 12:59:53 -0400 Subject: [PATCH 46/81] Format condition for filtering and add link to docs explaining need for filtering in the first case --- piker/brokers/kucoin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index e05d191a..3f4056a9 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -566,8 +566,12 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: case 'trade.ticker': trade_data = KucoinTrade(**msg.data) - # Filter for duplicate messages - if last_trade_data and trade_data.time == last_trade_data.time: + # XXX: Filter for duplicate messages as ws feed will send duplicate market state + # https://docs.kucoin.com/#level2-5-best-ask-bid-orders + if ( + last_trade_data + and trade_data.time == last_trade_data.time + ): continue last_trade_data = trade_data From bedbbc3025494d07424d1c733a8074f03f2bc23a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 11 Apr 2023 13:45:32 -0400 Subject: [PATCH 47/81] Only diff trade time --- piker/brokers/kucoin.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 3f4056a9..b6fef60f 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -546,7 +546,7 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 - last_trade_data: KucoinTrade | dict = {} + last_trade_ts = 0 while True: with trio.move_on_after(3) as cs: @@ -568,13 +568,10 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: # XXX: Filter for duplicate messages as ws feed will send duplicate market state # https://docs.kucoin.com/#level2-5-best-ask-bid-orders - if ( - last_trade_data - and trade_data.time == last_trade_data.time - ): + if trade_data.time == last_trade_ts: continue - last_trade_data = trade_data + last_trade_ts = trade_data.time yield 'trade', { 'symbol': sym, From d2f3a79c09db3c8d203cf77b05e717b147d9123d Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 12 Apr 2023 19:48:35 -0400 Subject: [PATCH 48/81] Use pendulum for header timestamp, type hint cleanup --- piker/brokers/binance.py | 2 +- piker/brokers/kucoin.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 37377136..64c69efb 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -485,7 +485,7 @@ async def stream_quotes( si['asset_type'] = 'crypto' symbol = symbols[0] - + breakpoint() init_msgs = { # pass back token, and bool, signalling if we're the writer # and that history has been written diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index b6fef60f..651b5d90 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -167,7 +167,7 @@ class BrokerConfig(Struct, frozen=True): def get_config() -> BrokerConfig | None: - conf, path = config.load() + conf, _= config.load() section = conf.get('kucoin') @@ -200,7 +200,7 @@ class Client: signature = base64.b64encode( hmac.new( - self._key_secret.encode('utf-8'), + self._config.key_secret.encode('utf-8'), str_to_sign.encode('utf-8'), hashlib.sha256, ).digest() @@ -216,7 +216,7 @@ class Client: return { 'KC-API-SIGN': signature, - 'KC-API-TIMESTAMP': str(now), + 'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000), 'KC-API-KEY': self._key_id, 'KC-API-PASSPHRASE': passphrase, # XXX: Even if using the v1 api - this stays the same @@ -422,7 +422,7 @@ async def stream_quotes( send_chan: trio.abc.SendChannel, symbols: list[str], feed_is_live: trio.Event, - loglevel: str = None, + loglevel: str = '', # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: @@ -523,7 +523,7 @@ async def stream_quotes( await send_chan.send({sym: msg}) -def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool]: +def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool] | None: match level: case 'l1': return { From ace04af21a9538d9c9ba218de57f62137decd6c2 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 12 Apr 2023 20:25:35 -0400 Subject: [PATCH 49/81] Use anext() in kucoin stream_quotes --- piker/brokers/kucoin.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 651b5d90..1aadfccd 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -38,6 +38,7 @@ from uuid import uuid4 import asks import tractor import trio +from trio_util import trio_async_generator from trio_typing import TaskStatus from fuzzywuzzy import process as fuzzy import pendulum @@ -454,7 +455,6 @@ async def stream_quotes( 'fqsn': sym, }, } - @acm async def subscribe(ws: wsproto.WSConnection): @acm @@ -505,16 +505,19 @@ async def stream_quotes( } ) - async with open_autorecon_ws( - f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]', - fixture=subscribe, - ) as ws: - msg_gen = stream_messages(ws, sym) - typ, quote = await msg_gen.__anext__() + async with ( + open_autorecon_ws( + f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]', + fixture=subscribe, + ) as ws, + stream_messages(ws, sym) as msg_gen, + ): + typ, quote = await anext(msg_gen) while typ != 'trade': # TODO: use ``anext()`` when it lands in 3.10! - typ, quote = await msg_gen.__anext__() + typ, quote = await anext(msg_gen) + task_status.started((init_msgs, quote)) feed_is_live.set() @@ -543,7 +546,7 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool] | None: 'response': True, } - +@trio_async_generator async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 last_trade_ts = 0 From ff0f8dfaca32c7c61ecc43ac8becdaf83aaf2a14 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 12 Apr 2023 20:37:10 -0400 Subject: [PATCH 50/81] Improve client._get_ws_token docstring --- piker/brokers/kucoin.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 1aadfccd..ff891f96 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -253,6 +253,8 @@ class Client: ''' Fetch ws token needed for sub access: https://docs.kucoin.com/#apply-connect-token + returns a token and the interval we must ping + the server at to keep the connection alive ''' token_type = 'private' if private else 'public' @@ -261,6 +263,7 @@ class Client: ) if data and 'token' in data: + # ping_interval is in ms ping_interval: int = data['instanceServers'][0]['pingInterval'] return data['token'], ping_interval elif data: From 2c82b2aba9fbec9e6b489c7596e720c1ab6efc5d Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 12 Apr 2023 20:43:28 -0400 Subject: [PATCH 51/81] Remove breakpoint in binance --- piker/brokers/binance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 64c69efb..11fce201 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -343,7 +343,7 @@ async def stream_messages( # https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams if msg.get('u'): - sym = msg['s'] + sym = msg['s']what does the trio_async_generator from trio_util do? bid = float(msg['b']) bsize = float(msg['B']) ask = float(msg['a']) @@ -485,7 +485,7 @@ async def stream_quotes( si['asset_type'] = 'crypto' symbol = symbols[0] - breakpoint() + init_msgs = { # pass back token, and bool, signalling if we're the writer # and that history has been written From 52a015d927afbdd292438196e7cc1deaa3be8796 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 12 Apr 2023 21:40:58 -0400 Subject: [PATCH 52/81] Remove typo in binance --- piker/brokers/binance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 11fce201..37377136 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -343,7 +343,7 @@ async def stream_messages( # https://binance-docs.github.io/apidocs/spot/en/#individual-symbol-book-ticker-streams if msg.get('u'): - sym = msg['s']what does the trio_async_generator from trio_util do? + sym = msg['s'] bid = float(msg['b']) bsize = float(msg['B']) ask = float(msg['a']) From b00abd0e51d80328f6fda0678f63eac0666f1664 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 21:48:17 -0400 Subject: [PATCH 53/81] Add a fail case ws token request --- piker/brokers/kucoin.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index ff891f96..39cd666c 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -258,9 +258,13 @@ class Client: ''' token_type = 'private' if private else 'public' - data: dict[str, Any] | None = await self._request( - 'POST', f'/bullet-{token_type}', 'v1' - ) + try: + data: dict[str, Any] | None = await self._request( + 'POST', f'/bullet-{token_type}', 'v1' + ) + except Exception as e: + log.error(f'Error making request for Kucoin ws token -> {str(e)}') + return None if data and 'token' in data: # ping_interval is in ms @@ -271,6 +275,7 @@ class Client: f'Error making request for Kucoin ws token -> {data.json()["msg"]}' ) + async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: From 92f372dcc88552c3539b04a32564e62d416115dd Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 21:52:40 -0400 Subject: [PATCH 54/81] Use proper value for init message --- piker/brokers/kucoin.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 39cd666c..5cf301a0 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -448,7 +448,8 @@ async def stream_quotes( for sym in symbols: token, ping_interval = await client._get_ws_token() pairs = await client.cache_pairs() - kucoin_sym = pairs[sym].symbol + pair = pairs[sym]: KucoinMktPair + kucoin_sym = pair.symbol init_msgs = { # pass back token, and bool, signalling if we're the writer @@ -456,8 +457,8 @@ async def stream_quotes( sym: { 'symbol_info': { 'asset_type': 'crypto', - 'price_tick_size': 0.0005, - 'lot_tick_size': 0.1, + 'price_tick_size': pair.baseIncrement, + 'lot_tick_size': pair.baseMinSize, }, 'shm_write_opts': {'sum_tick_vml': False}, 'fqsn': sym, From 63e34cf5957124d1d9cc962dad5f4d0a339f06dd Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 21:57:54 -0400 Subject: [PATCH 55/81] Typecast config, add type hint to pair in init message creation and turn init msg vals into floats --- piker/brokers/kucoin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 5cf301a0..64168367 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -176,7 +176,7 @@ def get_config() -> BrokerConfig | None: log.warning('No config section found for kucoin in config') return None - return BrokerConfig(**section) + return BrokerConfig(**section).typecast() class Client: @@ -448,7 +448,7 @@ async def stream_quotes( for sym in symbols: token, ping_interval = await client._get_ws_token() pairs = await client.cache_pairs() - pair = pairs[sym]: KucoinMktPair + pair: KucoinMktPair = pairs[sym] kucoin_sym = pair.symbol init_msgs = { @@ -457,8 +457,8 @@ async def stream_quotes( sym: { 'symbol_info': { 'asset_type': 'crypto', - 'price_tick_size': pair.baseIncrement, - 'lot_tick_size': pair.baseMinSize, + 'price_tick_size': float(pair.baseIncrement), + 'lot_tick_size': float(pair.baseMinSize), }, 'shm_write_opts': {'sum_tick_vml': False}, 'fqsn': sym, From 89bb1247283f3572c5ce84154485dc3cb85eead5 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:00:41 -0400 Subject: [PATCH 56/81] Remove old comments normalize arguents and improve pair fetching log --- piker/brokers/kucoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 64168367..6692d7f7 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -282,12 +282,11 @@ class Client: entries = await self._request('GET', '/symbols') syms = {kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item) for item in entries} - log.info('Kucoin market pairs fetched') + log.info(f' {syms.length} Kucoin market pairs fetched') return syms async def cache_pairs( self, - # normalize: bool = True, ) -> dict[str, KucoinMktPair]: ''' Get cached pairs and convert keyed symbols into fqsns if ya want From ebfd490a1a14b44ee99467b0f0a2d346f583bc10 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:02:13 -0400 Subject: [PATCH 57/81] Cache instead of get pairs in symbol search --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 6692d7f7..766cee62 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -302,7 +302,7 @@ class Client: pattern: str, limit: int = 30, ) -> dict[str, KucoinMktPair]: - data = await self._get_pairs() + data = await self.cache_pairs() matches = fuzzy.extractBests( pattern, data, score_cutoff=35, limit=limit) From 11bd2e2f6581e2a553cd1de9e9b45c3715467762 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:04:43 -0400 Subject: [PATCH 58/81] Use datetime | none instead of Optional[datetime] in get_bars --- piker/brokers/kucoin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 766cee62..596fcdb7 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -317,8 +317,8 @@ class Client: async def _get_bars( self, fqsn: str, - start_dt: Optional[datetime] = None, - end_dt: Optional[datetime] = None, + start_dt: datetime | None = None, + end_dt: datetime | None = None, limit: int = 1000, as_np: bool = True, type: str = '1min', From 9f5dfe8501abebc5beaad51f9f4d0d95abb5eddb Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:27:56 -0400 Subject: [PATCH 59/81] Remove anext() comment --- piker/brokers/kucoin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 596fcdb7..8037d457 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -282,7 +282,7 @@ class Client: entries = await self._request('GET', '/symbols') syms = {kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item) for item in entries} - log.info(f' {syms.length} Kucoin market pairs fetched') + log.info(f' {len(syms)} Kucoin market pairs fetched') return syms async def cache_pairs( @@ -342,7 +342,6 @@ class Client: kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) url = f'/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}' - bars = [] for i in range(10): data = await self._request( @@ -358,7 +357,7 @@ class Client: f'History call failed, backing off for {backoff_interval}s') await trio.sleep(backoff_interval) else: - bars = data + bars: list[list[str]] = data break # Map to OHLC values to dict then to np array @@ -465,6 +464,7 @@ async def stream_quotes( } @acm async def subscribe(ws: wsproto.WSConnection): + @acm async def open_ping_task(ws: wsproto.WSConnection): ''' @@ -523,7 +523,6 @@ async def stream_quotes( typ, quote = await anext(msg_gen) while typ != 'trade': - # TODO: use ``anext()`` when it lands in 3.10! typ, quote = await anext(msg_gen) From 1b1e35d32de965266cec47be7e6d034ab13a7219 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:28:44 -0400 Subject: [PATCH 60/81] Add comment explaining waiting for first trade quote --- piker/brokers/kucoin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 8037d457..62b811ca 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -523,6 +523,7 @@ async def stream_quotes( typ, quote = await anext(msg_gen) while typ != 'trade': + # take care to not unblock here until we get a real trade quote typ, quote = await anext(msg_gen) From f67ffeb70f057f16bd4d5e891c6748923b0a8aa4 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:34:04 -0400 Subject: [PATCH 61/81] Remove extra Noen check on msg.get --- piker/brokers/kucoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 62b811ca..3cd4b1ee 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -526,7 +526,6 @@ async def stream_quotes( # take care to not unblock here until we get a real trade quote typ, quote = await anext(msg_gen) - task_status.started((init_msgs, quote)) feed_is_live.set() @@ -571,7 +570,7 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: continue - if msg.get('subject') != None: + if msg.get('subject'): msg = KucoinMsg(**msg) match msg.subject: case 'trade.ticker': From 672c01f13a34736ae40ed4ea9e3a790770533665 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:35:21 -0400 Subject: [PATCH 62/81] Use trade_data_ts for trade message receival --- piker/brokers/kucoin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 3cd4b1ee..98bb32a8 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -586,13 +586,13 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: yield 'trade', { 'symbol': sym, 'last': trade_data.price, - 'brokerd_ts': trade_data.time, + 'brokerd_ts': trade_data_ts, 'ticks': [ { 'type': 'trade', 'price': float(trade_data.price), 'size': float(trade_data.size), - 'broker_ts': trade_data.time, + 'broker_ts': trade_data_ts, } ], } From 4f576b6f3604655f71aeab99b48382ad6523ae70 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 13 Apr 2023 22:37:17 -0400 Subject: [PATCH 63/81] Fix typo with ts vars --- piker/brokers/kucoin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 98bb32a8..fef7f5a3 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -586,13 +586,13 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: yield 'trade', { 'symbol': sym, 'last': trade_data.price, - 'brokerd_ts': trade_data_ts, + 'brokerd_ts': last_trade_ts, 'ticks': [ { 'type': 'trade', 'price': float(trade_data.price), 'size': float(trade_data.size), - 'broker_ts': trade_data_ts, + 'broker_ts': last_trade_ts, } ], } From a111819667784f5908dd1adf729ed722b4fdca2e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 14 Apr 2023 19:05:19 -0400 Subject: [PATCH 64/81] Few fixes after review to get running again B) - use `Struct.copy()` for frozen type - fix `BrokerConfig` delegation attr lookups - bit of linting according to `flake8` --- piker/brokers/kucoin.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index fef7f5a3..60315030 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -21,14 +21,12 @@ Kucoin broker backend from typing import ( Any, Callable, - Optional, Literal, AsyncGenerator ) from contextlib import asynccontextmanager as acm from datetime import datetime import time -import math import base64 import hmac import hashlib @@ -168,7 +166,7 @@ class BrokerConfig(Struct, frozen=True): def get_config() -> BrokerConfig | None: - conf, _= config.load() + conf, _ = config.load() section = conf.get('kucoin') @@ -176,7 +174,7 @@ def get_config() -> BrokerConfig | None: log.warning('No config section found for kucoin in config') return None - return BrokerConfig(**section).typecast() + return BrokerConfig(**section).copy() class Client: @@ -209,8 +207,8 @@ class Client: passphrase = base64.b64encode( hmac.new( - self._key_secret.encode('utf-8'), - self._key_passphrase.encode('utf-8'), + self._config.key_secret.encode('utf-8'), + self._config.key_passphrase.encode('utf-8'), hashlib.sha256, ).digest() ) @@ -218,7 +216,7 @@ class Client: return { 'KC-API-SIGN': signature, 'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000), - 'KC-API-KEY': self._key_id, + 'KC-API-KEY': self._config.key_id, 'KC-API-PASSPHRASE': passphrase, # XXX: Even if using the v1 api - this stays the same 'KC-API-KEY-VERSION': '2', @@ -249,7 +247,10 @@ class Client: f'Error making request to {api_url} -> {res.json()["msg"]}') return res.json()['msg'] - async def _get_ws_token(self, private: bool = False) -> tuple[str, int] | None: + async def _get_ws_token( + self, + private: bool = False, + ) -> tuple[str, int] | None: ''' Fetch ws token needed for sub access: https://docs.kucoin.com/#apply-connect-token @@ -275,7 +276,6 @@ class Client: f'Error making request for Kucoin ws token -> {data.json()["msg"]}' ) - async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: @@ -462,6 +462,7 @@ async def stream_quotes( 'fqsn': sym, }, } + @acm async def subscribe(ws: wsproto.WSConnection): @@ -553,6 +554,7 @@ def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool] | None: 'response': True, } + @trio_async_generator async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 @@ -569,7 +571,6 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: continue - if msg.get('subject'): msg = KucoinMsg(**msg) match msg.subject: From 8403d8a4827eee6f4ce2f54f04bed9a7e0f577e6 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 15 Apr 2023 21:05:04 -0400 Subject: [PATCH 65/81] Simplify numpy mapping logic --- piker/brokers/kucoin.py | 71 ++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 60315030..3142720c 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -327,6 +327,29 @@ class Client: Get OHLC data and convert to numpy array for perffff: https://docs.kucoin.com/#get-klines + Kucoin bar data format: + [ + "1545904980", //Start time of the candle cycle 0 + "0.058", //opening price 1 + "0.049", //closing price 2 + "0.058", //highest price 3 + "0.049", //lowest price 4 + "0.018", //Transaction volume 5 + "0.000945" //Transaction amount 6 + ], + + piker ohlc numpy array format: + [ + ('index', int), + ('time', int), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', float), + ('bar_wap', float), # will be zeroed by sampler if not filled + ] + ''' # Generate generic end and start time if values not passed # Currently gives us 12hrs of data @@ -360,34 +383,30 @@ class Client: bars: list[list[str]] = data break - # Map to OHLC values to dict then to np array new_bars = [] - for i, bar in enumerate(bars[::-1]): - data = { - 'index': i, - 'time': bar[0], - 'open': bar[1], - 'close': bar[2], - 'high': bar[3], - 'low': bar[4], - 'volume': bar[5], - 'amount': bar[6], - 'bar_wap': 0.0, - } + reversed_bars = bars[::-1] - row = [] - for _, (field_name, field_type) in enumerate(_ohlc_dtype): - value = data[field_name] - - match field_name: - case 'index': - row.append(int(value)) - case 'time': - row.append(value) - case _: - row.append(float(value)) - - new_bars.append(tuple(row)) + for i, bar in enumerate(reversed_bars): + new_bars.append( + ( + # index + i, + # time + int(bar[0]), + # open + float(bar[1]), + # high + float(bar[3]), + # low + float(bar[4]), + # close + float(bar[2]), + # volume + float(bar[5]), + # bar_wap + 0.0, + ) + ) array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars return array From 97068032202a1417b94780dee55cbd3827c7619e Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 16 Apr 2023 10:11:17 -0400 Subject: [PATCH 66/81] Refactor streaming logic to be less nested and readable --- piker/brokers/kucoin.py | 191 +++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 100 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 3142720c..6546b949 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -32,6 +32,7 @@ import hmac import hashlib import wsproto from uuid import uuid4 +from functools import partial import asks import tractor @@ -443,6 +444,28 @@ async def open_symbol_search( await stream.send(await client.search_symbols(pattern)) log.info('Kucoin symbol search opened') +@acm +async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): + ''' + Spawn a non-blocking task that pings the ws + server every ping_interval so Kucoin doesn't drop + our connection + + ''' + async with trio.open_nursery() as n: + # TODO: cache this task so it's only called once + async def ping_server(): + while True: + await trio.sleep((ping_interval - 1000) / 1000) + await ws.send_msg({'id': connect_id, 'type': 'ping'}) + + log.info(f'Starting ping task for kucoin ws connection') + n.start_soon(ping_server) + + yield ws + + n.cancel_scope.cancel() + async def stream_quotes( send_chan: trio.abc.SendChannel, @@ -457,121 +480,89 @@ async def stream_quotes( Where the rubber hits the road baby ''' - connect_id = str(uuid4()) - async with open_cached_client('kucoin') as client: - log.info('Starting up quote stream') - # loop through symbols and sub to feedz - for sym in symbols: - token, ping_interval = await client._get_ws_token() - pairs = await client.cache_pairs() - pair: KucoinMktPair = pairs[sym] - kucoin_sym = pair.symbol + token, ping_interval = await client._get_ws_token() + connect_id = str(uuid4()) + pairs = await client.cache_pairs() - init_msgs = { - # pass back token, and bool, signalling if we're the writer - # and that history has been written - sym: { - 'symbol_info': { - 'asset_type': 'crypto', - 'price_tick_size': float(pair.baseIncrement), - 'lot_tick_size': float(pair.baseMinSize), - }, - 'shm_write_opts': {'sum_tick_vml': False}, - 'fqsn': sym, - }, - } + # open ping task + async with ( + open_autorecon_ws( + f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]' + ) as ws, + open_ping_task(ws, ping_interval, connect_id) as ws, + ): + log.info('Starting up quote stream') + # loop through symbols and sub to feedz + for sym in symbols: + pair: KucoinMktPair = pairs[sym] + kucoin_sym = pair.symbol - @acm - async def subscribe(ws: wsproto.WSConnection): + init_msgs = { + # pass back token, and bool, signalling if we're the writer + # and that history has been written + sym: { + 'symbol_info': { + 'asset_type': 'crypto', + 'price_tick_size': float(pair.baseIncrement), + 'lot_tick_size': float(pair.baseMinSize), + }, + 'shm_write_opts': {'sum_tick_vml': False}, + 'fqsn': sym, + } + } - @acm - async def open_ping_task(ws: wsproto.WSConnection): - ''' - Spawn a non-blocking task that pings the ws - server every ping_interval so Kucoin doesn't drop - our connection - ''' - async with trio.open_nursery() as n: - # TODO: cache this task so it's only called once - async def ping_server(): - while True: - await trio.sleep((ping_interval - 1000) / 1000) - await ws.send_msg({'id': connect_id, 'type': 'ping'}) - - log.info(f'Starting ping task for {sym}') - n.start_soon(ping_server) - - yield ws - - n.cancel_scope.cancel() - - # Spawn the ping task here - async with open_ping_task(ws) as ws: - tasks = [] - tasks.append(make_sub(kucoin_sym, connect_id, level='l3')) - tasks.append(make_sub(kucoin_sym, connect_id, level='l1')) - - for task in tasks: - log.info( - f'Subscribing to {task["topic"]} feed for {sym}') - await ws.send_msg(task) - - yield - - # unsub - if ws.connected(): - log.info(f'Unsubscribing to {kucoin_sym} feed') - await ws.send_msg( - { - 'id': connect_id, - 'type': 'unsubscribe', - 'topic': f'/market/ticker:{sym}', - 'privateChannel': False, - 'response': True, - } - ) - - async with ( - open_autorecon_ws( - f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]', - fixture=subscribe, - ) as ws, - stream_messages(ws, sym) as msg_gen, - ): - typ, quote = await anext(msg_gen) - - while typ != 'trade': - # take care to not unblock here until we get a real trade quote + async with ( + subscribe(ws, connect_id, kucoin_sym), + stream_messages(ws, sym) as msg_gen, + ): typ, quote = await anext(msg_gen) + while typ != 'trade': + # take care to not unblock here until we get a real trade quote + typ, quote = await anext(msg_gen) - task_status.started((init_msgs, quote)) - feed_is_live.set() + task_status.started((init_msgs, quote)) + feed_is_live.set() - async for typ, msg in msg_gen: - await send_chan.send({sym: msg}) + async for typ, msg in msg_gen: + await send_chan.send({sym: msg}) +@acm +async def subscribe(ws: wsproto.WSConnection, connect_id, sym): + # breakpoint() + # level 2 sub + await ws.send_msg({ + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/spotMarket/level2Depth5:{sym}', + 'privateChannel': False, + 'response': True, + }) -def make_sub(sym, connect_id, level='l1') -> dict[str, str | bool] | None: - match level: - case 'l1': - return { + # watch trades + await ws.send_msg({ + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/market/ticker:{sym}', + 'privateChannel': False, + 'response': True, + }) + + yield + + # unsub + if ws.connected(): + log.info(f'Unsubscribing to {syn} feed') + await ws.send_msg( + { 'id': connect_id, - 'type': 'subscribe', - 'topic': f'/spotMarket/level2Depth5:{sym}', - 'privateChannel': False, - 'response': True, - } - - case 'l3': - return { - 'id': connect_id, - 'type': 'subscribe', + 'type': 'unsubscribe', 'topic': f'/market/ticker:{sym}', 'privateChannel': False, 'response': True, } + ) @trio_async_generator From dae56baeba6f54015ba5da6e87ed9c8c0dc166bf Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 16 Apr 2023 10:12:29 -0400 Subject: [PATCH 67/81] Refactor streaming logic to be less nested and readable --- piker/brokers/kucoin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 6546b949..4997a06a 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -530,7 +530,6 @@ async def stream_quotes( @acm async def subscribe(ws: wsproto.WSConnection, connect_id, sym): - # breakpoint() # level 2 sub await ws.send_msg({ 'id': connect_id, From 0e4095c947cc487661ff6e5560199829d429857e Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 16 Apr 2023 10:45:05 -0400 Subject: [PATCH 68/81] Don't yield ws from the ping task --- piker/brokers/kucoin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 4997a06a..020c9e1c 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -462,7 +462,7 @@ async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): log.info(f'Starting ping task for kucoin ws connection') n.start_soon(ping_server) - yield ws + yield n.cancel_scope.cancel() @@ -490,7 +490,7 @@ async def stream_quotes( open_autorecon_ws( f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]' ) as ws, - open_ping_task(ws, ping_interval, connect_id) as ws, + open_ping_task(ws, ping_interval, connect_id), ): log.info('Starting up quote stream') # loop through symbols and sub to feedz From b01771be1b6b0a0f624a7cefd457c78db987da8b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 16 Apr 2023 10:46:22 -0400 Subject: [PATCH 69/81] Add comments to kucoin->piker bar conversion --- piker/brokers/kucoin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 020c9e1c..8f00c97e 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -387,6 +387,7 @@ class Client: new_bars = [] reversed_bars = bars[::-1] + # Convert from kucoin format to piker format for i, bar in enumerate(reversed_bars): new_bars.append( ( From a109a8bf67a43fc91f3076c1981b81214cad97d3 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 18 Apr 2023 09:51:50 -0400 Subject: [PATCH 70/81] Add linting fixes --- piker/brokers/kucoin.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 8f00c97e..b1e94020 100644 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -32,7 +32,6 @@ import hmac import hashlib import wsproto from uuid import uuid4 -from functools import partial import asks import tractor @@ -281,7 +280,10 @@ class Client: self, ) -> dict[str, KucoinMktPair]: entries = await self._request('GET', '/symbols') - syms = {kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item) for item in entries} + syms = { + kucoin_sym_to_fqsn( + item['name']): KucoinMktPair( + **item) for item in entries} log.info(f' {len(syms)} Kucoin market pairs fetched') return syms @@ -445,6 +447,7 @@ async def open_symbol_search( await stream.send(await client.search_symbols(pattern)) log.info('Kucoin symbol search opened') + @acm async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): ''' @@ -513,14 +516,14 @@ async def stream_quotes( } } - async with ( subscribe(ws, connect_id, kucoin_sym), stream_messages(ws, sym) as msg_gen, ): typ, quote = await anext(msg_gen) while typ != 'trade': - # take care to not unblock here until we get a real trade quote + # take care to not unblock here until we get a real + # trade quote typ, quote = await anext(msg_gen) task_status.started((init_msgs, quote)) @@ -529,6 +532,7 @@ async def stream_quotes( async for typ, msg in msg_gen: await send_chan.send({sym: msg}) + @acm async def subscribe(ws: wsproto.WSConnection, connect_id, sym): # level 2 sub From 37ce04ca9ae1279f36275292e5209c493a45f05a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 18 Apr 2023 10:19:59 -0400 Subject: [PATCH 71/81] Linting fixes --- piker/brokers/kucoin.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) mode change 100644 => 100755 piker/brokers/kucoin.py diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py old mode 100644 new mode 100755 index b1e94020..19726ff6 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -11,7 +11,8 @@ # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . +# along with this program. If not, see +# . ''' Kucoin broker backend @@ -234,7 +235,8 @@ class Client: ''' if self._config: - headers = self._gen_auth_req_headers(action, endpoint, api_v) + headers = self._gen_auth_req_headers( + action, endpoint, api_v) api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' @@ -264,7 +266,8 @@ class Client: 'POST', f'/bullet-{token_type}', 'v1' ) except Exception as e: - log.error(f'Error making request for Kucoin ws token -> {str(e)}') + log.error( + f'Error making request for Kucoin ws token -> {str(e)}') return None if data and 'token' in data: @@ -360,7 +363,8 @@ class Client: end_dt = pendulum.now('UTC').add(minutes=1) if start_dt is None: - start_dt = end_dt.start_of('minute').subtract(minutes=limit) + start_dt = end_dt.start_of( + 'minute').subtract(minutes=limit) start_dt = int(start_dt.timestamp()) end_dt = int(end_dt.timestamp()) @@ -412,11 +416,14 @@ class Client: ) ) - array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + array = np.array( + new_bars, + dtype=_ohlc_dtype) if as_np else bars return array -def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: +def fqsn_to_kucoin_sym( + fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: pair_data = pairs[fqsn] return pair_data.baseCurrency + '-' + pair_data.quoteCurrency @@ -477,7 +484,8 @@ async def stream_quotes( feed_is_live: trio.Event, loglevel: str = '', # startup sync - task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, + task_status: TaskStatus[tuple[dict, dict] + ] = trio.TASK_STATUS_IGNORED, ) -> None: ''' Required piker api to stream real-time data. @@ -557,7 +565,7 @@ async def subscribe(ws: wsproto.WSConnection, connect_id, sym): # unsub if ws.connected(): - log.info(f'Unsubscribing to {syn} feed') + log.info(f'Unsubscribing to {sym} feed') await ws.send_msg( { 'id': connect_id, @@ -580,7 +588,8 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: if cs.cancelled_caught: timeouts += 1 if timeouts > 2: - log.error('kucoin feed is sh**ing the bed... rebooting...') + log.error( + 'kucoin feed is sh**ing the bed... rebooting...') await ws._connect() continue From 9fcfb8d7807326c77aaabd3fd21d83df8ccb5553 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 18 Apr 2023 10:39:47 -0400 Subject: [PATCH 72/81] More linting fixes --- piker/brokers/kucoin.py | 374 +++++++++++++++++++++------------------- 1 file changed, 194 insertions(+), 180 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 19726ff6..d326f376 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -11,20 +11,14 @@ # GNU Affero General Public License for more details. # You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see -# . +# along with this program. If not, see . -''' +""" Kucoin broker backend -''' +""" -from typing import ( - Any, - Callable, - Literal, - AsyncGenerator -) +from typing import Any, Callable, Literal, AsyncGenerator from contextlib import asynccontextmanager as acm from datetime import datetime import time @@ -56,23 +50,24 @@ from ..data._web_bs import ( log = get_logger(__name__) _ohlc_dtype = [ - ('index', int), - ('time', int), - ('open', float), - ('high', float), - ('low', float), - ('close', float), - ('volume', float), - ('bar_wap', float), # will be zeroed by sampler if not filled + ("index", int), + ("time", int), + ("open", float), + ("high", float), + ("low", float), + ("close", float), + ("volume", float), + ("bar_wap", float), # will be zeroed by sampler if not filled ] class KucoinMktPair(Struct, frozen=True): - ''' + """ Kucoin's pair format: https://docs.kucoin.com/#get-symbols-list - ''' + """ + baseCurrency: str baseIncrement: float baseMaxSize: float @@ -93,11 +88,12 @@ class KucoinMktPair(Struct, frozen=True): class AccountTrade(Struct, frozen=True): - ''' + """ Historical trade format: https://docs.kucoin.com/#get-account-ledgers - ''' + """ + id: str currency: str amount: float @@ -105,16 +101,17 @@ class AccountTrade(Struct, frozen=True): balance: float accountType: str bizType: str - direction: Literal['in', 'out'] + direction: Literal["in", "out"] createdAt: float context: list[str] class AccountResponse(Struct, frozen=True): - ''' + """ https://docs.kucoin.com/#get-account-ledgers - ''' + """ + currentPage: int pageSize: int totalNum: int @@ -123,11 +120,12 @@ class AccountResponse(Struct, frozen=True): class KucoinTrade(Struct, frozen=True): - ''' + """ Real-time trade format: https://docs.kucoin.com/#symbol-ticker - ''' + """ + bestAsk: float bestAskSize: float bestBid: float @@ -139,21 +137,23 @@ class KucoinTrade(Struct, frozen=True): class KucoinL2(Struct, frozen=True): - ''' + """ Real-time L2 order book format: https://docs.kucoin.com/#level2-5-best-ask-bid-orders - ''' + """ + asks: list[list[float]] bids: list[list[float]] timestamp: float class KucoinMsg(Struct, frozen=True): - ''' + """ Generic outer-wrapper for any Kucoin ws msg - ''' + """ + type: str topic: str subject: str @@ -169,10 +169,10 @@ class BrokerConfig(Struct, frozen=True): def get_config() -> BrokerConfig | None: conf, _ = config.load() - section = conf.get('kucoin') + section = conf.get("kucoin") if section is None: - log.warning('No config section found for kucoin in config') + log.warning("No config section found for kucoin in config") return None return BrokerConfig(**section).copy() @@ -186,118 +186,119 @@ class Client: def _gen_auth_req_headers( self, - action: Literal['POST', 'GET'], + action: Literal["POST", "GET"], endpoint: str, - api_v: str = 'v2', + api_v: str = "v2", ) -> dict[str, str | bytes]: - ''' + """ Generate authenticated request headers https://docs.kucoin.com/#authentication - ''' - str_to_sign = str(int(time.time() * 1000)) + \ - action + f'/api/{api_v}{endpoint}' + """ + str_to_sign = ( + str(int(time.time() * 1000)) + action + f"/api/{api_v}{endpoint}" + ) signature = base64.b64encode( hmac.new( - self._config.key_secret.encode('utf-8'), - str_to_sign.encode('utf-8'), + self._config.key_secret.encode("utf-8"), + str_to_sign.encode("utf-8"), hashlib.sha256, ).digest() ) passphrase = base64.b64encode( hmac.new( - self._config.key_secret.encode('utf-8'), - self._config.key_passphrase.encode('utf-8'), + self._config.key_secret.encode("utf-8"), + self._config.key_passphrase.encode("utf-8"), hashlib.sha256, ).digest() ) return { - 'KC-API-SIGN': signature, - 'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000), - 'KC-API-KEY': self._config.key_id, - 'KC-API-PASSPHRASE': passphrase, + "KC-API-SIGN": signature, + "KC-API-TIMESTAMP": str(pendulum.now().int_timestamp * 1000), + "KC-API-KEY": self._config.key_id, + "KC-API-PASSPHRASE": passphrase, # XXX: Even if using the v1 api - this stays the same - 'KC-API-KEY-VERSION': '2', + "KC-API-KEY-VERSION": "2", } async def _request( self, - action: Literal['POST', 'GET'], + action: Literal["POST", "GET"], endpoint: str, - api_v: str = 'v2', + api_v: str = "v2", headers: dict = {}, ) -> Any: - ''' + """ Generic request wrapper for Kucoin API - ''' + """ if self._config: - headers = self._gen_auth_req_headers( - action, endpoint, api_v) + headers = self._gen_auth_req_headers(action, endpoint, api_v) - api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' + api_url = f"https://api.kucoin.com/api/{api_v}{endpoint}" res = await asks.request(action, api_url, headers=headers) - if 'data' in res.json(): - return res.json()['data'] + if "data" in res.json(): + return res.json()["data"] else: log.error( - f'Error making request to {api_url} -> {res.json()["msg"]}') - return res.json()['msg'] + f'Error making request to {api_url} -> {res.json()["msg"]}' + ) + return res.json()["msg"] async def _get_ws_token( self, private: bool = False, ) -> tuple[str, int] | None: - ''' + """ Fetch ws token needed for sub access: https://docs.kucoin.com/#apply-connect-token returns a token and the interval we must ping the server at to keep the connection alive - ''' - token_type = 'private' if private else 'public' + """ + token_type = "private" if private else "public" try: data: dict[str, Any] | None = await self._request( - 'POST', f'/bullet-{token_type}', 'v1' + "POST", f"/bullet-{token_type}", "v1" ) except Exception as e: - log.error( - f'Error making request for Kucoin ws token -> {str(e)}') + log.error(f"Error making request for Kucoin ws token -> {str(e)}") return None - if data and 'token' in data: + if data and "token" in data: # ping_interval is in ms - ping_interval: int = data['instanceServers'][0]['pingInterval'] - return data['token'], ping_interval + ping_interval: int = data["instanceServers"][0]["pingInterval"] + return data["token"], ping_interval elif data: log.error( - f'Error making request for Kucoin ws token -> {data.json()["msg"]}' + 'Error making request for Kucoin ws token' + f'{data.json()["msg"]}' ) async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: - entries = await self._request('GET', '/symbols') + entries = await self._request("GET", "/symbols") syms = { - kucoin_sym_to_fqsn( - item['name']): KucoinMktPair( - **item) for item in entries} + kucoin_sym_to_fqsn(item["name"]): KucoinMktPair(**item) + for item in entries + } - log.info(f' {len(syms)} Kucoin market pairs fetched') + log.info(f" {len(syms)} Kucoin market pairs fetched") return syms async def cache_pairs( self, ) -> dict[str, KucoinMktPair]: - ''' + """ Get cached pairs and convert keyed symbols into fqsns if ya want - ''' + """ if not self._pairs: self._pairs = await self._get_pairs() @@ -311,12 +312,15 @@ class Client: data = await self.cache_pairs() matches = fuzzy.extractBests( - pattern, data, score_cutoff=35, limit=limit) + pattern, data, score_cutoff=35, limit=limit + ) # repack in dict form return {item[0].name: item[0] for item in matches} async def last_trades(self, sym: str) -> list[AccountTrade]: - trades = await self._request('GET', f'/accounts/ledgers?currency={sym}', 'v1') + trades = await self._request( + "GET", f"/accounts/ledgers?currency={sym}", "v1" + ) trades = AccountResponse(**trades) return trades.items @@ -327,9 +331,9 @@ class Client: end_dt: datetime | None = None, limit: int = 1000, as_np: bool = True, - type: str = '1min', + type: str = "1min", ) -> np.ndarray: - ''' + """ Get OHLC data and convert to numpy array for perffff: https://docs.kucoin.com/#get-klines @@ -356,35 +360,40 @@ class Client: ('bar_wap', float), # will be zeroed by sampler if not filled ] - ''' + """ # Generate generic end and start time if values not passed # Currently gives us 12hrs of data if end_dt is None: - end_dt = pendulum.now('UTC').add(minutes=1) + end_dt = pendulum.now("UTC").add(minutes=1) if start_dt is None: - start_dt = end_dt.start_of( - 'minute').subtract(minutes=limit) + start_dt = end_dt.start_of("minute").subtract(minutes=limit) start_dt = int(start_dt.timestamp()) end_dt = int(end_dt.timestamp()) kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) - url = f'/market/candles?type={type}&symbol={kucoin_sym}&startAt={start_dt}&endAt={end_dt}' + url = ( + f"/market/candles?type={type}" + f"&symbol={kucoin_sym}" + f"&startAt={start_dt}" + f"&endAt={end_dt}" + ) for i in range(10): data = await self._request( - 'GET', + "GET", url, - api_v='v1', + api_v="v1", ) if not isinstance(data, list): # Do a gradual backoff if Kucoin is rate limiting us backoff_interval = i log.warn( - f'History call failed, backing off for {backoff_interval}s') + f"History call failed, backing off for {backoff_interval}s" + ) await trio.sleep(backoff_interval) else: bars: list[list[str]] = data @@ -416,20 +425,17 @@ class Client: ) ) - array = np.array( - new_bars, - dtype=_ohlc_dtype) if as_np else bars + array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars return array -def fqsn_to_kucoin_sym( - fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: +def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: pair_data = pairs[fqsn] - return pair_data.baseCurrency + '-' + pair_data.quoteCurrency + return pair_data.baseCurrency + "-" + pair_data.quoteCurrency def kucoin_sym_to_fqsn(sym: str) -> str: - return sym.lower().replace('-', '') + return sym.lower().replace("-", "") @acm @@ -444,7 +450,7 @@ async def get_client() -> AsyncGenerator[Client, None]: async def open_symbol_search( ctx: tractor.Context, ) -> None: - async with open_cached_client('kucoin') as client: + async with open_cached_client("kucoin") as client: # load all symbols locally for fast search await client.cache_pairs() await ctx.started() @@ -452,25 +458,25 @@ async def open_symbol_search( async with ctx.open_stream() as stream: async for pattern in stream: await stream.send(await client.search_symbols(pattern)) - log.info('Kucoin symbol search opened') + log.info("Kucoin symbol search opened") @acm async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): - ''' + """ Spawn a non-blocking task that pings the ws server every ping_interval so Kucoin doesn't drop our connection - ''' + """ async with trio.open_nursery() as n: # TODO: cache this task so it's only called once async def ping_server(): while True: await trio.sleep((ping_interval - 1000) / 1000) - await ws.send_msg({'id': connect_id, 'type': 'ping'}) + await ws.send_msg({"id": connect_id, "type": "ping"}) - log.info(f'Starting ping task for kucoin ws connection') + log.info("Starting ping task for kucoin ws connection") n.start_soon(ping_server) yield @@ -482,29 +488,30 @@ async def stream_quotes( send_chan: trio.abc.SendChannel, symbols: list[str], feed_is_live: trio.Event, - loglevel: str = '', + loglevel: str = "", # startup sync - task_status: TaskStatus[tuple[dict, dict] - ] = trio.TASK_STATUS_IGNORED, + task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: - ''' + """ Required piker api to stream real-time data. Where the rubber hits the road baby - ''' - async with open_cached_client('kucoin') as client: + """ + async with open_cached_client("kucoin") as client: token, ping_interval = await client._get_ws_token() connect_id = str(uuid4()) pairs = await client.cache_pairs() + ws_url = ( + f"wss://ws-api-spot.kucoin.com/?" + f"token={token}&[connectId={connect_id}]" + ) - # open ping task + # open ping task async with ( - open_autorecon_ws( - f'wss://ws-api-spot.kucoin.com/?token={token}&[connectId={connect_id}]' - ) as ws, + open_autorecon_ws(ws_url) as ws, open_ping_task(ws, ping_interval, connect_id), ): - log.info('Starting up quote stream') + log.info("Starting up quote stream") # loop through symbols and sub to feedz for sym in symbols: pair: KucoinMktPair = pairs[sym] @@ -514,13 +521,13 @@ async def stream_quotes( # pass back token, and bool, signalling if we're the writer # and that history has been written sym: { - 'symbol_info': { - 'asset_type': 'crypto', - 'price_tick_size': float(pair.baseIncrement), - 'lot_tick_size': float(pair.baseMinSize), + "symbol_info": { + "asset_type": "crypto", + "price_tick_size": float(pair.baseIncrement), + "lot_tick_size": float(pair.baseMinSize), }, - 'shm_write_opts': {'sum_tick_vml': False}, - 'fqsn': sym, + "shm_write_opts": {"sum_tick_vml": False}, + "fqsn": sym, } } @@ -529,7 +536,7 @@ async def stream_quotes( stream_messages(ws, sym) as msg_gen, ): typ, quote = await anext(msg_gen) - while typ != 'trade': + while typ != "trade": # take care to not unblock here until we get a real # trade quote typ, quote = await anext(msg_gen) @@ -544,41 +551,47 @@ async def stream_quotes( @acm async def subscribe(ws: wsproto.WSConnection, connect_id, sym): # level 2 sub - await ws.send_msg({ - 'id': connect_id, - 'type': 'subscribe', - 'topic': f'/spotMarket/level2Depth5:{sym}', - 'privateChannel': False, - 'response': True, - }) + await ws.send_msg( + { + "id": connect_id, + "type": "subscribe", + "topic": f"/spotMarket/level2Depth5:{sym}", + "privateChannel": False, + "response": True, + } + ) # watch trades - await ws.send_msg({ - 'id': connect_id, - 'type': 'subscribe', - 'topic': f'/market/ticker:{sym}', - 'privateChannel': False, - 'response': True, - }) + await ws.send_msg( + { + "id": connect_id, + "type": "subscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, + } + ) yield # unsub if ws.connected(): - log.info(f'Unsubscribing to {sym} feed') + log.info(f"Unsubscribing to {sym} feed") await ws.send_msg( { - 'id': connect_id, - 'type': 'unsubscribe', - 'topic': f'/market/ticker:{sym}', - 'privateChannel': False, - 'response': True, + "id": connect_id, + "type": "unsubscribe", + "topic": f"/market/ticker:{sym}", + "privateChannel": False, + "response": True, } ) @trio_async_generator -async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: +async def stream_messages( + ws: NoBsWs, sym: str +) -> AsyncGenerator[NoBsWs, dict]: timeouts = 0 last_trade_ts = 0 @@ -588,65 +601,65 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: if cs.cancelled_caught: timeouts += 1 if timeouts > 2: - log.error( - 'kucoin feed is sh**ing the bed... rebooting...') + log.error("kucoin feed is sh**ing the bed... rebooting...") await ws._connect() continue - if msg.get('subject'): + if msg.get("subject"): msg = KucoinMsg(**msg) match msg.subject: - case 'trade.ticker': + case "trade.ticker": trade_data = KucoinTrade(**msg.data) - # XXX: Filter for duplicate messages as ws feed will send duplicate market state + # XXX: Filter for duplicate messages as ws feed will + # send duplicate market state # https://docs.kucoin.com/#level2-5-best-ask-bid-orders if trade_data.time == last_trade_ts: continue last_trade_ts = trade_data.time - yield 'trade', { - 'symbol': sym, - 'last': trade_data.price, - 'brokerd_ts': last_trade_ts, - 'ticks': [ + yield "trade", { + "symbol": sym, + "last": trade_data.price, + "brokerd_ts": last_trade_ts, + "ticks": [ { - 'type': 'trade', - 'price': float(trade_data.price), - 'size': float(trade_data.size), - 'broker_ts': last_trade_ts, + "type": "trade", + "price": float(trade_data.price), + "size": float(trade_data.size), + "broker_ts": last_trade_ts, } ], } - case 'level2': + case "level2": l2_data = KucoinL2(**msg.data) first_ask = l2_data.asks[0] first_bid = l2_data.bids[0] - yield 'l1', { - 'symbol': sym, - 'ticks': [ + yield "l1", { + "symbol": sym, + "ticks": [ { - 'type': 'bid', - 'price': float(first_bid[0]), - 'size': float(first_bid[1]), + "type": "bid", + "price": float(first_bid[0]), + "size": float(first_bid[1]), }, { - 'type': 'bsize', - 'price': float(first_bid[0]), - 'size': float(first_bid[1]), + "type": "bsize", + "price": float(first_bid[0]), + "size": float(first_bid[1]), }, { - 'type': 'ask', - 'price': float(first_ask[0]), - 'size': float(first_ask[1]), + "type": "ask", + "price": float(first_ask[0]), + "size": float(first_ask[1]), }, { - 'type': 'asize', - 'price': float(first_ask[0]), - 'size': float(first_ask[1]), + "type": "asize", + "price": float(first_ask[0]), + "size": float(first_ask[1]), }, ], } @@ -655,20 +668,20 @@ async def stream_messages(ws: NoBsWs, sym: str) -> AsyncGenerator[NoBsWs, dict]: @acm async def open_history_client( symbol: str, - type: str = '1m', + type: str = "1m", ) -> AsyncGenerator[Callable, None]: - async with open_cached_client('kucoin') as client: - - log.info('Attempting to open kucoin history client') + async with open_cached_client("kucoin") as client: + log.info("Attempting to open kucoin history client") async def get_ohlc_history( timeframe: float, end_dt: datetime | None = None, start_dt: datetime | None = None, - ) -> tuple[np.ndarray, datetime | None, datetime | None]: # start # end - + ) -> tuple[ + np.ndarray, datetime | None, datetime | None + ]: # start # end if timeframe != 60: - raise DataUnavailable('Only 1m bars are supported') + raise DataUnavailable("Only 1m bars are supported") array = await client._get_bars( symbol, @@ -676,13 +689,14 @@ async def open_history_client( end_dt=end_dt, ) - times = array['time'] + times = array["time"] if end_dt is None: inow = round(time.time()) print( - f'difference in time between load and processing {inow - times[-1]}' + f"difference in time between load and processing" + f"{inow - times[-1]}" ) if (inow - times[-1]) > 60: @@ -691,7 +705,7 @@ async def open_history_client( start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) - log.info('History succesfully fetched baby') + log.info("History succesfully fetched baby") return array, start_dt, end_dt From fcdddadec1aa8e4659a8c2ce44321152d4b9c37a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 18 Apr 2023 10:42:30 -0400 Subject: [PATCH 73/81] Use singlequotes --- piker/brokers/kucoin.py | 308 ++++++++++++++++++++-------------------- 1 file changed, 154 insertions(+), 154 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index d326f376..d1c3c1c7 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -13,10 +13,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -""" +''' Kucoin broker backend -""" +''' from typing import Any, Callable, Literal, AsyncGenerator from contextlib import asynccontextmanager as acm @@ -50,23 +50,23 @@ from ..data._web_bs import ( log = get_logger(__name__) _ohlc_dtype = [ - ("index", int), - ("time", int), - ("open", float), - ("high", float), - ("low", float), - ("close", float), - ("volume", float), - ("bar_wap", float), # will be zeroed by sampler if not filled + ('index', int), + ('time', int), + ('open', float), + ('high', float), + ('low', float), + ('close', float), + ('volume', float), + ('bar_wap', float), # will be zeroed by sampler if not filled ] class KucoinMktPair(Struct, frozen=True): - """ + ''' Kucoin's pair format: https://docs.kucoin.com/#get-symbols-list - """ + ''' baseCurrency: str baseIncrement: float @@ -88,11 +88,11 @@ class KucoinMktPair(Struct, frozen=True): class AccountTrade(Struct, frozen=True): - """ + ''' Historical trade format: https://docs.kucoin.com/#get-account-ledgers - """ + ''' id: str currency: str @@ -101,16 +101,16 @@ class AccountTrade(Struct, frozen=True): balance: float accountType: str bizType: str - direction: Literal["in", "out"] + direction: Literal['in', 'out'] createdAt: float context: list[str] class AccountResponse(Struct, frozen=True): - """ + ''' https://docs.kucoin.com/#get-account-ledgers - """ + ''' currentPage: int pageSize: int @@ -120,11 +120,11 @@ class AccountResponse(Struct, frozen=True): class KucoinTrade(Struct, frozen=True): - """ + ''' Real-time trade format: https://docs.kucoin.com/#symbol-ticker - """ + ''' bestAsk: float bestAskSize: float @@ -137,11 +137,11 @@ class KucoinTrade(Struct, frozen=True): class KucoinL2(Struct, frozen=True): - """ + ''' Real-time L2 order book format: https://docs.kucoin.com/#level2-5-best-ask-bid-orders - """ + ''' asks: list[list[float]] bids: list[list[float]] @@ -149,10 +149,10 @@ class KucoinL2(Struct, frozen=True): class KucoinMsg(Struct, frozen=True): - """ + ''' Generic outer-wrapper for any Kucoin ws msg - """ + ''' type: str topic: str @@ -169,10 +169,10 @@ class BrokerConfig(Struct, frozen=True): def get_config() -> BrokerConfig | None: conf, _ = config.load() - section = conf.get("kucoin") + section = conf.get('kucoin') if section is None: - log.warning("No config section found for kucoin in config") + log.warning('No config section found for kucoin in config') return None return BrokerConfig(**section).copy() @@ -186,94 +186,94 @@ class Client: def _gen_auth_req_headers( self, - action: Literal["POST", "GET"], + action: Literal['POST', 'GET'], endpoint: str, - api_v: str = "v2", + api_v: str = 'v2', ) -> dict[str, str | bytes]: - """ + ''' Generate authenticated request headers https://docs.kucoin.com/#authentication - """ + ''' str_to_sign = ( - str(int(time.time() * 1000)) + action + f"/api/{api_v}{endpoint}" + str(int(time.time() * 1000)) + action + f'/api/{api_v}{endpoint}' ) signature = base64.b64encode( hmac.new( - self._config.key_secret.encode("utf-8"), - str_to_sign.encode("utf-8"), + self._config.key_secret.encode('utf-8'), + str_to_sign.encode('utf-8'), hashlib.sha256, ).digest() ) passphrase = base64.b64encode( hmac.new( - self._config.key_secret.encode("utf-8"), - self._config.key_passphrase.encode("utf-8"), + self._config.key_secret.encode('utf-8'), + self._config.key_passphrase.encode('utf-8'), hashlib.sha256, ).digest() ) return { - "KC-API-SIGN": signature, - "KC-API-TIMESTAMP": str(pendulum.now().int_timestamp * 1000), - "KC-API-KEY": self._config.key_id, - "KC-API-PASSPHRASE": passphrase, + 'KC-API-SIGN': signature, + 'KC-API-TIMESTAMP': str(pendulum.now().int_timestamp * 1000), + 'KC-API-KEY': self._config.key_id, + 'KC-API-PASSPHRASE': passphrase, # XXX: Even if using the v1 api - this stays the same - "KC-API-KEY-VERSION": "2", + 'KC-API-KEY-VERSION': '2', } async def _request( self, - action: Literal["POST", "GET"], + action: Literal['POST', 'GET'], endpoint: str, - api_v: str = "v2", + api_v: str = 'v2', headers: dict = {}, ) -> Any: - """ + ''' Generic request wrapper for Kucoin API - """ + ''' if self._config: headers = self._gen_auth_req_headers(action, endpoint, api_v) - api_url = f"https://api.kucoin.com/api/{api_v}{endpoint}" + api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' res = await asks.request(action, api_url, headers=headers) - if "data" in res.json(): - return res.json()["data"] + if 'data' in res.json(): + return res.json()['data'] else: log.error( f'Error making request to {api_url} -> {res.json()["msg"]}' ) - return res.json()["msg"] + return res.json()['msg'] async def _get_ws_token( self, private: bool = False, ) -> tuple[str, int] | None: - """ + ''' Fetch ws token needed for sub access: https://docs.kucoin.com/#apply-connect-token returns a token and the interval we must ping the server at to keep the connection alive - """ - token_type = "private" if private else "public" + ''' + token_type = 'private' if private else 'public' try: data: dict[str, Any] | None = await self._request( - "POST", f"/bullet-{token_type}", "v1" + 'POST', f'/bullet-{token_type}', 'v1' ) except Exception as e: - log.error(f"Error making request for Kucoin ws token -> {str(e)}") + log.error(f'Error making request for Kucoin ws token -> {str(e)}') return None - if data and "token" in data: + if data and 'token' in data: # ping_interval is in ms - ping_interval: int = data["instanceServers"][0]["pingInterval"] - return data["token"], ping_interval + ping_interval: int = data['instanceServers'][0]['pingInterval'] + return data['token'], ping_interval elif data: log.error( 'Error making request for Kucoin ws token' @@ -283,22 +283,22 @@ class Client: async def _get_pairs( self, ) -> dict[str, KucoinMktPair]: - entries = await self._request("GET", "/symbols") + entries = await self._request('GET', '/symbols') syms = { - kucoin_sym_to_fqsn(item["name"]): KucoinMktPair(**item) + kucoin_sym_to_fqsn(item['name']): KucoinMktPair(**item) for item in entries } - log.info(f" {len(syms)} Kucoin market pairs fetched") + log.info(f' {len(syms)} Kucoin market pairs fetched') return syms async def cache_pairs( self, ) -> dict[str, KucoinMktPair]: - """ + ''' Get cached pairs and convert keyed symbols into fqsns if ya want - """ + ''' if not self._pairs: self._pairs = await self._get_pairs() @@ -319,7 +319,7 @@ class Client: async def last_trades(self, sym: str) -> list[AccountTrade]: trades = await self._request( - "GET", f"/accounts/ledgers?currency={sym}", "v1" + 'GET', f'/accounts/ledgers?currency={sym}', 'v1' ) trades = AccountResponse(**trades) return trades.items @@ -331,21 +331,21 @@ class Client: end_dt: datetime | None = None, limit: int = 1000, as_np: bool = True, - type: str = "1min", + type: str = '1min', ) -> np.ndarray: - """ + ''' Get OHLC data and convert to numpy array for perffff: https://docs.kucoin.com/#get-klines Kucoin bar data format: [ - "1545904980", //Start time of the candle cycle 0 - "0.058", //opening price 1 - "0.049", //closing price 2 - "0.058", //highest price 3 - "0.049", //lowest price 4 - "0.018", //Transaction volume 5 - "0.000945" //Transaction amount 6 + '1545904980', //Start time of the candle cycle 0 + '0.058', //opening price 1 + '0.049', //closing price 2 + '0.058', //highest price 3 + '0.049', //lowest price 4 + '0.018', //Transaction volume 5 + '0.000945' //Transaction amount 6 ], piker ohlc numpy array format: @@ -360,14 +360,14 @@ class Client: ('bar_wap', float), # will be zeroed by sampler if not filled ] - """ + ''' # Generate generic end and start time if values not passed # Currently gives us 12hrs of data if end_dt is None: - end_dt = pendulum.now("UTC").add(minutes=1) + end_dt = pendulum.now('UTC').add(minutes=1) if start_dt is None: - start_dt = end_dt.start_of("minute").subtract(minutes=limit) + start_dt = end_dt.start_of('minute').subtract(minutes=limit) start_dt = int(start_dt.timestamp()) end_dt = int(end_dt.timestamp()) @@ -375,24 +375,24 @@ class Client: kucoin_sym = fqsn_to_kucoin_sym(fqsn, self._pairs) url = ( - f"/market/candles?type={type}" - f"&symbol={kucoin_sym}" - f"&startAt={start_dt}" - f"&endAt={end_dt}" + f'/market/candles?type={type}' + f'&symbol={kucoin_sym}' + f'&startAt={start_dt}' + f'&endAt={end_dt}' ) for i in range(10): data = await self._request( - "GET", + 'GET', url, - api_v="v1", + api_v='v1', ) if not isinstance(data, list): # Do a gradual backoff if Kucoin is rate limiting us backoff_interval = i log.warn( - f"History call failed, backing off for {backoff_interval}s" + f'History call failed, backing off for {backoff_interval}s' ) await trio.sleep(backoff_interval) else: @@ -431,11 +431,11 @@ class Client: def fqsn_to_kucoin_sym(fqsn: str, pairs: dict[str, KucoinMktPair]) -> str: pair_data = pairs[fqsn] - return pair_data.baseCurrency + "-" + pair_data.quoteCurrency + return pair_data.baseCurrency + '-' + pair_data.quoteCurrency def kucoin_sym_to_fqsn(sym: str) -> str: - return sym.lower().replace("-", "") + return sym.lower().replace('-', '') @acm @@ -450,7 +450,7 @@ async def get_client() -> AsyncGenerator[Client, None]: async def open_symbol_search( ctx: tractor.Context, ) -> None: - async with open_cached_client("kucoin") as client: + async with open_cached_client('kucoin') as client: # load all symbols locally for fast search await client.cache_pairs() await ctx.started() @@ -458,25 +458,25 @@ async def open_symbol_search( async with ctx.open_stream() as stream: async for pattern in stream: await stream.send(await client.search_symbols(pattern)) - log.info("Kucoin symbol search opened") + log.info('Kucoin symbol search opened') @acm async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): - """ + ''' Spawn a non-blocking task that pings the ws server every ping_interval so Kucoin doesn't drop our connection - """ + ''' async with trio.open_nursery() as n: # TODO: cache this task so it's only called once async def ping_server(): while True: await trio.sleep((ping_interval - 1000) / 1000) - await ws.send_msg({"id": connect_id, "type": "ping"}) + await ws.send_msg({'id': connect_id, 'type': 'ping'}) - log.info("Starting ping task for kucoin ws connection") + log.info('Starting ping task for kucoin ws connection') n.start_soon(ping_server) yield @@ -488,22 +488,22 @@ async def stream_quotes( send_chan: trio.abc.SendChannel, symbols: list[str], feed_is_live: trio.Event, - loglevel: str = "", + loglevel: str = '', # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: - """ + ''' Required piker api to stream real-time data. Where the rubber hits the road baby - """ - async with open_cached_client("kucoin") as client: + ''' + async with open_cached_client('kucoin') as client: token, ping_interval = await client._get_ws_token() connect_id = str(uuid4()) pairs = await client.cache_pairs() ws_url = ( - f"wss://ws-api-spot.kucoin.com/?" - f"token={token}&[connectId={connect_id}]" + f'wss://ws-api-spot.kucoin.com/?' + f'token={token}&[connectId={connect_id}]' ) # open ping task @@ -511,7 +511,7 @@ async def stream_quotes( open_autorecon_ws(ws_url) as ws, open_ping_task(ws, ping_interval, connect_id), ): - log.info("Starting up quote stream") + log.info('Starting up quote stream') # loop through symbols and sub to feedz for sym in symbols: pair: KucoinMktPair = pairs[sym] @@ -521,13 +521,13 @@ async def stream_quotes( # pass back token, and bool, signalling if we're the writer # and that history has been written sym: { - "symbol_info": { - "asset_type": "crypto", - "price_tick_size": float(pair.baseIncrement), - "lot_tick_size": float(pair.baseMinSize), + 'symbol_info': { + 'asset_type': 'crypto', + 'price_tick_size': float(pair.baseIncrement), + 'lot_tick_size': float(pair.baseMinSize), }, - "shm_write_opts": {"sum_tick_vml": False}, - "fqsn": sym, + 'shm_write_opts': {'sum_tick_vml': False}, + 'fqsn': sym, } } @@ -536,7 +536,7 @@ async def stream_quotes( stream_messages(ws, sym) as msg_gen, ): typ, quote = await anext(msg_gen) - while typ != "trade": + while typ != 'trade': # take care to not unblock here until we get a real # trade quote typ, quote = await anext(msg_gen) @@ -553,22 +553,22 @@ async def subscribe(ws: wsproto.WSConnection, connect_id, sym): # level 2 sub await ws.send_msg( { - "id": connect_id, - "type": "subscribe", - "topic": f"/spotMarket/level2Depth5:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/spotMarket/level2Depth5:{sym}', + 'privateChannel': False, + 'response': True, } ) # watch trades await ws.send_msg( { - "id": connect_id, - "type": "subscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'subscribe', + 'topic': f'/market/ticker:{sym}', + 'privateChannel': False, + 'response': True, } ) @@ -576,14 +576,14 @@ async def subscribe(ws: wsproto.WSConnection, connect_id, sym): # unsub if ws.connected(): - log.info(f"Unsubscribing to {sym} feed") + log.info(f'Unsubscribing to {sym} feed') await ws.send_msg( { - "id": connect_id, - "type": "unsubscribe", - "topic": f"/market/ticker:{sym}", - "privateChannel": False, - "response": True, + 'id': connect_id, + 'type': 'unsubscribe', + 'topic': f'/market/ticker:{sym}', + 'privateChannel': False, + 'response': True, } ) @@ -601,15 +601,15 @@ async def stream_messages( if cs.cancelled_caught: timeouts += 1 if timeouts > 2: - log.error("kucoin feed is sh**ing the bed... rebooting...") + log.error('kucoin feed is sh**ing the bed... rebooting...') await ws._connect() continue - if msg.get("subject"): + if msg.get('subject'): msg = KucoinMsg(**msg) match msg.subject: - case "trade.ticker": + case 'trade.ticker': trade_data = KucoinTrade(**msg.data) # XXX: Filter for duplicate messages as ws feed will @@ -620,46 +620,46 @@ async def stream_messages( last_trade_ts = trade_data.time - yield "trade", { - "symbol": sym, - "last": trade_data.price, - "brokerd_ts": last_trade_ts, - "ticks": [ + yield 'trade', { + 'symbol': sym, + 'last': trade_data.price, + 'brokerd_ts': last_trade_ts, + 'ticks': [ { - "type": "trade", - "price": float(trade_data.price), - "size": float(trade_data.size), - "broker_ts": last_trade_ts, + 'type': 'trade', + 'price': float(trade_data.price), + 'size': float(trade_data.size), + 'broker_ts': last_trade_ts, } ], } - case "level2": + case 'level2': l2_data = KucoinL2(**msg.data) first_ask = l2_data.asks[0] first_bid = l2_data.bids[0] - yield "l1", { - "symbol": sym, - "ticks": [ + yield 'l1', { + 'symbol': sym, + 'ticks': [ { - "type": "bid", - "price": float(first_bid[0]), - "size": float(first_bid[1]), + 'type': 'bid', + 'price': float(first_bid[0]), + 'size': float(first_bid[1]), }, { - "type": "bsize", - "price": float(first_bid[0]), - "size": float(first_bid[1]), + 'type': 'bsize', + 'price': float(first_bid[0]), + 'size': float(first_bid[1]), }, { - "type": "ask", - "price": float(first_ask[0]), - "size": float(first_ask[1]), + 'type': 'ask', + 'price': float(first_ask[0]), + 'size': float(first_ask[1]), }, { - "type": "asize", - "price": float(first_ask[0]), - "size": float(first_ask[1]), + 'type': 'asize', + 'price': float(first_ask[0]), + 'size': float(first_ask[1]), }, ], } @@ -668,10 +668,10 @@ async def stream_messages( @acm async def open_history_client( symbol: str, - type: str = "1m", + type: str = '1m', ) -> AsyncGenerator[Callable, None]: - async with open_cached_client("kucoin") as client: - log.info("Attempting to open kucoin history client") + async with open_cached_client('kucoin') as client: + log.info('Attempting to open kucoin history client') async def get_ohlc_history( timeframe: float, @@ -681,7 +681,7 @@ async def open_history_client( np.ndarray, datetime | None, datetime | None ]: # start # end if timeframe != 60: - raise DataUnavailable("Only 1m bars are supported") + raise DataUnavailable('Only 1m bars are supported') array = await client._get_bars( symbol, @@ -689,14 +689,14 @@ async def open_history_client( end_dt=end_dt, ) - times = array["time"] + times = array['time'] if end_dt is None: inow = round(time.time()) print( - f"difference in time between load and processing" - f"{inow - times[-1]}" + f'difference in time between load and processing' + f'{inow - times[-1]}' ) if (inow - times[-1]) > 60: @@ -705,7 +705,7 @@ async def open_history_client( start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) - log.info("History succesfully fetched baby") + log.info('History succesfully fetched baby') return array, start_dt, end_dt From d07a73cf7034280e620e7780a435ed743f9e97ba Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 19 Apr 2023 14:47:19 -0400 Subject: [PATCH 74/81] Add type annotation for open_ping_task' --- piker/brokers/kucoin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index d1c3c1c7..d997e8f6 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -462,7 +462,10 @@ async def open_symbol_search( @acm -async def open_ping_task(ws: wsproto.WSConnection, ping_interval, connect_id): +async def open_ping_task( + ws: wsproto.WSConnection, + ping_interval, connect_id +) -> AsyncGenerator[None, None]: ''' Spawn a non-blocking task that pings the ws server every ping_interval so Kucoin doesn't drop @@ -678,7 +681,9 @@ async def open_history_client( end_dt: datetime | None = None, start_dt: datetime | None = None, ) -> tuple[ - np.ndarray, datetime | None, datetime | None + np.ndarray, datetime | + None, datetime | + None ]: # start # end if timeframe != 60: raise DataUnavailable('Only 1m bars are supported') @@ -699,9 +704,6 @@ async def open_history_client( f'{inow - times[-1]}' ) - if (inow - times[-1]) > 60: - await tractor.breakpoint() - start_dt = pendulum.from_timestamp(times[0]) end_dt = pendulum.from_timestamp(times[-1]) From 6f91c2932d7cb17f8be1853dd93f26bf27908ed4 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 19 Apr 2023 14:49:28 -0400 Subject: [PATCH 75/81] Type bars data dict --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index d997e8f6..e9861b91 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -382,7 +382,7 @@ class Client: ) for i in range(10): - data = await self._request( + data: list[list[str]] | dict = await self._request( 'GET', url, api_v='v1', From d772fe45c0d07171b76f71276d2cd17bcc14721b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 19 Apr 2023 14:55:58 -0400 Subject: [PATCH 76/81] Comment out unused args --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index e9861b91..35d9ac22 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -491,7 +491,7 @@ async def stream_quotes( send_chan: trio.abc.SendChannel, symbols: list[str], feed_is_live: trio.Event, - loglevel: str = '', + # loglevel: str = '', # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: From efad49ec5be6f065b2bf530a7dda6827b7a5d98b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Wed, 19 Apr 2023 14:58:28 -0400 Subject: [PATCH 77/81] Raise ValueError if no config is found when sending authenticated headers --- piker/brokers/kucoin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 35d9ac22..f826856e 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -195,6 +195,10 @@ class Client: https://docs.kucoin.com/#authentication ''' + + if not self._config: + raise ValueError('No config found when trying to send authenticated request') + str_to_sign = ( str(int(time.time() * 1000)) + action + f'/api/{api_v}{endpoint}' ) From a69c8a8b442be6db11532c32be487c7764cca6b9 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 20 Apr 2023 18:51:13 -0400 Subject: [PATCH 78/81] Uncomment loglevel --- piker/brokers/kucoin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index f826856e..fc97b78c 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -495,7 +495,7 @@ async def stream_quotes( send_chan: trio.abc.SendChannel, symbols: list[str], feed_is_live: trio.Event, - # loglevel: str = '', + loglevel: str = '', # startup sync task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, ) -> None: From a06a4f67cc6dc004f22e277474f2787bfd9651b9 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 21 Apr 2023 17:17:47 -0400 Subject: [PATCH 79/81] Remove unused timeframe var from open_history_client --- piker/brokers/kucoin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index fc97b78c..849901cf 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -467,7 +467,7 @@ async def open_symbol_search( @acm async def open_ping_task( - ws: wsproto.WSConnection, + ws: wsproto.WSConnection, ping_interval, connect_id ) -> AsyncGenerator[None, None]: ''' @@ -675,7 +675,6 @@ async def stream_messages( @acm async def open_history_client( symbol: str, - type: str = '1m', ) -> AsyncGenerator[Callable, None]: async with open_cached_client('kucoin') as client: log.info('Attempting to open kucoin history client') From ae3f6696a7fe618b2d36a455810d0565b9001833 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 21 Apr 2023 20:40:23 -0400 Subject: [PATCH 80/81] Fix type hinting for stream_messages return type --- piker/brokers/kucoin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 849901cf..85ee7de6 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -598,7 +598,7 @@ async def subscribe(ws: wsproto.WSConnection, connect_id, sym): @trio_async_generator async def stream_messages( ws: NoBsWs, sym: str -) -> AsyncGenerator[NoBsWs, dict]: +) -> AsyncGenerator[tuple[str, dict], None]: timeouts = 0 last_trade_ts = 0 @@ -612,7 +612,6 @@ async def stream_messages( await ws._connect() continue - if msg.get('subject'): msg = KucoinMsg(**msg) match msg.subject: @@ -672,6 +671,8 @@ async def stream_messages( } + + @acm async def open_history_client( symbol: str, From 3836f7d458601f979076e15781c85156a2357ee4 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 21 Apr 2023 21:16:14 -0400 Subject: [PATCH 81/81] Run autopep8, add default case for message stream match case --- piker/brokers/kucoin.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/piker/brokers/kucoin.py b/piker/brokers/kucoin.py index 85ee7de6..743a78c2 100755 --- a/piker/brokers/kucoin.py +++ b/piker/brokers/kucoin.py @@ -197,10 +197,12 @@ class Client: ''' if not self._config: - raise ValueError('No config found when trying to send authenticated request') + raise ValueError( + 'No config found when trying to send authenticated request') str_to_sign = ( - str(int(time.time() * 1000)) + action + f'/api/{api_v}{endpoint}' + str(int(time.time() * 1000)) + + action + f'/api/{api_v}{endpoint}' ) signature = base64.b64encode( @@ -240,7 +242,8 @@ class Client: ''' if self._config: - headers = self._gen_auth_req_headers(action, endpoint, api_v) + headers = self._gen_auth_req_headers( + action, endpoint, api_v) api_url = f'https://api.kucoin.com/api/{api_v}{endpoint}' @@ -271,7 +274,8 @@ class Client: 'POST', f'/bullet-{token_type}', 'v1' ) except Exception as e: - log.error(f'Error making request for Kucoin ws token -> {str(e)}') + log.error( + f'Error making request for Kucoin ws token -> {str(e)}') return None if data and 'token' in data: @@ -371,7 +375,8 @@ class Client: end_dt = pendulum.now('UTC').add(minutes=1) if start_dt is None: - start_dt = end_dt.start_of('minute').subtract(minutes=limit) + start_dt = end_dt.start_of( + 'minute').subtract(minutes=limit) start_dt = int(start_dt.timestamp()) end_dt = int(end_dt.timestamp()) @@ -429,7 +434,8 @@ class Client: ) ) - array = np.array(new_bars, dtype=_ohlc_dtype) if as_np else bars + array = np.array( + new_bars, dtype=_ohlc_dtype) if as_np else bars return array @@ -497,7 +503,8 @@ async def stream_quotes( feed_is_live: trio.Event, loglevel: str = '', # startup sync - task_status: TaskStatus[tuple[dict, dict]] = trio.TASK_STATUS_IGNORED, + task_status: TaskStatus[tuple[dict, dict] + ] = trio.TASK_STATUS_IGNORED, ) -> None: ''' Required piker api to stream real-time data. @@ -556,7 +563,7 @@ async def stream_quotes( @acm -async def subscribe(ws: wsproto.WSConnection, connect_id, sym): +async def subscribe(ws: wsproto.WSConnection, connect_id, sym) -> AsyncGenerator[None, None]: # level 2 sub await ws.send_msg( { @@ -608,7 +615,8 @@ async def stream_messages( if cs.cancelled_caught: timeouts += 1 if timeouts > 2: - log.error('kucoin feed is sh**ing the bed... rebooting...') + log.error( + 'kucoin feed is sh**ing the bed... rebooting...') await ws._connect() continue @@ -670,7 +678,8 @@ async def stream_messages( ], } - + case _: + log.warn(f'Unhandled message: {msg}') @acm @@ -685,9 +694,9 @@ async def open_history_client( end_dt: datetime | None = None, start_dt: datetime | None = None, ) -> tuple[ - np.ndarray, datetime | - None, datetime | - None + np.ndarray, datetime + | None, datetime + | None ]: # start # end if timeframe != 60: raise DataUnavailable('Only 1m bars are supported')