Compare commits

...

4 Commits

Author SHA1 Message Date
Tyler Goodlet 44b8c70521 Change type-annots to use `httpx.Response` 2024-05-20 12:55:45 -04:00
Tyler Goodlet e6af97c596 Port `kucoin` backend to `httpx` 2024-05-20 11:09:30 -04:00
Tyler Goodlet 95ace5acb8 Port `kraken` backend to `httpx` 2024-05-20 11:09:10 -04:00
Tyler Goodlet 129cf58d41 Bump deps for Py3.12, go PyQt6, tweak ruff rules
Code base is already ported for `Qt6` so this removes the pyqt5 dep,
adds latest pyqt6 as well as buncha other updates:

- add `xonsh` and ptk as dev deps for those of us using wacky shells ;P
- bump compiled deps as needed for python 3.12 (`numpy`, `numba`)
- add `httpx` and drop `asks` since the latter is zombied and not compat
  with other libs on 3.12.
- add `ruff` linting ignore rules for the new `.ui.qt` shim mod layer.
- few other deps updates to latest versions.
- add in the `keywords` and `classifiers` sections from the old
  `setup.py`.
2024-05-20 11:07:27 -04:00
6 changed files with 196 additions and 131 deletions

View File

@ -50,7 +50,7 @@ __brokers__: list[str] = [
'binance', 'binance',
'ib', 'ib',
'kraken', 'kraken',
'kucoin' 'kucoin',
# broken but used to work # broken but used to work
# 'questrade', # 'questrade',
@ -71,7 +71,7 @@ def get_brokermod(brokername: str) -> ModuleType:
Return the imported broker module by name. Return the imported broker module by name.
''' '''
module = import_module('.' + brokername, 'piker.brokers') module: ModuleType = import_module('.' + brokername, 'piker.brokers')
# we only allow monkeying because it's for internal keying # we only allow monkeying because it's for internal keying
module.name = module.__name__.split('.')[-1] module.name = module.__name__.split('.')[-1]
return module return module

View File

@ -18,10 +18,11 @@
Handy cross-broker utils. Handy cross-broker utils.
""" """
from __future__ import annotations
from functools import partial from functools import partial
import json import json
import asks import httpx
import logging import logging
from ..log import ( from ..log import (
@ -60,11 +61,11 @@ class NoData(BrokerError):
def __init__( def __init__(
self, self,
*args, *args,
info: dict, info: dict|None = None,
) -> None: ) -> None:
super().__init__(*args) super().__init__(*args)
self.info: dict = info self.info: dict|None = info
# when raised, machinery can check if the backend # when raised, machinery can check if the backend
# set a "frame size" for doing datetime calcs. # set a "frame size" for doing datetime calcs.
@ -90,16 +91,18 @@ class DataThrottle(BrokerError):
def resproc( def resproc(
resp: asks.response_objects.Response, resp: httpx.Response,
log: logging.Logger, log: logging.Logger,
return_json: bool = True, return_json: bool = True,
log_resp: bool = False, log_resp: bool = False,
) -> asks.response_objects.Response: ) -> httpx.Response:
"""Process response and return its json content. '''
Process response and return its json content.
Raise the appropriate error on non-200 OK responses. Raise the appropriate error on non-200 OK responses.
"""
'''
if not resp.status_code == 200: if not resp.status_code == 200:
raise BrokerError(resp.body) raise BrokerError(resp.body)
try: try:

View File

@ -27,8 +27,8 @@ from typing import (
) )
import time import time
import httpx
import pendulum import pendulum
import asks
import numpy as np import numpy as np
import urllib.parse import urllib.parse
import hashlib import hashlib
@ -60,6 +60,11 @@ log = get_logger('piker.brokers.kraken')
# <uri>/<version>/ # <uri>/<version>/
_url = 'https://api.kraken.com/0' _url = 'https://api.kraken.com/0'
_headers: dict[str, str] = {
'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
}
# TODO: this is the only backend providing this right? # TODO: this is the only backend providing this right?
# in which case we should drop it from the defaults and # in which case we should drop it from the defaults and
# instead make a custom fields descr in this module! # instead make a custom fields descr in this module!
@ -135,16 +140,15 @@ class Client:
def __init__( def __init__(
self, self,
config: dict[str, str], config: dict[str, str],
httpx_client: httpx.AsyncClient,
name: str = '', name: str = '',
api_key: str = '', api_key: str = '',
secret: str = '' secret: str = ''
) -> None: ) -> None:
self._sesh = asks.Session(connections=4)
self._sesh.base_location = _url self._sesh: httpx.AsyncClient = httpx_client
self._sesh.headers.update({
'User-Agent':
'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
})
self._name = name self._name = name
self._api_key = api_key self._api_key = api_key
self._secret = secret self._secret = secret
@ -166,10 +170,9 @@ class Client:
method: str, method: str,
data: dict, data: dict,
) -> dict[str, Any]: ) -> dict[str, Any]:
resp = await self._sesh.post( resp: httpx.Response = await self._sesh.post(
path=f'/public/{method}', url=f'/public/{method}',
json=data, json=data,
timeout=float('inf')
) )
return resproc(resp, log) return resproc(resp, log)
@ -180,18 +183,18 @@ class Client:
uri_path: str uri_path: str
) -> dict[str, Any]: ) -> dict[str, Any]:
headers = { headers = {
'Content-Type': 'Content-Type': 'application/x-www-form-urlencoded',
'application/x-www-form-urlencoded', 'API-Key': self._api_key,
'API-Key': 'API-Sign': get_kraken_signature(
self._api_key, uri_path,
'API-Sign': data,
get_kraken_signature(uri_path, data, self._secret) self._secret,
),
} }
resp = await self._sesh.post( resp: httpx.Response = await self._sesh.post(
path=f'/private/{method}', url=f'/private/{method}',
data=data, data=data,
headers=headers, headers=headers,
timeout=float('inf')
) )
return resproc(resp, log) return resproc(resp, log)
@ -665,24 +668,36 @@ class Client:
@acm @acm
async def get_client() -> Client: async def get_client() -> Client:
conf = get_config() conf: dict[str, Any] = get_config()
if conf: async with httpx.AsyncClient(
client = Client( base_url=_url,
conf, headers=_headers,
# TODO: don't break these up and just do internal # TODO: is there a way to numerate this?
# conf lookups instead.. # https://www.python-httpx.org/advanced/clients/#why-use-a-client
name=conf['key_descr'], # connections=4
api_key=conf['api_key'], ) as trio_client:
secret=conf['secret'] if conf:
) client = Client(
else: conf,
client = Client({}) httpx_client=trio_client,
# at startup, load all symbols, and asset info in # TODO: don't break these up and just do internal
# batch requests. # conf lookups instead..
async with trio.open_nursery() as nurse: name=conf['key_descr'],
nurse.start_soon(client.get_assets) api_key=conf['api_key'],
await client.get_mkt_pairs() secret=conf['secret']
)
else:
client = Client(
conf={},
httpx_client=trio_client,
)
yield client # at startup, load all symbols, and asset info in
# batch requests.
async with trio.open_nursery() as nurse:
nurse.start_soon(client.get_assets)
await client.get_mkt_pairs()
yield client

View File

@ -612,18 +612,18 @@ async def open_trade_dialog(
# enter relay loop # enter relay loop
await handle_order_updates( await handle_order_updates(
client, client=client,
ws, ws=ws,
stream, ws_stream=stream,
ems_stream, ems_stream=ems_stream,
apiflows, apiflows=apiflows,
ids, ids=ids,
reqids2txids, reqids2txids=reqids2txids,
acnt, acnt=acnt,
api_trans, ledger=ledger,
acctid, acctid=acctid,
acc_name, acc_name=acc_name,
token, token=token,
) )
@ -639,7 +639,8 @@ async def handle_order_updates(
# transaction records which will be updated # transaction records which will be updated
# on new trade clearing events (aka order "fills") # on new trade clearing events (aka order "fills")
ledger_trans: dict[str, Transaction], ledger: TransactionLedger,
# ledger_trans: dict[str, Transaction],
acctid: str, acctid: str,
acc_name: str, acc_name: str,
token: str, token: str,
@ -699,7 +700,8 @@ async def handle_order_updates(
# if tid not in ledger_trans # if tid not in ledger_trans
} }
for tid, trade in trades.items(): for tid, trade in trades.items():
assert tid not in ledger_trans # assert tid not in ledger_trans
assert tid not in ledger
txid = trade['ordertxid'] txid = trade['ordertxid']
reqid = trade.get('userref') reqid = trade.get('userref')
@ -747,11 +749,17 @@ async def handle_order_updates(
client, client,
api_name_set='wsname', api_name_set='wsname',
) )
ppmsgs = trades2pps( ppmsgs: list[BrokerdPosition] = trades2pps(
acnt, acnt=acnt,
acctid, ledger=ledger,
new_trans, acctid=acctid,
new_trans=new_trans,
) )
# ppmsgs = trades2pps(
# acnt,
# acctid,
# new_trans,
# )
for pp_msg in ppmsgs: for pp_msg in ppmsgs:
await ems_stream.send(pp_msg) await ems_stream.send(pp_msg)

View File

@ -16,10 +16,9 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>. # along with this program. If not, see <https://www.gnu.org/licenses/>.
''' '''
Kucoin broker backend Kucoin cex API backend.
''' '''
from contextlib import ( from contextlib import (
asynccontextmanager as acm, asynccontextmanager as acm,
aclosing, aclosing,
@ -42,7 +41,7 @@ import wsproto
from uuid import uuid4 from uuid import uuid4
from trio_typing import TaskStatus from trio_typing import TaskStatus
import asks import httpx
from bidict import bidict from bidict import bidict
import numpy as np import numpy as np
import pendulum import pendulum
@ -212,8 +211,12 @@ def get_config() -> BrokerConfig | None:
class Client: class Client:
def __init__(self) -> None: def __init__(
self._config: BrokerConfig | None = get_config() self,
httpx_client: httpx.AsyncClient,
) -> None:
self._http: httpx.AsyncClient = httpx_client
self._config: BrokerConfig|None = get_config()
self._pairs: dict[str, KucoinMktPair] = {} self._pairs: dict[str, KucoinMktPair] = {}
self._fqmes2mktids: bidict[str, str] = bidict() self._fqmes2mktids: bidict[str, str] = bidict()
self._bars: list[list[float]] = [] self._bars: list[list[float]] = []
@ -227,18 +230,24 @@ class Client:
) -> dict[str, str | bytes]: ) -> dict[str, str | bytes]:
''' '''
Generate authenticated request headers Generate authenticated request headers:
https://docs.kucoin.com/#authentication https://docs.kucoin.com/#authentication
https://www.kucoin.com/docs/basic-info/connection-method/authentication/creating-a-request
https://www.kucoin.com/docs/basic-info/connection-method/authentication/signing-a-message
''' '''
if not self._config: if not self._config:
raise ValueError( raise ValueError(
'No config found when trying to send authenticated request') 'No config found when trying to send authenticated request'
)
str_to_sign = ( str_to_sign = (
str(int(time.time() * 1000)) str(int(time.time() * 1000))
+ action + f'/api/{api}/{endpoint.lstrip("/")}' +
action
+
f'/api/{api}/{endpoint.lstrip("/")}'
) )
signature = base64.b64encode( signature = base64.b64encode(
@ -249,6 +258,7 @@ class Client:
).digest() ).digest()
) )
# TODO: can we cache this between calls?
passphrase = base64.b64encode( passphrase = base64.b64encode(
hmac.new( hmac.new(
self._config.key_secret.encode('utf-8'), self._config.key_secret.encode('utf-8'),
@ -270,8 +280,10 @@ class Client:
self, self,
action: Literal['POST', 'GET'], action: Literal['POST', 'GET'],
endpoint: str, endpoint: str,
api: str = 'v2', api: str = 'v2',
headers: dict = {}, headers: dict = {},
) -> Any: ) -> Any:
''' '''
Generic request wrapper for Kucoin API Generic request wrapper for Kucoin API
@ -284,13 +296,17 @@ class Client:
api, api,
) )
api_url = f'https://api.kucoin.com/api/{api}/{endpoint}' req_meth: Callable = getattr(
self._http,
res = await asks.request(action, api_url, headers=headers) action.lower(),
)
json = res.json() res = await req_meth(
if 'data' in json: url=f'/{api}/{endpoint}',
return json['data'] headers=headers,
)
json: dict = res.json()
if data := json.get('data'):
return data
else: else:
log.error( log.error(
f'Error making request to {api_url} ->\n' f'Error making request to {api_url} ->\n'
@ -311,7 +327,7 @@ class Client:
''' '''
token_type = 'private' if private else 'public' token_type = 'private' if private else 'public'
try: try:
data: dict[str, Any] | None = await self._request( data: dict[str, Any]|None = await self._request(
'POST', 'POST',
endpoint=f'bullet-{token_type}', endpoint=f'bullet-{token_type}',
api='v1' api='v1'
@ -349,8 +365,8 @@ class Client:
currencies: dict[str, Currency] = {} currencies: dict[str, Currency] = {}
entries: list[dict] = await self._request( entries: list[dict] = await self._request(
'GET', 'GET',
api='v1',
endpoint='currencies', endpoint='currencies',
api='v1',
) )
for entry in entries: for entry in entries:
curr = Currency(**entry).copy() curr = Currency(**entry).copy()
@ -366,7 +382,10 @@ class Client:
dict[str, KucoinMktPair], dict[str, KucoinMktPair],
bidict[str, KucoinMktPair], bidict[str, KucoinMktPair],
]: ]:
entries = await self._request('GET', 'symbols') entries = await self._request(
'GET',
endpoint='symbols',
)
log.info(f' {len(entries)} Kucoin market pairs fetched') log.info(f' {len(entries)} Kucoin market pairs fetched')
pairs: dict[str, KucoinMktPair] = {} pairs: dict[str, KucoinMktPair] = {}
@ -567,13 +586,21 @@ def fqme_to_kucoin_sym(
@acm @acm
async def get_client() -> AsyncGenerator[Client, None]: async def get_client() -> AsyncGenerator[Client, None]:
client = Client() '''
Load an API `Client` preconfigured from user settings
async with trio.open_nursery() as n: '''
n.start_soon(client.get_mkt_pairs) async with (
await client.get_currencies() httpx.AsyncClient(
base_url=f'https://api.kucoin.com/api',
) as trio_client,
):
client = Client(httpx_client=trio_client)
async with trio.open_nursery() as tn:
tn.start_soon(client.get_mkt_pairs)
await client.get_currencies()
yield client yield client
@tractor.context @tractor.context

View File

@ -20,40 +20,31 @@ build-backend = "poetry.core.masonry.api"
# ------ - ------ # ------ - ------
[tool.ruff.lint]
# https://docs.astral.sh/ruff/settings/#lint_ignore
ignore = []
# https://docs.astral.sh/ruff/settings/#lint_per-file-ignores
"piker/ui/qt.py" = [
"E402",
'F401', # unused imports (without __all__ or blah as blah)
# "F841", # unused variable rules
]
# ignore-init-module-imports = false
# ------ - ------
[tool.poetry] [tool.poetry]
name = "piker" name = "piker"
version = "0.1.0.alpha0.dev0" version = "0.1.0.alpha0.dev0"
description = "trading gear for hackers" description = "trading gear for hackers"
authors = ["Tyler Goodlet <jgbt@protonmail.com>"] authors = ["Tyler Goodlet <goodboy_foss@protonmail.com>"]
license = "AGPLv3" license = "AGPLv3"
readme = "README.rst" readme = "README.rst"
# TODO: add meta-data from setup.py
# keywords=[
# "async",
# "trading",
# "finance",
# "quant",
# "charting",
# ],
# classifiers=[
# 'Development Status :: 3 - Alpha',
# 'License :: OSI Approved :: ',
# 'Operating System :: POSIX :: Linux',
# "Programming Language :: Python :: Implementation :: CPython",
# "Programming Language :: Python :: 3 :: Only",
# "Programming Language :: Python :: 3.10",
# "Programming Language :: Python :: 3.11",
# 'Intended Audience :: Financial and Insurance Industry',
# 'Intended Audience :: Science/Research',
# 'Intended Audience :: Developers',
# 'Intended Audience :: Education',
# ],
# ------ - ------ # ------ - ------
[tool.poetry.dependencies] [tool.poetry.dependencies]
asks = "^3.0.0"
async-generator = "^1.10" async-generator = "^1.10"
attrs = "^23.1.0" attrs = "^23.1.0"
bidict = "^0.22.1" bidict = "^0.22.1"
@ -63,41 +54,40 @@ cython = "^3.0.0"
greenback = "^1.1.1" greenback = "^1.1.1"
ib-insync = "^0.9.86" ib-insync = "^0.9.86"
msgspec = "^0.18.0" msgspec = "^0.18.0"
numba = "^0.57.1" numba = "^0.59.0"
numpy = "1.24" numpy = "^1.25"
pendulum = "^2.1.2"
polars = "^0.18.13" polars = "^0.18.13"
pygments = "^2.16.1" pygments = "^2.16.1"
python = "^3.10" python = ">=3.11, <3.13"
rich = "^13.5.2" rich = "^13.5.2"
# setuptools = "^68.0.0" # setuptools = "^68.0.0"
tomli = "^2.0.1" tomli = "^2.0.1"
tomli-w = "^1.0.0" tomli-w = "^1.0.0"
trio = "^0.22.2"
trio-util = "^0.7.0" trio-util = "^0.7.0"
trio-websocket = "^0.10.3" trio-websocket = "^0.10.3"
typer = "^0.9.0" typer = "^0.9.0"
rapidfuzz = "^3.5.2"
pdbp = "^1.5.0"
trio = "^0.24"
pendulum = "^3.0.0"
httpx = "^0.27.0"
[tool.poetry.dependencies.tractor]
develop = true
git = 'https://github.com/goodboy/tractor.git'
branch = 'asyncio_debugger_support'
# path = "../tractor"
[tool.poetry.dependencies.asyncvnc] [tool.poetry.dependencies.asyncvnc]
git = 'https://github.com/pikers/asyncvnc.git' git = 'https://github.com/pikers/asyncvnc.git'
branch = 'main' branch = 'main'
[tool.poetry.dependencies.tomlkit] [tool.poetry.dependencies.tomlkit]
develop = true
git = 'https://github.com/pikers/tomlkit.git' git = 'https://github.com/pikers/tomlkit.git'
branch = 'piker_pin' branch = 'piker_pin'
develop = true
# path = "../tomlkit/" # path = "../tomlkit/"
[tool.poetry.dependencies.tractor]
git = 'https://github.com/goodboy/tractor.git'
branch = 'asyncio_debugger_support'
# branch = 'piker_pin'
develop = true
# path = '../tractor/'
# ------ - ------
[tool.poetry.group.uis] [tool.poetry.group.uis]
optional = true optional = true
[tool.poetry.group.uis.dependencies] [tool.poetry.group.uis.dependencies]
@ -106,11 +96,10 @@ optional = true
# rapidfuzz = {extras = ["speedup"], version = "^0.18.0"} # rapidfuzz = {extras = ["speedup"], version = "^0.18.0"}
rapidfuzz = "^3.2.0" rapidfuzz = "^3.2.0"
qdarkstyle = ">=3.0.2" qdarkstyle = ">=3.0.2"
pyqt5 = "^5.15.9"
pyqtgraph = { git = 'https://github.com/pikers/pyqtgraph.git' } pyqtgraph = { git = 'https://github.com/pikers/pyqtgraph.git' }
pyqt6 = "^6.5.2"
# ------ - ------ # ------ - ------
pyqt6 = "^6.7.0"
[tool.poetry.group.dev] [tool.poetry.group.dev]
optional = true optional = true
@ -118,6 +107,8 @@ optional = true
# testing / CI # testing / CI
pytest = "^6.0.0" pytest = "^6.0.0"
elasticsearch = "^8.9.0" elasticsearch = "^8.9.0"
xonsh = "^0.14.2"
prompt-toolkit = "3.0.40"
# console ehancements and eventually remote debugging # console ehancements and eventually remote debugging
# extras/helpers. # extras/helpers.
@ -126,8 +117,6 @@ elasticsearch = "^8.9.0"
# - xonsh + xxh # - xonsh + xxh
# - rsyscall + pdbp # - rsyscall + pdbp
# - actor runtime control console like BEAM/OTP # - actor runtime control console like BEAM/OTP
xonsh = "^0.14.0" # XXX: explicit env install for shell use w nix
prompt-toolkit = "^3.0.39"
# ------ - ------ # ------ - ------
@ -140,3 +129,26 @@ prompt-toolkit = "^3.0.39"
piker = 'piker.cli:cli' piker = 'piker.cli:cli'
pikerd = 'piker.cli:pikerd' pikerd = 'piker.cli:pikerd'
ledger = 'piker.accounting.cli:ledger' ledger = 'piker.accounting.cli:ledger'
[project]
keywords=[
"async",
"trading",
"finance",
"quant",
"charting",
]
classifiers=[
'Development Status :: 3 - Alpha',
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
'Operating System :: POSIX :: Linux',
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
'Intended Audience :: Financial and Insurance Industry',
'Intended Audience :: Science/Research',
'Intended Audience :: Developers',
'Intended Audience :: Education',
]