# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
Core (web) API client
'''
from contextlib import asynccontextmanager as acm
from datetime import datetime
import itertools
from typing import (
Any,
Union,
)
import time
import httpx
import pendulum
import numpy as np
import urllib.parse
import hashlib
import hmac
import base64
import trio
from piker import config
from piker.data import (
def_iohlcv_fields,
match_from_pairs,
)
from piker.accounting._mktinfo import (
Asset,
digits_to_dec,
dec_digits,
)
from piker.brokers._util import (
resproc,
SymbolNotFound,
BrokerError,
DataThrottle,
)
from piker.accounting import Transaction
from piker.log import get_logger
from .symbols import Pair
log = get_logger('piker.brokers.kraken')
# <uri>/<version>/
_url = 'https://api.kraken.com/0'
_headers: dict[str, str] = {
'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)'
}
# TODO: this is the only backend providing this right?
# in which case we should drop it from the defaults and
# instead make a custom fields descr in this module!
_show_wap_in_history = True
_symbol_info_translation: dict[str, str] = {
'tick_decimals': 'pair_decimals',
}
def get_config() -> dict[str, Any]:
'''
Load our section from `piker/brokers.toml`.
'''
conf, path = config.load(
conf_name='brokers',
touch_if_dne=True,
)
if (section := conf.get('kraken')) is None:
log.warning(
f'No config section found for kraken in {path}'
)
return {}
return section
def get_kraken_signature(
urlpath: str,
data: dict[str, Any],
secret: str
) -> str:
postdata = urllib.parse.urlencode(data)
encoded = (str(data['nonce']) + postdata).encode()
message = urlpath.encode() + hashlib.sha256(encoded).digest()
mac = hmac.new(base64.b64decode(secret), message, hashlib.sha512)
sigdigest = base64.b64encode(mac.digest())
return sigdigest.decode()
class InvalidKey(ValueError):
'''
EAPI:Invalid key
This error is returned when the API key used for the call is
either expired or disabled, please review the API key in your
Settings -> API tab of account management or generate a new one
and update your application.
'''
class Client:
# assets and mkt pairs are key-ed by kraken's ReST response
# symbol-bs_mktids (we call them "X-keys" like fricking
# "XXMRZEUR"). these keys used directly since ledger endpoints
# return transaction sets keyed with the same set!
_Assets: dict[str, Asset] = {}
_AssetPairs: dict[str, Pair] = {}
# offer lookup tables for all .altname and .wsname
# to the equivalent .xname so that various symbol-schemas
# can be mapped to `Pair`s in the tables above.
_altnames: dict[str, str] = {}
_wsnames: dict[str, str] = {}
# key-ed by `Pair.bs_fqme: str`, and thus used for search
# allowing for lookup using piker's own FQME symbology sys.
_pairs: dict[str, Pair] = {}
_assets: dict[str, Asset] = {}
def __init__(
self,
config: dict[str, str],
httpx_client: httpx.AsyncClient,
name: str = '',
api_key: str = '',
secret: str = ''
) -> None:
self._sesh: httpx.AsyncClient = httpx_client
self._name = name
self._api_key = api_key
self._secret = secret
self.conf: dict[str, str] = config
@property
def pairs(self) -> dict[str, Pair]:
if self._pairs is None:
raise RuntimeError(
"Client didn't run `.get_mkt_pairs()` on startup?!"
)
return self._pairs
async def _public(
self,
method: str,
data: dict,
) -> dict[str, Any]:
resp: httpx.Response = await self._sesh.post(
url=f'/public/{method}',
json=data,
)
return resproc(resp, log)
async def _private(
self,
method: str,
data: dict,
uri_path: str
) -> dict[str, Any]:
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
'API-Key': self._api_key,
'API-Sign': get_kraken_signature(
uri_path,
data,
self._secret,
),
}
resp: httpx.Response = await self._sesh.post(
url=f'/private/{method}',
data=data,
headers=headers,
)
return resproc(resp, log)
async def endpoint(
self,
method: str,
data: dict[str, Any]
) -> dict[str, Any]:
uri_path = f'/0/private/{method}'
data['nonce'] = str(int(1000*time.time()))
return await self._private(method, data, uri_path)
async def get_balances(
self,
) -> dict[str, float]:
'''
Return the set of asset balances for this account
by symbol.
'''
resp = await self.endpoint(
'Balance',
{},
)
by_bsmktid: dict[str, dict] = resp['result']
balances: dict = {}
for xname, bal in by_bsmktid.items():
asset: Asset = self._Assets[xname]
# TODO: which KEY should we use? it's used to index
# the `Account.pps: dict` ..
key: str = asset.name.lower()
# TODO: should we just return a `Decimal` here
# or is the rounded version ok?
balances[key] = round(
float(bal),
ndigits=dec_digits(asset.tx_tick)
)
return balances
async def get_assets(
self,
reload: bool = False,
) -> dict[str, Asset]:
'''
Load and cache all asset infos and pack into
our native ``Asset`` struct.
https://docs.kraken.com/rest/#tag/Market-Data/operation/getAssetInfo
return msg:
"asset1": {
"aclass": "string",
"altname": "string",
"decimals": 0,
"display_decimals": 0,
"collateral_value": 0,
"status": "string"
}
'''
if (
not self._assets
or reload
):
resp = await self._public('Assets', {})
assets: dict[str, dict] = resp['result']
for bs_mktid, info in assets.items():
altname: str = info['altname']
aclass: str = info['aclass']
asset = Asset(
name=altname,
atype=f'crypto_{aclass}',
tx_tick=digits_to_dec(info['decimals']),
info=info,
)
# NOTE: yes we keep 2 sets since kraken insists on
# keeping 3 frickin sets bc apparently they have
# no sane data engineers whol all like different
# keys for their fricking symbology sets..
self._Assets[bs_mktid] = asset
self._assets[altname.lower()] = asset
self._assets[altname] = asset
# we return the "most native" set merged with our preferred
# naming (which i guess is the "altname" one) since that's
# what the symcache loader will be storing, and we need the
# keys that are easiest to match against in any trade
# records.
return self._Assets | self._assets
async def get_trades(
self,
fetch_limit: int | None = None,
) -> dict[str, Any]:
'''
Get the trades (aka cleared orders) history from the rest endpoint:
https://docs.kraken.com/rest/#operation/getTradeHistory
'''
ofs = 0
trades_by_id: dict[str, Any] = {}
for i in itertools.count():
if (
fetch_limit
and i >= fetch_limit
):
break
# increment 'ofs' pagination offset
ofs = i*50
resp = await self.endpoint(
'TradesHistory',
{'ofs': ofs},
)
by_id = resp['result']['trades']
trades_by_id.update(by_id)
# can get up to 50 results per query, see:
# https://docs.kraken.com/rest/#tag/User-Data/operation/getTradeHistory
if (
len(by_id) < 50
):
err = resp.get('error')
if err:
raise BrokerError(err)
# we know we received the max amount of
# trade results so there may be more history.
# catch the end of the trades
count = resp['result']['count']
break
# santity check on update
assert count == len(trades_by_id.values())
return trades_by_id
async def get_xfers(
self,
asset: str,
src_asset: str = '',
) -> dict[str, Transaction]:
'''
Get asset balance transfer transactions.
Currently only withdrawals are supported.
'''
resp = await self.endpoint(
'WithdrawStatus',
{'asset': asset},
)
try:
xfers: list[dict] = resp['result']
except KeyError:
log.exception(f'Kraken suxxx: {resp}')
return []
# eg. resp schema:
# 'result': [{'method': 'Bitcoin', 'aclass': 'currency', 'asset':
# 'XXBT', 'refid': 'AGBJRMB-JHD2M4-NDI3NR', 'txid':
# 'b95d66d3bb6fd76cbccb93f7639f99a505cb20752c62ea0acc093a0e46547c44',
# 'info': 'bc1qc8enqjekwppmw3g80p56z5ns7ze3wraqk5rl9z',
# 'amount': '0.00300726', 'fee': '0.00001000', 'time':
# 1658347714, 'status': 'Success'}]}
if xfers:
import tractor
await tractor.pp()
trans: dict[str, Transaction] = {}
for entry in xfers:
# look up the normalized name and asset info
asset_key: str = entry['asset']
asset: Asset = self._Assets[asset_key]
asset_key: str = asset.name.lower()
# XXX: this is in the asset units (likely) so it isn't
# quite the same as a commisions cost necessarily..)
# TODO: also round this based on `Pair` cost precision info?
cost = float(entry['fee'])
# fqme: str = asset_key + '.kraken'
tx = Transaction(
fqme=asset_key, # this must map to an entry in .assets!
tid=entry['txid'],
dt=pendulum.from_timestamp(entry['time']),
bs_mktid=f'{asset_key}{src_asset}',
size=-1*(
float(entry['amount'])
+
cost
),
# since this will be treated as a "sell" it
# shouldn't be needed to compute the be price.
price='NaN',
# XXX: see note above
cost=cost,
# not a trade but a withdrawal or deposit on the
# asset (chain) system.
etype='transfer',
)
trans[tx.tid] = tx
return trans
async def submit_limit(
self,
symbol: str,
price: float,
action: str,
size: float,
reqid: str = None,
validate: bool = False # set True test call without a real submission
) -> dict:
'''
Place an order and return integer request id provided by client.
'''
# Build common data dict for common keys from both endpoints
data = {
"pair": symbol,
"price": str(price),
"validate": validate
}
if reqid is None:
# Build order data for kraken api
data |= {
"ordertype": "limit",
"type": action,
"volume": str(size),
}
return await self.endpoint('AddOrder', data)
else:
# Edit order data for kraken api
data["txid"] = reqid
return await self.endpoint('EditOrder', data)
async def submit_cancel(
self,
reqid: str,
) -> dict:
'''
Send cancel request for order id ``reqid``.
'''
# txid is a transaction id given by kraken
return await self.endpoint('CancelOrder', {"txid": reqid})
async def asset_pairs(
self,
pair_patt: str | None = None,
) -> dict[str, Pair] | Pair:
'''
Query for a tradeable asset pair (info), or all if no input
pattern is provided.
https://docs.kraken.com/rest/#tag/Market-Data/operation/getTradableAssetPairs
'''
if not self._AssetPairs:
# get all pairs by default, or filter
# to whatever pattern is provided as input.
req_pairs: dict[str, str] | None = None
if pair_patt is not None:
req_pairs = {'pair': pair_patt}
resp = await self._public(
'AssetPairs',
req_pairs,
)
err = resp['error']
if err:
raise SymbolNotFound(pair_patt)
# NOTE: we try to key pairs by our custom defined
# `.bs_fqme` field since we want to offer search over
# this pattern set, callers should fill out lookup
# tables for kraken's bs_mktid keys to map to these
# keys!
# XXX: FURTHER kraken's data eng team decided to offer
# 3 frickin market-pair-symbol key sets depending on
# which frickin API is being used.
# Example for the trading pair 'LTC<EUR'
# - the "X-key" from rest eps 'XLTCZEUR'
# - the "websocket key" from ws msgs is 'LTC/EUR'
# - the "altname key" also delivered in pair info is 'LTCEUR'
for xkey, data in resp['result'].items():
# NOTE: always cache in pairs tables for faster lookup
pair = Pair(xname=xkey, **data)
# register the above `Pair` structs for all
# key-sets/monikers: a set of 4 (frickin) tables
# acting as a combined surjection of all possible
# (and stupid) kraken names to their `Pair` obj.
self._AssetPairs[xkey] = pair
self._pairs[pair.bs_fqme] = pair
self._altnames[pair.altname] = pair
self._wsnames[pair.wsname] = pair
if pair_patt is not None:
return next(iter(self._pairs.items()))[1]
return self._AssetPairs
async def get_mkt_pairs(
self,
reload: bool = False,
) -> dict:
'''
Load all market pair info build and cache it for downstream
use.
Multiple pair info lookup tables (like ``._altnames:
dict[str, str]``) are created for looking up the
piker-native `Pair`-struct from any input of the three
(yes, it's that idiotic..) available symbol/pair-key-sets
that kraken frickin offers depending on the API including
the .altname, .wsname and the weird ass default set they
return in ReST responses .xname..
'''
if (
not self._pairs
or reload
):
await self.asset_pairs()
return self._AssetPairs
async def search_symbols(
self,
pattern: str,
) -> dict[str, Any]:
'''
Search for a symbol by "alt name"..
It is expected that the ``Client._pairs`` table
gets populated before conducting the underlying fuzzy-search
over the pair-key set.
'''
if not len(self._pairs):
await self.get_mkt_pairs()
assert self._pairs, '`Client.get_mkt_pairs()` was never called!?'
matches: dict[str, Pair] = match_from_pairs(
pairs=self._pairs,
query=pattern.upper(),
score_cutoff=50,
)
# repack in .altname-keyed output table
return {
pair.altname: pair
for pair in matches.values()
}
async def bars(
self,
symbol: str = 'XBTUSD',
# UTC 2017-07-02 12:53:20
since: Union[int, datetime] | None = None,
count: int = 720, # <- max allowed per query
as_np: bool = True,
) -> dict:
if since is None:
since = pendulum.now('UTC').start_of('minute').subtract(
minutes=count).timestamp()
elif isinstance(since, int):
since = pendulum.from_timestamp(since).timestamp()
else: # presumably a pendulum datetime
since = since.timestamp()
# UTC 2017-07-02 12:53:20 is oldest seconds value
since = str(max(1499000000, int(since)))
json = await self._public(
'OHLC',
data={
'pair': symbol,
'since': since,
},
)
try:
res = json['result']
res.pop('last')
bars = next(iter(res.values()))
new_bars = []
first = bars[0]
last_nz_vwap = first[-3]
if last_nz_vwap == 0:
# use close if vwap is zero
last_nz_vwap = first[-4]
# convert all fields to native types
for i, bar in enumerate(bars):
# normalize weird zero-ed vwap values..cmon kraken..
# indicates vwap didn't change since last bar
vwap = float(bar.pop(-3))
if vwap != 0:
last_nz_vwap = vwap
if vwap == 0:
vwap = last_nz_vwap
# re-insert vwap as the last of the fields
bar.append(vwap)
new_bars.append(
(i,) + tuple(
ftype(bar[j]) for j, (name, ftype) in enumerate(
def_iohlcv_fields[1:]
)
)
)
array = np.array(new_bars, dtype=def_iohlcv_fields) if as_np else bars
return array
except KeyError:
errmsg = json['error'][0]
if 'not found' in errmsg:
raise SymbolNotFound(errmsg + f': {symbol}')
elif 'Too many requests' in errmsg:
raise DataThrottle(f'{symbol}')
else:
raise BrokerError(errmsg)
@classmethod
def to_bs_fqme(
cls,
pair_str: str
) -> str:
'''
Normalize symbol names to to a 3x3 pair from the global
definition map which we build out from the data retreived from
the 'AssetPairs' endpoint, see methods above.
'''
try:
return cls._altnames[pair_str.upper()].bs_fqme
except KeyError as ke:
raise SymbolNotFound(f'kraken has no {ke.args[0]}')
@acm
async def get_client() -> Client:
conf: dict[str, Any] = get_config()
async with httpx.AsyncClient(
base_url=_url,
headers=_headers,
# TODO: is there a way to numerate this?
# https://www.python-httpx.org/advanced/clients/#why-use-a-client
# connections=4
) as trio_client:
if conf:
client = Client(
conf,
httpx_client=trio_client,
# TODO: don't break these up and just do internal
# conf lookups instead..
name=conf['key_descr'],
api_key=conf['api_key'],
secret=conf['secret']
)
else:
client = Client(
conf={},
httpx_client=trio_client,
)
# at startup, load all symbols, and asset info in
# batch requests.
async with trio.open_nursery() as nurse:
nurse.start_soon(client.get_assets)
await client.get_mkt_pairs()
yield client