From b67d020e2328b78fbea543c646f92dd2ff7dac59 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Fri, 13 Jan 2023 01:14:47 +0000 Subject: [PATCH 01/62] add basic func to load paper_trades file --- piker/clearing/_paper_engine.py | 16 +++++++++++++++- piker/config.py | 5 +++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 33ca5761..4f5cfee8 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -30,11 +30,12 @@ from typing import ( Callable, ) import uuid - +import os.path from bidict import bidict import pendulum import trio import tractor +import logging from .. import data from ..data._source import Symbol @@ -56,6 +57,7 @@ from ._messages import ( BrokerdError, ) +from ..config import load log = get_logger(__name__) @@ -84,6 +86,17 @@ class PaperBoi(Struct): last_ask: tuple[float, float] = (float('inf'), 0) # price, size last_bid: tuple[float, float] = (0, 0) + + def record_paper_trade(self): + try: + #create paper trades record + + print('RECORDING PAPER TRADE') + config, path = load('paper_trades') + except Exception as Arguement: + logging.exception('ERROR RECORDING PAPER TRADE') + pass + async def submit_limit( self, oid: str, # XXX: see return value @@ -271,6 +284,7 @@ class PaperBoi(Struct): dt=pendulum.from_timestamp(fill_time_s), bsuid=key, ) + self.record_paper_trade() pp.add_clear(t) pp_msg = BrokerdPosition( diff --git a/piker/config.py b/piker/config.py index cb250386..59532eac 100644 --- a/piker/config.py +++ b/piker/config.py @@ -115,6 +115,7 @@ _conf_names: set[str] = { 'pps', 'trades', 'watchlists', + 'paper_trades' } _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json') @@ -212,6 +213,10 @@ def load( # if one exists. if os.path.isfile(template): shutil.copyfile(template, path) + else: + # create an empty file + with open(path, 'x'): + pass else: with open(path, 'r'): pass # touch it From a4bd51a01b4488143e325cf8d81930f198f6c6a1 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Thu, 9 Feb 2023 15:19:50 -0500 Subject: [PATCH 02/62] change open_trade_ledger typing to return a Generator type --- piker/clearing/_paper_engine.py | 26 +++++++++++--------------- piker/pp.py | 14 +++++++++----- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 4f5cfee8..2e5c94c8 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -43,6 +43,7 @@ from ..data.types import Struct from ..pp import ( Position, Transaction, + open_trade_ledger ) from ..data._normalize import iterticks from ..data._source import unpack_fqsn @@ -86,17 +87,6 @@ class PaperBoi(Struct): last_ask: tuple[float, float] = (float('inf'), 0) # price, size last_bid: tuple[float, float] = (0, 0) - - def record_paper_trade(self): - try: - #create paper trades record - - print('RECORDING PAPER TRADE') - config, path = load('paper_trades') - except Exception as Arguement: - logging.exception('ERROR RECORDING PAPER TRADE') - pass - async def submit_limit( self, oid: str, # XXX: see return value @@ -246,9 +236,7 @@ class PaperBoi(Struct): ) log.info(f'Fake filling order:\n{fill_msg}') await self.ems_trades_stream.send(fill_msg) - - self._trade_ledger.update(fill_msg.to_dict()) - + if order_complete: msg = BrokerdStatus( reqid=reqid, @@ -284,7 +272,15 @@ class PaperBoi(Struct): dt=pendulum.from_timestamp(fill_time_s), bsuid=key, ) - self.record_paper_trade() + + # Transacupdate ledger per trade + ledger_entry = {} + ledger_entry[oid] = t.to_dict() + self._trade_ledger.update(ledger_entry) + with open_trade_ledger('paper', 'paper', self._trade_ledger) as ledger: + log.info(ledger) + + pp.add_clear(t) pp_msg = BrokerdPosition( diff --git a/piker/pp.py b/piker/pp.py index 6c5a60d8..92052a71 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -32,13 +32,13 @@ from typing import ( Iterator, Optional, Union, + Generator ) import pendulum from pendulum import datetime, now import tomli import toml - from . import config from .brokers import get_brokermod from .clearing._messages import BrokerdPosition, Status @@ -53,8 +53,8 @@ log = get_logger(__name__) def open_trade_ledger( broker: str, account: str, - -) -> dict: + trades: dict[str, Any] +) -> Generator[dict, None, None]: ''' Indempotently create and read in a trade log file from the ``/ledgers/`` directory. @@ -82,7 +82,8 @@ def open_trade_ledger( ledger = tomli.load(cf) print(f'Ledger load took {time.time() - start}s') cpy = ledger.copy() - + if trades: + cpy.update(trades) try: yield cpy finally: @@ -91,10 +92,13 @@ def open_trade_ledger( # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries print(f'Updating ledger for {tradesfile}:\n') ledger.update(cpy) - + + print("updated ledger") + print(ledger) # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) + return class Transaction(Struct, frozen=True): From 9c28d7086e4475149e899f959f81ef1839874ef4 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Fri, 27 Jan 2023 18:39:00 -0500 Subject: [PATCH 03/62] Add Generator as return type of open_trade_ledger --- piker/clearing/_paper_engine.py | 22 +++++++++++++++------- piker/pp.py | 19 ++++++++++--------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 2e5c94c8..cd4eccf9 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -30,12 +30,10 @@ from typing import ( Callable, ) import uuid -import os.path from bidict import bidict import pendulum import trio import tractor -import logging from .. import data from ..data._source import Symbol @@ -43,7 +41,8 @@ from ..data.types import Struct from ..pp import ( Position, Transaction, - open_trade_ledger + open_trade_ledger, + open_pps ) from ..data._normalize import iterticks from ..data._source import unpack_fqsn @@ -82,11 +81,12 @@ class PaperBoi(Struct): _reqids: bidict _positions: dict[str, Position] _trade_ledger: dict[str, Any] + _txn_dict: dict[str, Transaction] = {} # init edge case L1 spread last_ask: tuple[float, float] = (float('inf'), 0) # price, size last_bid: tuple[float, float] = (0, 0) - + async def submit_limit( self, oid: str, # XXX: see return value @@ -273,13 +273,21 @@ class PaperBoi(Struct): bsuid=key, ) - # Transacupdate ledger per trade + # Update in memory ledger per trade ledger_entry = {} ledger_entry[oid] = t.to_dict() + + # Store txn in state for PP update + self._txn_dict[oid] = t self._trade_ledger.update(ledger_entry) - with open_trade_ledger('paper', 'paper', self._trade_ledger) as ledger: - log.info(ledger) + + # Write to ledger toml + with open_trade_ledger(self.broker, 'paper') as ledger: + ledger.update(self._trade_ledger) + # Write to pps toml + with open_pps(self.broker, 'paper-id') as table: + table.update_from_trans(self._txn_dict) pp.add_clear(t) diff --git a/piker/pp.py b/piker/pp.py index 92052a71..133f706b 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -20,6 +20,7 @@ that doesn't try to cuk most humans who prefer to not lose their moneys.. (looking at you `ib` and dirt-bird friends) ''' +from __future__ import annotations from contextlib import contextmanager as cm from pprint import pformat import os @@ -34,11 +35,13 @@ from typing import ( Union, Generator ) +from typing import Generator import pendulum from pendulum import datetime, now import tomli import toml + from . import config from .brokers import get_brokermod from .clearing._messages import BrokerdPosition, Status @@ -56,7 +59,7 @@ def open_trade_ledger( trades: dict[str, Any] ) -> Generator[dict, None, None]: ''' - Indempotently create and read in a trade log file from the + Indempotently creat0616cbd1e and read in a trade log file from the ``/ledgers/`` directory. Files are named per broker account of the form @@ -82,8 +85,6 @@ def open_trade_ledger( ledger = tomli.load(cf) print(f'Ledger load took {time.time() - start}s') cpy = ledger.copy() - if trades: - cpy.update(trades) try: yield cpy finally: @@ -93,8 +94,6 @@ def open_trade_ledger( print(f'Updating ledger for {tradesfile}:\n') ledger.update(cpy) - print("updated ledger") - print(ledger) # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) @@ -543,15 +542,16 @@ class PpTable(Struct): self, trans: dict[str, Transaction], cost_scalar: float = 2, - + write_now: bool = False, ) -> dict[str, Position]: pps = self.pps updated: dict[str, Position] = {} - + print('TRANSACTIONS') + print(trans.items) # lifo update all pps from records for tid, t in trans.items(): - + print(t) pp = pps.setdefault( t.bsuid, @@ -587,6 +587,7 @@ class PpTable(Struct): # update clearing table pp.add_clear(t) updated[t.bsuid] = pp + # minimize clears tables and update sizing. for bsuid, pp in updated.items(): @@ -885,7 +886,7 @@ def open_pps( acctid: str, write_on_exit: bool = True, -) -> PpTable: +) -> Generator[PpTable, None, None]: ''' Read out broker-specific position entries from incremental update file: ``pps.toml``. From 8cd2354d7390508a9f727c18707ba1d4ce26dff9 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:14:59 -0500 Subject: [PATCH 04/62] ensure that paper pps are pulled on open --- piker/clearing/_paper_engine.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index cd4eccf9..3c8ad233 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -288,6 +288,8 @@ class PaperBoi(Struct): # Write to pps toml with open_pps(self.broker, 'paper-id') as table: table.update_from_trans(self._txn_dict) + # save pps in local state + self._positions.update(table.pps) pp.add_clear(t) @@ -551,9 +553,16 @@ async def trades_dialogue( ) as feed, ): + with open_pps(broker, 'paper-id') as table: + # save pps in local state + _positions.update(table.pps) + + pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' + print("POSITIONS::") + print(_positions.items()) for token, pos in _positions.items(): pp_msgs.append(BrokerdPosition( broker=broker, From 1b2fce430f503657e2f1c90a8b37951932def5e6 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:21:34 -0500 Subject: [PATCH 05/62] remove logs, unused args --- piker/pp.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index 133f706b..6db4ed54 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -542,16 +542,12 @@ class PpTable(Struct): self, trans: dict[str, Transaction], cost_scalar: float = 2, - write_now: bool = False, ) -> dict[str, Position]: pps = self.pps updated: dict[str, Position] = {} - print('TRANSACTIONS') - print(trans.items) # lifo update all pps from records for tid, t in trans.items(): - print(t) pp = pps.setdefault( t.bsuid, From 97627a49766df1c3baa8ebb346ee0b42b10ab7c6 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:23:42 -0500 Subject: [PATCH 06/62] remove more logs --- piker/clearing/_paper_engine.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 3c8ad233..3f63dd99 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -561,8 +561,7 @@ async def trades_dialogue( pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' - print("POSITIONS::") - print(_positions.items()) + for token, pos in _positions.items(): pp_msgs.append(BrokerdPosition( broker=broker, From 41bb0445e0fc87a681afb655db0be25ddd2c13fb Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:25:37 -0500 Subject: [PATCH 07/62] remove unnecessary return --- piker/pp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/pp.py b/piker/pp.py index 6db4ed54..a7fd60aa 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -97,7 +97,6 @@ def open_trade_ledger( # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) - return class Transaction(Struct, frozen=True): From 6126c4f438a549b985b11f735eb0fd893bde3fb5 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:27:35 -0500 Subject: [PATCH 08/62] restore spacing --- piker/pp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/pp.py b/piker/pp.py index a7fd60aa..0cbd88b3 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -541,12 +541,15 @@ class PpTable(Struct): self, trans: dict[str, Transaction], cost_scalar: float = 2, + ) -> dict[str, Position]: pps = self.pps updated: dict[str, Position] = {} + # lifo update all pps from records for tid, t in trans.items(): + pp = pps.setdefault( t.bsuid, From 3028a8b1f85b38e394bd1fef126428724f09428b Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sat, 28 Jan 2023 18:28:59 -0500 Subject: [PATCH 09/62] restore spacing --- piker/clearing/_paper_engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 3f63dd99..818d57ec 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -557,7 +557,6 @@ async def trades_dialogue( # save pps in local state _positions.update(table.pps) - pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' From 5bb93ccc5f57bb427bcde0dfc80ef689440ce3e5 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sun, 29 Jan 2023 18:50:54 -0500 Subject: [PATCH 10/62] change id to 'piker-paper' --- piker/clearing/_paper_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 818d57ec..589a201b 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -286,7 +286,7 @@ class PaperBoi(Struct): ledger.update(self._trade_ledger) # Write to pps toml - with open_pps(self.broker, 'paper-id') as table: + with open_pps(self.broker, 'piker-paper') as table: table.update_from_trans(self._txn_dict) # save pps in local state self._positions.update(table.pps) From 86b438652257e20ee27d0a9d2e17738a430b2116 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Thu, 9 Feb 2023 14:53:57 -0500 Subject: [PATCH 11/62] minor changes, prepare for rebase of overlays branch --- piker/clearing/_messages.py | 2 +- piker/clearing/_paper_engine.py | 16 ++-- pytest.ini | 3 + tests/test_paper.py | 143 ++++++++++++++++++++++++++++++++ tests/test_services.py | 14 +++- 5 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/test_paper.py diff --git a/piker/clearing/_messages.py b/piker/clearing/_messages.py index fe4d426f..c7693b9f 100644 --- a/piker/clearing/_messages.py +++ b/piker/clearing/_messages.py @@ -100,7 +100,7 @@ class Order(Struct): price: float size: float # -ve is "sell", +ve is "buy" - brokers: Optional[list[str]] = [] + brokers: list[str] = [] class Cancel(Struct): diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 589a201b..d530125f 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -280,7 +280,7 @@ class PaperBoi(Struct): # Store txn in state for PP update self._txn_dict[oid] = t self._trade_ledger.update(ledger_entry) - + # Write to ledger toml with open_trade_ledger(self.broker, 'paper') as ledger: ledger.update(self._trade_ledger) @@ -330,6 +330,7 @@ async def simulate_fills( # https://github.com/quantopian/zipline/blob/master/zipline/finance/ledger.py # this stream may eventually contain multiple symbols + async for quotes in quote_stream: for sym, quote in quotes.items(): for tick in iterticks( @@ -423,9 +424,9 @@ async def simulate_fills( # simulated live orders prematurely. case _: continue - - # iterate all potentially clearable book prices - # in FIFO order per side. + + # iterate alcl potentially clearable book prices + # in FIFO order per side.c for order_info, pred in iter_entries: (our_price, size, reqid, action) = order_info @@ -438,6 +439,8 @@ async def simulate_fills( 'sell': sells }[action].inverse.pop(order_info) + log.warning(f'order_info: {order_info}') + # clearing price would have filled entirely await client.fake_fill( fqsn=sym, @@ -553,10 +556,11 @@ async def trades_dialogue( ) as feed, ): + with open_pps(broker, 'paper-id') as table: # save pps in local state _positions.update(table.pps) - + pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' @@ -589,7 +593,6 @@ async def trades_dialogue( _reqids=_reqids, - # TODO: load paper positions from ``positions.toml`` _positions=_positions, # TODO: load postions from ledger file @@ -628,7 +631,6 @@ async def open_paperboi( # (we likely don't need more then one proc for basic # simulated order clearing) if portal is None: - log.info('Starting new paper-engine actor') portal = await tn.start_actor( service_name, enable_modules=[__name__] diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..ffadaba4 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +trio_mode=True +log_cli=1 diff --git a/tests/test_paper.py b/tests/test_paper.py new file mode 100644 index 00000000..b1918698 --- /dev/null +++ b/tests/test_paper.py @@ -0,0 +1,143 @@ +from datetime import datetime +import time +import trio +import pytest +import tractor +import math +from piker.log import get_logger +from piker.clearing._messages import ( + Order +) +from typing import AsyncContextManager + +from piker.pp import ( + Position, + Transaction, + open_trade_ledger, + open_pps +) + +from piker._daemon import ( + find_service, + check_for_service, + Services, +) +from piker.data import ( + open_feed, +) +from piker.clearing import ( + open_ems, +) +from piker.clearing._messages import ( + BrokerdPosition, + Status, +) +from piker.clearing._client import ( + OrderBook, +) + +log = get_logger(__name__) + + +def test_paper_trade( + open_test_pikerd: AsyncContextManager +): + + async def main(): + # type declares + book: OrderBook + trades_stream: tractor.MsgStream + pps: dict[str, list[BrokerdPosition]] + accounts: list[str] + dialogs: dict[str, Status] + + async with ( + open_test_pikerd() as (_, _, _, services), + + open_ems( + 'xbtusdt.kraken', + mode='paper', + ) as ( + book, + trades_stream, + pps, + accounts, + dialogs, + ), + ): + + test_exec_mode='live' + test_action = 'buy' + test_oid = '560beac8-b1b1-4dee-bd1e-6604a704c9ea' + test_account = 'paper' + test_size = 1 + test_price = 30000 + test_broker = 'kraken' + test_brokers = [test_broker] + test_symbol = 'xbtusdt' + test_fqsn = f'{test_symbol}.{test_broker}' + test_pp_account = 'piker-paper' + + order = Order( + exec_mode=test_exec_mode, + action=test_action, + oid=test_oid, + account=test_account, + size=test_size, + symbol=test_fqsn, + price=test_price, + brokers=test_brokers + ) + + book.send(order) + + await trio.sleep(1) + + cleared_ledger_entry = {} + # check if trades have been updated in in ledge and pp + with open_trade_ledger(test_broker, test_account) as ledger: + log.warning(f'ledger: {ledger}') + cleared_ledger_entry = ledger[test_oid] + assert list(ledger.keys())[0] == test_oid + assert cleared_ledger_entry['size'] == test_size + assert cleared_ledger_entry['fqsn'] == test_fqsn + + with open_pps(test_broker, test_pp_account) as table: + # save pps in local state + assert table.brokername == test_broker + assert table.acctid == test_pp_account +# assert cleared_ledger_entry['price'] == table.conf.clears[0].price + pp_price = table.conf[test_broker][test_pp_account][test_fqsn]["ppu"] + assert math.isclose(pp_price, cleared_ledger_entry['size'], rel_tol=1) + + raise KeyboardInterrupt + + + with pytest.raises( + trio.MultiError + ) as exc_info: + trio.run(main) + + +def test_trades_persist( + open_test_pikerd: AsyncContextManager +): + + async def main(): + async with ( + open_test_pikerd() as (_, _, _, services), + + open_ems( + 'xbtusdt.kraken', + mode='paper', + ) as ( + book, + trades_stream, + pps, + accounts, + dialogs, + ), + ): + print(f'pps: {pps}') + + trio.run(main) diff --git a/tests/test_services.py b/tests/test_services.py index bdce6aa2..3acd81ad 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -9,6 +9,13 @@ import pytest import trio import tractor +from datetime import datetime +import time +from piker.log import get_logger +from piker.clearing._messages import ( + BrokerdFill +) + from piker._daemon import ( find_service, Services, @@ -27,6 +34,8 @@ from piker.clearing._client import ( OrderBook, ) +log = get_logger(__name__) + def test_runtime_boot( open_test_pikerd: AsyncContextManager @@ -174,5 +183,8 @@ def test_ensure_ems_in_paper_actors( ) as exc_info: trio.run(main) - cancel_msg: str = '`_emsd_main()` was remotely cancelled by its caller' + cancel_msg: str = '_emsd_main was remotely cancelled by its caller' assert cancel_msg in exc_info.value.args[0] + + + From 84cd1e005953b918065baf85af910e06ceaa748d Mon Sep 17 00:00:00 2001 From: algorandpa Date: Thu, 9 Feb 2023 15:51:58 -0500 Subject: [PATCH 12/62] initial commit on copy --- piker/pp.py | 1 - tests/test_paper.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index 0cbd88b3..26eb1f1d 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -56,7 +56,6 @@ log = get_logger(__name__) def open_trade_ledger( broker: str, account: str, - trades: dict[str, Any] ) -> Generator[dict, None, None]: ''' Indempotently creat0616cbd1e and read in a trade log file from the diff --git a/tests/test_paper.py b/tests/test_paper.py index b1918698..0c920986 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -138,6 +138,6 @@ def test_trades_persist( dialogs, ), ): - print(f'pps: {pps}') + assert pps.len() trio.run(main) From 68a196218b465b9992477d959de71c715304216b Mon Sep 17 00:00:00 2001 From: algorandpa Date: Thu, 9 Feb 2023 16:26:22 -0500 Subject: [PATCH 13/62] force change branch name --- piker/clearing/_paper_engine.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index d530125f..ce25243f 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -557,14 +557,15 @@ async def trades_dialogue( ): - with open_pps(broker, 'paper-id') as table: + with open_pps(broker, 'piker-paper') as table: + log.warning(f'pp table: {table}') # save pps in local state _positions.update(table.pps) pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' - + log.warning(f'local _positions: {_positions}') for token, pos in _positions.items(): pp_msgs.append(BrokerdPosition( broker=broker, @@ -573,7 +574,7 @@ async def trades_dialogue( size=pos.size, avg_price=pos.ppu, )) - + log.warning(f'pp_msgs: {pp_msgs}') # TODO: load paper positions per broker from .toml config file # and pass as symbol to position data mapping: ``dict[str, dict]`` await ctx.started(( From df868cec35cf495ee1c757c42b100549a132fe93 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Thu, 9 Feb 2023 18:14:41 -0500 Subject: [PATCH 14/62] Assert that trades persist in ems after teardown and startup --- tests/test_paper.py | 152 +++++++++++++++++++++++--------------------- 1 file changed, 81 insertions(+), 71 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 0c920986..b91547ae 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -8,8 +8,12 @@ from piker.log import get_logger from piker.clearing._messages import ( Order ) -from typing import AsyncContextManager +from typing import ( + AsyncContextManager, + Any, +) +from functools import partial from piker.pp import ( Position, Transaction, @@ -38,21 +42,38 @@ from piker.clearing._client import ( log = get_logger(__name__) +_clearing_price: float + def test_paper_trade( open_test_pikerd: AsyncContextManager ): + _cleared_price: float + test_exec_mode='live' + test_action = 'buy' + test_oid = '560beac8-b1b1-4dee-bd1e-6604a704c9ea' + test_account = 'paper' + test_size = 1 + test_price = 30000 + test_broker = 'kraken' + test_brokers = [test_broker] + test_symbol = 'xbtusdt' + test_fqsn = f'{test_symbol}.{test_broker}' + test_pp_account = 'piker-paper' - async def main(): + + async def open( + open_pikerd: AsyncContextManager, + send_order: bool = False, + assert_entries: bool = False, + teardown: bool = True, + ) -> Any: # type declares book: OrderBook - trades_stream: tractor.MsgStream - pps: dict[str, list[BrokerdPosition]] - accounts: list[str] - dialogs: dict[str, Status] + global _cleared_price async with ( - open_test_pikerd() as (_, _, _, services), + open_pikerd() as (_, _, _, services), open_ems( 'xbtusdt.kraken', @@ -66,78 +87,67 @@ def test_paper_trade( ), ): - test_exec_mode='live' - test_action = 'buy' - test_oid = '560beac8-b1b1-4dee-bd1e-6604a704c9ea' - test_account = 'paper' - test_size = 1 - test_price = 30000 - test_broker = 'kraken' - test_brokers = [test_broker] - test_symbol = 'xbtusdt' - test_fqsn = f'{test_symbol}.{test_broker}' - test_pp_account = 'piker-paper' - order = Order( - exec_mode=test_exec_mode, - action=test_action, - oid=test_oid, - account=test_account, - size=test_size, - symbol=test_fqsn, - price=test_price, - brokers=test_brokers - ) + if send_order: + order = Order( + exec_mode=test_exec_mode, + action=test_action, + oid=test_oid, + account=test_account, + size=test_size, + symbol=test_fqsn, + price=test_price, + brokers=test_brokers + ) - book.send(order) + book.send(order) + + await trio.sleep(1) - await trio.sleep(1) + if assert_entries: - cleared_ledger_entry = {} - # check if trades have been updated in in ledge and pp - with open_trade_ledger(test_broker, test_account) as ledger: - log.warning(f'ledger: {ledger}') - cleared_ledger_entry = ledger[test_oid] - assert list(ledger.keys())[0] == test_oid - assert cleared_ledger_entry['size'] == test_size - assert cleared_ledger_entry['fqsn'] == test_fqsn - - with open_pps(test_broker, test_pp_account) as table: - # save pps in local state - assert table.brokername == test_broker - assert table.acctid == test_pp_account -# assert cleared_ledger_entry['price'] == table.conf.clears[0].price - pp_price = table.conf[test_broker][test_pp_account][test_fqsn]["ppu"] - assert math.isclose(pp_price, cleared_ledger_entry['size'], rel_tol=1) + cleared_ledger_entry = {} + # check if trades have been updated in in ledge and pp + with open_trade_ledger(test_broker, test_account) as ledger: + cleared_ledger_entry = ledger[test_oid] + _cleared_price = cleared_ledger_entry["price"] + assert list(ledger.keys())[0] == test_oid + assert cleared_ledger_entry['size'] == test_size + assert cleared_ledger_entry['fqsn'] == test_fqsn - raise KeyboardInterrupt - + + with open_pps(test_broker, test_pp_account) as table: + # save pps in local state + assert table.brokername == test_broker + assert table.acctid == test_pp_account + # assert cleared_ledger_entry['price'] == table.conf.clears[0].price + pp_price = table.conf[test_broker][test_pp_account][test_fqsn]["ppu"] + assert math.isclose(pp_price, cleared_ledger_entry['size'], rel_tol=1) + + if teardown: + raise KeyboardInterrupt + return pps + + async def open_and_assert_pps(): + pps = await open(open_test_pikerd) + assert pps(test_broker, test_account)[0] == _cleared_price with pytest.raises( trio.MultiError ) as exc_info: - trio.run(main) + # run initial time and send sent and assert trade + trio.run(partial(open, + open_pikerd=open_test_pikerd, + send_order=True, + assert_entries=True, + ) + ) + + # Run again just to boot piker + trio.run(partial(open, + open_pikerd=open_test_pikerd, + ) + ) + trio.run(open_and_assert_pps) -def test_trades_persist( - open_test_pikerd: AsyncContextManager -): - - async def main(): - async with ( - open_test_pikerd() as (_, _, _, services), - - open_ems( - 'xbtusdt.kraken', - mode='paper', - ) as ( - book, - trades_stream, - pps, - accounts, - dialogs, - ), - ): - - assert pps.len() - trio.run(main) From 95b9dacb7a29e32ca6fd700a8d8f0759f347b5e7 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Sun, 12 Feb 2023 17:04:49 -0500 Subject: [PATCH 15/62] Break test into steps --- piker/clearing/_client.py | 7 +- piker/clearing/_ems.py | 5 +- piker/pp.py | 1 + tests/test_paper.py | 146 +++++++++++++++++++------------------- 4 files changed, 80 insertions(+), 79 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 0a40b548..4537a03b 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -236,8 +236,9 @@ async def open_ems( or mode == 'paper' ): mode = 'paper' - + from ._ems import _emsd_main + async with ( # connect to emsd portal.open_context( @@ -255,7 +256,6 @@ async def open_ems( dialogs, ) ), - # open 2-way trade command stream ctx.open_stream() as trades_stream, ): @@ -265,8 +265,7 @@ async def open_ems( relay_order_cmds_from_sync_code, fqsn, trades_stream - ) - + ) yield ( book, trades_stream, diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 477da310..3af462a0 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -1351,7 +1351,7 @@ async def maybe_open_trade_relays( loglevel, ) yield relay, feed, client_ready - + print("ABOUT TO OPEN CACHED MNGR") async with tractor.trionics.maybe_open_context( acm_func=cached_mngr, kwargs={ @@ -1365,6 +1365,7 @@ async def maybe_open_trade_relays( cache_hit, (relay, feed, client_ready) ): + print("YIELDING RELAY") yield relay, feed, client_ready @@ -1452,7 +1453,7 @@ async def _emsd_main( brokerd_stream = relay.brokerd_stream dark_book = _router.get_dark_book(broker) - + # signal to client that we're started and deliver # all known pps and accounts for this ``brokerd``. await ems_ctx.started(( diff --git a/piker/pp.py b/piker/pp.py index 26eb1f1d..637dab53 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -978,6 +978,7 @@ def open_pps( pp.ensure_state() try: +# breakpoint() yield table finally: if write_on_exit: diff --git a/tests/test_paper.py b/tests/test_paper.py index b91547ae..b0ef404b 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -1,5 +1,3 @@ -from datetime import datetime -import time import trio import pytest import tractor @@ -8,76 +6,72 @@ from piker.log import get_logger from piker.clearing._messages import ( Order ) +from uuid import uuid4 from typing import ( AsyncContextManager, Any, + Literal, ) from functools import partial from piker.pp import ( - Position, - Transaction, open_trade_ledger, - open_pps + open_pps, + PpTable ) -from piker._daemon import ( - find_service, - check_for_service, - Services, -) -from piker.data import ( - open_feed, -) from piker.clearing import ( open_ems, ) -from piker.clearing._messages import ( - BrokerdPosition, - Status, -) + from piker.clearing._client import ( OrderBook, ) +from piker.clearing._messages import ( + BrokerdPosition +) + +from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -_clearing_price: float - - def test_paper_trade( open_test_pikerd: AsyncContextManager ): - _cleared_price: float + + cleared_price: float test_exec_mode='live' - test_action = 'buy' - test_oid = '560beac8-b1b1-4dee-bd1e-6604a704c9ea' test_account = 'paper' test_size = 1 - test_price = 30000 test_broker = 'kraken' test_brokers = [test_broker] test_symbol = 'xbtusdt' test_fqsn = f'{test_symbol}.{test_broker}' test_pp_account = 'piker-paper' + positions: dict[ + # brokername, acctid + tuple[str, str], + list[BrokerdPosition], + ] - - async def open( + async def _async_main( open_pikerd: AsyncContextManager, - send_order: bool = False, + action: Literal['buy', 'sell'] | None = None, + price: int = 30000, assert_entries: bool = False, - teardown: bool = True, ) -> Any: - # type declares - book: OrderBook - global _cleared_price + oid: str = str(uuid4()) + book: OrderBook + global cleared_price + global positions + + # Set up piker and EMS async with ( open_pikerd() as (_, _, _, services), - open_ems( - 'xbtusdt.kraken', - mode='paper', + test_fqsn, + mode=test_account, ) as ( book, trades_stream, @@ -86,68 +80,74 @@ def test_paper_trade( dialogs, ), ): - - if send_order: + # Send order to EMS + if action: order = Order( exec_mode=test_exec_mode, - action=test_action, - oid=test_oid, + action=action, + oid=oid, account=test_account, size=test_size, symbol=test_fqsn, - price=test_price, + price=price, brokers=test_brokers ) book.send(order) - await trio.sleep(1) - - if assert_entries: + await trio.sleep(2) + # Assert entries are made in both ledger and PPS + if assert_entries: cleared_ledger_entry = {} - # check if trades have been updated in in ledge and pp with open_trade_ledger(test_broker, test_account) as ledger: - cleared_ledger_entry = ledger[test_oid] - _cleared_price = cleared_ledger_entry["price"] - assert list(ledger.keys())[0] == test_oid + cleared_ledger_entry = ledger[oid] + cleared_price = cleared_ledger_entry["price"] + + assert list(ledger.keys())[-1] == oid assert cleared_ledger_entry['size'] == test_size assert cleared_ledger_entry['fqsn'] == test_fqsn - with open_pps(test_broker, test_pp_account) as table: - # save pps in local state - assert table.brokername == test_broker - assert table.acctid == test_pp_account - # assert cleared_ledger_entry['price'] == table.conf.clears[0].price pp_price = table.conf[test_broker][test_pp_account][test_fqsn]["ppu"] assert math.isclose(pp_price, cleared_ledger_entry['size'], rel_tol=1) - - if teardown: - raise KeyboardInterrupt - return pps + assert table.brokername == test_broker + assert table.acctid == test_pp_account - async def open_and_assert_pps(): - pps = await open(open_test_pikerd) - assert pps(test_broker, test_account)[0] == _cleared_price + positions = pps + + # Close piker like a user would + raise KeyboardInterrupt + + # Open piker and ensure last pps price is the same as ledger entry + async def _open_and_assert_pps(): + await _async_main(open_test_pikerd) + assert positions(test_broker, test_account)[-1] == cleared_price + + # Close position and assert empty position in pps + async def _close_pp_and_assert(): + await _async_main(open_test_pikerd, 'sell', 1) + with open_pps(test_broker, test_pp_account) as table: + assert len(table.pps) == 0 + + # run initial time and send sent and assert trade + with pytest.raises( + BaseExceptionGroup + ) as exc_info: + trio.run(partial(_async_main, + open_pikerd=open_test_pikerd, + action='buy', + ) + ) with pytest.raises( - trio.MultiError + BaseExceptionGroup ) as exc_info: - # run initial time and send sent and assert trade - trio.run(partial(open, - open_pikerd=open_test_pikerd, - send_order=True, - assert_entries=True, - ) - ) - - # Run again just to boot piker - trio.run(partial(open, - open_pikerd=open_test_pikerd, - ) - ) - trio.run(open_and_assert_pps) + trio.run(_open_and_assert_pps) + with pytest.raises( + BaseExceptionGroup + ) as exc_info: + trio.run(_close_pp_and_assert) From b180602a3edb77c33c1bfbddb20142e4c5991402 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Mon, 13 Feb 2023 11:01:02 -0500 Subject: [PATCH 16/62] Make config grab _testing dir in pytest env, - Remove print statements --- piker/clearing/_ems.py | 3 +-- piker/config.py | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 3af462a0..9895b533 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -1351,7 +1351,7 @@ async def maybe_open_trade_relays( loglevel, ) yield relay, feed, client_ready - print("ABOUT TO OPEN CACHED MNGR") + async with tractor.trionics.maybe_open_context( acm_func=cached_mngr, kwargs={ @@ -1365,7 +1365,6 @@ async def maybe_open_trade_relays( cache_hit, (relay, feed, client_ready) ): - print("YIELDING RELAY") yield relay, feed, client_ready diff --git a/piker/config.py b/piker/config.py index 59532eac..ff2f822f 100644 --- a/piker/config.py +++ b/piker/config.py @@ -71,10 +71,12 @@ def get_app_dir(app_name, roaming=True, force_posix=False): dot instead of the XDG config home or darwin's application support folder. """ - def _posixify(name): return "-".join(name.split()).lower() + if "pytest" in sys.modules: + app_name += '/_testing' + # if WIN: if platform.system() == 'Windows': key = "APPDATA" if roaming else "LOCALAPPDATA" @@ -94,7 +96,6 @@ def get_app_dir(app_name, roaming=True, force_posix=False): _posixify(app_name), ) - _config_dir = _click_config_dir = get_app_dir('piker') _parent_user = os.environ.get('SUDO_USER') From dff8abd6ada76e28ecc49b46fbd2c12d69303288 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 12:07:42 -0500 Subject: [PATCH 17/62] Minor reformatting --- tests/test_paper.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index b0ef404b..2cd4817e 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -9,7 +9,6 @@ from piker.clearing._messages import ( from uuid import uuid4 from typing import ( AsyncContextManager, - Any, Literal, ) @@ -17,7 +16,6 @@ from functools import partial from piker.pp import ( open_trade_ledger, open_pps, - PpTable ) from piker.clearing import ( @@ -39,7 +37,6 @@ def test_paper_trade( open_test_pikerd: AsyncContextManager ): - cleared_price: float test_exec_mode='live' test_account = 'paper' test_size = 1 @@ -53,13 +50,20 @@ def test_paper_trade( tuple[str, str], list[BrokerdPosition], ] + cleared_price: float + async def _async_main( open_pikerd: AsyncContextManager, action: Literal['buy', 'sell'] | None = None, price: int = 30000, assert_entries: bool = False, - ) -> Any: + ) -> None: + """Spawn a paper piper actor, place a trade and assert entries are present + in both trade ledger and pps tomls. Then restart piker and ensure + that pps from previous trade exists in the ems pps. + Finally close the position and ensure that the position in pps.toml is closed. + """ oid: str = str(uuid4()) book: OrderBook @@ -134,7 +138,7 @@ def test_paper_trade( # run initial time and send sent and assert trade with pytest.raises( BaseExceptionGroup - ) as exc_info: + ): trio.run(partial(_async_main, open_pikerd=open_test_pikerd, action='buy', @@ -143,11 +147,11 @@ def test_paper_trade( with pytest.raises( BaseExceptionGroup - ) as exc_info: + ): trio.run(_open_and_assert_pps) with pytest.raises( BaseExceptionGroup - ) as exc_info: + ): trio.run(_close_pp_and_assert) From 7142a6a7cae76d98ec02740332118bcd59bcb299 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 17:06:48 -0500 Subject: [PATCH 18/62] Add hacky cleanup solution for _testng data --- piker/config.py | 7 +++++-- tests/test_paper.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/piker/config.py b/piker/config.py index ff2f822f..e4c2705f 100644 --- a/piker/config.py +++ b/piker/config.py @@ -33,7 +33,6 @@ from .log import get_logger log = get_logger('broker-config') - # taken from ``click`` since apparently they have some # super weirdness with sigint and sudo..no clue def get_app_dir(app_name, roaming=True, force_posix=False): @@ -73,7 +72,11 @@ def get_app_dir(app_name, roaming=True, force_posix=False): """ def _posixify(name): return "-".join(name.split()).lower() - + + # TODO: This is a hacky way to a) determine we're testing + # and b) creating a test dir. We should aim to set a variable + # within the tractor runtimes and store testing config data + # outside of the users filesystem if "pytest" in sys.modules: app_name += '/_testing' diff --git a/tests/test_paper.py b/tests/test_paper.py index 2cd4817e..38b3dd78 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -2,6 +2,9 @@ import trio import pytest import tractor import math +import os +from shutil import rmtree +from piker.config import get_app_dir from piker.log import get_logger from piker.clearing._messages import ( Order @@ -11,30 +14,34 @@ from typing import ( AsyncContextManager, Literal, ) - from functools import partial from piker.pp import ( open_trade_ledger, open_pps, ) - from piker.clearing import ( open_ems, ) - from piker.clearing._client import ( OrderBook, ) - from piker.clearing._messages import ( BrokerdPosition ) - from exceptiongroup import BaseExceptionGroup + log = get_logger(__name__) +@pytest.fixture +def paper_cleanup(): + yield + app_dir = get_app_dir('piker') + rmtree(app_dir) + assert not os.path.isfile(app_dir) + def test_paper_trade( - open_test_pikerd: AsyncContextManager + open_test_pikerd: AsyncContextManager, + paper_cleanup: None ): test_exec_mode='live' From e5cefeb44bac6cf107f283314ece3a8c39dc246f Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 19:00:49 -0500 Subject: [PATCH 19/62] Format to prep for PR --- piker/clearing/_paper_engine.py | 12 ++++-------- piker/pp.py | 8 +++----- tests/test_paper.py | 3 +-- tests/test_services.py | 10 ---------- 4 files changed, 8 insertions(+), 25 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index ce25243f..b7fcc411 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -425,8 +425,8 @@ async def simulate_fills( case _: continue - # iterate alcl potentially clearable book prices - # in FIFO order per side.c + # iterate all potentially clearable book prices + # in FIFO order per side. for order_info, pred in iter_entries: (our_price, size, reqid, action) = order_info @@ -438,9 +438,7 @@ async def simulate_fills( 'buy': buys, 'sell': sells }[action].inverse.pop(order_info) - - log.warning(f'order_info: {order_info}') - + # clearing price would have filled entirely await client.fake_fill( fqsn=sym, @@ -558,14 +556,12 @@ async def trades_dialogue( ): with open_pps(broker, 'piker-paper') as table: - log.warning(f'pp table: {table}') # save pps in local state _positions.update(table.pps) pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' - log.warning(f'local _positions: {_positions}') for token, pos in _positions.items(): pp_msgs.append(BrokerdPosition( broker=broker, @@ -574,7 +570,6 @@ async def trades_dialogue( size=pos.size, avg_price=pos.ppu, )) - log.warning(f'pp_msgs: {pp_msgs}') # TODO: load paper positions per broker from .toml config file # and pass as symbol to position data mapping: ``dict[str, dict]`` await ctx.started(( @@ -632,6 +627,7 @@ async def open_paperboi( # (we likely don't need more then one proc for basic # simulated order clearing) if portal is None: + log.info('Starting new paper-engine actor') portal = await tn.start_actor( service_name, enable_modules=[__name__] diff --git a/piker/pp.py b/piker/pp.py index 637dab53..e71045d1 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -58,7 +58,7 @@ def open_trade_ledger( account: str, ) -> Generator[dict, None, None]: ''' - Indempotently creat0616cbd1e and read in a trade log file from the + Indempotently create and read in a trade log file from the ``/ledgers/`` directory. Files are named per broker account of the form @@ -91,8 +91,7 @@ def open_trade_ledger( # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries print(f'Updating ledger for {tradesfile}:\n') - ledger.update(cpy) - + ledger.update(cpy) # we write on close the mutated ledger data with open(tradesfile, 'w') as cf: toml.dump(ledger, cf) @@ -544,8 +543,7 @@ class PpTable(Struct): ) -> dict[str, Position]: pps = self.pps - updated: dict[str, Position] = {} - + updated: dict[str, Position] = {} # lifo update all pps from records for tid, t in trans.items(): diff --git a/tests/test_paper.py b/tests/test_paper.py index 38b3dd78..16785afb 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -44,6 +44,7 @@ def test_paper_trade( paper_cleanup: None ): + cleared_price: float test_exec_mode='live' test_account = 'paper' test_size = 1 @@ -57,8 +58,6 @@ def test_paper_trade( tuple[str, str], list[BrokerdPosition], ] - cleared_price: float - async def _async_main( open_pikerd: AsyncContextManager, diff --git a/tests/test_services.py b/tests/test_services.py index 3acd81ad..6b67d3c5 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -9,12 +9,7 @@ import pytest import trio import tractor -from datetime import datetime -import time from piker.log import get_logger -from piker.clearing._messages import ( - BrokerdFill -) from piker._daemon import ( find_service, @@ -34,8 +29,6 @@ from piker.clearing._client import ( OrderBook, ) -log = get_logger(__name__) - def test_runtime_boot( open_test_pikerd: AsyncContextManager @@ -185,6 +178,3 @@ def test_ensure_ems_in_paper_actors( cancel_msg: str = '_emsd_main was remotely cancelled by its caller' assert cancel_msg in exc_info.value.args[0] - - - From 730906a072bf8a2c282726eb6b86b222d5093b60 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 19:02:43 -0500 Subject: [PATCH 20/62] Minor formatting --- piker/clearing/_paper_engine.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index b7fcc411..4444306a 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -423,8 +423,7 @@ async def simulate_fills( # below unecessarily and further don't want to pop # simulated live orders prematurely. case _: - continue - + continue # iterate all potentially clearable book prices # in FIFO order per side. for order_info, pred in iter_entries: From acc86ae6db74dbf66be5a95286b9b0ae31e5d59c Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 19:17:23 -0500 Subject: [PATCH 21/62] more formatting --- piker/clearing/_client.py | 4 +--- piker/clearing/_ems.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 4537a03b..08b0f4c8 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -235,10 +235,8 @@ async def open_ems( not getattr(mod, 'trades_dialogue', None) or mode == 'paper' ): - mode = 'paper' - + mode = 'paper' from ._ems import _emsd_main - async with ( # connect to emsd portal.open_context( diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 9895b533..1f3c52cb 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -1451,8 +1451,7 @@ async def _emsd_main( ) as (relay, feed, client_ready): brokerd_stream = relay.brokerd_stream - dark_book = _router.get_dark_book(broker) - + dark_book = _router.get_dark_book(broker) # signal to client that we're started and deliver # all known pps and accounts for this ``brokerd``. await ems_ctx.started(( From 0dec2b9c89eb7938ef3c43cdae8676062d47dc87 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 19:46:09 -0500 Subject: [PATCH 22/62] Enable backpressure during data-feed layer startup to avoid overruns --- piker/data/feed.py | 2 +- tests/test_services.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 906f4bb4..b045ad41 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -1467,7 +1467,7 @@ async def maybe_open_feed( 'tick_throttle': kwargs.get('tick_throttle'), # XXX: super critical to have bool defaults here XD - 'backpressure': kwargs.get('backpressure', True), + # 'backpressure': kwargs.get('backpressure', True), 'start_stream': kwargs.get('start_stream', True), }, key=fqsn, diff --git a/tests/test_services.py b/tests/test_services.py index 6b67d3c5..11f0db7c 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -176,5 +176,5 @@ def test_ensure_ems_in_paper_actors( ) as exc_info: trio.run(main) - cancel_msg: str = '_emsd_main was remotely cancelled by its caller' + cancel_msg: str = '_emsd_main()` was remotely cancelled by its caller' assert cancel_msg in exc_info.value.args[0] From 4b6d3fe1380aa3260014320008c890070ee3f080 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 19:53:55 -0500 Subject: [PATCH 23/62] Scope cleanup fixture to module --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 16785afb..bdd91080 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -32,7 +32,7 @@ from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -@pytest.fixture +@pytest.fixture(scope="module") def paper_cleanup(): yield app_dir = get_app_dir('piker') From 316ead577dcb5f241dc0dfd36d85b00494bfeda9 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 20:00:10 -0500 Subject: [PATCH 24/62] Remove scoping --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index bdd91080..16785afb 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -32,7 +32,7 @@ from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -@pytest.fixture(scope="module") +@pytest.fixture def paper_cleanup(): yield app_dir = get_app_dir('piker') From 9acbfacd4cd40182a4bb9a36fc8a1ddb7debf049 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 20:02:36 -0500 Subject: [PATCH 25/62] only clean up if _testing file exists --- tests/test_paper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 16785afb..36ab2e91 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -36,8 +36,9 @@ log = get_logger(__name__) def paper_cleanup(): yield app_dir = get_app_dir('piker') - rmtree(app_dir) - assert not os.path.isfile(app_dir) + if os.path.isfile(app_dir) + rmtree(app_dir) + assert not os.path.isfile(app_dir) def test_paper_trade( open_test_pikerd: AsyncContextManager, From 2c366d7349796248889ba87501a210a490edaef1 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Tue, 14 Feb 2023 20:06:05 -0500 Subject: [PATCH 26/62] Fix type --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 36ab2e91..d2f6efd2 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -36,7 +36,7 @@ log = get_logger(__name__) def paper_cleanup(): yield app_dir = get_app_dir('piker') - if os.path.isfile(app_dir) + if os.path.isfile(app_dir): rmtree(app_dir) assert not os.path.isfile(app_dir) From 7e87dc52eb200e87234b6405bc040ddfe6f3966c Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 13:47:23 -0500 Subject: [PATCH 27/62] Scope fixture to session --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index d2f6efd2..91fa0d07 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -32,7 +32,7 @@ from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -@pytest.fixture +@pytest.fixture(scope="session") def paper_cleanup(): yield app_dir = get_app_dir('piker') From 8122e6c86f4d0729e7101e9654cd05e68c2d0520 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 13:55:39 -0500 Subject: [PATCH 28/62] Disable cleanup to see if CI passes --- tests/test_paper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 91fa0d07..8efd2281 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -32,13 +32,13 @@ from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -@pytest.fixture(scope="session") +@pytest.fixture def paper_cleanup(): yield app_dir = get_app_dir('piker') - if os.path.isfile(app_dir): - rmtree(app_dir) - assert not os.path.isfile(app_dir) + # if os.path.isfile(app_dir): + # rmtree(app_dir) + # assert not os.path.isfile(app_dir) def test_paper_trade( open_test_pikerd: AsyncContextManager, From 7bd8019876ebab28ed2774882a8fded459fbad40 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 19:12:26 -0500 Subject: [PATCH 29/62] Add back cleanup fixture --- piker/config.py | 4 ++-- tests/test_paper.py | 8 ++++---- tests/test_services.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/piker/config.py b/piker/config.py index e4c2705f..1e69b218 100644 --- a/piker/config.py +++ b/piker/config.py @@ -30,7 +30,7 @@ from bidict import bidict import toml from .log import get_logger - +from tests.test_services import get_test_app_dir log = get_logger('broker-config') # taken from ``click`` since apparently they have some @@ -78,7 +78,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # within the tractor runtimes and store testing config data # outside of the users filesystem if "pytest" in sys.modules: - app_name += '/_testing' + return get_test_app_dir() # if WIN: if platform.system() == 'Windows': diff --git a/tests/test_paper.py b/tests/test_paper.py index 8efd2281..91fa0d07 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -32,13 +32,13 @@ from exceptiongroup import BaseExceptionGroup log = get_logger(__name__) -@pytest.fixture +@pytest.fixture(scope="session") def paper_cleanup(): yield app_dir = get_app_dir('piker') - # if os.path.isfile(app_dir): - # rmtree(app_dir) - # assert not os.path.isfile(app_dir) + if os.path.isfile(app_dir): + rmtree(app_dir) + assert not os.path.isfile(app_dir) def test_paper_trade( open_test_pikerd: AsyncContextManager, diff --git a/tests/test_services.py b/tests/test_services.py index 11f0db7c..e44a21f4 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -177,4 +177,4 @@ def test_ensure_ems_in_paper_actors( trio.run(main) cancel_msg: str = '_emsd_main()` was remotely cancelled by its caller' - assert cancel_msg in exc_info.value.args[0] + assert cancel_msg in exc_info.value.args[0] From 8c9c165e0a847a85c73845a8ea23cfc696bb4164 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 19:17:17 -0500 Subject: [PATCH 30/62] Remove broken import --- piker/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/config.py b/piker/config.py index 1e69b218..e4c2705f 100644 --- a/piker/config.py +++ b/piker/config.py @@ -30,7 +30,7 @@ from bidict import bidict import toml from .log import get_logger -from tests.test_services import get_test_app_dir + log = get_logger('broker-config') # taken from ``click`` since apparently they have some @@ -78,7 +78,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # within the tractor runtimes and store testing config data # outside of the users filesystem if "pytest" in sys.modules: - return get_test_app_dir() + app_name += '/_testing' # if WIN: if platform.system() == 'Windows': From 3bc54e308f0d2d3f284c143e063f9271439b0cd5 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 19:35:37 -0500 Subject: [PATCH 31/62] Use Path.mkdir instead of os.mkdir --- piker/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/config.py b/piker/config.py index e4c2705f..b45011d1 100644 --- a/piker/config.py +++ b/piker/config.py @@ -25,7 +25,7 @@ from os import path from os.path import dirname import shutil from typing import Optional - +from pathlib import Path from bidict import bidict import toml @@ -72,7 +72,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): """ def _posixify(name): return "-".join(name.split()).lower() - + # TODO: This is a hacky way to a) determine we're testing # and b) creating a test dir. We should aim to set a variable # within the tractor runtimes and store testing config data @@ -92,7 +92,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): os.path.expanduser("~/.{}".format(_posixify(app_name)))) if sys.platform == "darwin": return os.path.join( - os.path.expanduser("~/Library/Application Support"), app_name + os.path.expanduser("~/Los.mkdir(_config_dir)ibrary/Application Support"), app_name ) return os.path.join( os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), @@ -203,7 +203,7 @@ def load( path = path or get_conf_path(conf_name) if not os.path.isdir(_config_dir): - os.mkdir(_config_dir) + Path(_config_dir).mkdir(parents=True, exist_ok=True) if not os.path.isfile(path): fn = _conf_fn_w_ext(conf_name) From db2e2ed78f9620d576d32b5472134b9751364488 Mon Sep 17 00:00:00 2001 From: algorandpa Date: Wed, 15 Feb 2023 19:48:39 -0500 Subject: [PATCH 32/62] Use constants value for test config dir path --- piker/config.py | 4 ++-- piker/testing/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/config.py b/piker/config.py index b45011d1..a8f5f494 100644 --- a/piker/config.py +++ b/piker/config.py @@ -28,7 +28,7 @@ from typing import Optional from pathlib import Path from bidict import bidict import toml - +from piker.testing import TEST_CONFIG_DIR_PATH from .log import get_logger log = get_logger('broker-config') @@ -78,7 +78,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # within the tractor runtimes and store testing config data # outside of the users filesystem if "pytest" in sys.modules: - app_name += '/_testing' + app_name += TEST_CONFIG_DIR_PATH # if WIN: if platform.system() == 'Windows': diff --git a/piker/testing/__init__.py b/piker/testing/__init__.py index e69de29b..5e3ac93a 100644 --- a/piker/testing/__init__.py +++ b/piker/testing/__init__.py @@ -0,0 +1 @@ +TEST_CONFIG_DIR_PATH = '_testing' From c99381216dbf09a2b8a96ccc2767104bb6ba8624 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Thu, 23 Feb 2023 15:21:10 -0500 Subject: [PATCH 33/62] Ensure actual pp is sent to ems ensure not to write pp header on startup Comment out pytest settings Add comments explaining delete_testing_dir fixture use nonlocal instead of global for test state Add unpacking get_fqsn method Format test_paper Add comments explaining sync/async book.send calls --- piker/clearing/_paper_engine.py | 50 +++----- pytest.ini | 6 +- tests/test_paper.py | 201 +++++++++++++++++++------------- tests/test_services.py | 3 +- 4 files changed, 141 insertions(+), 119 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 4444306a..c53e02e5 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -36,7 +36,6 @@ import trio import tractor from .. import data -from ..data._source import Symbol from ..data.types import Struct from ..pp import ( Position, @@ -86,7 +85,7 @@ class PaperBoi(Struct): # init edge case L1 spread last_ask: tuple[float, float] = (float('inf'), 0) # price, size last_bid: tuple[float, float] = (0, 0) - + async def submit_limit( self, oid: str, # XXX: see return value @@ -236,7 +235,7 @@ class PaperBoi(Struct): ) log.info(f'Fake filling order:\n{fill_msg}') await self.ems_trades_stream.send(fill_msg) - + if order_complete: msg = BrokerdStatus( reqid=reqid, @@ -251,18 +250,6 @@ class PaperBoi(Struct): # lookup any existing position key = fqsn.rstrip(f'.{self.broker}') - pp = self._positions.setdefault( - fqsn, - Position( - Symbol( - key=key, - broker_info={self.broker: {}}, - ), - size=size, - ppu=price, - bsuid=key, - ) - ) t = Transaction( fqsn=fqsn, tid=oid, @@ -274,24 +261,24 @@ class PaperBoi(Struct): ) # Update in memory ledger per trade - ledger_entry = {} - ledger_entry[oid] = t.to_dict() - + ledger_entry = {oid: t.to_dict()} + # Store txn in state for PP update self._txn_dict[oid] = t self._trade_ledger.update(ledger_entry) - # Write to ledger toml + # Write to ledger toml right now with open_trade_ledger(self.broker, 'paper') as ledger: - ledger.update(self._trade_ledger) + ledger.update(self._trade_ledger) - # Write to pps toml - with open_pps(self.broker, 'piker-paper') as table: + # Write to pps toml right now + with open_pps(self.broker, 'piker-paper') as table: table.update_from_trans(self._txn_dict) # save pps in local state - self._positions.update(table.pps) + self._positions = table.pps - pp.add_clear(t) + # Ensure we have the latest positioning data when sending pp_msg + pp = self._positions[key] pp_msg = BrokerdPosition( broker=self.broker, @@ -300,7 +287,7 @@ class PaperBoi(Struct): # TODO: we need to look up the asset currency from # broker info. i guess for crypto this can be # inferred from the pair? - currency='', + currency=key, size=pp.size, avg_price=pp.ppu, ) @@ -330,7 +317,6 @@ async def simulate_fills( # https://github.com/quantopian/zipline/blob/master/zipline/finance/ledger.py # this stream may eventually contain multiple symbols - async for quotes in quote_stream: for sym, quote in quotes.items(): for tick in iterticks( @@ -423,7 +409,8 @@ async def simulate_fills( # below unecessarily and further don't want to pop # simulated live orders prematurely. case _: - continue + continue + # iterate all potentially clearable book prices # in FIFO order per side. for order_info, pred in iter_entries: @@ -437,7 +424,7 @@ async def simulate_fills( 'buy': buys, 'sell': sells }[action].inverse.pop(order_info) - + # clearing price would have filled entirely await client.fake_fill( fqsn=sym, @@ -554,10 +541,10 @@ async def trades_dialogue( ): - with open_pps(broker, 'piker-paper') as table: + with open_pps(broker, 'piker-paper', False) as table: # save pps in local state _positions.update(table.pps) - + pp_msgs: list[BrokerdPosition] = [] pos: Position token: str # f'{symbol}.{self.broker}' @@ -569,8 +556,7 @@ async def trades_dialogue( size=pos.size, avg_price=pos.ppu, )) - # TODO: load paper positions per broker from .toml config file - # and pass as symbol to position data mapping: ``dict[str, dict]`` + await ctx.started(( pp_msgs, ['paper'], diff --git a/pytest.ini b/pytest.ini index ffadaba4..19d9d41b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ -[pytest] -trio_mode=True -log_cli=1 +#[pytest] +#trio_mode=True +#log_cli=1 diff --git a/tests/test_paper.py b/tests/test_paper.py index 91fa0d07..943fb9d5 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -1,20 +1,26 @@ +""" +Paper-mode testing +""" + import trio -import pytest -import tractor import math -import os from shutil import rmtree -from piker.config import get_app_dir -from piker.log import get_logger -from piker.clearing._messages import ( - Order -) -from uuid import uuid4 +from exceptiongroup import BaseExceptionGroup from typing import ( AsyncContextManager, Literal, ) +from pathlib import Path + +import pytest +import tractor +from tractor._exceptions import ContextCancelled +from uuid import uuid4 from functools import partial + +from piker.config import get_app_dir +from piker.log import get_logger +from piker.clearing._messages import Order from piker.pp import ( open_trade_ledger, open_pps, @@ -25,35 +31,36 @@ from piker.clearing import ( from piker.clearing._client import ( OrderBook, ) -from piker.clearing._messages import ( - BrokerdPosition -) -from exceptiongroup import BaseExceptionGroup +from piker._cacheables import open_cached_client +from piker.clearing._messages import BrokerdPosition log = get_logger(__name__) -@pytest.fixture(scope="session") -def paper_cleanup(): +@pytest.fixture(scope="module") +def delete_testing_dir(): + '''This fixture removes the temp directory + used for storing all config/ledger/pp data + created during testing sessions + ''' yield - app_dir = get_app_dir('piker') - if os.path.isfile(app_dir): - rmtree(app_dir) - assert not os.path.isfile(app_dir) + app_dir = Path(get_app_dir("piker")).resolve() + if app_dir.is_dir(): + rmtree(str(app_dir)) + assert not app_dir.is_dir() -def test_paper_trade( - open_test_pikerd: AsyncContextManager, - paper_cleanup: None -): +def get_fqsn(broker, symbol): + fqsn = f"{symbol}.{broker}" + return (fqsn, symbol, broker) + +def test_paper_trade(open_test_pikerd: AsyncContextManager): cleared_price: float - test_exec_mode='live' - test_account = 'paper' - test_size = 1 - test_broker = 'kraken' - test_brokers = [test_broker] - test_symbol = 'xbtusdt' - test_fqsn = f'{test_symbol}.{test_broker}' - test_pp_account = 'piker-paper' + test_exec_mode = "live" + test_account = "paper" + test_size = 1 + (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") + brokers = [broker] + test_pp_account = "piker-paper" positions: dict[ # brokername, acctid tuple[str, str], @@ -62,28 +69,25 @@ def test_paper_trade( async def _async_main( open_pikerd: AsyncContextManager, - action: Literal['buy', 'sell'] | None = None, + action: Literal["buy", "sell"] | None = None, price: int = 30000, assert_entries: bool = False, ) -> None: - """Spawn a paper piper actor, place a trade and assert entries are present - in both trade ledger and pps tomls. Then restart piker and ensure - that pps from previous trade exists in the ems pps. - Finally close the position and ensure that the position in pps.toml is closed. - """ + '''Spawn a paper piper actor, place a trade and assert entries are present + in both trade ledger and pps tomls. Then restart piker and ensure + that pps from previous trade exists in the ems pps. + Finally close the position and ensure that the position in pps.toml is closed. + ''' oid: str = str(uuid4()) book: OrderBook - global cleared_price - global positions - + nonlocal cleared_price + nonlocal positions + # Set up piker and EMS async with ( open_pikerd() as (_, _, _, services), - open_ems( - test_fqsn, - mode=test_account, - ) as ( + open_ems(fqsn, mode="paper") as ( book, trades_stream, pps, @@ -91,74 +95,107 @@ def test_paper_trade( dialogs, ), ): - # Send order to EMS - if action: + if action: order = Order( exec_mode=test_exec_mode, action=action, oid=oid, account=test_account, size=test_size, - symbol=test_fqsn, + symbol=fqsn, price=price, - brokers=test_brokers + brokers=brokers, ) - + # This is actually a syncronous call to push a message + # to the async ems clue - hence why we call trio.sleep afterwards book.send(order) - + await trio.sleep(2) # Assert entries are made in both ledger and PPS if assert_entries: cleared_ledger_entry = {} - with open_trade_ledger(test_broker, test_account) as ledger: + with open_trade_ledger(broker, test_account) as ledger: cleared_ledger_entry = ledger[oid] cleared_price = cleared_ledger_entry["price"] assert list(ledger.keys())[-1] == oid - assert cleared_ledger_entry['size'] == test_size - assert cleared_ledger_entry['fqsn'] == test_fqsn - - with open_pps(test_broker, test_pp_account) as table: - pp_price = table.conf[test_broker][test_pp_account][test_fqsn]["ppu"] - assert math.isclose(pp_price, cleared_ledger_entry['size'], rel_tol=1) - assert table.brokername == test_broker - assert table.acctid == test_pp_account - + assert cleared_ledger_entry["size"] == test_size + assert cleared_ledger_entry["fqsn"] == fqsn + + with open_pps(broker, test_pp_account) as table: + pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + # Ensure the price-per-unit (breakeven) price is close to our clearing price + assert math.isclose( + pp_price, cleared_ledger_entry["size"], rel_tol=1 + ) + assert table.brokername == broker + assert table.acctid == test_pp_account + positions = pps # Close piker like a user would raise KeyboardInterrupt - # Open piker and ensure last pps price is the same as ledger entry - async def _open_and_assert_pps(): + # Open piker load pps locally + # and ensure last pps price is the same as ledger entry + async def _open_and_assert_pps(): await _async_main(open_test_pikerd) - assert positions(test_broker, test_account)[-1] == cleared_price - + assert positions(broker, test_account)[-1] == cleared_price + # Close position and assert empty position in pps async def _close_pp_and_assert(): - await _async_main(open_test_pikerd, 'sell', 1) - with open_pps(test_broker, test_pp_account) as table: + await _async_main(open_test_pikerd, "sell", 1) + with open_pps(broker, test_pp_account) as table: assert len(table.pps) == 0 - # run initial time and send sent and assert trade - with pytest.raises( - BaseExceptionGroup - ): - trio.run(partial(_async_main, - open_pikerd=open_test_pikerd, - action='buy', - ) - ) + def _run_test_and_check(exception, fn): + with pytest.raises(exception) as exc_info: + trio.run(fn) - with pytest.raises( - BaseExceptionGroup - ): - trio.run(_open_and_assert_pps) + for exception in exc_info.value.exceptions: + assert isinstance(exception, KeyboardInterrupt) or isinstance( + exception, ContextCancelled + ) - with pytest.raises( - BaseExceptionGroup + # Send and execute a trade and assert trade + _run_test_and_check( + BaseExceptionGroup, + partial( + _async_main, + open_pikerd=open_test_pikerd, + action="buy", + ), + ) + _run_test_and_check(BaseExceptionGroup, _open_and_assert_pps) + _run_test_and_check(BaseExceptionGroup, _close_pp_and_assert) + + +def test_paper_client( + open_test_pikerd: AsyncContextManager +): + + async def _async_main( + open_pikerd: AsyncContextManager, ): - trio.run(_close_pp_and_assert) + (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") + async with ( + open_pikerd() as (_, _, _, services), + open_ems(fqsn, mode="paper") as ( + book, + trades_stream, + pps, + accounts, + dialogs, + ), + ): + async with open_cached_client(broker) as client: + symbol_info = await client.symbol_info() + print(f'client: {symbol_info["XBTUSDT"]}') + trio.run(partial( + _async_main, + open_pikerd=open_test_pikerd, + ), + ) diff --git a/tests/test_services.py b/tests/test_services.py index e44a21f4..763b438e 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -10,7 +10,6 @@ import trio import tractor from piker.log import get_logger - from piker._daemon import ( find_service, Services, @@ -177,4 +176,4 @@ def test_ensure_ems_in_paper_actors( trio.run(main) cancel_msg: str = '_emsd_main()` was remotely cancelled by its caller' - assert cancel_msg in exc_info.value.args[0] + assert cancel_msg in exc_info.value.args[0] From e54d928405dbbbb8b18b75b988cc59e68c30f795 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 13:42:44 -0500 Subject: [PATCH 34/62] Reformat fake fill in paper engine, Ensure tests pass, refactor test wrapper --- piker/clearing/_paper_engine.py | 24 ++---- piker/config.py | 3 +- tests/test_paper.py | 146 ++++++++++++++++++-------------- 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index c53e02e5..32e829ed 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -260,22 +260,16 @@ class PaperBoi(Struct): bsuid=key, ) - # Update in memory ledger per trade - ledger_entry = {oid: t.to_dict()} - - # Store txn in state for PP update - self._txn_dict[oid] = t - self._trade_ledger.update(ledger_entry) - # Write to ledger toml right now - with open_trade_ledger(self.broker, 'paper') as ledger: - ledger.update(self._trade_ledger) - - # Write to pps toml right now - with open_pps(self.broker, 'piker-paper') as table: - table.update_from_trans(self._txn_dict) - # save pps in local state - self._positions = table.pps + with ( + open_trade_ledger(self.broker, 'paper') as ledger, + open_pps(self.broker, 'piker-paper') as table + ): + ledger.update({oid: t.to_dict()}) + # Write to pps toml right now + table.update_from_trans({oid: t}) + # save pps in local state + self._positions.update(table.pps) # Ensure we have the latest positioning data when sending pp_msg pp = self._positions[key] diff --git a/piker/config.py b/piker/config.py index a8f5f494..c11ca3f1 100644 --- a/piker/config.py +++ b/piker/config.py @@ -78,7 +78,8 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # within the tractor runtimes and store testing config data # outside of the users filesystem if "pytest" in sys.modules: - app_name += TEST_CONFIG_DIR_PATH + log.info("TESTING") + os.path.join(app_name, TEST_CONFIG_DIR_PATH) # if WIN: if platform.system() == 'Windows': diff --git a/tests/test_paper.py b/tests/test_paper.py index 943fb9d5..ac916f2c 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -1,6 +1,6 @@ -""" +''' Paper-mode testing -""" +''' import trio import math @@ -22,6 +22,7 @@ from piker.config import get_app_dir from piker.log import get_logger from piker.clearing._messages import Order from piker.pp import ( + PpTable, open_trade_ledger, open_pps, ) @@ -36,31 +37,33 @@ from piker.clearing._messages import BrokerdPosition log = get_logger(__name__) -@pytest.fixture(scope="module") + +@pytest.fixture(scope='module') def delete_testing_dir(): '''This fixture removes the temp directory used for storing all config/ledger/pp data created during testing sessions ''' yield - app_dir = Path(get_app_dir("piker")).resolve() + app_dir = Path(get_app_dir('piker')).resolve() if app_dir.is_dir(): rmtree(str(app_dir)) assert not app_dir.is_dir() + def get_fqsn(broker, symbol): - fqsn = f"{symbol}.{broker}" + fqsn = f'{symbol}.{broker}' return (fqsn, symbol, broker) def test_paper_trade(open_test_pikerd: AsyncContextManager): - cleared_price: float - test_exec_mode = "live" - test_account = "paper" + oid = '' + test_exec_mode = 'live' + test_account = 'paper' test_size = 1 - (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") + (fqsn, symbol, broker) = get_fqsn('kraken', 'xbtusdt') brokers = [broker] - test_pp_account = "piker-paper" + test_pp_account = 'piker-paper' positions: dict[ # brokername, acctid tuple[str, str], @@ -69,7 +72,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): async def _async_main( open_pikerd: AsyncContextManager, - action: Literal["buy", "sell"] | None = None, + action: Literal['buy', 'sell'] | None = None, price: int = 30000, assert_entries: bool = False, ) -> None: @@ -78,16 +81,14 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): that pps from previous trade exists in the ems pps. Finally close the position and ensure that the position in pps.toml is closed. ''' - - oid: str = str(uuid4()) + nonlocal oid book: OrderBook - nonlocal cleared_price nonlocal positions # Set up piker and EMS async with ( open_pikerd() as (_, _, _, services), - open_ems(fqsn, mode="paper") as ( + open_ems(fqsn, mode='paper') as ( book, trades_stream, pps, @@ -97,6 +98,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): ): # Send order to EMS if action: + oid = str(uuid4()) order = Order( exec_mode=test_exec_mode, action=action, @@ -118,17 +120,15 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): cleared_ledger_entry = {} with open_trade_ledger(broker, test_account) as ledger: cleared_ledger_entry = ledger[oid] - cleared_price = cleared_ledger_entry["price"] - assert list(ledger.keys())[-1] == oid - assert cleared_ledger_entry["size"] == test_size - assert cleared_ledger_entry["fqsn"] == fqsn + assert cleared_ledger_entry['size'] == test_size + assert cleared_ledger_entry['fqsn'] == fqsn with open_pps(broker, test_pp_account) as table: - pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + pp_price = table.conf[broker][test_pp_account][fqsn]['ppu'] # Ensure the price-per-unit (breakeven) price is close to our clearing price assert math.isclose( - pp_price, cleared_ledger_entry["size"], rel_tol=1 + pp_price, cleared_ledger_entry['size'], rel_tol=1 ) assert table.brokername == broker assert table.acctid == test_pp_account @@ -140,62 +140,78 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # Open piker load pps locally # and ensure last pps price is the same as ledger entry - async def _open_and_assert_pps(): - await _async_main(open_test_pikerd) - assert positions(broker, test_account)[-1] == cleared_price + def _assert_pps(ledger, table): + return ( + positions[(broker, test_account)][-1]['avg_price'] == ledger[oid]['price'] + ) + + def _assert_no_pps(ledger, table): + return len(table.pps) == 0 # Close position and assert empty position in pps - async def _close_pp_and_assert(): - await _async_main(open_test_pikerd, "sell", 1) - with open_pps(broker, test_pp_account) as table: - assert len(table.pps) == 0 - - def _run_test_and_check(exception, fn): + def _run_test_and_check(exception, fn, assert_cb=None): with pytest.raises(exception) as exc_info: trio.run(fn) - for exception in exc_info.value.exceptions: - assert isinstance(exception, KeyboardInterrupt) or isinstance( - exception, ContextCancelled - ) + with ( + open_trade_ledger(broker, test_account) as ledger, + open_pps(broker, test_pp_account) as table, + ): + if assert_cb: + assert assert_cb(ledger, table) - # Send and execute a trade and assert trade + for exception in exc_info.value.exceptions: + assert isinstance(exception, KeyboardInterrupt) or isinstance( + exception, ContextCancelled + ) + + # Setablend and execute a trade and assert trade _run_test_and_check( BaseExceptionGroup, partial( _async_main, open_pikerd=open_test_pikerd, - action="buy", + action='buy', ), ) - _run_test_and_check(BaseExceptionGroup, _open_and_assert_pps) - _run_test_and_check(BaseExceptionGroup, _close_pp_and_assert) - -def test_paper_client( - open_test_pikerd: AsyncContextManager -): - - async def _async_main( - open_pikerd: AsyncContextManager, - ): - (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") - async with ( - open_pikerd() as (_, _, _, services), - open_ems(fqsn, mode="paper") as ( - book, - trades_stream, - pps, - accounts, - dialogs, - ), - ): - async with open_cached_client(broker) as client: - symbol_info = await client.symbol_info() - print(f'client: {symbol_info["XBTUSDT"]}') - - trio.run(partial( - _async_main, - open_pikerd=open_test_pikerd, - ), + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, open_pikerd=open_test_pikerd), + _assert_pps, ) + + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, open_pikerd=open_test_pikerd, action='sell', price=1), + _assert_no_pps, + ) + + +# def test_paper_client( +# open_test_pikerd: AsyncContextManager +# ): +# +# async def _async_main( +# open_pikerd: AsyncContextManager, +# ): +# (fqsn, symbol, broker) = get_fqsn('kraken', 'xbtusdt') +# async with ( +# open_pikerd() as (_, _, _, services), +# open_ems(fqsn, mode='paper') as ( +# book, +# trades_stream, +# pps, +# accounts, +# dialogs, +# ), +# ): +# async with open_cached_client(broker) as client: +# symbol_info = await client.symbol_info() +# print(f'client: {symbol_info['XBTUSDT']}') +# +# trio.run(partial( +# _async_main, +# open_pikerd=open_test_pikerd, +# ), +# ) From 2d25d1f048f0fde6a24f51e325acbdbe9bb0e764 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 17:23:08 -0500 Subject: [PATCH 35/62] Push failing assert no pps test --- piker/clearing/_paper_engine.py | 35 ++++++++--------- piker/pp.py | 1 - tests/test_paper.py | 70 +++++++++++++++++---------------- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 32e829ed..5b1453a8 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -41,7 +41,8 @@ from ..pp import ( Position, Transaction, open_trade_ledger, - open_pps + open_pps, + load_pps_from_ledger ) from ..data._normalize import iterticks from ..data._source import unpack_fqsn @@ -260,7 +261,6 @@ class PaperBoi(Struct): bsuid=key, ) - # Write to ledger toml right now with ( open_trade_ledger(self.broker, 'paper') as ledger, open_pps(self.broker, 'piker-paper') as table @@ -268,25 +268,22 @@ class PaperBoi(Struct): ledger.update({oid: t.to_dict()}) # Write to pps toml right now table.update_from_trans({oid: t}) - # save pps in local state - self._positions.update(table.pps) + load_pps_from_ledger(self.broker, 'piker-paper') - # Ensure we have the latest positioning data when sending pp_msg - pp = self._positions[key] + pp = table.pps[key] + pp_msg = BrokerdPosition( + broker=self.broker, + account='paper', + symbol=fqsn, + # TODO: we need to look up the asset currency from + # broker info. i guess for crypto this can be + # inferred from the pair? + currency=key, + size=pp.size, + avg_price=pp.ppu, + ) - pp_msg = BrokerdPosition( - broker=self.broker, - account='paper', - symbol=fqsn, - # TODO: we need to look up the asset currency from - # broker info. i guess for crypto this can be - # inferred from the pair? - currency=key, - size=pp.size, - avg_price=pp.ppu, - ) - - await self.ems_trades_stream.send(pp_msg) + await self.ems_trades_stream.send(pp_msg) async def simulate_fills( diff --git a/piker/pp.py b/piker/pp.py index e71045d1..756d9811 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -680,7 +680,6 @@ class PpTable(Struct): # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries print(f'Updating ``pps.toml`` for {path}:\n') - # active, closed_pp_objs = table.dump_active() pp_entries = self.to_toml() self.conf[self.brokername][self.acctid] = pp_entries diff --git a/tests/test_paper.py b/tests/test_paper.py index ac916f2c..1002df3e 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -1,6 +1,6 @@ -''' +""" Paper-mode testing -''' +""" import trio import math @@ -38,32 +38,32 @@ from piker.clearing._messages import BrokerdPosition log = get_logger(__name__) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def delete_testing_dir(): - '''This fixture removes the temp directory + """This fixture removes the temp directory used for storing all config/ledger/pp data created during testing sessions - ''' + """ yield - app_dir = Path(get_app_dir('piker')).resolve() + app_dir = Path(get_app_dir("piker")).resolve() if app_dir.is_dir(): rmtree(str(app_dir)) assert not app_dir.is_dir() def get_fqsn(broker, symbol): - fqsn = f'{symbol}.{broker}' + fqsn = f"{symbol}.{broker}" return (fqsn, symbol, broker) def test_paper_trade(open_test_pikerd: AsyncContextManager): - oid = '' - test_exec_mode = 'live' - test_account = 'paper' + oid = "" + test_exec_mode = "live" + test_account = "paper" test_size = 1 - (fqsn, symbol, broker) = get_fqsn('kraken', 'xbtusdt') + (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") brokers = [broker] - test_pp_account = 'piker-paper' + test_pp_account = "piker-paper" positions: dict[ # brokername, acctid tuple[str, str], @@ -72,15 +72,15 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): async def _async_main( open_pikerd: AsyncContextManager, - action: Literal['buy', 'sell'] | None = None, + action: Literal["buy", "sell"] | None = None, price: int = 30000, assert_entries: bool = False, ) -> None: - '''Spawn a paper piper actor, place a trade and assert entries are present + """Spawn a paper piper actor, place a trade and assert entries are present in both trade ledger and pps tomls. Then restart piker and ensure that pps from previous trade exists in the ems pps. Finally close the position and ensure that the position in pps.toml is closed. - ''' + """ nonlocal oid book: OrderBook nonlocal positions @@ -88,7 +88,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # Set up piker and EMS async with ( open_pikerd() as (_, _, _, services), - open_ems(fqsn, mode='paper') as ( + open_ems(fqsn, mode="paper") as ( book, trades_stream, pps, @@ -121,14 +121,14 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): with open_trade_ledger(broker, test_account) as ledger: cleared_ledger_entry = ledger[oid] assert list(ledger.keys())[-1] == oid - assert cleared_ledger_entry['size'] == test_size - assert cleared_ledger_entry['fqsn'] == fqsn + assert cleared_ledger_entry["size"] == test_size + assert cleared_ledger_entry["fqsn"] == fqsn with open_pps(broker, test_pp_account) as table: - pp_price = table.conf[broker][test_pp_account][fqsn]['ppu'] + pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] # Ensure the price-per-unit (breakeven) price is close to our clearing price assert math.isclose( - pp_price, cleared_ledger_entry['size'], rel_tol=1 + pp_price, cleared_ledger_entry["size"], rel_tol=1 ) assert table.brokername == broker assert table.acctid == test_pp_account @@ -142,11 +142,13 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # and ensure last pps price is the same as ledger entry def _assert_pps(ledger, table): return ( - positions[(broker, test_account)][-1]['avg_price'] == ledger[oid]['price'] + positions[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] ) def _assert_no_pps(ledger, table): - return len(table.pps) == 0 + print(f"positions: {positions}") + return not bool(table) + # return len(table.pps) == 0 # Close position and assert empty position in pps def _run_test_and_check(exception, fn, assert_cb=None): @@ -171,7 +173,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): partial( _async_main, open_pikerd=open_test_pikerd, - action='buy', + action="buy", ), ) @@ -183,22 +185,19 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): _run_test_and_check( BaseExceptionGroup, - partial(_async_main, open_pikerd=open_test_pikerd, action='sell', price=1), + partial(_async_main, open_pikerd=open_test_pikerd, action="sell", price=1), _assert_no_pps, ) -# def test_paper_client( -# open_test_pikerd: AsyncContextManager -# ): -# +# def test_paper_client(open_test_pikerd: AsyncContextManager): # async def _async_main( # open_pikerd: AsyncContextManager, # ): -# (fqsn, symbol, broker) = get_fqsn('kraken', 'xbtusdt') +# (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") # async with ( # open_pikerd() as (_, _, _, services), -# open_ems(fqsn, mode='paper') as ( +# open_ems(fqsn, mode="paper") as ( # book, # trades_stream, # pps, @@ -206,11 +205,14 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # dialogs, # ), # ): -# async with open_cached_client(broker) as client: -# symbol_info = await client.symbol_info() -# print(f'client: {symbol_info['XBTUSDT']}') +# # async with open_cached_client(broker) as client: +# # symbol_info = await client.symbol_info() +# # print(f'client: {symbol_info['XBTUSDT']}') +# with (open_pps(broker, "piker-paper") as table,): +# print(f"table: {table}") # -# trio.run(partial( +# trio.run( +# partial( # _async_main, # open_pikerd=open_test_pikerd, # ), From 3fcad162988b88f5996f976b6c152df8e8229571 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 17:34:58 -0500 Subject: [PATCH 36/62] Ensure not to write to pps when asserting? --- tests/test_paper.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 1002df3e..8f05503d 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -137,17 +137,34 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # Close piker like a user would raise KeyboardInterrupt + + # def _assert_entries(): + # cleared_ledger_entry = {} + # with open_trade_ledger(broker, test_account) as ledger: + # cleared_ledger_entry = ledger[oid] + # assert list(ledger.keys())[-1] == oid + # assert cleared_ledger_entry["size"] == test_size + # assert cleared_ledger_entry["fqsn"] == fqsn + # + # with open_pps(broker, test_pp_account, False) as table: + # pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + # # Ensure the price-per-unit (breakeven) price is close to our clearing price + # assert math.isclose( + # pp_price, cleared_ledger_entry["size"], rel_tol=1 + # ) + # assert table.brokername == broker + # assert table.acctid == test_pp_account + # # Open piker load pps locally # and ensure last pps price is the same as ledger entry def _assert_pps(ledger, table): - return ( - positions[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] - ) + assert positions[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] + def _assert_no_pps(ledger, table): print(f"positions: {positions}") - return not bool(table) + assert not bool(table) # return len(table.pps) == 0 # Close position and assert empty position in pps @@ -160,7 +177,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): open_pps(broker, test_pp_account) as table, ): if assert_cb: - assert assert_cb(ledger, table) + assert_cb(ledger, table) for exception in exc_info.value.exceptions: assert isinstance(exception, KeyboardInterrupt) or isinstance( @@ -175,6 +192,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): open_pikerd=open_test_pikerd, action="buy", ), + # _assert_entries ) _run_test_and_check( From 1e748f11ef2d59012678ca2c99367e39da7f77c8 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 17:50:09 -0500 Subject: [PATCH 37/62] Ensure config path is being updated with _testing correctly during testing --- piker/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/config.py b/piker/config.py index c11ca3f1..b1b2eedc 100644 --- a/piker/config.py +++ b/piker/config.py @@ -79,7 +79,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # outside of the users filesystem if "pytest" in sys.modules: log.info("TESTING") - os.path.join(app_name, TEST_CONFIG_DIR_PATH) + app_name = os.path.join(app_name, TEST_CONFIG_DIR_PATH) # if WIN: if platform.system() == 'Windows': From 4c2e776e01362348d8006ce477bd89fdbf3bee51 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 18:05:17 -0500 Subject: [PATCH 38/62] Ensure to cleanup by passing fixture in paper_test signature --- piker/config.py | 1 - tests/test_paper.py | 12 +++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/piker/config.py b/piker/config.py index b1b2eedc..f7b1e588 100644 --- a/piker/config.py +++ b/piker/config.py @@ -78,7 +78,6 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # within the tractor runtimes and store testing config data # outside of the users filesystem if "pytest" in sys.modules: - log.info("TESTING") app_name = os.path.join(app_name, TEST_CONFIG_DIR_PATH) # if WIN: diff --git a/tests/test_paper.py b/tests/test_paper.py index 8f05503d..66e85893 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -38,7 +38,7 @@ from piker.clearing._messages import BrokerdPosition log = get_logger(__name__) -@pytest.fixture(scope="module") +@pytest.fixture(scope="session") def delete_testing_dir(): """This fixture removes the temp directory used for storing all config/ledger/pp data @@ -56,7 +56,7 @@ def get_fqsn(broker, symbol): return (fqsn, symbol, broker) -def test_paper_trade(open_test_pikerd: AsyncContextManager): +def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): oid = "" test_exec_mode = "live" test_account = "paper" @@ -71,7 +71,6 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): ] async def _async_main( - open_pikerd: AsyncContextManager, action: Literal["buy", "sell"] | None = None, price: int = 30000, assert_entries: bool = False, @@ -87,7 +86,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): # Set up piker and EMS async with ( - open_pikerd() as (_, _, _, services), + open_test_pikerd() as (_, _, _, services), open_ems(fqsn, mode="paper") as ( book, trades_stream, @@ -189,7 +188,6 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): BaseExceptionGroup, partial( _async_main, - open_pikerd=open_test_pikerd, action="buy", ), # _assert_entries @@ -197,13 +195,13 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager): _run_test_and_check( BaseExceptionGroup, - partial(_async_main, open_pikerd=open_test_pikerd), + partial(_async_main), _assert_pps, ) _run_test_and_check( BaseExceptionGroup, - partial(_async_main, open_pikerd=open_test_pikerd, action="sell", price=1), + partial(_async_main, action="sell", price=1), _assert_no_pps, ) From 76736a54417197c0c1bfd45b96d49b40a06e62cc Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Fri, 24 Feb 2023 19:09:36 -0500 Subject: [PATCH 39/62] Refactor to avoid global state while testing --- tests/test_paper.py | 115 ++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 67 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 66e85893..ded87b74 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -74,6 +74,8 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): action: Literal["buy", "sell"] | None = None, price: int = 30000, assert_entries: bool = False, + assert_pps: bool = False, + assert_zeroed_pps: bool = False, ) -> None: """Spawn a paper piper actor, place a trade and assert entries are present in both trade ledger and pps tomls. Then restart piker and ensure @@ -115,95 +117,74 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): await trio.sleep(2) # Assert entries are made in both ledger and PPS - if assert_entries: - cleared_ledger_entry = {} - with open_trade_ledger(broker, test_account) as ledger: - cleared_ledger_entry = ledger[oid] - assert list(ledger.keys())[-1] == oid - assert cleared_ledger_entry["size"] == test_size - assert cleared_ledger_entry["fqsn"] == fqsn - - with open_pps(broker, test_pp_account) as table: - pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] - # Ensure the price-per-unit (breakeven) price is close to our clearing price - assert math.isclose( - pp_price, cleared_ledger_entry["size"], rel_tol=1 - ) - assert table.brokername == broker - assert table.acctid == test_pp_account - - positions = pps + if assert_entries or assert_pps or assert_zeroed_pps: + _assert(assert_entries, assert_pps, assert_zeroed_pps, pps) # Close piker like a user would raise KeyboardInterrupt - - # def _assert_entries(): - # cleared_ledger_entry = {} - # with open_trade_ledger(broker, test_account) as ledger: - # cleared_ledger_entry = ledger[oid] - # assert list(ledger.keys())[-1] == oid - # assert cleared_ledger_entry["size"] == test_size - # assert cleared_ledger_entry["fqsn"] == fqsn - # - # with open_pps(broker, test_pp_account, False) as table: - # pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] - # # Ensure the price-per-unit (breakeven) price is close to our clearing price - # assert math.isclose( - # pp_price, cleared_ledger_entry["size"], rel_tol=1 - # ) - # assert table.brokername == broker - # assert table.acctid == test_pp_account - # - - # Open piker load pps locally - # and ensure last pps price is the same as ledger entry - def _assert_pps(ledger, table): - assert positions[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] - - - def _assert_no_pps(ledger, table): - print(f"positions: {positions}") - assert not bool(table) - # return len(table.pps) == 0 - - # Close position and assert empty position in pps - def _run_test_and_check(exception, fn, assert_cb=None): - with pytest.raises(exception) as exc_info: - trio.run(fn) + def _assert(assert_entries: bool, assert_pps: bool, assert_zerod_pps, pps): with ( open_trade_ledger(broker, test_account) as ledger, open_pps(broker, test_pp_account) as table, ): - if assert_cb: - assert_cb(ledger, table) + # assert that entires are have been written + if assert_entries: + cleared_ledger_entry = ledger[oid] + assert list(ledger.keys())[-1] == oid + assert cleared_ledger_entry["size"] == test_size + assert cleared_ledger_entry["fqsn"] == fqsn + pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + # Ensure the price-per-unit (breakeven) price is close to our clearing price + assert math.isclose(pp_price, cleared_ledger_entry["size"], rel_tol=1) + assert table.brokername == broker + assert table.acctid == test_pp_account - for exception in exc_info.value.exceptions: - assert isinstance(exception, KeyboardInterrupt) or isinstance( - exception, ContextCancelled + # assert that the last pps price is the same as the ledger price + if assert_pps: + assert ( + pps[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] ) + if assert_zerod_pps: + # assert that positions are present + assert not bool(table) + + # Close position and assert empty position in pps + def _run_test_and_check(exception, fn): + with pytest.raises(exception) as exc_info: + trio.run(fn) + + for exception in exc_info.value.exceptions: + assert isinstance(exception, KeyboardInterrupt) or isinstance( + exception, ContextCancelled + ) + # Setablend and execute a trade and assert trade + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, action="buy", assert_entries=True), + ) + + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, assert_pps=True), + ) + _run_test_and_check( BaseExceptionGroup, partial( - _async_main, - action="buy", + _async_main, action="sell", price=1 ), - # _assert_entries ) _run_test_and_check( BaseExceptionGroup, - partial(_async_main), - _assert_pps, + partial( + _async_main, assert_zeroed_pps=True + ), ) - _run_test_and_check( - BaseExceptionGroup, - partial(_async_main, action="sell", price=1), - _assert_no_pps, - ) # def test_paper_client(open_test_pikerd: AsyncContextManager): From 15525c2b4695218876aed27398fd36b886a73199 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 25 Feb 2023 15:59:14 -0500 Subject: [PATCH 40/62] Add functionality and tests for executing mutliple orders --- tests/test_paper.py | 157 ++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 73 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index ded87b74..b5a74e40 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -11,6 +11,7 @@ from typing import ( Literal, ) from pathlib import Path +from operator import attrgetter import pytest import tractor @@ -60,7 +61,6 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): oid = "" test_exec_mode = "live" test_account = "paper" - test_size = 1 (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") brokers = [broker] test_pp_account = "piker-paper" @@ -76,16 +76,19 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): assert_entries: bool = False, assert_pps: bool = False, assert_zeroed_pps: bool = False, + assert_precision: bool = False, + executions: int = 1, + size: float = 0.01, ) -> None: - """Spawn a paper piper actor, place a trade and assert entries are present + """Start piker, place a trade and assert entries are present in both trade ledger and pps tomls. Then restart piker and ensure that pps from previous trade exists in the ems pps. Finally close the position and ensure that the position in pps.toml is closed. """ nonlocal oid - book: OrderBook nonlocal positions - + book: OrderBook + msg = () # Set up piker and EMS async with ( open_test_pikerd() as (_, _, _, services), @@ -99,56 +102,80 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): ): # Send order to EMS if action: - oid = str(uuid4()) - order = Order( - exec_mode=test_exec_mode, - action=action, - oid=oid, - account=test_account, - size=test_size, - symbol=fqsn, - price=price, - brokers=brokers, - ) - # This is actually a syncronous call to push a message - # to the async ems clue - hence why we call trio.sleep afterwards - book.send(order) - - await trio.sleep(2) + for x in range(executions): + print(f"Sending {action} order num {x}") + oid = str(uuid4()) + order = Order( + exec_mode=test_exec_mode, + action=action, + oid=oid, + account=test_account, + size=size, + symbol=fqsn, + price=price, + brokers=brokers, + ) + # This is actually a syncronous call to push a message + # to the async ems clue - hence why we call trio.sleep afterwards + book.send(order) + async for msg in trades_stream: + msg = await trades_stream.receive() + try: + if msg["name"] == "position": + break + except (NameError, AttributeError): + pass + # Do nothing, message isn't a position + await trio.sleep(1) # Assert entries are made in both ledger and PPS - if assert_entries or assert_pps or assert_zeroed_pps: - _assert(assert_entries, assert_pps, assert_zeroed_pps, pps) + if assert_entries or assert_pps or assert_zeroed_pps or assert_precision: + _assert( + assert_entries, + assert_pps, + assert_zeroed_pps, + assert_precision, + pps, + msg, + ) # Close piker like a user would raise KeyboardInterrupt - def _assert(assert_entries: bool, assert_pps: bool, assert_zerod_pps, pps): + def _assert( + assert_entries, assert_pps, assert_zerod_pps, assert_precision, pps, msg + ): with ( open_trade_ledger(broker, test_account) as ledger, open_pps(broker, test_pp_account) as table, ): + # TODO: Assert between msg and pp, ledger and pp, ledger and message + # for proper values + print(f"assertion msg: {msg}") # assert that entires are have been written if assert_entries: - cleared_ledger_entry = ledger[oid] - assert list(ledger.keys())[-1] == oid - assert cleared_ledger_entry["size"] == test_size - assert cleared_ledger_entry["fqsn"] == fqsn + latest_ledger_entry = ledger[oid] + latest_position = pps[(broker, test_account)][-1] pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + # assert most + assert list(ledger.keys())[-1] == oid + assert latest_ledger_entry["size"] == test_size + assert latest_ledger_entry["fqsn"] == fqsn + # Ensure the price-per-unit (breakeven) price is close to our clearing price - assert math.isclose(pp_price, cleared_ledger_entry["size"], rel_tol=1) + assert math.isclose(pp_price, latest_ledger_entry["size"], rel_tol=1) assert table.brokername == broker assert table.acctid == test_pp_account # assert that the last pps price is the same as the ledger price if assert_pps: - assert ( - pps[(broker, test_account)][-1]["avg_price"] == ledger[oid]["price"] - ) + latest_ledger_entry = ledger[oid] + latest_position = pps[(broker, test_account)][-1] + assert latest_position["avg_price"] == latest_ledger_entry["price"] if assert_zerod_pps: # assert that positions are present - assert not bool(table) + assert not bool(table.pps) # Close position and assert empty position in pps def _run_test_and_check(exception, fn): @@ -156,61 +183,45 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): trio.run(fn) for exception in exc_info.value.exceptions: - assert isinstance(exception, KeyboardInterrupt) or isinstance( - exception, ContextCancelled + assert ( + isinstance(exception, KeyboardInterrupt) + or isinstance(exception, ContextCancelled) + or isinstance(exception, KeyError) ) - # Setablend and execute a trade and assert trade + # Enter a trade and assert entries are made in pps and ledger files _run_test_and_check( BaseExceptionGroup, partial(_async_main, action="buy", assert_entries=True), ) + # Open ems and assert existence of pps entries _run_test_and_check( BaseExceptionGroup, partial(_async_main, assert_pps=True), ) + # Sell position + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, action="sell", price=1), + ) + + # Ensure pps are zeroed + _run_test_and_check( + BaseExceptionGroup, + partial(_async_main, assert_zeroed_pps=True), + ) + + # Make 5 market limit buy orders + _run_test_and_check( + BaseExceptionGroup, partial(_async_main, action="buy", executions=5) + ) + + # Sell 5 slots at the same price, assert cleared positions _run_test_and_check( BaseExceptionGroup, partial( - _async_main, action="sell", price=1 + _async_main, action="sell", executions=5, price=1, assert_zeroed_pps=True ), ) - - _run_test_and_check( - BaseExceptionGroup, - partial( - _async_main, assert_zeroed_pps=True - ), - ) - - - -# def test_paper_client(open_test_pikerd: AsyncContextManager): -# async def _async_main( -# open_pikerd: AsyncContextManager, -# ): -# (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") -# async with ( -# open_pikerd() as (_, _, _, services), -# open_ems(fqsn, mode="paper") as ( -# book, -# trades_stream, -# pps, -# accounts, -# dialogs, -# ), -# ): -# # async with open_cached_client(broker) as client: -# # symbol_info = await client.symbol_info() -# # print(f'client: {symbol_info['XBTUSDT']}') -# with (open_pps(broker, "piker-paper") as table,): -# print(f"table: {table}") -# -# trio.run( -# partial( -# _async_main, -# open_pikerd=open_test_pikerd, -# ), -# ) From 85ad23a1e95c5de2ae4eb8fd885671662c1286b6 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 25 Feb 2023 16:01:44 -0500 Subject: [PATCH 41/62] Remove uneeded assert_precision arg --- tests/test_paper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index b5a74e40..0d208ce0 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -76,7 +76,6 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): assert_entries: bool = False, assert_pps: bool = False, assert_zeroed_pps: bool = False, - assert_precision: bool = False, executions: int = 1, size: float = 0.01, ) -> None: @@ -129,12 +128,11 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): # Do nothing, message isn't a position await trio.sleep(1) # Assert entries are made in both ledger and PPS - if assert_entries or assert_pps or assert_zeroed_pps or assert_precision: + if assert_entries or assert_pps or assert_zeroed_pps: _assert( assert_entries, assert_pps, assert_zeroed_pps, - assert_precision, pps, msg, ) @@ -143,7 +141,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): raise KeyboardInterrupt def _assert( - assert_entries, assert_pps, assert_zerod_pps, assert_precision, pps, msg + assert_entries, assert_pps, assert_zerod_pps, pps, msg ): with ( open_trade_ledger(broker, test_account) as ledger, From 3a6fbabaf893d1bbb282a7c2f6704e379786ceec Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 25 Feb 2023 16:09:16 -0500 Subject: [PATCH 42/62] Minor formatting --- piker/clearing/_client.py | 7 +++++-- piker/config.py | 2 +- tests/test_paper.py | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index 08b0f4c8..0a40b548 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -235,7 +235,8 @@ async def open_ems( not getattr(mod, 'trades_dialogue', None) or mode == 'paper' ): - mode = 'paper' + mode = 'paper' + from ._ems import _emsd_main async with ( # connect to emsd @@ -254,6 +255,7 @@ async def open_ems( dialogs, ) ), + # open 2-way trade command stream ctx.open_stream() as trades_stream, ): @@ -263,7 +265,8 @@ async def open_ems( relay_order_cmds_from_sync_code, fqsn, trades_stream - ) + ) + yield ( book, trades_stream, diff --git a/piker/config.py b/piker/config.py index f7b1e588..abcbb652 100644 --- a/piker/config.py +++ b/piker/config.py @@ -76,7 +76,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): # TODO: This is a hacky way to a) determine we're testing # and b) creating a test dir. We should aim to set a variable # within the tractor runtimes and store testing config data - # outside of the users filesystem + # outside of the users filesystem if "pytest" in sys.modules: app_name = os.path.join(app_name, TEST_CONFIG_DIR_PATH) diff --git a/tests/test_paper.py b/tests/test_paper.py index 0d208ce0..f6183497 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -126,6 +126,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): except (NameError, AttributeError): pass # Do nothing, message isn't a position + await trio.sleep(1) # Assert entries are made in both ledger and PPS if assert_entries or assert_pps or assert_zeroed_pps: From 3e83764b5b3a7a64906eec922ed56a96e12c46b1 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 25 Feb 2023 16:21:30 -0500 Subject: [PATCH 43/62] Remove whitespace, uneeded comments --- piker/clearing/_ems.py | 3 ++- piker/clearing/_paper_engine.py | 2 +- piker/config.py | 3 ++- piker/pp.py | 6 ++++-- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/piker/clearing/_ems.py b/piker/clearing/_ems.py index 1f3c52cb..477da310 100644 --- a/piker/clearing/_ems.py +++ b/piker/clearing/_ems.py @@ -1451,7 +1451,8 @@ async def _emsd_main( ) as (relay, feed, client_ready): brokerd_stream = relay.brokerd_stream - dark_book = _router.get_dark_book(broker) + dark_book = _router.get_dark_book(broker) + # signal to client that we're started and deliver # all known pps and accounts for this ``brokerd``. await ems_ctx.started(( diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 5b1453a8..4b226c1c 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -30,6 +30,7 @@ from typing import ( Callable, ) import uuid + from bidict import bidict import pendulum import trio @@ -249,7 +250,6 @@ class PaperBoi(Struct): ) await self.ems_trades_stream.send(msg) - # lookup any existing position key = fqsn.rstrip(f'.{self.broker}') t = Transaction( fqsn=fqsn, diff --git a/piker/config.py b/piker/config.py index abcbb652..6a76410e 100644 --- a/piker/config.py +++ b/piker/config.py @@ -33,6 +33,7 @@ from .log import get_logger log = get_logger('broker-config') + # taken from ``click`` since apparently they have some # super weirdness with sigint and sudo..no clue def get_app_dir(app_name, roaming=True, force_posix=False): @@ -92,7 +93,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): os.path.expanduser("~/.{}".format(_posixify(app_name)))) if sys.platform == "darwin": return os.path.join( - os.path.expanduser("~/Los.mkdir(_config_dir)ibrary/Application Support"), app_name + os.path.expanduser("~/Library/Application Support"), app_name ) return os.path.join( os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), diff --git a/piker/pp.py b/piker/pp.py index 756d9811..33086ba1 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -35,7 +35,6 @@ from typing import ( Union, Generator ) -from typing import Generator import pendulum from pendulum import datetime, now @@ -56,6 +55,7 @@ log = get_logger(__name__) def open_trade_ledger( broker: str, account: str, + ) -> Generator[dict, None, None]: ''' Indempotently create and read in a trade log file from the @@ -84,6 +84,7 @@ def open_trade_ledger( ledger = tomli.load(cf) print(f'Ledger load took {time.time() - start}s') cpy = ledger.copy() + try: yield cpy finally: @@ -544,6 +545,7 @@ class PpTable(Struct): pps = self.pps updated: dict[str, Position] = {} + # lifo update all pps from records for tid, t in trans.items(): @@ -680,6 +682,7 @@ class PpTable(Struct): # TODO: show diff output? # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries print(f'Updating ``pps.toml`` for {path}:\n') + # active, closed_pp_objs = table.dump_active() pp_entries = self.to_toml() self.conf[self.brokername][self.acctid] = pp_entries @@ -975,7 +978,6 @@ def open_pps( pp.ensure_state() try: -# breakpoint() yield table finally: if write_on_exit: From fcd8b8eb78c3137ec6b468272fc5331bd1653e2b Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sat, 25 Feb 2023 17:45:21 -0500 Subject: [PATCH 44/62] Remove breaking call to load pps from ledger --- piker/clearing/_paper_engine.py | 5 ++--- piker/pp.py | 2 +- tests/test_paper.py | 27 +++++++++++++++++++-------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 4b226c1c..4fe1788f 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -43,7 +43,6 @@ from ..pp import ( Transaction, open_trade_ledger, open_pps, - load_pps_from_ledger ) from ..data._normalize import iterticks from ..data._source import unpack_fqsn @@ -250,6 +249,7 @@ class PaperBoi(Struct): ) await self.ems_trades_stream.send(msg) + # lookup any existing position key = fqsn.rstrip(f'.{self.broker}') t = Transaction( fqsn=fqsn, @@ -263,12 +263,11 @@ class PaperBoi(Struct): with ( open_trade_ledger(self.broker, 'paper') as ledger, - open_pps(self.broker, 'piker-paper') as table + open_pps(self.broker, 'paper') as table ): ledger.update({oid: t.to_dict()}) # Write to pps toml right now table.update_from_trans({oid: t}) - load_pps_from_ledger(self.broker, 'piker-paper') pp = table.pps[key] pp_msg = BrokerdPosition( diff --git a/piker/pp.py b/piker/pp.py index 33086ba1..d9f09d8e 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -730,7 +730,7 @@ def load_pps_from_ledger( return {} mod = get_brokermod(brokername) - src_records: dict[str, Transaction] = mod.norm_trade_records(ledger) + src_records: dict[str, Transaction] = mod.norm_tr_records(ledger) if filter_by: records = {} diff --git a/tests/test_paper.py b/tests/test_paper.py index f6183497..1c3134a9 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -63,7 +63,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): test_account = "paper" (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") brokers = [broker] - test_pp_account = "piker-paper" + account = "paper" positions: dict[ # brokername, acctid tuple[str, str], @@ -76,6 +76,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): assert_entries: bool = False, assert_pps: bool = False, assert_zeroed_pps: bool = False, + assert_msg: bool = True, executions: int = 1, size: float = 0.01, ) -> None: @@ -87,7 +88,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): nonlocal oid nonlocal positions book: OrderBook - msg = () + msg = {} # Set up piker and EMS async with ( open_test_pikerd() as (_, _, _, services), @@ -129,24 +130,26 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): await trio.sleep(1) # Assert entries are made in both ledger and PPS - if assert_entries or assert_pps or assert_zeroed_pps: + if assert_entries or assert_pps or assert_zeroed_pps or assert_msg: _assert( assert_entries, assert_pps, assert_zeroed_pps, + assert_msg, pps, msg, + size, ) # Close piker like a user would raise KeyboardInterrupt def _assert( - assert_entries, assert_pps, assert_zerod_pps, pps, msg + assert_entries, assert_pps, assert_zerod_pps, assert_msg, pps, msg, size ): with ( open_trade_ledger(broker, test_account) as ledger, - open_pps(broker, test_pp_account) as table, + open_pps(broker, test_account) as table, ): # TODO: Assert between msg and pp, ledger and pp, ledger and message # for proper values @@ -155,7 +158,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): if assert_entries: latest_ledger_entry = ledger[oid] latest_position = pps[(broker, test_account)][-1] - pp_price = table.conf[broker][test_pp_account][fqsn]["ppu"] + pp_price = table.conf[broker][account][fqsn]["ppu"] # assert most assert list(ledger.keys())[-1] == oid assert latest_ledger_entry["size"] == test_size @@ -164,7 +167,7 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): # Ensure the price-per-unit (breakeven) price is close to our clearing price assert math.isclose(pp_price, latest_ledger_entry["size"], rel_tol=1) assert table.brokername == broker - assert table.acctid == test_pp_account + assert table.acctid == account # assert that the last pps price is the same as the ledger price if assert_pps: @@ -173,9 +176,17 @@ def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): assert latest_position["avg_price"] == latest_ledger_entry["price"] if assert_zerod_pps: - # assert that positions are present + # assert that positions are not present assert not bool(table.pps) + if assert_msg and msg["name"] == "position": + latest_position = pps[(broker, test_account)][-1] + breakpoint() + assert msg["broker"] == broker + assert msg["account"]== test_account + assert msg["symbol"] == fqsn + assert msg["avg_price"]== latest_position["avg_price"] + # Close position and assert empty position in pps def _run_test_and_check(exception, fn): with pytest.raises(exception) as exc_info: From 36f466fff8cbfbd9864d108c319316775669780c Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 26 Feb 2023 15:59:55 -0500 Subject: [PATCH 45/62] Ensure tests are running and working up until asserting pps --- piker/clearing/_paper_engine.py | 2 +- tests/conftest.py | 50 +++++ tests/test_paper.py | 323 +++++++++++++++----------------- 3 files changed, 198 insertions(+), 177 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 4fe1788f..7cf60726 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -531,7 +531,7 @@ async def trades_dialogue( ): - with open_pps(broker, 'piker-paper', False) as table: + with open_pps(broker, 'paper', False) as table: # save pps in local state _positions.update(table.pps) diff --git a/tests/conftest.py b/tests/conftest.py index 75c8a92d..beb649f3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,9 @@ from contextlib import asynccontextmanager as acm from functools import partial import os +from typing import AsyncContextManager +from pathlib import Path +from shutil import rmtree import pytest import tractor @@ -11,6 +14,7 @@ from piker import ( from piker._daemon import ( Services, ) +from piker.clearing._client import open_ems def pytest_addoption(parser): @@ -132,3 +136,49 @@ def open_test_pikerd( # - no leaked subprocs or shm buffers # - all requested container service are torn down # - certain ``tractor`` runtime state? + + +@acm +async def _open_test_pikerd_and_ems( + fqsn, + mode, + loglevel, + open_test_pikerd +): + async with ( + open_test_pikerd() as (_, _, _, services), + open_ems( + fqsn, + mode=mode, + loglevel=loglevel, + ) as ems_services): + yield (services, ems_services) + + + +@pytest.fixture +def open_test_pikerd_and_ems( + open_test_pikerd, + fqsn: str = 'xbtusdt.kraken', + mode: str = 'paper', + loglevel: str = 'info', +): + yield partial( + _open_test_pikerd_and_ems, + fqsn, + mode, + loglevel, + open_test_pikerd + ) + +@pytest.fixture(scope='session') +def delete_testing_dir(): + '''This fixture removes the temp directory + used for storing all config/ledger/pp data + created during testing sessions + ''' + yield + app_dir = Path(config.get_app_dir('piker')).resolve() + if app_dir.is_dir(): + rmtree(str(app_dir)) + assert not app_dir.is_dir() diff --git a/tests/test_paper.py b/tests/test_paper.py index 1c3134a9..1a0fdd2f 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -1,17 +1,13 @@ -""" +''' Paper-mode testing -""" +''' import trio -import math -from shutil import rmtree from exceptiongroup import BaseExceptionGroup from typing import ( AsyncContextManager, Literal, ) -from pathlib import Path -from operator import attrgetter import pytest import tractor @@ -19,219 +15,194 @@ from tractor._exceptions import ContextCancelled from uuid import uuid4 from functools import partial -from piker.config import get_app_dir from piker.log import get_logger from piker.clearing._messages import Order from piker.pp import ( - PpTable, open_trade_ledger, open_pps, ) -from piker.clearing import ( - open_ems, -) -from piker.clearing._client import ( - OrderBook, -) -from piker._cacheables import open_cached_client -from piker.clearing._messages import BrokerdPosition log = get_logger(__name__) -@pytest.fixture(scope="session") -def delete_testing_dir(): - """This fixture removes the temp directory - used for storing all config/ledger/pp data - created during testing sessions - """ - yield - app_dir = Path(get_app_dir("piker")).resolve() - if app_dir.is_dir(): - rmtree(str(app_dir)) - assert not app_dir.is_dir() - - def get_fqsn(broker, symbol): - fqsn = f"{symbol}.{broker}" + fqsn = f'{symbol}.{broker}' return (fqsn, symbol, broker) -def test_paper_trade(open_test_pikerd: AsyncContextManager, delete_testing_dir): - oid = "" - test_exec_mode = "live" - test_account = "paper" - (fqsn, symbol, broker) = get_fqsn("kraken", "xbtusdt") - brokers = [broker] - account = "paper" - positions: dict[ - # brokername, acctid - tuple[str, str], - list[BrokerdPosition], - ] +oid = '' +test_exec_mode = 'live' +(fqsn, symbol, broker) = get_fqsn('kraken', 'xbtusdt') +brokers = [broker] +account = 'paper' - async def _async_main( - action: Literal["buy", "sell"] | None = None, - price: int = 30000, - assert_entries: bool = False, - assert_pps: bool = False, - assert_zeroed_pps: bool = False, - assert_msg: bool = True, - executions: int = 1, - size: float = 0.01, - ) -> None: - """Start piker, place a trade and assert entries are present - in both trade ledger and pps tomls. Then restart piker and ensure - that pps from previous trade exists in the ems pps. - Finally close the position and ensure that the position in pps.toml is closed. - """ - nonlocal oid - nonlocal positions - book: OrderBook - msg = {} - # Set up piker and EMS - async with ( - open_test_pikerd() as (_, _, _, services), - open_ems(fqsn, mode="paper") as ( - book, - trades_stream, - pps, - accounts, - dialogs, - ), - ): - # Send order to EMS - if action: - for x in range(executions): - print(f"Sending {action} order num {x}") - oid = str(uuid4()) - order = Order( - exec_mode=test_exec_mode, - action=action, - oid=oid, - account=test_account, - size=size, - symbol=fqsn, - price=price, - brokers=brokers, - ) - # This is actually a syncronous call to push a message - # to the async ems clue - hence why we call trio.sleep afterwards - book.send(order) - async for msg in trades_stream: - msg = await trades_stream.receive() - try: - if msg["name"] == "position": - break - except (NameError, AttributeError): - pass - # Do nothing, message isn't a position - - await trio.sleep(1) - # Assert entries are made in both ledger and PPS - if assert_entries or assert_pps or assert_zeroed_pps or assert_msg: - _assert( - assert_entries, - assert_pps, - assert_zeroed_pps, - assert_msg, - pps, - msg, - size, - ) - - # Close piker like a user would - raise KeyboardInterrupt - - def _assert( - assert_entries, assert_pps, assert_zerod_pps, assert_msg, pps, msg, size +async def _async_main( + open_test_pikerd_and_ems: AsyncContextManager, + action: Literal['buy', 'sell'] | None = None, + price: int = 30000, + assert_entries: bool = False, + assert_pps: bool = False, + assert_zeroed_pps: bool = False, + assert_msg: bool = False, + executions: int = 1, + size: float = 0.01, +) -> None: + '''Start piker, place a trade and assert data in pps stream, ledger and position table. Then restart piker and ensure + that pps from previous trade exists in the ems pps. + Finally close the position and ensure that the position in pps.toml is closed. + ''' + oid: str = '' + last_msg = {} + async with open_test_pikerd_and_ems() as ( + services, + (book, trades_stream, pps, accounts, dialogs), ): - with ( - open_trade_ledger(broker, test_account) as ledger, - open_pps(broker, test_account) as table, - ): - # TODO: Assert between msg and pp, ledger and pp, ledger and message - # for proper values - print(f"assertion msg: {msg}") - # assert that entires are have been written - if assert_entries: - latest_ledger_entry = ledger[oid] - latest_position = pps[(broker, test_account)][-1] - pp_price = table.conf[broker][account][fqsn]["ppu"] - # assert most - assert list(ledger.keys())[-1] == oid - assert latest_ledger_entry["size"] == test_size - assert latest_ledger_entry["fqsn"] == fqsn + # Set up piker and EMS + # Send order to EMS + if action: + for x in range(executions): + oid = str(uuid4()) + order = Order( + exec_mode=test_exec_mode, + action=action, + oid=oid, + account=account, + size=size, + symbol=fqsn, + price=price, + brokers=brokers, + ) + # This is actually a syncronous call to push a message + # to the async ems clue - hence why we call trio.sleep afterwards + book.send(order) - # Ensure the price-per-unit (breakeven) price is close to our clearing price - assert math.isclose(pp_price, latest_ledger_entry["size"], rel_tol=1) - assert table.brokername == broker - assert table.acctid == account + async for msg in trades_stream: + last_msg = msg + match msg: + case {'name': 'position'}: + break - # assert that the last pps price is the same as the ledger price - if assert_pps: - latest_ledger_entry = ledger[oid] - latest_position = pps[(broker, test_account)][-1] - assert latest_position["avg_price"] == latest_ledger_entry["price"] - - if assert_zerod_pps: - # assert that positions are not present - assert not bool(table.pps) - - if assert_msg and msg["name"] == "position": - latest_position = pps[(broker, test_account)][-1] - breakpoint() - assert msg["broker"] == broker - assert msg["account"]== test_account - assert msg["symbol"] == fqsn - assert msg["avg_price"]== latest_position["avg_price"] - - # Close position and assert empty position in pps - def _run_test_and_check(exception, fn): - with pytest.raises(exception) as exc_info: - trio.run(fn) - - for exception in exc_info.value.exceptions: - assert ( - isinstance(exception, KeyboardInterrupt) - or isinstance(exception, ContextCancelled) - or isinstance(exception, KeyError) + if assert_entries or assert_pps or assert_zeroed_pps or assert_msg: + _assert( + assert_entries, + assert_pps, + assert_zeroed_pps, + pps, + last_msg, + size, + executions, ) + # Teardown piker like a user would + raise KeyboardInterrupt + + +def _assert( + assert_entries, + assert_pps, + assert_zerod_pps, + pps, + last_msg, + size, + executions, +): + with ( + open_trade_ledger(broker, account) as ledger, + open_pps(broker, account) as table, + ): + ''' + Assert multiple cases including pps, ledger and final position message state + ''' + if assert_entries: + assert last_msg['broker'] == broker + assert last_msg['account'] == account + assert last_msg['symbol'] == fqsn + assert last_msg['size'] == size * executions + assert last_msg['currency'] == symbol + assert last_msg['avg_price'] == table.pps[symbol].ppu + + if assert_pps: + last_ppu = pps[(broker, account)][-1] + assert last_ppu['avg_price'] == table.pps[symbol].ppu + + if assert_zerod_pps: + assert not bool(table.pps) + + +# Close position and assert empty position in pps +def _run_test_and_check(fn): + with pytest.raises(BaseExceptionGroup) as exc_info: + trio.run(fn) + + for exception in exc_info.value.exceptions: + assert isinstance(exception, KeyboardInterrupt) or isinstance( + exception, ContextCancelled + ) + + +def test_buy(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Enter a trade and assert entries are made in pps and ledger files _run_test_and_check( - BaseExceptionGroup, - partial(_async_main, action="buy", assert_entries=True), + partial( + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + action='buy', + assert_entries=True, + ), ) # Open ems and assert existence of pps entries _run_test_and_check( - BaseExceptionGroup, - partial(_async_main, assert_pps=True), + partial( + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + assert_pps=True, + ), ) + +def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Sell position _run_test_and_check( - BaseExceptionGroup, - partial(_async_main, action="sell", price=1), + partial( + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + action='sell', + price=1, + ), ) # Ensure pps are zeroed _run_test_and_check( - BaseExceptionGroup, - partial(_async_main, assert_zeroed_pps=True), + partial( + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + assert_zeroed_pps=True, + ), ) + +# +def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Make 5 market limit buy orders _run_test_and_check( - BaseExceptionGroup, partial(_async_main, action="buy", executions=5) + partial( + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + action='buy', + executions=5, + ), ) # Sell 5 slots at the same price, assert cleared positions _run_test_and_check( - BaseExceptionGroup, partial( - _async_main, action="sell", executions=5, price=1, assert_zeroed_pps=True + _async_main, + open_test_pikerd_and_ems=open_test_pikerd_and_ems, + action='sell', + executions=5, + price=1, + assert_zeroed_pps=True, ), ) From 61296bbdfcea987c4ad60391a4d93c03704d9a10 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 26 Feb 2023 16:05:02 -0500 Subject: [PATCH 46/62] Minor formatting, removing whitespace --- piker/clearing/_paper_engine.py | 1 - piker/config.py | 2 ++ piker/pp.py | 5 ++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 7cf60726..14361c53 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -81,7 +81,6 @@ class PaperBoi(Struct): _reqids: bidict _positions: dict[str, Position] _trade_ledger: dict[str, Any] - _txn_dict: dict[str, Transaction] = {} # init edge case L1 spread last_ask: tuple[float, float] = (float('inf'), 0) # price, size diff --git a/piker/config.py b/piker/config.py index 6a76410e..8bf14759 100644 --- a/piker/config.py +++ b/piker/config.py @@ -71,6 +71,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): dot instead of the XDG config home or darwin's application support folder. """ + def _posixify(name): return "-".join(name.split()).lower() @@ -100,6 +101,7 @@ def get_app_dir(app_name, roaming=True, force_posix=False): _posixify(app_name), ) + _config_dir = _click_config_dir = get_app_dir('piker') _parent_user = os.environ.get('SUDO_USER') diff --git a/piker/pp.py b/piker/pp.py index d9f09d8e..d48661a0 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -544,7 +544,7 @@ class PpTable(Struct): ) -> dict[str, Position]: pps = self.pps - updated: dict[str, Position] = {} + updated: dict[str, Position] = {} # lifo update all pps from records for tid, t in trans.items(): @@ -583,8 +583,7 @@ class PpTable(Struct): # update clearing table pp.add_clear(t) - updated[t.bsuid] = pp - + updated[t.bsuid] = pp # minimize clears tables and update sizing. for bsuid, pp in updated.items(): From 4b72d3ba999b1bdbf38f6587207a77b4734604fb Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 26 Feb 2023 16:11:28 -0500 Subject: [PATCH 47/62] Add backpressure setting back as it wasn't altering test behaviour --- piker/data/feed.py | 2 +- tests/test_paper.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index b045ad41..906f4bb4 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -1467,7 +1467,7 @@ async def maybe_open_feed( 'tick_throttle': kwargs.get('tick_throttle'), # XXX: super critical to have bool defaults here XD - # 'backpressure': kwargs.get('backpressure', True), + 'backpressure': kwargs.get('backpressure', True), 'start_stream': kwargs.get('start_stream', True), }, key=fqsn, diff --git a/tests/test_paper.py b/tests/test_paper.py index 1a0fdd2f..cf84dd6a 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -41,25 +41,26 @@ async def _async_main( open_test_pikerd_and_ems: AsyncContextManager, action: Literal['buy', 'sell'] | None = None, price: int = 30000, + executions: int = 1, + size: float = 0.01, + # Assert options assert_entries: bool = False, assert_pps: bool = False, assert_zeroed_pps: bool = False, assert_msg: bool = False, - executions: int = 1, - size: float = 0.01, -) -> None: - '''Start piker, place a trade and assert data in pps stream, ledger and position table. Then restart piker and ensure - that pps from previous trade exists in the ems pps. - Finally close the position and ensure that the position in pps.toml is closed. + ) -> None: ''' + Start piker, place a trade and assert data in + pps stream, ledger and position table. + ''' + oid: str = '' last_msg = {} + async with open_test_pikerd_and_ems() as ( services, (book, trades_stream, pps, accounts, dialogs), ): - # Set up piker and EMS - # Send order to EMS if action: for x in range(executions): oid = str(uuid4()) @@ -80,6 +81,7 @@ async def _async_main( async for msg in trades_stream: last_msg = msg match msg: + # Wait for position message before moving on case {'name': 'position'}: break @@ -183,7 +185,6 @@ def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir) ) -# def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Make 5 market limit buy orders _run_test_and_check( From 55253c8469c81dcfba8fda0e00c6dfa98bed53fa Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 26 Feb 2023 16:15:00 -0500 Subject: [PATCH 48/62] Remove whitespace and correct typo --- piker/pp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/pp.py b/piker/pp.py index d48661a0..74c710d4 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -583,7 +583,7 @@ class PpTable(Struct): # update clearing table pp.add_clear(t) - updated[t.bsuid] = pp + updated[t.bsuid] = pp # minimize clears tables and update sizing. for bsuid, pp in updated.items(): @@ -729,7 +729,7 @@ def load_pps_from_ledger( return {} mod = get_brokermod(brokername) - src_records: dict[str, Transaction] = mod.norm_tr_records(ledger) + src_records: dict[str, Transaction] = mod.norm_trade_records(ledger) if filter_by: records = {} From 342aec648ba5f75d49d5bc7990445135821a5b60 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Sun, 26 Feb 2023 16:41:06 -0500 Subject: [PATCH 49/62] Skip zero test and change use Path when creating a config folder in marketstore --- piker/data/marketstore.py | 3 ++- tests/test_paper.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/data/marketstore.py b/piker/data/marketstore.py index 236bcfaf..190667d6 100644 --- a/piker/data/marketstore.py +++ b/piker/data/marketstore.py @@ -35,6 +35,7 @@ from typing import ( ) import time from math import isnan +from pathlib import Path from bidict import bidict from msgspec.msgpack import encode, decode @@ -134,7 +135,7 @@ def start_marketstore( # create dirs when dne if not os.path.isdir(config._config_dir): - os.mkdir(config._config_dir) + Path(config._config_dir).mkdir(parents=True, exist_ok=True) if not os.path.isdir(mktsdir): os.mkdir(mktsdir) diff --git a/tests/test_paper.py b/tests/test_paper.py index cf84dd6a..9422b434 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -184,7 +184,7 @@ def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir) ), ) - +@pytest.mark.skip(reason="Due to precision issues, this test will currently fail") def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Make 5 market limit buy orders _run_test_and_check( From e4e368923d1e44534a19e23420b36407dd18f6c8 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 12:58:39 -0500 Subject: [PATCH 50/62] Add specific kwarg key to open_pps call when starting paperboi --- piker/clearing/_paper_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 14361c53..79ef55f4 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -530,7 +530,7 @@ async def trades_dialogue( ): - with open_pps(broker, 'paper', False) as table: + with open_pps(broker, 'paper', write_on_exit=False) as table: # save pps in local state _positions.update(table.pps) From 802af306acfb91bb00168b5f4b8e1677ddef6468 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:01:42 -0500 Subject: [PATCH 51/62] Add specific location of _testing dir in delete_testing_dir fixture --- tests/conftest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index beb649f3..3e8f34a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -171,11 +171,14 @@ def open_test_pikerd_and_ems( open_test_pikerd ) -@pytest.fixture(scope='session') +@pytest.fixture(scope='session') def delete_testing_dir(): - '''This fixture removes the temp directory + ''' + This fixture removes the temp directory used for storing all config/ledger/pp data - created during testing sessions + created during testing sessions. During test runs + this file can be found in .config/piker/_testing + ''' yield app_dir = Path(config.get_app_dir('piker')).resolve() From 6f15d470128684be536bcfb8b7cc39aeea9e7f00 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:05:57 -0500 Subject: [PATCH 52/62] Add space in docstrings, remove duplicate import --- tests/test_paper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 9422b434..07444401 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -10,7 +10,6 @@ from typing import ( ) import pytest -import tractor from tractor._exceptions import ContextCancelled from uuid import uuid4 from functools import partial @@ -52,6 +51,7 @@ async def _async_main( ''' Start piker, place a trade and assert data in pps stream, ledger and position table. + ''' oid: str = '' @@ -115,6 +115,7 @@ def _assert( ): ''' Assert multiple cases including pps, ledger and final position message state + ''' if assert_entries: assert last_msg['broker'] == broker From ecb22dda1a909ea20fb18f5096642424940284db Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:16:26 -0500 Subject: [PATCH 53/62] Remove whitespace, remove stale comments --- tests/test_paper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 07444401..e5c7fffc 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -47,10 +47,10 @@ async def _async_main( assert_pps: bool = False, assert_zeroed_pps: bool = False, assert_msg: bool = False, - ) -> None: +) -> None: ''' - Start piker, place a trade and assert data in - pps stream, ledger and position table. + Start piker, place a trade and assert data in + pps stream, ledger and position table. ''' @@ -75,7 +75,6 @@ async def _async_main( brokers=brokers, ) # This is actually a syncronous call to push a message - # to the async ems clue - hence why we call trio.sleep afterwards book.send(order) async for msg in trades_stream: @@ -185,7 +184,8 @@ def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir) ), ) -@pytest.mark.skip(reason="Due to precision issues, this test will currently fail") + +@pytest.mark.skip(reason='Due to precision issues, this test will currently fail') def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): # Make 5 market limit buy orders _run_test_and_check( From 87eb9c5772ffe0f71f63139ff92d722426b59186 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:19:21 -0500 Subject: [PATCH 54/62] Format assertion conditions --- tests/test_paper.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index e5c7fffc..8850e2a2 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -84,7 +84,12 @@ async def _async_main( case {'name': 'position'}: break - if assert_entries or assert_pps or assert_zeroed_pps or assert_msg: + if ( + assert_entries + or assert_pps + or assert_zeroed_pps + or assert_msg + ): _assert( assert_entries, assert_pps, From d7317c371001a3d02d5112d8d5092fc145bc0c1a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:26:37 -0500 Subject: [PATCH 55/62] Shorten assertion docstring --- tests/test_paper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 8850e2a2..e7fea452 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -114,11 +114,11 @@ def _assert( executions, ): with ( - open_trade_ledger(broker, account) as ledger, open_pps(broker, account) as table, ): ''' - Assert multiple cases including pps, ledger and final position message state + Assert multiple cases including pps, + ledger and final position message state ''' if assert_entries: From 973c068e964fe0a303942bd2d35cac5928ad7854 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:31:17 -0500 Subject: [PATCH 56/62] Assert conditions like a nerd --- tests/test_paper.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index e7fea452..8e053772 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -122,12 +122,15 @@ def _assert( ''' if assert_entries: - assert last_msg['broker'] == broker - assert last_msg['account'] == account - assert last_msg['symbol'] == fqsn - assert last_msg['size'] == size * executions - assert last_msg['currency'] == symbol - assert last_msg['avg_price'] == table.pps[symbol].ppu + for key, val in [ + ('broker', broker), + ('account', account), + ('symbol', fqsn), + ('size', size * executions), + ('currency', symbol), + ('avg_price', table.pps[symbol].ppu) + ]: + assert last_msg[key] == val if assert_pps: last_ppu = pps[(broker, account)][-1] From a6257ae61571b8978d0e17841ac9ed5ef077847c Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:39:13 -0500 Subject: [PATCH 57/62] Add docstrings to test cases, format function calls --- tests/test_paper.py | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 8e053772..4be9d176 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -140,8 +140,11 @@ def _assert( assert not bool(table.pps) -# Close position and assert empty position in pps def _run_test_and_check(fn): + ''' + Close position and assert empty position in pps + + ''' with pytest.raises(BaseExceptionGroup) as exc_info: trio.run(fn) @@ -151,8 +154,14 @@ def _run_test_and_check(fn): ) -def test_buy(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): - # Enter a trade and assert entries are made in pps and ledger files +def test_buy( + open_test_pikerd_and_ems: AsyncContextManager, + delete_testing_dir +): + ''' + Enter a trade and assert entries are made in pps and ledger files. + + ''' _run_test_and_check( partial( _async_main, @@ -172,8 +181,14 @@ def test_buy(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): ) -def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): - # Sell position +def test_sell( + open_test_pikerd_and_ems: AsyncContextManager, + delete_testing_dir +): + ''' + Sell position ensure pps are zeroed. + + ''' _run_test_and_check( partial( _async_main, @@ -183,7 +198,6 @@ def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir) ), ) - # Ensure pps are zeroed _run_test_and_check( partial( _async_main, @@ -194,8 +208,16 @@ def test_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir) @pytest.mark.skip(reason='Due to precision issues, this test will currently fail') -def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir): - # Make 5 market limit buy orders +def test_multi_sell( + open_test_pikerd_and_ems: AsyncContextManager, + delete_testing_dir +): + ''' + Make 5 market limit buy orders and + then sell 5 slots at the same price. + Finally, assert cleared positions. + + ''' _run_test_and_check( partial( _async_main, @@ -205,7 +227,6 @@ def test_multi_sell(open_test_pikerd_and_ems: AsyncContextManager, delete_testin ), ) - # Sell 5 slots at the same price, assert cleared positions _run_test_and_check( partial( _async_main, From 882032e3a39a33d96bf04b1edf994ccb469e97ab Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:40:21 -0500 Subject: [PATCH 58/62] Change skip to xfail --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 4be9d176..5737f418 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -207,7 +207,7 @@ def test_sell( ) -@pytest.mark.skip(reason='Due to precision issues, this test will currently fail') +@pytest.mark.xskip(reason='Due to precision issues, this test will currently fail') def test_multi_sell( open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir From 1323981cc454b280afcf92bdac81def4dd3eb60e Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:42:36 -0500 Subject: [PATCH 59/62] Format lines in conftest Add extra line in conftest --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3e8f34a2..e25c8840 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -155,7 +155,6 @@ async def _open_test_pikerd_and_ems( yield (services, ems_services) - @pytest.fixture def open_test_pikerd_and_ems( open_test_pikerd, @@ -171,6 +170,7 @@ def open_test_pikerd_and_ems( open_test_pikerd ) + @pytest.fixture(scope='session') def delete_testing_dir(): ''' From 05fdc9dd60ac15fcea4231d7bade0eaa72ba233a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 13:55:12 -0500 Subject: [PATCH 60/62] Add xfail --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index 5737f418..fbc06033 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -207,7 +207,7 @@ def test_sell( ) -@pytest.mark.xskip(reason='Due to precision issues, this test will currently fail') +@pytest.mark.xfail(reason='Due to precision issues, this test will currently fail') def test_multi_sell( open_test_pikerd_and_ems: AsyncContextManager, delete_testing_dir From 41f81eb70108ace72c29e81ee5869dba2dc1ee9a Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 14:14:05 -0500 Subject: [PATCH 61/62] Make write on exit default false --- piker/clearing/_paper_engine.py | 4 ++-- piker/pp.py | 2 +- tests/conftest.py | 2 +- tests/test_paper.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/clearing/_paper_engine.py b/piker/clearing/_paper_engine.py index 79ef55f4..fc37f1e4 100644 --- a/piker/clearing/_paper_engine.py +++ b/piker/clearing/_paper_engine.py @@ -262,7 +262,7 @@ class PaperBoi(Struct): with ( open_trade_ledger(self.broker, 'paper') as ledger, - open_pps(self.broker, 'paper') as table + open_pps(self.broker, 'paper', True) as table ): ledger.update({oid: t.to_dict()}) # Write to pps toml right now @@ -530,7 +530,7 @@ async def trades_dialogue( ): - with open_pps(broker, 'paper', write_on_exit=False) as table: + with open_pps(broker, 'paper') as table: # save pps in local state _positions.update(table.pps) diff --git a/piker/pp.py b/piker/pp.py index 74c710d4..b4ab2d0c 100644 --- a/piker/pp.py +++ b/piker/pp.py @@ -880,7 +880,7 @@ class PpsEncoder(toml.TomlEncoder): def open_pps( brokername: str, acctid: str, - write_on_exit: bool = True, + write_on_exit: bool = False, ) -> Generator[PpTable, None, None]: ''' diff --git a/tests/conftest.py b/tests/conftest.py index e25c8840..8218ec16 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -171,7 +171,7 @@ def open_test_pikerd_and_ems( ) -@pytest.fixture(scope='session') +@pytest.fixture(scope='module') def delete_testing_dir(): ''' This fixture removes the temp directory diff --git a/tests/test_paper.py b/tests/test_paper.py index fbc06033..d1168502 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -114,7 +114,7 @@ def _assert( executions, ): with ( - open_pps(broker, account) as table, + open_pps(broker, account, write_on_exit=False) as table, ): ''' Assert multiple cases including pps, From c83fe5aaa7c530f8272f7c2ef075fb4a1bc38060 Mon Sep 17 00:00:00 2001 From: jaredgoldman Date: Tue, 28 Feb 2023 14:22:24 -0500 Subject: [PATCH 62/62] Fix typo in test docstring --- tests/test_paper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_paper.py b/tests/test_paper.py index d1168502..3339db6c 100644 --- a/tests/test_paper.py +++ b/tests/test_paper.py @@ -186,7 +186,7 @@ def test_sell( delete_testing_dir ): ''' - Sell position ensure pps are zeroed. + Sell position and ensure pps are zeroed. ''' _run_test_and_check(