TOSQASH 6f92c6b5: xdotool trycatch

Our issue was not having the correct value set on each
`Symbol.lot_tick_size`.. and then doing PPU calcs with the default set
for legacy mkts..

Also,
- actually write `pps.toml` on broker mode exit.
- drop `get_likely_pair()` and import from pp module.

piker/accounting/__init__.py

@@ -0,0 +1,92 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+"Accounting for degens": count dem numberz that tracks how much you got
+for tendiez.
+
+'''
+from ..log import get_logger
+
+from ._pos import (
+    Transaction,
+    open_trade_ledger,
+    PpTable,
+)
+from ._pos import (
+    open_pps,
+    load_pps_from_ledger,
+    Position,
+)
+
+log = get_logger(__name__)
+
+__all__ = [
+    'Transaction',
+    'open_trade_ledger',
+    'PpTable',
+    'open_pps',
+    'load_pps_from_ledger',
+    'Position',
+]
+
+
+def get_likely_pair(
+    src: str,
+    dst: str,
+    bsuid: str,
+
+) -> str:
+    '''
+    Attempt to get the likely trading pair matching a given destination
+    asset `dst: str`.
+
+    '''
+    try:
+        src_name_start = bsuid.rindex(src)
+    except (
+        ValueError,   # substr not found
+    ):
+        # TODO: handle nested positions..(i.e.
+        # positions where the src fiat was used to
+        # buy some other dst which was furhter used
+        # to buy another dst..)
+        log.warning(
+            f'No src fiat {src} found in {bsuid}?'
+        )
+        return
+
+    likely_dst = bsuid[:src_name_start]
+    if likely_dst == dst:
+        return bsuid
+
+
+if __name__ == '__main__':
+    import sys
+    from pprint import pformat
+
+    args = sys.argv
+    assert len(args) > 1, 'Specifiy account(s) from `brokers.toml`'
+    args = args[1:]
+    for acctid in args:
+        broker, name = acctid.split('.')
+        trans, updated_pps = load_pps_from_ledger(broker, name)
+        print(
+            f'Processing transactions into pps for {broker}:{acctid}\n'
+            f'{pformat(trans)}\n\n'
+            f'{pformat(updated_pps)}'
+        )

piker/accounting/_ledger.py

@@ -0,0 +1,125 @@
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+from contextlib import contextmanager as cm
+import os
+from os import path
+import time
+from typing import (
+    Any,
+    Iterator,
+    Union,
+    Generator
+)
+
+from pendulum import (
+    datetime,
+)
+import tomli
+import toml
+
+from .. import config
+from ..data._source import Symbol
+from ..data.types import Struct
+from ..log import get_logger
+
+log = get_logger(__name__)
+
+
+@cm
+def open_trade_ledger(
+    broker: str,
+    account: str,
+
+) -> Generator[dict, None, None]:
+    '''
+    Indempotently create and read in a trade log file from the
+    ``<configuration_dir>/ledgers/`` directory.
+
+    Files are named per broker account of the form
+    ``<brokername>_<accountname>.toml``. The ``accountname`` here is the
+    name as defined in the user's ``brokers.toml`` config.
+
+    '''
+    ldir = path.join(config._config_dir, 'ledgers')
+    if not path.isdir(ldir):
+        os.makedirs(ldir)
+
+    fname = f'trades_{broker}_{account}.toml'
+    tradesfile = path.join(ldir, fname)
+
+    if not path.isfile(tradesfile):
+        log.info(
+            f'Creating new local trades ledger: {tradesfile}'
+        )
+        with open(tradesfile, 'w') as cf:
+            pass  # touch
+    with open(tradesfile, 'rb') as cf:
+        start = time.time()
+        ledger = tomli.load(cf)
+        log.info(f'Ledger load took {time.time() - start}s')
+        cpy = ledger.copy()
+
+    try:
+        yield cpy
+    finally:
+        if cpy != ledger:
+
+            # TODO: show diff output?
+            # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
+            log.info(f'Updating ledger for {tradesfile}:\n')
+            ledger.update(cpy)
+
+            # we write on close the mutated ledger data
+            with open(tradesfile, 'w') as cf:
+                toml.dump(ledger, cf)
+
+
+class Transaction(Struct, frozen=True):
+    # TODO: should this be ``.to`` (see below)?
+    fqsn: str
+
+    sym: Symbol
+    tid: Union[str, int]  # unique transaction id
+    size: float
+    price: float
+    cost: float  # commisions or other additional costs
+    dt: datetime
+    expiry: datetime | None = None
+
+    # optional key normally derived from the broker
+    # backend which ensures the instrument-symbol this record
+    # is for is truly unique.
+    bsuid: Union[str, int] | None = None
+
+    # optional fqsn for the source "asset"/money symbol?
+    # from: Optional[str] = None
+
+
+def iter_by_dt(
+    clears: dict[str, Any],
+) -> Iterator[tuple[str, dict]]:
+    '''
+    Iterate entries of a ``clears: dict`` table sorted by entry recorded
+    datetime presumably set at the ``'dt'`` field in each entry.
+
+    '''
+    for tid, data in sorted(
+        list(clears.items()),
+        key=lambda item: item[1]['dt'],
+    ):
+        yield tid, data

piker/accounting/_pos.py

@@ -14,20 +14,18 @@
 # You should have received a copy of the GNU Affero General Public License
 
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 '''
 Personal/Private position parsing, calculating, summarizing in a way
 that doesn't try to cuk most humans who prefer to not lose their moneys..
+
 (looking at you `ib` and dirt-bird friends)
 
 '''
 from __future__ import annotations
 from contextlib import contextmanager as cm
-from pprint import pformat
-import os
-from os import path
 from math import copysign
 import re
-import time
 from typing import (
     Any,
     Iterator,
@@ -38,104 +36,23 @@ from typing import (
 
 import pendulum
 from pendulum import datetime, now
-import tomli
 import toml
 
-from . import config
-from .brokers import get_brokermod
-from .clearing._messages import BrokerdPosition, Status
-from .data._source import Symbol, unpack_fqsn
-from .log import get_logger
-from .data.types import Struct
+from ._ledger import (
+    Transaction,
+    iter_by_dt,
+    open_trade_ledger,
+)
+from .. import config
+from ..brokers import get_brokermod
+from ..clearing._messages import BrokerdPosition, Status
+from ..data._source import Symbol, unpack_fqsn
+from ..data.types import Struct
+from ..log import get_logger
 
 log = get_logger(__name__)
 
 
-@cm
-def open_trade_ledger(
-    broker: str,
-    account: str,
-
-) -> Generator[dict, None, None]:
-    '''
-    Indempotently create and read in a trade log file from the
-    ``<configuration_dir>/ledgers/`` directory.
-
-    Files are named per broker account of the form
-    ``<brokername>_<accountname>.toml``. The ``accountname`` here is the
-    name as defined in the user's ``brokers.toml`` config.
-
-    '''
-    ldir = path.join(config._config_dir, 'ledgers')
-    if not path.isdir(ldir):
-        os.makedirs(ldir)
-
-    fname = f'trades_{broker}_{account}.toml'
-    tradesfile = path.join(ldir, fname)
-
-    if not path.isfile(tradesfile):
-        log.info(
-            f'Creating new local trades ledger: {tradesfile}'
-        )
-        with open(tradesfile, 'w') as cf:
-            pass  # touch
-    with open(tradesfile, 'rb') as cf:
-        start = time.time()
-        ledger = tomli.load(cf)
-        log.info(f'Ledger load took {time.time() - start}s')
-        cpy = ledger.copy()
-
-    try:
-        yield cpy
-    finally:
-        if cpy != ledger:
-
-            # TODO: show diff output?
-            # https://stackoverflow.com/questions/12956957/print-diff-of-python-dictionaries
-            log.info(f'Updating ledger for {tradesfile}:\n')
-            ledger.update(cpy)
-
-            # we write on close the mutated ledger data
-            with open(tradesfile, 'w') as cf:
-                toml.dump(ledger, cf)
-
-
-class Transaction(Struct, frozen=True):
-    # TODO: should this be ``.to`` (see below)?
-    fqsn: str
-
-    sym: Symbol
-    tid: Union[str, int]  # unique transaction id
-    size: float
-    price: float
-    cost: float  # commisions or other additional costs
-    dt: datetime
-    expiry: datetime | None = None
-
-    # optional key normally derived from the broker
-    # backend which ensures the instrument-symbol this record
-    # is for is truly unique.
-    bsuid: Union[str, int] | None = None
-
-    # optional fqsn for the source "asset"/money symbol?
-    # from: Optional[str] = None
-
-
-def iter_by_dt(
-    clears: dict[str, Any],
-) -> Iterator[tuple[str, dict]]:
-    '''
-    Iterate entries of a ``clears: dict`` table sorted by entry recorded
-    datetime presumably set at the ``'dt'`` field in each entry.
-
-    '''
-    for tid, data in sorted(
-        list(clears.items()),
-        key=lambda item: item[1]['dt'],
-    ):
-        yield tid, data
-
-
 class Position(Struct):
     '''
     Basic pp (personal/piker position) model with attached clearing
@@ -484,7 +401,9 @@ class Position(Struct):
         if self.split_ratio is not None:
             size = round(size * self.split_ratio)
 
-        return float(self.symbol.quantize_size(size))
+        return float(
+            self.symbol.quantize_size(size),
+        )
 
     def minimize_clears(
         self,
@@ -564,9 +483,13 @@ class PpTable(Struct):
         pps = self.pps
         updated: dict[str, Position] = {}
 
-        # lifo update all pps from records
-        for tid, t in trans.items():
-
+        # lifo update all pps from records, ensuring
+        # we compute the PPU and size sorted in time!
+        for t in sorted(
+            trans.values(),
+            key=lambda t: t.dt,
+            reverse=True,
+        ):
             pp = pps.setdefault(
                 t.bsuid,
 
@@ -590,7 +513,10 @@ class PpTable(Struct):
                 # included in the current pps state.
                 if (
                     t.tid in clears
-                    or first_clear_dt and t.dt < first_clear_dt
+                    or (
+                        first_clear_dt
+                        and t.dt < first_clear_dt
+                    )
                 ):
                     # NOTE: likely you'll see repeats of the same
                     # ``Transaction`` passed in here if/when you are restarting
@@ -607,6 +533,8 @@ class PpTable(Struct):
         for bsuid, pp in updated.items():
             pp.ensure_state()
 
+        # deliver only the position entries that were actually updated
+        # (modified the state) from the input transaction set.
         return updated
 
     def dump_active(
@@ -701,8 +629,10 @@ class PpTable(Struct):
         # active, closed_pp_objs = table.dump_active()
         pp_entries = self.to_toml()
         if pp_entries:
-            log.info(f'Updating ``pps.toml`` for {path}:\n')
-            log.info(f'Current positions:\n{pp_entries}')
+            log.info(
+                f'Updating ``pps.toml``:\n'
+                f'Current positions:\n{pp_entries}'
+            )
             self.conf[self.brokername][self.acctid] = pp_entries
 
         elif (
@@ -1029,19 +959,3 @@ def open_pps(
     finally:
         if write_on_exit:
             table.write_config()
-
-
-if __name__ == '__main__':
-    import sys
-
-    args = sys.argv
-    assert len(args) > 1, 'Specifiy account(s) from `brokers.toml`'
-    args = args[1:]
-    for acctid in args:
-        broker, name = acctid.split('.')
-        trans, updated_pps = load_pps_from_ledger(broker, name)
-        print(
-            f'Processing transactions into pps for {broker}:{acctid}\n'
-            f'{pformat(trans)}\n\n'
-            f'{pformat(updated_pps)}'
-        )

piker/brokers/ib/_util.py

@@ -24,6 +24,10 @@ import subprocess
 
 import tractor
 
+from piker.log import get_logger
+
+log = get_logger(__name__)
+
 
 _reset_tech: Literal[
     'vnc',
@@ -134,54 +138,54 @@ def i3ipc_xdotool_manual_click_hack() -> None:
         # 'IB',  # gw running in i3 (newer version?)
     ]
 
-    for name in win_names:
-        results = t.find_titled(name)
-        print(f'results for {name}: {results}')
-        if results:
-            con = results[0]
-            print(f'Resetting data feed for {name}')
-            win_id = str(con.window)
-            w, h = con.rect.width, con.rect.height
+    try:
+        for name in win_names:
+            results = t.find_titled(name)
+            print(f'results for {name}: {results}')
+            if results:
+                con = results[0]
+                print(f'Resetting data feed for {name}')
+                win_id = str(con.window)
+                w, h = con.rect.width, con.rect.height
 
-            # TODO: seems to be a few libs for python but not sure
-            # if they support all the sub commands we need, order of
-            # most recent commit history:
-            # https://github.com/rr-/pyxdotool
-            # https://github.com/ShaneHutter/pyxdotool
-            # https://github.com/cphyc/pyxdotool
+                # TODO: seems to be a few libs for python but not sure
+                # if they support all the sub commands we need, order of
+                # most recent commit history:
+                # https://github.com/rr-/pyxdotool
+                # https://github.com/ShaneHutter/pyxdotool
+                # https://github.com/cphyc/pyxdotool
 
-            # TODO: only run the reconnect (2nd) kc on a detected
-            # disconnect?
-            for key_combo, timeout in [
-                # only required if we need a connection reset.
-                # ('ctrl+alt+r', 12),
-                # data feed reset.
-                ('ctrl+alt+f', 6)
-            ]:
-                subprocess.call([
-                    'xdotool',
-                    'windowactivate', '--sync', win_id,
+                # TODO: only run the reconnect (2nd) kc on a detected
+                # disconnect?
+                for key_combo, timeout in [
+                    # only required if we need a connection reset.
+                    # ('ctrl+alt+r', 12),
+                    # data feed reset.
+                    ('ctrl+alt+f', 6)
+                ]:
+                    subprocess.call([
+                        'xdotool',
+                        'windowactivate', '--sync', win_id,
 
-                    # move mouse to bottom left of window (where there should
-                    # be nothing to click).
-                    'mousemove_relative', '--sync', str(w-4), str(h-4),
+                        # move mouse to bottom left of window (where
+                        # there should be nothing to click).
+                        'mousemove_relative', '--sync', str(w-4), str(h-4),
 
-                    # NOTE: we may need to stick a `--retry 3` in here..
-                    'click', '--window', win_id,
-                    '--repeat', '3', '1',
+                        # NOTE: we may need to stick a `--retry 3` in here..
+                        'click', '--window', win_id,
+                        '--repeat', '3', '1',
 
-                    # hackzorzes
-                    'key', key_combo,
-                    ],
-                    timeout=timeout,
-                )
+                        # hackzorzes
+                        'key', key_combo,
+                        ],
+                        timeout=timeout,
+                    )
 
     # re-activate and focus original window
-    try:
         subprocess.call([
             'xdotool',
             'windowactivate', '--sync', str(orig_win_id),
             'click', '--window', str(orig_win_id), '1',
         ])
     except subprocess.TimeoutExpired:
-        log.exception(f'xdotool timed out?')
+        log.exception('xdotool timed out?')

piker/brokers/ib/broker.py

@@ -51,7 +51,7 @@ from ib_insync.objects import Position as IbPosition
 import pendulum
 
 from piker import config
-from piker.pp import (
+from piker.accounting import (
     Position,
     Transaction,
     open_trade_ledger,
@@ -1153,7 +1153,7 @@ def norm_trade_records(
 
         # special handling of symbol extraction from
         # flex records using some ad-hoc schema parsing.
-        asset_type: str = record.get('assetCategory') or record['secType']
+        asset_type: str = record.get('assetCategory') or record.get('secType', 'STK')
 
         # TODO: XXX: WOA this is kinda hacky.. probably
         # should figure out the correct future pair key more

piker/brokers/kraken/api.py

@@ -20,6 +20,7 @@ Kraken web API wrapping.
 '''
 from contextlib import asynccontextmanager as acm
 from datetime import datetime
+from decimal import Decimal
 import itertools
 from typing import (
     Any,
@@ -48,7 +49,7 @@ from piker.brokers._util import (
     BrokerError,
     DataThrottle,
 )
-from piker.pp import Transaction
+from piker.accounting import Transaction
 from . import log
 
 # <uri>/<version>/
@@ -248,6 +249,9 @@ class Client:
             {},
         )
         by_bsuid = resp['result']
+
+        # TODO: we need to pull out the "asset" decimals
+        # data and return a `decimal.Decimal` instead here!
         return {
             self._atable[sym].lower(): float(bal)
             for sym, bal in by_bsuid.items()

piker/brokers/kraken/broker.py

@@ -21,7 +21,6 @@ Order api and machinery
 from collections import ChainMap, defaultdict
 from contextlib import (
     asynccontextmanager as acm,
-    contextmanager as cm,
 )
 from functools import partial
 from itertools import count
@@ -41,14 +40,18 @@ import pendulum
 import trio
 import tractor
 
-from piker.pp import (
+from piker.accounting import (
     Position,
     PpTable,
     Transaction,
     open_trade_ledger,
     open_pps,
+    get_likely_pair,
+)
+from piker.data._source import (
+    Symbol,
+    digits_to_dec,
 )
-from piker.data._source import Symbol
 from piker.clearing._messages import (
     Order,
     Status,
@@ -470,12 +473,14 @@ async def trades_dialogue(
         with (
             open_pps(
                 'kraken',
-                acctid
+                acctid,
+                write_on_exit=True,
+
             ) as table,
 
             open_trade_ledger(
                 'kraken',
-                acctid
+                acctid,
             ) as ledger_dict,
         ):
             # transaction-ify the ledger entries
@@ -494,7 +499,10 @@ async def trades_dialogue(
             # what amount of trades-transactions need
             # to be reloaded.
             balances = await client.get_balances()
+            # await tractor.breakpoint()
+
             for dst, size in balances.items():
+
                 # we don't care about tracking positions
                 # in the user's source fiat currency.
                 if (
@@ -508,45 +516,20 @@ async def trades_dialogue(
                     )
                     continue
 
-                def get_likely_pair(
-                    dst: str,
-                    bsuid: str,
-                    src_fiat: str = src_fiat
-
-                ) -> str:
-                    '''
-                    Attempt to get the likely trading pair masting
-                    a given destination asset `dst: str`.
-
-                    '''
-                    try:
-                        src_name_start = bsuid.rindex(src_fiat)
-                    except (
-                        ValueError,   # substr not found
-                    ):
-                        # TODO: handle nested positions..(i.e.
-                        # positions where the src fiat was used to
-                        # buy some other dst which was furhter used
-                        # to buy another dst..)
-                        log.warning(
-                            f'No src fiat {src_fiat} found in {bsuid}?'
-                        )
-                        return
-
-                    likely_dst = bsuid[:src_name_start]
-                    if likely_dst == dst:
-                        return bsuid
-
                 def has_pp(
                     dst: str,
                     size: float,
 
-                ) -> Position | bool:
+                ) -> Position | None:
 
                     src2dst: dict[str, str] = {}
 
                     for bsuid in table.pps:
-                        likely_pair = get_likely_pair(dst, bsuid)
+                        likely_pair = get_likely_pair(
+                            src_fiat,
+                            dst,
+                            bsuid,
+                        )
                         if likely_pair:
                             src2dst[src_fiat] = dst
 
@@ -574,7 +557,7 @@ async def trades_dialogue(
                             )
                             return pp
 
-                    return False
+                    return None  # signal no entry
 
                 pos = has_pp(dst, size)
                 if not pos:
@@ -602,7 +585,11 @@ async def trades_dialogue(
                             # yet and thus this likely pair grabber will
                             # likely fail.
                             for bsuid in table.pps:
-                                likely_pair = get_likely_pair(dst, bsuid)
+                                likely_pair = get_likely_pair(
+                                    src_fiat,
+                                    dst,
+                                    bsuid,
+                                )
                                 if likely_pair:
                                     break
                             else:
@@ -724,8 +711,8 @@ async def handle_order_updates(
     '''
     Main msg handling loop for all things order management.
 
-    This code is broken out to make the context explicit and state variables
-    defined in the signature clear to the reader.
+    This code is broken out to make the context explicit and state
+    variables defined in the signature clear to the reader.
 
     '''
     async for msg in ws_stream:
@@ -1204,7 +1191,13 @@ def norm_trade_records(
             fqsn,
             info={
                 'lot_size_digits': pair_info.lot_decimals,
+                'lot_tick_size': digits_to_dec(
+                    pair_info.lot_decimals,
+                ),
                 'tick_size_digits': pair_info.pair_decimals,
+                'price_tick_size': digits_to_dec(
+                    pair_info.pair_decimals,
+                ),
                 'asset_type': 'crypto',
             },
         )

piker/clearing/_allocate.py

@@ -25,7 +25,7 @@ from bidict import bidict
 
 from ..data._source import Symbol
 from ..data.types import Struct
-from ..pp import Position
+from ..accounting import Position
 
 
 _size_units = bidict({

piker/clearing/_paper_engine.py

@@ -39,7 +39,7 @@ import tractor
 from .. import data
 from ..data.types import Struct
 from ..data._source import Symbol
-from ..pp import (
+from ..accounting import (
     Position,
     Transaction,
     open_trade_ledger,
@@ -58,8 +58,6 @@ from ._messages import (
     BrokerdError,
 )
 
-from ..config import load
-
 log = get_logger(__name__)
 
 

piker/data/_source.py

@@ -90,6 +90,21 @@ def float_digits(
     return int(-Decimal(str(value)).as_tuple().exponent)
 
 
+def digits_to_dec(
+    ndigits: int,
+) -> Decimal:
+    '''
+    Return the minimum float value for an input integer value.
+
+    eg. 3 -> 0.001
+
+    '''
+    if ndigits == 0:
+        return Decimal('0')
+
+    return Decimal('0.' + '0'*(ndigits-1) + '1')
+
+
 def ohlc_zeros(length: int) -> np.ndarray:
     """Construct an OHLC field formatted structarray.
 
@@ -213,10 +228,13 @@ class Symbol(Struct):
 
         return Symbol(
             key=symbol,
+
             tick_size=tick_size,
             lot_tick_size=lot_size,
+
             tick_size_digits=float_digits(tick_size),
             lot_size_digits=float_digits(lot_size),
+
             suffix=suffix,
             broker_info={broker: info},
         )

piker/ui/_position.py

@@ -47,7 +47,7 @@ from ..calc import (
     puterize,
 )
 from ..clearing._allocate import Allocator
-from ..pp import Position
+from ..accounting import Position
 from ..data._normalize import iterticks
 from ..data.feed import (
     Feed,

piker/ui/order_mode.py

@@ -37,7 +37,7 @@ import trio
 from PyQt5.QtCore import Qt
 
 from .. import config
-from ..pp import Position
+from ..accounting import Position
 from ..clearing._client import open_ems, OrderBook
 from ..clearing._allocate import (
     mk_allocator,

tests/test_paper.py

@@ -16,7 +16,7 @@ from functools import partial
 
 from piker.log import get_logger
 from piker.clearing._messages import Order
-from piker.pp import (
+from piker.accounting import (
     open_pps,
 )