Merge pull request #367 from pikers/livenpaper

`ib`: live & paper accounts together, infra refinements

dockering/ib/docker-compose.yml

@@ -3,11 +3,12 @@
 version: "3.5"
 
 services:
-  ib-gateway:
+  ib_gw_paper:
     # other image tags available:
     # https://github.com/waytrade/ib-gateway-docker#supported-tags
-    image: waytrade/ib-gateway:981.3j
+    # image: waytrade/ib-gateway:981.3j
-    restart: always
+    image: waytrade/ib-gateway:1012.2i
+    restart: always  # restart whenev there's a crash or user clicsk
     network_mode: 'host'
 
     volumes:
@@ -39,14 +40,12 @@ services:
     # this compose file which looks something like:
     # TWS_USERID='myuser'
     # TWS_PASSWORD='guest'
-    # TRADING_MODE=paper (or live)
-    # VNC_SERVER_PASSWORD='diggity'
-
     environment:
       TWS_USERID: ${TWS_USERID}
       TWS_PASSWORD: ${TWS_PASSWORD}
-      TRADING_MODE: ${TRADING_MODE:-paper}
+      TRADING_MODE: 'paper'
-      VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
+      VNC_SERVER_PASSWORD: 'doggy'
+      VNC_SERVER_PORT: '3003'
 
     # ports:
     #   - target: 4002
@@ -62,3 +61,40 @@ services:
       # - "127.0.0.1:4001:4001"
       # - "127.0.0.1:4002:4002"
       # - "127.0.0.1:5900:5900"
+
+  ib_gw_live:
+    image: waytrade/ib-gateway:1012.2i
+    restart: always
+    network_mode: 'host'
+
+    volumes:
+      - type: bind
+        source: ./jts_live.ini
+        target: /root/jts/jts.ini
+        # don't let ibc clobber this file for
+        # the main reason of not having a stupid
+        # timezone set..
+        read_only: true
+
+      # force our own ibc config
+      - type: bind
+        source: ./ibc.ini
+        target: /root/ibc/config.ini
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        source: ./fork_ports_delayed.sh
+        target: /root/scripts/fork_ports_delayed.sh
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        source: ./run_x11_vnc.sh
+        target: /root/scripts/run_x11_vnc.sh
+        read_only: true
+
+    # NOTE: to fill these out, define an `.env` file in the same dir as
+    # this compose file which looks something like:
+    environment:
+      TRADING_MODE: 'live'
+      VNC_SERVER_PASSWORD: 'doggy'
+      VNC_SERVER_PORT: '3004'

dockering/ib/ibc.ini

@@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes
 #
 # The default value is 60.
 
-LoginDialogDisplayTimeout = 60
+LoginDialogDisplayTimeout=20
 
 
 
@@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary
 # be set dynamically at run-time: most users will never need it,
 # so don't use it unless you know you need it.
 
-OverrideTwsApiPort=4002
+; OverrideTwsApiPort=4002
 
 
 # Read-only Login

dockering/ib/jts_live.ini

@@ -0,0 +1,33 @@
+[IBGateway]
+ApiOnly=true
+LocalServerPort=4001
+# NOTE: must be set if using IBC's "reject" mode
+TrustedIPs=127.0.0.1
+; RemoteHostOrderRouting=ndc1.ibllc.com
+; WriteDebug=true
+; RemotePortOrderRouting=4001
+; useRemoteSettings=false
+; tradingMode=p
+; Steps=8
+; colorPalletName=dark
+
+# window geo, this may be useful for sending `xdotool` commands?
+; MainWindow.Width=1986
+; screenHeight=3960
+
+
+[Logon]
+Locale=en
+# most markets are oriented around this zone
+# so might as well hard code it.
+TimeZone=America/New_York
+UseSSL=true
+displayedproxymsg=1
+os_titlebar=true
+s3store=true
+useRemoteSettings=false
+
+[Communication]
+ctciAutoEncrypt=true
+Region=usr
+; Peer=cdc1.ibllc.com:4001

dockering/ib/run_x11_vnc.sh

@@ -1,20 +1,35 @@
 #!/bin/sh
+# start vnc server and listen for connections
+# on port specced in `$VNC_SERVER_PORT`
 
-# start VNC server
 x11vnc \
     -listen 127.0.0.1 \
     -allow 127.0.0.1 \
-    -autoport 3003 \
+    -rfbport "${VNC_SERVER_PORT}" \
-    -no6 \
-    -noipv6 \
     -display :1 \
-    -bg \
     -forever \
     -shared \
-    -logappend /var/log/x11vnc.log \
+    -bg \
-    -ncache_cr \
+    -nowf \
-    -ncache \
+    -noxdamage \
+    -noxfixes \
+    -no6 \
+    -noipv6 \
 
-    # can't use this because of ``asyncvnc`` issue:
+
+    # -nowcr \
+    # TODO: can't use this because of ``asyncvnc`` issue:
     # https://github.com/barneygale/asyncvnc/issues/1
     # -passwd 'ibcansmbz'
+
+    # XXX: optional graphics caching flags that seem to rekt the overlay
+    # of the 2 gw windows? When running a single gateway
+    # this seems to maybe optimize some memory usage?
+    # -ncache_cr \
+    # -ncache \
+
+    # NOTE: this will prevent logs from going to the console.
+    # -logappend /var/log/x11vnc.log \
+
+    # where to start allocating ports
+    # -autoport "${VNC_SERVER_PORT}" \

piker/brokers/ib/README.rst

@@ -0,0 +1,134 @@
+``ib`` backend
+--------------
+more or less the "everything broker" for traditional and international
+markets. they are the "go to" provider for automatic retail trading
+and we interface to their APIs using the `ib_insync` project.
+
+status
+******
+current support is *production grade* and both real-time data and order
+management should be correct and fast. this backend is used by core devs
+for live trading.
+
+currently there is not yet full support for:
+- options charting and trading
+- paxos based crypto rt feeds and trading
+
+
+config
+******
+In order to get order mode support your ``brokers.toml``
+needs to have something like the following:
+
+.. code:: toml
+
+   [ib]
+   hosts = [
+    "127.0.0.1",
+   ]
+   # TODO: when we eventually spawn gateways in our
+   # container, we can just dynamically allocate these
+   # using IBC.
+   ports = [
+       4002,
+       4003,
+       4006,
+       4001,
+       7497,
+   ]
+
+   # XXX: for a paper account the flex web query service
+   # is not supported so you have to manually download
+   # and XML report and put it in a location that can be
+   # accessed by the ``brokerd.ib`` backend code for parsing.
+   flex_token = '1111111111111111'
+   flex_trades_query_id = '6969696'  # live accounts only?
+
+   # 3rd party web-api token
+   # (XXX: not sure if this works yet)
+   trade_log_token = '111111111111111'
+
+   # when clients are being scanned this determines
+   # which clients are preferred to be used for data feeds
+   # based on account names which are detected as active
+   # on each client.
+   prefer_data_account = [
+       # this has to be first in order to make data work with dual paper + live
+       'main',
+       'algopaper',
+   ]
+
+   [ib.accounts]
+   main = 'U69696969'
+   algopaper = 'DU9696969'
+
+
+If everything works correctly you should see any current positions
+loaded in the pps pane on chart load and you should also be able to
+check your trade records in the file::
+
+    <pikerk_conf_dir>/ledgers/trades_ib_algopaper.toml
+
+
+An example ledger file will have entries written verbatim from the
+trade events schema:
+
+.. code:: toml
+
+    ["0000e1a7.630f5e5a.01.01"]
+    secType = "FUT"
+    conId = 515416577
+    symbol = "MNQ"
+    lastTradeDateOrContractMonth = "20221216"
+    strike = 0.0
+    right = ""
+    multiplier = "2"
+    exchange = "GLOBEX"
+    primaryExchange = ""
+    currency = "USD"
+    localSymbol = "MNQZ2"
+    tradingClass = "MNQ"
+    includeExpired = false
+    secIdType = ""
+    secId = ""
+    comboLegsDescrip = ""
+    comboLegs = []
+    execId = "0000e1a7.630f5e5a.01.01"
+    time = 1661972086.0
+    acctNumber = "DU69696969"
+    side = "BOT"
+    shares = 1.0
+    price = 12372.75
+    permId = 441472655
+    clientId = 6116
+    orderId = 985
+    liquidation = 0
+    cumQty = 1.0
+    avgPrice = 12372.75
+    orderRef = ""
+    evRule = ""
+    evMultiplier = 0.0
+    modelCode = ""
+    lastLiquidity = 1
+    broker_time = 1661972086.0
+    name = "ib"
+    commission = 0.57
+    realizedPNL = 243.41
+    yield_ = 0.0
+    yieldRedemptionDate = 0
+    listingExchange = "GLOBEX"
+    date = "2022-08-31T18:54:46+00:00"
+
+
+your ``pps.toml`` file will have position entries like,
+
+.. code:: toml
+
+    [ib.algopaper."mnq.globex.20221216"]
+    size = -1.0
+    ppu = 12423.630576923071
+    bsuid = 515416577
+    expiry = "2022-12-16T00:00:00+00:00"
+    clears = [
+     { dt = "2022-08-31T18:54:46+00:00", ppu = 12423.630576923071, accum_size = -19.0, price = 12372.75, size = 1.0, cost = 0.57, tid = "0000e1a7.630f5e5a.01.01" },
+    ]

piker/brokers/ib/api.py

@@ -678,6 +678,13 @@ class Client:
             con = ibis.Commodity(**con_kwargs)
             con.bars_kwargs = bars_kwargs
 
+        # crypto$
+        elif exch == 'PAXOS':  # btc.paxos
+            con = ibis.Crypto(
+                symbol=symbol,
+                currency=currency,
+            )
+
         # stonks
         else:
             # TODO: metadata system for all these exchange rules..

piker/brokers/ib/feed.py

@@ -426,6 +426,7 @@ asset_type_map = {
     'WAR': 'warrant',
     'IOPT': 'warran',
     'BAG': 'bag',
+    'CRYPTO': 'crypto',  # bc it's diff then fiat?
     # 'NEWS': 'news',
 }
 
@@ -576,7 +577,6 @@ def normalize(
 
     # check for special contract types
     con = ticker.contract
-
     fqsn, calc_price = con2fqsn(con)
 
     # convert named tuples to dicts so we send usable keys
@@ -722,7 +722,8 @@ async def stream_quotes(
             isnan(first_ticker.last)
             and type(first_ticker.contract) not in (
                 ibis.Commodity,
-                ibis.Forex
+                ibis.Forex,
+                ibis.Crypto,
             )
         ):
             task_status.started((init_msgs, first_quote))
@@ -866,15 +867,31 @@ async def open_symbol_search(
     # TODO: load user defined symbol set locally for fast search?
     await ctx.started({})
 
-    async with open_data_client() as proxy:
+    async with (
+        open_client_proxies() as (proxies, clients),
+        open_data_client() as data_proxy,
+    ):
         async with ctx.open_stream() as stream:
 
-            last = time.time()
+            # select a non-history client for symbol search to lighten
+            # the load in the main data node.
+            proxy = data_proxy
+            for name, proxy in proxies.items():
+                if proxy is data_proxy:
+                    continue
+                break
 
+            ib_client = proxy._aio_ns.ib
+            log.info(f'Using {ib_client} for symbol search')
+
+            last = time.time()
             async for pattern in stream:
-                log.debug(f'received {pattern}')
+                log.info(f'received {pattern}')
                 now = time.time()
 
+                # this causes tractor hang...
+                # assert 0
+
                 assert pattern, 'IB can not accept blank search pattern'
 
                 # throttle search requests to no faster then 1Hz
@@ -902,7 +919,7 @@ async def open_symbol_search(
 
                     continue
 
-                log.debug(f'searching for {pattern}')
+                log.info(f'searching for {pattern}')
 
                 last = time.time()
 
@@ -913,17 +930,27 @@ async def open_symbol_search(
                 async def stash_results(target: Awaitable[list]):
                     stock_results.extend(await target)
 
-                async with trio.open_nursery() as sn:
+                for i in range(10):
-                    sn.start_soon(
+                    with trio.move_on_after(3) as cs:
-                        stash_results,
+                        async with trio.open_nursery() as sn:
-                        proxy.search_symbols(
+                            sn.start_soon(
-                            pattern=pattern,
+                                stash_results,
-                            upto=5,
+                                proxy.search_symbols(
-                        ),
+                                    pattern=pattern,
-                    )
+                                    upto=5,
+                                ),
+                            )
 
-                    # trigger async request
+                            # trigger async request
-                    await trio.sleep(0)
+                            await trio.sleep(0)
+
+                    if cs.cancelled_caught:
+                        log.warning(
+                            f'Search timeout? {proxy._aio_ns.ib.client}'
+                        )
+                        continue
+                    else:
+                        break
 
                     # # match against our ad-hoc set immediately
                     # adhoc_matches = fuzzy.extractBests(

piker/ui/_overlay.py

@@ -80,8 +80,8 @@ class ComposedGridLayout:
     ``<axis_name>i`` in the layout.
 
     The ``item: PlotItem`` passed to the constructor's grid layout is
-    used verbatim as the "main plot" who's view box is give precedence
+    used verbatim as the "main plot" who's view box is given precedence
-    for input handling. The main plot's axes are removed from it's
+    for input handling. The main plot's axes are removed from its
     layout and placed in the surrounding exterior layouts to allow for
     re-ordering if desired.