POC using paper-in-docker gw for symbol search

This is like, super first-draft-y (and ideally we move to offering an
`piker.data._ahab` super for this) but, it's a start at allowing easy
setup of both paper and live `ib-gw` container spawning. We Expect the
user to input creds for the live account manually and the vnc server is
(hackily) only run inside the paper instance which most of the time
seems to make it possible click on the live gui window and input creds
manually.

We also add extra files for the live instance:
- a `dockering/ib/run_x11_vnc_live.sh` which is is a blank script
  that avoids running an `x11vnc` server in the line account cntr.
- a `dockering/ib/jts_live.ini` config which manually sets the live
  gw to use the `4001` port for api connections.

Further config tweaks:
- IBC: drop the api dynamic port override, decrease login display
  timeout to the riskier but likely to be faster 20s.
- `x11vnc` cmd: got back to using `rfbport` instead of `autoport`, drop
  `-logappend` so we see logging on docker console again, drop the
  frame cacheing flags and add in some x-hack disable flags.

dockering/ib/docker-compose.yml

@@ -3,11 +3,12 @@
 version: "3.5"
 
 services:
-  ib-gateway:
+  ib_gw_paper:
     # other image tags available:
     # https://github.com/waytrade/ib-gateway-docker#supported-tags
-    image: waytrade/ib-gateway:981.3j
+    # image: waytrade/ib-gateway:981.3j
-    restart: always
+    image: waytrade/ib-gateway:1012.2i
+    restart: always  # restart whenev there's a crash or user clicsk
     network_mode: 'host'
 
     volumes:
@@ -39,14 +40,12 @@ services:
     # this compose file which looks something like:
     # TWS_USERID='myuser'
     # TWS_PASSWORD='guest'
-    # TRADING_MODE=paper (or live)
-    # VNC_SERVER_PASSWORD='diggity'
-
     environment:
       TWS_USERID: ${TWS_USERID}
       TWS_PASSWORD: ${TWS_PASSWORD}
-      TRADING_MODE: ${TRADING_MODE:-paper}
+      TRADING_MODE: 'paper'
-      VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-}
+      VNC_SERVER_PASSWORD: 'doggy'
+      VNC_SERVER_PORT: '3003'
 
     # ports:
     #   - target: 4002
@@ -62,3 +61,39 @@ services:
       # - "127.0.0.1:4001:4001"
       # - "127.0.0.1:4002:4002"
       # - "127.0.0.1:5900:5900"
+
+  ib_gw_live:
+    image: waytrade/ib-gateway:1012.2i
+    restart: always
+    network_mode: 'host'
+
+    volumes:
+      - type: bind
+        source: ./jts_live.ini
+        target: /root/jts/jts.ini
+        # don't let ibc clobber this file for
+        # the main reason of not having a stupid
+        # timezone set..
+        read_only: true
+
+      # force our own ibc config
+      - type: bind
+        source: ./ibc.ini
+        target: /root/ibc/config.ini
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        source: ./fork_ports_delayed.sh
+        target: /root/scripts/fork_ports_delayed.sh
+
+      # force our noop script - socat isn't needed in host mode.
+      - type: bind
+        # source: ./run_x11_vnc.sh
+        source: ./run_x11_vnc_live.sh
+        target: /root/scripts/run_x11_vnc.sh
+        read_only: true
+
+    # NOTE: to fill these out, define an `.env` file in the same dir as
+    # this compose file which looks something like:
+    environment:
+      TRADING_MODE: 'live'

dockering/ib/ibc.ini

@@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes
 #
 # The default value is 60.
 
-LoginDialogDisplayTimeout = 60
+LoginDialogDisplayTimeout=20
 
 
 
@@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary
 # be set dynamically at run-time: most users will never need it,
 # so don't use it unless you know you need it.
 
-OverrideTwsApiPort=4002
+; OverrideTwsApiPort=4002
 
 
 # Read-only Login

dockering/ib/jts_live.ini

@@ -0,0 +1,33 @@
+[IBGateway]
+ApiOnly=true
+LocalServerPort=4001
+# NOTE: must be set if using IBC's "reject" mode
+TrustedIPs=127.0.0.1
+; RemoteHostOrderRouting=ndc1.ibllc.com
+; WriteDebug=true
+; RemotePortOrderRouting=4001
+; useRemoteSettings=false
+; tradingMode=p
+; Steps=8
+; colorPalletName=dark
+
+# window geo, this may be useful for sending `xdotool` commands?
+; MainWindow.Width=1986
+; screenHeight=3960
+
+
+[Logon]
+Locale=en
+# most markets are oriented around this zone
+# so might as well hard code it.
+TimeZone=America/New_York
+UseSSL=true
+displayedproxymsg=1
+os_titlebar=true
+s3store=true
+useRemoteSettings=false
+
+[Communication]
+ctciAutoEncrypt=true
+Region=usr
+; Peer=cdc1.ibllc.com:4001

dockering/ib/run_x11_vnc.sh

@@ -1,20 +1,35 @@
 #!/bin/sh
+# start vnc server and listen for connections
+# on port specced in `$VNC_SERVER_PORT`
 
-# start VNC server
 x11vnc \
     -listen 127.0.0.1 \
     -allow 127.0.0.1 \
-    -autoport 3003 \
+    -rfbport "${VNC_SERVER_PORT}" \
-    -no6 \
-    -noipv6 \
     -display :1 \
-    -bg \
     -forever \
     -shared \
-    -logappend /var/log/x11vnc.log \
+    -bg \
-    -ncache_cr \
+    -nowf \
-    -ncache \
+    -noxdamage \
+    -noxfixes \
 
-    # can't use this because of ``asyncvnc`` issue:
+
+    # -nowcr \
+    # TODO: can't use this because of ``asyncvnc`` issue:
     # https://github.com/barneygale/asyncvnc/issues/1
     # -passwd 'ibcansmbz'
+
+    # XXX: optional graphics caching flags that seem to rekt the overlay
+    # of the 2 gw windows? When running a single gateway
+    # this seems to maybe optimize some memory usage?
+    # -ncache_cr \
+    # -ncache \
+    # -no6 \
+    # -noipv6 \
+
+    # NOTE: this will prevent logs from going to the console.
+    # -logappend /var/log/x11vnc.log \
+
+    # where to start allocating ports
+    # -autoport "${VNC_SERVER_PORT}" \

dockering/ib/run_x11_vnc_live.sh

@@ -0,0 +1 @@
+#!/bin/sh

piker/brokers/ib/api.py

@@ -678,6 +678,13 @@ class Client:
             con = ibis.Commodity(**con_kwargs)
             con.bars_kwargs = bars_kwargs
 
+        # crypto$
+        elif exch == 'PAXOS':  # btc.paxos
+            con = ibis.Crypto(
+                symbol=symbol,
+                currency=currency,
+            )
+
         # stonks
         else:
             # TODO: metadata system for all these exchange rules..

piker/brokers/ib/feed.py

@@ -426,6 +426,7 @@ asset_type_map = {
     'WAR': 'warrant',
     'IOPT': 'warran',
     'BAG': 'bag',
+    'CRYPTO': 'crypto',  # bc it's diff then fiat?
     # 'NEWS': 'news',
 }
 
@@ -576,7 +577,6 @@ def normalize(
 
     # check for special contract types
     con = ticker.contract
-
     fqsn, calc_price = con2fqsn(con)
 
     # convert named tuples to dicts so we send usable keys
@@ -722,7 +722,8 @@ async def stream_quotes(
             isnan(first_ticker.last)
             and type(first_ticker.contract) not in (
                 ibis.Commodity,
-                ibis.Forex
+                ibis.Forex,
+                ibis.Crypto,
             )
         ):
             task_status.started((init_msgs, first_quote))
@@ -866,15 +867,24 @@ async def open_symbol_search(
     # TODO: load user defined symbol set locally for fast search?
     await ctx.started({})
 
-    async with open_data_client() as proxy:
+    # async with open_data_client() as proxy:
+    async with (
+        open_client_proxies() as (proxies, clients),
+    ):
         async with ctx.open_stream() as stream:
 
+            # await tractor.breakpoint()
+            proxy = proxies['ib.algopaper']
+
             last = time.time()
 
             async for pattern in stream:
-                log.debug(f'received {pattern}')
+                log.info(f'received {pattern}')
                 now = time.time()
 
+                # this causes tractor hang...
+                # assert 0
+
                 assert pattern, 'IB can not accept blank search pattern'
 
                 # throttle search requests to no faster then 1Hz
@@ -902,7 +912,7 @@ async def open_symbol_search(
 
                     continue
 
-                log.debug(f'searching for {pattern}')
+                log.info(f'searching for {pattern}')
 
                 last = time.time()
 
@@ -913,17 +923,25 @@ async def open_symbol_search(
                 async def stash_results(target: Awaitable[list]):
                     stock_results.extend(await target)
 
-                async with trio.open_nursery() as sn:
+                for i in range(10):
-                    sn.start_soon(
+                    with trio.move_on_after(3) as cs:
-                        stash_results,
+                        async with trio.open_nursery() as sn:
-                        proxy.search_symbols(
+                            sn.start_soon(
-                            pattern=pattern,
+                                stash_results,
-                            upto=5,
+                                proxy.search_symbols(
-                        ),
+                                    pattern=pattern,
-                    )
+                                    upto=5,
+                                ),
+                            )
 
-                    # trigger async request
+                            # trigger async request
-                    await trio.sleep(0)
+                            await trio.sleep(0)
+
+                    if cs.cancelled_caught:
+                        log.warning(f'Search timeout? {proxy._aio_ns.ib.client}')
+                        continue
+                    else:
+                        break
 
                     # # match against our ad-hoc set immediately
                     # adhoc_matches = fuzzy.extractBests(