From 3a9bc8058fe7b18102107f34b0059b05de81afaf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 20 Aug 2022 17:49:06 -0400 Subject: [PATCH 1/8] Spawn a live account gateway alongside paper This is like, super first-draft-y (and ideally we move to offering an `piker.data._ahab` super for this) but, it's a start at allowing easy setup of both paper and live `ib-gw` container spawning. We Expect the user to input creds for the live account manually and the vnc server is (hackily) only run inside the paper instance which most of the time seems to make it possible click on the live gui window and input creds manually. We also add extra files for the live instance: - a `dockering/ib/run_x11_vnc_live.sh` which is is a blank script that avoids running an `x11vnc` server in the line account cntr. - a `dockering/ib/jts_live.ini` config which manually sets the live gw to use the `4001` port for api connections. Further config tweaks: - IBC: drop the api dynamic port override, decrease login display timeout to the riskier but likely to be faster 20s. - `x11vnc` cmd: got back to using `rfbport` instead of `autoport`, drop `-logappend` so we see logging on docker console again, drop the frame cacheing flags and add in some x-hack disable flags. --- dockering/ib/docker-compose.yml | 51 +++++++++++++++++++++++++++----- dockering/ib/ibc.ini | 4 +-- dockering/ib/jts_live.ini | 33 +++++++++++++++++++++ dockering/ib/run_x11_vnc.sh | 33 +++++++++++++++------ dockering/ib/run_x11_vnc_live.sh | 1 + 5 files changed, 103 insertions(+), 19 deletions(-) create mode 100644 dockering/ib/jts_live.ini create mode 100755 dockering/ib/run_x11_vnc_live.sh diff --git a/dockering/ib/docker-compose.yml b/dockering/ib/docker-compose.yml index de09119b..a5d13cc0 100644 --- a/dockering/ib/docker-compose.yml +++ b/dockering/ib/docker-compose.yml @@ -3,11 +3,12 @@ version: "3.5" services: - ib-gateway: + ib_gw_paper: # other image tags available: # https://github.com/waytrade/ib-gateway-docker#supported-tags - image: waytrade/ib-gateway:981.3j - restart: always + # image: waytrade/ib-gateway:981.3j + image: waytrade/ib-gateway:1012.2i + restart: always # restart whenev there's a crash or user clicsk network_mode: 'host' volumes: @@ -39,14 +40,12 @@ services: # this compose file which looks something like: # TWS_USERID='myuser' # TWS_PASSWORD='guest' - # TRADING_MODE=paper (or live) - # VNC_SERVER_PASSWORD='diggity' - environment: TWS_USERID: ${TWS_USERID} TWS_PASSWORD: ${TWS_PASSWORD} - TRADING_MODE: ${TRADING_MODE:-paper} - VNC_SERVER_PASSWORD: ${VNC_SERVER_PASSWORD:-} + TRADING_MODE: 'paper' + VNC_SERVER_PASSWORD: 'doggy' + VNC_SERVER_PORT: '3003' # ports: # - target: 4002 @@ -62,3 +61,39 @@ services: # - "127.0.0.1:4001:4001" # - "127.0.0.1:4002:4002" # - "127.0.0.1:5900:5900" + + ib_gw_live: + image: waytrade/ib-gateway:1012.2i + restart: always + network_mode: 'host' + + volumes: + - type: bind + source: ./jts_live.ini + target: /root/jts/jts.ini + # don't let ibc clobber this file for + # the main reason of not having a stupid + # timezone set.. + read_only: true + + # force our own ibc config + - type: bind + source: ./ibc.ini + target: /root/ibc/config.ini + + # force our noop script - socat isn't needed in host mode. + - type: bind + source: ./fork_ports_delayed.sh + target: /root/scripts/fork_ports_delayed.sh + + # force our noop script - socat isn't needed in host mode. + - type: bind + # source: ./run_x11_vnc.sh + source: ./run_x11_vnc_live.sh + target: /root/scripts/run_x11_vnc.sh + read_only: true + + # NOTE: to fill these out, define an `.env` file in the same dir as + # this compose file which looks something like: + environment: + TRADING_MODE: 'live' diff --git a/dockering/ib/ibc.ini b/dockering/ib/ibc.ini index adbb9843..c9c06658 100644 --- a/dockering/ib/ibc.ini +++ b/dockering/ib/ibc.ini @@ -188,7 +188,7 @@ AcceptNonBrokerageAccountWarning=yes # # The default value is 60. -LoginDialogDisplayTimeout = 60 +LoginDialogDisplayTimeout=20 @@ -292,7 +292,7 @@ ExistingSessionDetectedAction=primary # be set dynamically at run-time: most users will never need it, # so don't use it unless you know you need it. -OverrideTwsApiPort=4002 +; OverrideTwsApiPort=4002 # Read-only Login diff --git a/dockering/ib/jts_live.ini b/dockering/ib/jts_live.ini new file mode 100644 index 00000000..291a1e7d --- /dev/null +++ b/dockering/ib/jts_live.ini @@ -0,0 +1,33 @@ +[IBGateway] +ApiOnly=true +LocalServerPort=4001 +# NOTE: must be set if using IBC's "reject" mode +TrustedIPs=127.0.0.1 +; RemoteHostOrderRouting=ndc1.ibllc.com +; WriteDebug=true +; RemotePortOrderRouting=4001 +; useRemoteSettings=false +; tradingMode=p +; Steps=8 +; colorPalletName=dark + +# window geo, this may be useful for sending `xdotool` commands? +; MainWindow.Width=1986 +; screenHeight=3960 + + +[Logon] +Locale=en +# most markets are oriented around this zone +# so might as well hard code it. +TimeZone=America/New_York +UseSSL=true +displayedproxymsg=1 +os_titlebar=true +s3store=true +useRemoteSettings=false + +[Communication] +ctciAutoEncrypt=true +Region=usr +; Peer=cdc1.ibllc.com:4001 diff --git a/dockering/ib/run_x11_vnc.sh b/dockering/ib/run_x11_vnc.sh index 1005fb41..d7e2a658 100755 --- a/dockering/ib/run_x11_vnc.sh +++ b/dockering/ib/run_x11_vnc.sh @@ -1,20 +1,35 @@ #!/bin/sh +# start vnc server and listen for connections +# on port specced in `$VNC_SERVER_PORT` -# start VNC server x11vnc \ -listen 127.0.0.1 \ -allow 127.0.0.1 \ - -autoport 3003 \ - -no6 \ - -noipv6 \ + -rfbport "${VNC_SERVER_PORT}" \ -display :1 \ - -bg \ -forever \ -shared \ - -logappend /var/log/x11vnc.log \ - -ncache_cr \ - -ncache \ + -bg \ + -nowf \ + -noxdamage \ + -noxfixes \ - # can't use this because of ``asyncvnc`` issue: + + # -nowcr \ + # TODO: can't use this because of ``asyncvnc`` issue: # https://github.com/barneygale/asyncvnc/issues/1 # -passwd 'ibcansmbz' + + # XXX: optional graphics caching flags that seem to rekt the overlay + # of the 2 gw windows? When running a single gateway + # this seems to maybe optimize some memory usage? + # -ncache_cr \ + # -ncache \ + # -no6 \ + # -noipv6 \ + + # NOTE: this will prevent logs from going to the console. + # -logappend /var/log/x11vnc.log \ + + # where to start allocating ports + # -autoport "${VNC_SERVER_PORT}" \ diff --git a/dockering/ib/run_x11_vnc_live.sh b/dockering/ib/run_x11_vnc_live.sh new file mode 100755 index 00000000..1a248525 --- /dev/null +++ b/dockering/ib/run_x11_vnc_live.sh @@ -0,0 +1 @@ +#!/bin/sh From 38b190e59873171f80d25e5df452ea3ab8918c45 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 20 Aug 2022 18:59:53 -0400 Subject: [PATCH 2/8] Add `ib` `Crypto` contract support --- piker/brokers/ib/api.py | 7 +++++++ piker/brokers/ib/feed.py | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/piker/brokers/ib/api.py b/piker/brokers/ib/api.py index c18125f4..f5d61879 100644 --- a/piker/brokers/ib/api.py +++ b/piker/brokers/ib/api.py @@ -678,6 +678,13 @@ class Client: con = ibis.Commodity(**con_kwargs) con.bars_kwargs = bars_kwargs + # crypto$ + elif exch == 'PAXOS': # btc.paxos + con = ibis.Crypto( + symbol=symbol, + currency=currency, + ) + # stonks else: # TODO: metadata system for all these exchange rules.. diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 4ec214bc..294ebc99 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -426,6 +426,7 @@ asset_type_map = { 'WAR': 'warrant', 'IOPT': 'warran', 'BAG': 'bag', + 'CRYPTO': 'crypto', # bc it's diff then fiat? # 'NEWS': 'news', } @@ -722,7 +723,8 @@ async def stream_quotes( isnan(first_ticker.last) and type(first_ticker.contract) not in ( ibis.Commodity, - ibis.Forex + ibis.Forex, + ibis.Crypto, ) ): task_status.started((init_msgs, first_quote)) From 848e3453641a1f60933bc54d9279a34203b3dc51 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 21 Jul 2022 10:30:23 -0400 Subject: [PATCH 3/8] POC using paper-in-docker gw for symbol search --- piker/brokers/ib/feed.py | 44 +++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 294ebc99..85634c18 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -577,7 +577,6 @@ def normalize( # check for special contract types con = ticker.contract - fqsn, calc_price = con2fqsn(con) # convert named tuples to dicts so we send usable keys @@ -868,15 +867,24 @@ async def open_symbol_search( # TODO: load user defined symbol set locally for fast search? await ctx.started({}) - async with open_data_client() as proxy: + # async with open_data_client() as proxy: + async with ( + open_client_proxies() as (proxies, clients), + ): async with ctx.open_stream() as stream: + # await tractor.breakpoint() + proxy = proxies['ib.algopaper'] + last = time.time() async for pattern in stream: - log.debug(f'received {pattern}') + log.info(f'received {pattern}') now = time.time() + # this causes tractor hang... + # assert 0 + assert pattern, 'IB can not accept blank search pattern' # throttle search requests to no faster then 1Hz @@ -904,7 +912,7 @@ async def open_symbol_search( continue - log.debug(f'searching for {pattern}') + log.info(f'searching for {pattern}') last = time.time() @@ -915,17 +923,25 @@ async def open_symbol_search( async def stash_results(target: Awaitable[list]): stock_results.extend(await target) - async with trio.open_nursery() as sn: - sn.start_soon( - stash_results, - proxy.search_symbols( - pattern=pattern, - upto=5, - ), - ) + for i in range(10): + with trio.move_on_after(3) as cs: + async with trio.open_nursery() as sn: + sn.start_soon( + stash_results, + proxy.search_symbols( + pattern=pattern, + upto=5, + ), + ) - # trigger async request - await trio.sleep(0) + # trigger async request + await trio.sleep(0) + + if cs.cancelled_caught: + log.warning(f'Search timeout? {proxy._aio_ns.ib.client}') + continue + else: + break # # match against our ad-hoc set immediately # adhoc_matches = fuzzy.extractBests( From de9f215c832859a286c68f9a0566e5393e07fb3e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 22 Aug 2022 17:52:47 -0400 Subject: [PATCH 4/8] If more then one `ib` api client is available use next available for search --- piker/brokers/ib/feed.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/piker/brokers/ib/feed.py b/piker/brokers/ib/feed.py index 85634c18..18981f60 100644 --- a/piker/brokers/ib/feed.py +++ b/piker/brokers/ib/feed.py @@ -867,17 +867,24 @@ async def open_symbol_search( # TODO: load user defined symbol set locally for fast search? await ctx.started({}) - # async with open_data_client() as proxy: async with ( open_client_proxies() as (proxies, clients), + open_data_client() as data_proxy, ): async with ctx.open_stream() as stream: - # await tractor.breakpoint() - proxy = proxies['ib.algopaper'] + # select a non-history client for symbol search to lighten + # the load in the main data node. + proxy = data_proxy + for name, proxy in proxies.items(): + if proxy is data_proxy: + continue + break + + ib_client = proxy._aio_ns.ib + log.info(f'Using {ib_client} for symbol search') last = time.time() - async for pattern in stream: log.info(f'received {pattern}') now = time.time() @@ -938,7 +945,9 @@ async def open_symbol_search( await trio.sleep(0) if cs.cancelled_caught: - log.warning(f'Search timeout? {proxy._aio_ns.ib.client}') + log.warning( + f'Search timeout? {proxy._aio_ns.ib.client}' + ) continue else: break From 75a5f3795a16bf3b1cbc19c18ac1adac15dd8002 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 24 Aug 2022 15:27:44 -0400 Subject: [PATCH 5/8] I guess go back to doing vnc servers on both? --- dockering/ib/docker-compose.yml | 6 ++++-- dockering/ib/run_x11_vnc.sh | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dockering/ib/docker-compose.yml b/dockering/ib/docker-compose.yml index a5d13cc0..c058a367 100644 --- a/dockering/ib/docker-compose.yml +++ b/dockering/ib/docker-compose.yml @@ -88,8 +88,8 @@ services: # force our noop script - socat isn't needed in host mode. - type: bind - # source: ./run_x11_vnc.sh - source: ./run_x11_vnc_live.sh + source: ./run_x11_vnc.sh + # source: ./run_x11_vnc_live.sh target: /root/scripts/run_x11_vnc.sh read_only: true @@ -97,3 +97,5 @@ services: # this compose file which looks something like: environment: TRADING_MODE: 'live' + VNC_SERVER_PASSWORD: 'doggy' + VNC_SERVER_PORT: '3004' diff --git a/dockering/ib/run_x11_vnc.sh b/dockering/ib/run_x11_vnc.sh index d7e2a658..f74f95d5 100755 --- a/dockering/ib/run_x11_vnc.sh +++ b/dockering/ib/run_x11_vnc.sh @@ -13,6 +13,8 @@ x11vnc \ -nowf \ -noxdamage \ -noxfixes \ + -no6 \ + -noipv6 \ # -nowcr \ @@ -25,8 +27,6 @@ x11vnc \ # this seems to maybe optimize some memory usage? # -ncache_cr \ # -ncache \ - # -no6 \ - # -noipv6 \ # NOTE: this will prevent logs from going to the console. # -logappend /var/log/x11vnc.log \ From 7f224f0342c9c2fb1195fbba6e844c7e35e803d2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Aug 2022 18:48:35 -0400 Subject: [PATCH 6/8] Doc string typos --- piker/ui/_overlay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_overlay.py b/piker/ui/_overlay.py index 65ec2364..0bbba413 100644 --- a/piker/ui/_overlay.py +++ b/piker/ui/_overlay.py @@ -80,8 +80,8 @@ class ComposedGridLayout: ``i`` in the layout. The ``item: PlotItem`` passed to the constructor's grid layout is - used verbatim as the "main plot" who's view box is give precedence - for input handling. The main plot's axes are removed from it's + used verbatim as the "main plot" who's view box is given precedence + for input handling. The main plot's axes are removed from its layout and placed in the surrounding exterior layouts to allow for re-ordering if desired. From f9217570ab108eed1f23611aec60bf4d4fc71371 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 31 Aug 2022 17:38:24 -0400 Subject: [PATCH 7/8] Add intiial `ib` backend readme --- piker/brokers/ib/README.rst | 134 ++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 piker/brokers/ib/README.rst diff --git a/piker/brokers/ib/README.rst b/piker/brokers/ib/README.rst new file mode 100644 index 00000000..c8661317 --- /dev/null +++ b/piker/brokers/ib/README.rst @@ -0,0 +1,134 @@ +``ib`` backend +-------------- +more or less the "everything broker" for traditional and international +markets. they are the "go to" provider for automatic retail trading +and we interface to their APIs using the `ib_insync` project. + +status +****** +current support is *production grade* and both real-time data and order +management should be correct and fast. this backend is used by core devs +for live trading. + +currently there is not yet full support for: +- options charting and trading +- paxos based crypto rt feeds and trading + + +config +****** +In order to get order mode support your ``brokers.toml`` +needs to have something like the following: + +.. code:: toml + + [ib] + hosts = [ + "127.0.0.1", + ] + # TODO: when we eventually spawn gateways in our + # container, we can just dynamically allocate these + # using IBC. + ports = [ + 4002, + 4003, + 4006, + 4001, + 7497, + ] + + # XXX: for a paper account the flex web query service + # is not supported so you have to manually download + # and XML report and put it in a location that can be + # accessed by the ``brokerd.ib`` backend code for parsing. + flex_token = '1111111111111111' + flex_trades_query_id = '6969696' # live accounts only? + + # 3rd party web-api token + # (XXX: not sure if this works yet) + trade_log_token = '111111111111111' + + # when clients are being scanned this determines + # which clients are preferred to be used for data feeds + # based on account names which are detected as active + # on each client. + prefer_data_account = [ + # this has to be first in order to make data work with dual paper + live + 'main', + 'algopaper', + ] + + [ib.accounts] + main = 'U69696969' + algopaper = 'DU9696969' + + +If everything works correctly you should see any current positions +loaded in the pps pane on chart load and you should also be able to +check your trade records in the file:: + + /ledgers/trades_ib_algopaper.toml + + +An example ledger file will have entries written verbatim from the +trade events schema: + +.. code:: toml + + ["0000e1a7.630f5e5a.01.01"] + secType = "FUT" + conId = 515416577 + symbol = "MNQ" + lastTradeDateOrContractMonth = "20221216" + strike = 0.0 + right = "" + multiplier = "2" + exchange = "GLOBEX" + primaryExchange = "" + currency = "USD" + localSymbol = "MNQZ2" + tradingClass = "MNQ" + includeExpired = false + secIdType = "" + secId = "" + comboLegsDescrip = "" + comboLegs = [] + execId = "0000e1a7.630f5e5a.01.01" + time = 1661972086.0 + acctNumber = "DU69696969" + side = "BOT" + shares = 1.0 + price = 12372.75 + permId = 441472655 + clientId = 6116 + orderId = 985 + liquidation = 0 + cumQty = 1.0 + avgPrice = 12372.75 + orderRef = "" + evRule = "" + evMultiplier = 0.0 + modelCode = "" + lastLiquidity = 1 + broker_time = 1661972086.0 + name = "ib" + commission = 0.57 + realizedPNL = 243.41 + yield_ = 0.0 + yieldRedemptionDate = 0 + listingExchange = "GLOBEX" + date = "2022-08-31T18:54:46+00:00" + + +your ``pps.toml`` file will have position entries like, + +.. code:: toml + + [ib.algopaper."mnq.globex.20221216"] + size = -1.0 + ppu = 12423.630576923071 + bsuid = 515416577 + expiry = "2022-12-16T00:00:00+00:00" + clears = [ + { dt = "2022-08-31T18:54:46+00:00", ppu = 12423.630576923071, accum_size = -19.0, price = 12372.75, size = 1.0, cost = 0.57, tid = "0000e1a7.630f5e5a.01.01" }, + ] From 653f5c824b6e1bba6145de55b0f4acf1662ff435 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 31 Aug 2022 17:45:02 -0400 Subject: [PATCH 8/8] Drop empty vnc server script idea for live account --- dockering/ib/docker-compose.yml | 1 - dockering/ib/run_x11_vnc_live.sh | 1 - 2 files changed, 2 deletions(-) delete mode 100755 dockering/ib/run_x11_vnc_live.sh diff --git a/dockering/ib/docker-compose.yml b/dockering/ib/docker-compose.yml index c058a367..f3a28d66 100644 --- a/dockering/ib/docker-compose.yml +++ b/dockering/ib/docker-compose.yml @@ -89,7 +89,6 @@ services: # force our noop script - socat isn't needed in host mode. - type: bind source: ./run_x11_vnc.sh - # source: ./run_x11_vnc_live.sh target: /root/scripts/run_x11_vnc.sh read_only: true diff --git a/dockering/ib/run_x11_vnc_live.sh b/dockering/ib/run_x11_vnc_live.sh deleted file mode 100755 index 1a248525..00000000 --- a/dockering/ib/run_x11_vnc_live.sh +++ /dev/null @@ -1 +0,0 @@ -#!/bin/sh