Variety of IB backend improvements

- Move to new shared mem system only writing on the first (by process)
  entry to `stream_quotes()`.
- Deliver bars before first quote arrives so that chart can populate and
  then wait for initial arrival.
- Allow caching clients per actor.
- Load bars using the same (cached) client that starts the quote stream
  thus speeding up initialization.

piker/brokers/ib.py

@@ -26,8 +26,10 @@ import tractor
 
 from ..log import get_logger, get_console_log
 from ..data import (
-    maybe_spawn_brokerd, iterticks, attach_shared_array,
+    maybe_spawn_brokerd,
-    incr_buffer,
+    iterticks,
+    attach_shm_array,
+    get_shm_token
 )
 from ..ui._source import from_df
 
@@ -145,7 +147,7 @@ class Client:
             # durationStr='1 D',
 
             # time length calcs
-            durationStr='{count} S'.format(count=5000 * 5),
+            durationStr='{count} S'.format(count=1000 * 5),
             barSizeSetting='5 secs',
 
             # always use extended hours
@@ -311,6 +313,8 @@ class Client:
 _tws_port: int = 7497
 _gw_port: int = 4002
 _try_ports = [_tws_port, _gw_port]
+_client_ids = itertools.count()
+_client_cache = {}
 
 
 @asynccontextmanager
@@ -321,17 +325,18 @@ async def _aio_get_client(
 ) -> Client:
     """Return an ``ib_insync.IB`` instance wrapped in our client API.
     """
+    # first check cache for existing client
+
+    try:
+        yield _client_cache[(host, port)]
+    except KeyError:
+        # TODO: in case the arbiter has no record
+        # of existing brokerd we need to broadcast for one.
+
         if client_id is None:
             # if this is a persistent brokerd, try to allocate a new id for
             # each client
-        try:
+            client_id = next(_client_ids)
-            ss = tractor.current_actor().statespace
-            client_id = next(ss.setdefault('client_ids', itertools.count()))
-            # TODO: in case the arbiter has no record
-            # of existing brokerd we need to broadcase for one.
-        except RuntimeError:
-            # tractor likely isn't running
-            client_id = 1
 
         ib = NonShittyIB()
         ports = _try_ports if port is None else [port]
@@ -347,7 +352,9 @@ async def _aio_get_client(
             raise ConnectionRefusedError(_err)
 
         try:
-        yield Client(ib)
+            client = Client(ib)
+            _client_cache[(host, port)] = client
+            yield client
         except BaseException:
             ib.disconnect()
             raise
@@ -489,7 +496,7 @@ def normalize(
 # @tractor.msg.pub
 async def stream_quotes(
     symbols: List[str],
-    shared_array_token: Tuple[str, str],
+    shm_token: Tuple[str, str, List[tuple]],
     loglevel: str = None,
     # compat for @tractor.msg.pub
     topics: Any = None,
@@ -506,16 +513,35 @@ async def stream_quotes(
     # TODO: support multiple subscriptions
     sym = symbols[0]
 
-    stream = await tractor.to_asyncio.run_task(
+    stream = await _trio_run_client_method(
-        _trio_run_client_method,
         method='stream_ticker',
         symbol=sym,
     )
 
-    async with get_client() as client:
-        bars = await client.bars(symbol=sym)
-
     async with aclosing(stream):
+
+        # maybe load historical ohlcv in to shared mem
+        # check if shm has already been created by previous
+        # feed initialization
+        writer_exists = get_shm_token(shm_token['shm_name'])
+
+        if not writer_exists:
+            shm = attach_shm_array(
+                token=shm_token,
+                # we are writer
+                readonly=False,
+            )
+            bars = await _trio_run_client_method(
+                method='bars',
+                symbol=sym,
+            )
+
+            shm.push(bars)
+            shm_token = shm.token
+
+        # pass back token, and bool, signalling if we're the writer
+        yield shm_token, not writer_exists
+
         # first quote can be ignored as a 2nd with newer data is sent?
         first_ticker = await stream.__anext__()
         quote = normalize(first_ticker)
@@ -538,7 +564,7 @@ async def stream_quotes(
                 else:
                     log.debug("Received first real volume tick")
                     # ugh, clear ticks since we've consumed them
-                    # (ahem, ib_insync is stateful trash)
+                    # (ahem, ib_insync is truly stateful trash)
                     ticker.ticks = []
 
                     # XXX: this works because we don't use
@@ -555,27 +581,8 @@ async def stream_quotes(
         topic = '.'.join((con['symbol'], con[suffix])).lower()
         first_quote = {topic: quote}
         ticker.ticks = []
-
+        # yield first quote asap
-        # load historical ohlcv in to shared mem
+        yield first_quote
-        ss = tractor.current_actor().statespace
-        existing_shm = ss.get(f'ib_shm.{sym}')
-        if not existing_shm:
-            readonly = False
-        else:
-            readonly = True
-            shm = existing_shm
-
-        with attach_shared_array(
-            token=shared_array_token,
-            readonly=readonly
-        ) as shm:
-            if not existing_shm:
-                shm.push(bars)
-                ss[f'ib_shm.{sym}'] = shm
-
-                yield (first_quote, shm.token)
-            else:
-                yield (first_quote, None)
 
         async for ticker in stream:
             quote = normalize(
@@ -589,8 +596,9 @@ async def stream_quotes(
             # at the yield such that the array write isn't delayed
             # while another consumer is serviced..
 
-                # if we are the lone tick writer
+            # if we are the lone tick writer start writing
-                if not existing_shm:
+            # the buffer with appropriate trade data
+            if not writer_exists:
                 for tick in iterticks(quote, type='trade'):
                     last = tick['price']
                     # print(f'broker last: {tick}')