Add better shared mem writer task checking
If you have a common broker feed daemon then likely you don't want to create superfluous shared mem buffers for the same symbol. This adds an ad hoc little context manger which keeps a bool state of whether a buffer writer task currently is running in this process. Before we were checking the shared array token cache and **not** clearing it when the writer task exited, resulting in incorrect writer/loader logic on the next entry.. Really, we need a better set of SC semantics around the shared mem stuff presuming there's only ever one writer per shared buffer at given time. Hopefully that will come soon!bar_select
							parent
							
								
									e0613675c7
								
							
						
					
					
						commit
						454b445b4b
					
				|  | @ -5,7 +5,7 @@ Note the client runs under an ``asyncio`` loop (since ``ib_insync`` is | ||||||
| built on it) and thus actor aware API calls must be spawned with | built on it) and thus actor aware API calls must be spawned with | ||||||
| ``infected_aio==True``. | ``infected_aio==True``. | ||||||
| """ | """ | ||||||
| from contextlib import asynccontextmanager | from contextlib import asynccontextmanager, contextmanager | ||||||
| from dataclasses import asdict | from dataclasses import asdict | ||||||
| from functools import partial | from functools import partial | ||||||
| from typing import List, Dict, Any, Tuple, Optional, AsyncIterator, Callable | from typing import List, Dict, Any, Tuple, Optional, AsyncIterator, Callable | ||||||
|  | @ -292,7 +292,7 @@ class Client: | ||||||
|         ticker: Ticker = self.ib.reqMktData(contract, ','.join(opts)) |         ticker: Ticker = self.ib.reqMktData(contract, ','.join(opts)) | ||||||
| 
 | 
 | ||||||
|         def push(t): |         def push(t): | ||||||
|             log.debug(t) |             # log.debug(t) | ||||||
|             try: |             try: | ||||||
|                 to_trio.send_nowait(t) |                 to_trio.send_nowait(t) | ||||||
|             except trio.BrokenResourceError: |             except trio.BrokenResourceError: | ||||||
|  | @ -497,6 +497,21 @@ def normalize( | ||||||
|     return data |     return data | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
|  | _local_buffer_writers = {} | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | @contextmanager | ||||||
|  | def activate_writer(key: str): | ||||||
|  |     try: | ||||||
|  |         writer_already_exists = _local_buffer_writers.get(key, False) | ||||||
|  |         if not writer_already_exists: | ||||||
|  |             _local_buffer_writers[key] = True | ||||||
|  | 
 | ||||||
|  |         yield writer_already_exists | ||||||
|  |     finally: | ||||||
|  |         _local_buffer_writers.pop(key, None) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
| # TODO: figure out how to share quote feeds sanely despite | # TODO: figure out how to share quote feeds sanely despite | ||||||
| # the wacky ``ib_insync`` api. | # the wacky ``ib_insync`` api. | ||||||
| # @tractor.msg.pub | # @tractor.msg.pub | ||||||
|  | @ -528,15 +543,19 @@ async def stream_quotes( | ||||||
| 
 | 
 | ||||||
|     async with aclosing(stream): |     async with aclosing(stream): | ||||||
| 
 | 
 | ||||||
|  |         # check if a writer already is alive in a streaming task, | ||||||
|  |         # otherwise start one and mark it as now existing | ||||||
|  |         with activate_writer(shm_token['shm_name']) as writer_already_exists: | ||||||
|  | 
 | ||||||
|             # maybe load historical ohlcv in to shared mem |             # maybe load historical ohlcv in to shared mem | ||||||
|             # check if shm has already been created by previous |             # check if shm has already been created by previous | ||||||
|             # feed initialization |             # feed initialization | ||||||
|         writer_exists = get_shm_token(shm_token['shm_name']) |             if not writer_already_exists: | ||||||
| 
 | 
 | ||||||
|         if not writer_exists: |  | ||||||
|                 shm = attach_shm_array( |                 shm = attach_shm_array( | ||||||
|                     token=shm_token, |                     token=shm_token, | ||||||
|                 # we are writer | 
 | ||||||
|  |                     # we are the buffer writer | ||||||
|                     readonly=False, |                     readonly=False, | ||||||
|                 ) |                 ) | ||||||
|                 bars = await _trio_run_client_method( |                 bars = await _trio_run_client_method( | ||||||
|  | @ -544,6 +563,7 @@ async def stream_quotes( | ||||||
|                     symbol=sym, |                     symbol=sym, | ||||||
|                 ) |                 ) | ||||||
| 
 | 
 | ||||||
|  |                 # write historical data to buffer | ||||||
|                 shm.push(bars) |                 shm.push(bars) | ||||||
|                 shm_token = shm.token |                 shm_token = shm.token | ||||||
| 
 | 
 | ||||||
|  | @ -552,7 +572,7 @@ async def stream_quotes( | ||||||
|                 subscribe_ohlc_for_increment(shm, delay_s) |                 subscribe_ohlc_for_increment(shm, delay_s) | ||||||
| 
 | 
 | ||||||
|             # pass back token, and bool, signalling if we're the writer |             # pass back token, and bool, signalling if we're the writer | ||||||
|         await ctx.send_yield((shm_token, not writer_exists)) |             await ctx.send_yield((shm_token, not writer_already_exists)) | ||||||
| 
 | 
 | ||||||
|             # first quote can be ignored as a 2nd with newer data is sent? |             # first quote can be ignored as a 2nd with newer data is sent? | ||||||
|             first_ticker = await stream.__anext__() |             first_ticker = await stream.__anext__() | ||||||
|  | @ -612,7 +632,7 @@ async def stream_quotes( | ||||||
| 
 | 
 | ||||||
|                 # if we are the lone tick writer start writing |                 # if we are the lone tick writer start writing | ||||||
|                 # the buffer with appropriate trade data |                 # the buffer with appropriate trade data | ||||||
|             if not writer_exists: |                 if not writer_already_exists: | ||||||
|                     for tick in iterticks(quote, type='trade'): |                     for tick in iterticks(quote, type='trade'): | ||||||
|                         last = tick['price'] |                         last = tick['price'] | ||||||
|                         # print(f'broker last: {tick}') |                         # print(f'broker last: {tick}') | ||||||
|  |  | ||||||
|  | @ -167,6 +167,7 @@ async def open_feed( | ||||||
|     # Attempt to allocate (or attach to) shm array for this broker/symbol |     # Attempt to allocate (or attach to) shm array for this broker/symbol | ||||||
|     shm, opened = maybe_open_shm_array( |     shm, opened = maybe_open_shm_array( | ||||||
|         key=sym_to_shm_key(name, symbols[0]), |         key=sym_to_shm_key(name, symbols[0]), | ||||||
|  | 
 | ||||||
|         # use any broker defined ohlc dtype: |         # use any broker defined ohlc dtype: | ||||||
|         dtype=getattr(mod, '_ohlc_dtype', base_ohlc_dtype), |         dtype=getattr(mod, '_ohlc_dtype', base_ohlc_dtype), | ||||||
| 
 | 
 | ||||||
|  | @ -193,12 +194,13 @@ async def open_feed( | ||||||
|         # tests are in? |         # tests are in? | ||||||
|         shm_token, is_writer = await stream.receive() |         shm_token, is_writer = await stream.receive() | ||||||
| 
 | 
 | ||||||
|  |         if opened: | ||||||
|  |             assert is_writer | ||||||
|  |             log.info("Started shared mem bar writer") | ||||||
|  | 
 | ||||||
|         shm_token['dtype_descr'] = list(shm_token['dtype_descr']) |         shm_token['dtype_descr'] = list(shm_token['dtype_descr']) | ||||||
|         assert shm_token == shm.token  # sanity |         assert shm_token == shm.token  # sanity | ||||||
| 
 | 
 | ||||||
|         if is_writer: |  | ||||||
|             log.info("Started shared mem bar writer") |  | ||||||
| 
 |  | ||||||
|         yield Feed( |         yield Feed( | ||||||
|             name=name, |             name=name, | ||||||
|             stream=stream, |             stream=stream, | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue