Do tsdb backloading to shm concurrently
Not only improves startup latency but also avoids a bug where the rt buffer was being tsdb-history prepended *before* the backfilling of recent data from the backend was complete resulting in our of order frames in shm.ib_1m_hist
parent
4ca7817735
commit
dc1edeecda
|
@ -552,6 +552,7 @@ async def tsdb_backfill(
|
||||||
last_tsdb_dt,
|
last_tsdb_dt,
|
||||||
None,
|
None,
|
||||||
None,
|
None,
|
||||||
|
bf_done,
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
@ -561,6 +562,7 @@ async def tsdb_backfill(
|
||||||
last_tsdb_dt,
|
last_tsdb_dt,
|
||||||
latest_start_dt,
|
latest_start_dt,
|
||||||
latest_end_dt,
|
latest_end_dt,
|
||||||
|
bf_done,
|
||||||
)
|
)
|
||||||
|
|
||||||
# if len(hist_shm.array) < 2:
|
# if len(hist_shm.array) < 2:
|
||||||
|
@ -580,6 +582,18 @@ async def tsdb_backfill(
|
||||||
shms[1],
|
shms[1],
|
||||||
))
|
))
|
||||||
|
|
||||||
|
async def back_load_from_tsdb(
|
||||||
|
timeframe: int,
|
||||||
|
shm: ShmArray,
|
||||||
|
):
|
||||||
|
(
|
||||||
|
tsdb_history,
|
||||||
|
last_tsdb_dt,
|
||||||
|
latest_start_dt,
|
||||||
|
latest_end_dt,
|
||||||
|
bf_done,
|
||||||
|
) = dts_per_tf[timeframe]
|
||||||
|
|
||||||
# sync to backend history task's query/load completion
|
# sync to backend history task's query/load completion
|
||||||
await bf_done.wait()
|
await bf_done.wait()
|
||||||
|
|
||||||
|
@ -588,13 +602,6 @@ async def tsdb_backfill(
|
||||||
# TODO: eventually it'd be nice to not require a shm array/buffer
|
# TODO: eventually it'd be nice to not require a shm array/buffer
|
||||||
# to accomplish this.. maybe we can do some kind of tsdb direct to
|
# to accomplish this.. maybe we can do some kind of tsdb direct to
|
||||||
# graphics format eventually in a child-actor?
|
# graphics format eventually in a child-actor?
|
||||||
for timeframe, shm in shms.items():
|
|
||||||
(
|
|
||||||
tsdb_history,
|
|
||||||
last_tsdb_dt,
|
|
||||||
latest_start_dt,
|
|
||||||
latest_end_dt,
|
|
||||||
) = dts_per_tf[timeframe]
|
|
||||||
|
|
||||||
# do diff against last start frame of history and only fill
|
# do diff against last start frame of history and only fill
|
||||||
# in from the tsdb an allotment that allows for most recent
|
# in from the tsdb an allotment that allows for most recent
|
||||||
|
@ -688,6 +695,17 @@ async def tsdb_backfill(
|
||||||
|
|
||||||
# TODO: write new data to tsdb to be ready to for next read.
|
# TODO: write new data to tsdb to be ready to for next read.
|
||||||
|
|
||||||
|
# backload from db (concurrently per timeframe) once backfilling of
|
||||||
|
# recent dat a loaded from the backend provider (see
|
||||||
|
# ``bf_done.wait()`` call).
|
||||||
|
async with trio.open_nursery() as nurse:
|
||||||
|
for timeframe, shm in shms.items():
|
||||||
|
nurse.start_soon(
|
||||||
|
back_load_from_tsdb,
|
||||||
|
timeframe,
|
||||||
|
shm,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def manage_history(
|
async def manage_history(
|
||||||
mod: ModuleType,
|
mod: ModuleType,
|
||||||
|
|
Loading…
Reference in New Issue