Do tsdb backloading to shm concurrently

Not only improves startup latency but also avoids a bug where the rt
buffer was being tsdb-history prepended *before* the backfilling of
recent data from the backend was complete resulting in our of order
frames in shm.
ib_1m_hist
Tyler Goodlet 2022-10-26 09:52:51 -04:00
parent 4ca7817735
commit dc1edeecda
1 changed files with 28 additions and 10 deletions

View File

@ -552,6 +552,7 @@ async def tsdb_backfill(
last_tsdb_dt, last_tsdb_dt,
None, None,
None, None,
bf_done,
) )
continue continue
@ -561,6 +562,7 @@ async def tsdb_backfill(
last_tsdb_dt, last_tsdb_dt,
latest_start_dt, latest_start_dt,
latest_end_dt, latest_end_dt,
bf_done,
) )
# if len(hist_shm.array) < 2: # if len(hist_shm.array) < 2:
@ -580,6 +582,18 @@ async def tsdb_backfill(
shms[1], shms[1],
)) ))
async def back_load_from_tsdb(
timeframe: int,
shm: ShmArray,
):
(
tsdb_history,
last_tsdb_dt,
latest_start_dt,
latest_end_dt,
bf_done,
) = dts_per_tf[timeframe]
# sync to backend history task's query/load completion # sync to backend history task's query/load completion
await bf_done.wait() await bf_done.wait()
@ -588,13 +602,6 @@ async def tsdb_backfill(
# TODO: eventually it'd be nice to not require a shm array/buffer # TODO: eventually it'd be nice to not require a shm array/buffer
# to accomplish this.. maybe we can do some kind of tsdb direct to # to accomplish this.. maybe we can do some kind of tsdb direct to
# graphics format eventually in a child-actor? # graphics format eventually in a child-actor?
for timeframe, shm in shms.items():
(
tsdb_history,
last_tsdb_dt,
latest_start_dt,
latest_end_dt,
) = dts_per_tf[timeframe]
# do diff against last start frame of history and only fill # do diff against last start frame of history and only fill
# in from the tsdb an allotment that allows for most recent # in from the tsdb an allotment that allows for most recent
@ -688,6 +695,17 @@ async def tsdb_backfill(
# TODO: write new data to tsdb to be ready to for next read. # TODO: write new data to tsdb to be ready to for next read.
# backload from db (concurrently per timeframe) once backfilling of
# recent dat a loaded from the backend provider (see
# ``bf_done.wait()`` call).
async with trio.open_nursery() as nurse:
for timeframe, shm in shms.items():
nurse.start_soon(
back_load_from_tsdb,
timeframe,
shm,
)
async def manage_history( async def manage_history(
mod: ModuleType, mod: ModuleType,