Bleh, fix ib's `Client.bars()` recursion..
Turns out this was the main source of all sorts of gaps and overlaps in history frame backfilling. The original idea was that when a gap causes not enough (1m) bars to be delivered (like over a weekend or holiday) when we just implicitly do another frame query to try and at least fill out the default duration (normally 1-2 days). Doing the recursion sloppily was causing all sorts of stupid problems.. It's kinda obvious now what was wrong in hindsight: - always pass the sampling period (timeframe) when recursing - adjust the logic to not be mutex with the no-data case (since it already is mutex..) - pack to the `numpy` array BEFORE the recursive call to ensure the `end_dt: DateTime` is selected and passed correctly! Toss in some other helpfuls: - more explicit `pendulum` typing imports - some masked out sorted-diffing checks (that can be enabled when debugging out-of-order frame issues) - always error log about less-than time step mismatches since we should never have time-diff steps **smaller** then specified in the `sample_period_s`!distribute_dis
parent
b03eceebef
commit
8e4d1a48ed
|
@ -49,7 +49,12 @@ from bidict import bidict
|
||||||
import trio
|
import trio
|
||||||
import tractor
|
import tractor
|
||||||
from tractor import to_asyncio
|
from tractor import to_asyncio
|
||||||
import pendulum
|
from pendulum import (
|
||||||
|
from_timestamp,
|
||||||
|
DateTime,
|
||||||
|
Duration,
|
||||||
|
duration as mk_duration,
|
||||||
|
)
|
||||||
from eventkit import Event
|
from eventkit import Event
|
||||||
from ib_insync import (
|
from ib_insync import (
|
||||||
client as ib_client,
|
client as ib_client,
|
||||||
|
@ -221,16 +226,20 @@ def bars_to_np(bars: list) -> np.ndarray:
|
||||||
# https://interactivebrokers.github.io/tws-api/historical_limitations.html#non-available_hd
|
# https://interactivebrokers.github.io/tws-api/historical_limitations.html#non-available_hd
|
||||||
_samplings: dict[int, tuple[str, str]] = {
|
_samplings: dict[int, tuple[str, str]] = {
|
||||||
1: (
|
1: (
|
||||||
|
# ib strs
|
||||||
'1 secs',
|
'1 secs',
|
||||||
f'{int(2e3)} S',
|
f'{int(2e3)} S',
|
||||||
pendulum.duration(seconds=2e3),
|
|
||||||
|
mk_duration(seconds=2e3),
|
||||||
),
|
),
|
||||||
# TODO: benchmark >1 D duration on query to see if
|
# TODO: benchmark >1 D duration on query to see if
|
||||||
# throughput can be made faster during backfilling.
|
# throughput can be made faster during backfilling.
|
||||||
60: (
|
60: (
|
||||||
|
# ib strs
|
||||||
'1 min',
|
'1 min',
|
||||||
'2 D',
|
'2 D',
|
||||||
pendulum.duration(days=2),
|
|
||||||
|
mk_duration(days=2),
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -315,7 +324,7 @@ class Client:
|
||||||
|
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
|
||||||
) -> tuple[BarDataList, np.ndarray, pendulum.Duration]:
|
) -> tuple[BarDataList, np.ndarray, Duration]:
|
||||||
'''
|
'''
|
||||||
Retreive OHLCV bars for a fqme over a range to the present.
|
Retreive OHLCV bars for a fqme over a range to the present.
|
||||||
|
|
||||||
|
@ -324,11 +333,20 @@ class Client:
|
||||||
# https://interactivebrokers.github.io/tws-api/historical_data.html
|
# https://interactivebrokers.github.io/tws-api/historical_data.html
|
||||||
bars_kwargs = {'whatToShow': 'TRADES'}
|
bars_kwargs = {'whatToShow': 'TRADES'}
|
||||||
bars_kwargs.update(kwargs)
|
bars_kwargs.update(kwargs)
|
||||||
bar_size, duration, dt_duration = _samplings[sample_period_s]
|
(
|
||||||
|
bar_size,
|
||||||
|
ib_duration_str,
|
||||||
|
default_dt_duration,
|
||||||
|
) = _samplings[sample_period_s]
|
||||||
|
|
||||||
|
dt_duration: DateTime = (
|
||||||
|
duration
|
||||||
|
or default_dt_duration
|
||||||
|
)
|
||||||
|
|
||||||
global _enters
|
global _enters
|
||||||
log.info(
|
log.info(
|
||||||
f"REQUESTING {duration}'s worth {bar_size} BARS\n"
|
f"REQUESTING {ib_duration_str}'s worth {bar_size} BARS\n"
|
||||||
f'{_enters} @ end={end_dt}"'
|
f'{_enters} @ end={end_dt}"'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -353,7 +371,7 @@ class Client:
|
||||||
|
|
||||||
# time history length values format:
|
# time history length values format:
|
||||||
# ``durationStr=integer{SPACE}unit (S|D|W|M|Y)``
|
# ``durationStr=integer{SPACE}unit (S|D|W|M|Y)``
|
||||||
durationStr=duration,
|
durationStr=ib_duration_str,
|
||||||
|
|
||||||
# always use extended hours
|
# always use extended hours
|
||||||
useRTH=False,
|
useRTH=False,
|
||||||
|
@ -383,16 +401,23 @@ class Client:
|
||||||
# => we recursively call this method until we get at least
|
# => we recursively call this method until we get at least
|
||||||
# as many bars such that they sum in aggregate to the the
|
# as many bars such that they sum in aggregate to the the
|
||||||
# desired total time (duration) at most.
|
# desired total time (duration) at most.
|
||||||
elif (
|
if end_dt:
|
||||||
end_dt
|
nparr: np.ndarray = bars_to_np(bars)
|
||||||
and (
|
times: np.ndarray = nparr['time']
|
||||||
(len(bars) * sample_period_s) < dt_duration.in_seconds()
|
first: float = times[0]
|
||||||
)
|
tdiff: float = times[-1] - first
|
||||||
|
|
||||||
|
if (
|
||||||
|
# len(bars) * sample_period_s) < dt_duration.in_seconds()
|
||||||
|
tdiff < dt_duration.in_seconds()
|
||||||
):
|
):
|
||||||
|
end_dt: DateTime = from_timestamp(first)
|
||||||
log.warning(
|
log.warning(
|
||||||
f'Recursing to get more bars from {end_dt} for {dt_duration}'
|
f'Frame result was shorter then {dt_duration}!?\n'
|
||||||
|
'Recursing for more bars:\n'
|
||||||
|
f'end_dt: {end_dt}\n'
|
||||||
|
f'dt_duration: {dt_duration}\n'
|
||||||
)
|
)
|
||||||
end_dt -= dt_duration
|
|
||||||
(
|
(
|
||||||
r_bars,
|
r_bars,
|
||||||
r_arr,
|
r_arr,
|
||||||
|
@ -401,11 +426,30 @@ class Client:
|
||||||
fqme,
|
fqme,
|
||||||
start_dt=start_dt,
|
start_dt=start_dt,
|
||||||
end_dt=end_dt,
|
end_dt=end_dt,
|
||||||
|
sample_period_s=sample_period_s,
|
||||||
|
|
||||||
|
# TODO: make a table for Duration to
|
||||||
|
# the ib str values in order to use this?
|
||||||
|
# duration=duration,
|
||||||
)
|
)
|
||||||
r_bars.extend(bars)
|
r_bars.extend(bars)
|
||||||
bars = r_bars
|
bars = r_bars
|
||||||
|
|
||||||
nparr = bars_to_np(bars)
|
nparr = bars_to_np(bars)
|
||||||
|
|
||||||
|
# timestep should always be at least as large as the
|
||||||
|
# period step.
|
||||||
|
tdiff: np.ndarray = np.diff(nparr['time'])
|
||||||
|
to_short: np.ndarray = tdiff < sample_period_s
|
||||||
|
if (to_short).any():
|
||||||
|
# raise ValueError(
|
||||||
|
log.error(
|
||||||
|
f'OHLC frame for {sample_period_s} has {to_short.size} '
|
||||||
|
'time steps which are shorter then expected?!"'
|
||||||
|
)
|
||||||
|
# OOF: this will break teardown?
|
||||||
|
# breakpoint()
|
||||||
|
|
||||||
return bars, nparr, dt_duration
|
return bars, nparr, dt_duration
|
||||||
|
|
||||||
async def con_deats(
|
async def con_deats(
|
||||||
|
|
|
@ -20,7 +20,7 @@ Order and trades endpoints for use with ``piker``'s EMS.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from contextlib import ExitStack
|
from contextlib import ExitStack
|
||||||
from collections import ChainMap
|
# from collections import ChainMap
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from pprint import pformat
|
from pprint import pformat
|
||||||
import time
|
import time
|
||||||
|
|
|
@ -196,10 +196,8 @@ async def open_history_client(
|
||||||
f'mean: {mean}'
|
f'mean: {mean}'
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
|
||||||
out is None
|
|
||||||
):
|
|
||||||
# could be trying to retreive bars over weekend
|
# could be trying to retreive bars over weekend
|
||||||
|
if out is None:
|
||||||
log.error(f"Can't grab bars starting at {end_dt}!?!?")
|
log.error(f"Can't grab bars starting at {end_dt}!?!?")
|
||||||
raise NoData(
|
raise NoData(
|
||||||
f'{end_dt}',
|
f'{end_dt}',
|
||||||
|
@ -213,7 +211,24 @@ async def open_history_client(
|
||||||
):
|
):
|
||||||
raise DataUnavailable(f'First timestamp is {head_dt}')
|
raise DataUnavailable(f'First timestamp is {head_dt}')
|
||||||
|
|
||||||
bars, bars_array, first_dt, last_dt = out
|
# also see return type for `get_bars()`
|
||||||
|
bars: ibis.objects.BarDataList
|
||||||
|
bars_array: np.ndarray
|
||||||
|
first_dt: datetime
|
||||||
|
last_dt: datetime
|
||||||
|
(
|
||||||
|
bars,
|
||||||
|
bars_array,
|
||||||
|
first_dt,
|
||||||
|
last_dt,
|
||||||
|
) = out
|
||||||
|
|
||||||
|
# TODO: audit the sampling period here as well?
|
||||||
|
# timestep should always be at least as large as the
|
||||||
|
# period step.
|
||||||
|
# tdiff: np.ndarray = np.diff(bars_array['time'])
|
||||||
|
# if (tdiff < timeframe).any():
|
||||||
|
# await tractor.pause()
|
||||||
|
|
||||||
# volume cleaning since there's -ve entries,
|
# volume cleaning since there's -ve entries,
|
||||||
# wood luv to know what crookery that is..
|
# wood luv to know what crookery that is..
|
||||||
|
|
Loading…
Reference in New Issue