Add venue-closure gap-detection in `.ib.api.Client.bars()`
With all detection logic coming from our new `.ib.venues` helpers allowing use to verify IB's OHLC bars frames don't contain unexpected time-gaps. `Client.bars()` new checking deats, - add `is_venue_open()`, `has_weekend()`, `sesh_times()`, and `is_venue_closure()` checks when `last_dt < end_dt` - always calc gap-period in local tz via `ContractDetails.timeZoneId`. - log warnings on invalid non-closure gaps, debug on closures for now. - change recursion case to just `log.error()` + `breakpoint()`; we might end up tossing it since i don't think i could ever get it to be reliable.. * mask-out recursive `.bars()` call (likely unnecessary). - flip `start_dt`/`end_dt` param defaults to `None` vs epoch `str`. - update docstring to clarify no `start_dt` support by IB - add mod level `_iso8601_epoch_in_est` const to keep track of orig param default value. - add multiline style to return type-annot, type all `pendulum` objects. Also, - uppercase `Crypto.symbol` for PAXOS contracts in `.find_contracts()`, tho now we're getting a weird new API error i left in a todo-comment.. (this commit msg was generated in some part by [`claude-code`][claude-code-gh]) [claude-code-gh]: https://github.com/anthropics/claude-codecont_hist_fixes
parent
f484726a8c
commit
ba575d93ea
|
|
@ -50,10 +50,11 @@ import tractor
|
||||||
from tractor import to_asyncio
|
from tractor import to_asyncio
|
||||||
from tractor import trionics
|
from tractor import trionics
|
||||||
from pendulum import (
|
from pendulum import (
|
||||||
from_timestamp,
|
|
||||||
DateTime,
|
DateTime,
|
||||||
Duration,
|
Duration,
|
||||||
duration as mk_duration,
|
duration as mk_duration,
|
||||||
|
from_timestamp,
|
||||||
|
Interval,
|
||||||
)
|
)
|
||||||
from eventkit import Event
|
from eventkit import Event
|
||||||
from ib_insync import (
|
from ib_insync import (
|
||||||
|
|
@ -260,6 +261,16 @@ def remove_handler_on_err(
|
||||||
event.disconnect(handler)
|
event.disconnect(handler)
|
||||||
|
|
||||||
|
|
||||||
|
# (originally?) i thot that,
|
||||||
|
# > "EST in ISO 8601 format is required.."
|
||||||
|
#
|
||||||
|
# XXX, but see `ib_async`'s impl,
|
||||||
|
# - `ib_async.ib.IB.reqHistoricalDataAsync()`
|
||||||
|
# - `ib_async.util.formatIBDatetime()`
|
||||||
|
# below is EPOCH.
|
||||||
|
_iso8601_epoch_in_est: str = "1970-01-01T00:00:00.000000-05:00"
|
||||||
|
|
||||||
|
|
||||||
class Client:
|
class Client:
|
||||||
'''
|
'''
|
||||||
IB wrapped for our broker backend API.
|
IB wrapped for our broker backend API.
|
||||||
|
|
@ -333,9 +344,11 @@ class Client:
|
||||||
self,
|
self,
|
||||||
fqme: str,
|
fqme: str,
|
||||||
|
|
||||||
# EST in ISO 8601 format is required... below is EPOCH
|
# EST in ISO 8601 format is required..
|
||||||
start_dt: datetime|str = "1970-01-01T00:00:00.000000-05:00",
|
# XXX, see `ib_async.ib.IB.reqHistoricalDataAsync()`
|
||||||
end_dt: datetime|str = "",
|
# below is EPOCH.
|
||||||
|
start_dt: datetime|None = None, # _iso8601_epoch_in_est,
|
||||||
|
end_dt: datetime|None = None,
|
||||||
|
|
||||||
# ohlc sample period in seconds
|
# ohlc sample period in seconds
|
||||||
sample_period_s: int = 1,
|
sample_period_s: int = 1,
|
||||||
|
|
@ -346,9 +359,17 @@ class Client:
|
||||||
|
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
|
||||||
) -> tuple[BarDataList, np.ndarray, Duration]:
|
) -> tuple[
|
||||||
|
BarDataList,
|
||||||
|
np.ndarray,
|
||||||
|
Duration,
|
||||||
|
]:
|
||||||
'''
|
'''
|
||||||
Retreive OHLCV bars for a fqme over a range to the present.
|
Retreive the `fqme`'s OHLCV-bars for the time-range "until `end_dt`".
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- IB's api doesn't support a `start_dt` (which is why default
|
||||||
|
is null) so we only use it for bar-frame duration checking.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
# See API docs here:
|
# See API docs here:
|
||||||
|
|
@ -363,13 +384,19 @@ class Client:
|
||||||
|
|
||||||
dt_duration: Duration = (
|
dt_duration: Duration = (
|
||||||
duration
|
duration
|
||||||
or default_dt_duration
|
or
|
||||||
|
default_dt_duration
|
||||||
)
|
)
|
||||||
|
|
||||||
# TODO: maybe remove all this?
|
# TODO: maybe remove all this?
|
||||||
global _enters
|
global _enters
|
||||||
if not end_dt:
|
if end_dt is None:
|
||||||
end_dt = ''
|
end_dt: str = ''
|
||||||
|
|
||||||
|
else:
|
||||||
|
est_end_dt = end_dt.in_tz('EST')
|
||||||
|
if est_end_dt != end_dt:
|
||||||
|
breakpoint()
|
||||||
|
|
||||||
_enters += 1
|
_enters += 1
|
||||||
|
|
||||||
|
|
@ -438,58 +465,127 @@ class Client:
|
||||||
+ query_info
|
+ query_info
|
||||||
)
|
)
|
||||||
|
|
||||||
# TODO: we could maybe raise ``NoData`` instead if we
|
# TODO: we could maybe raise `NoData` instead if we
|
||||||
# rewrite the method in the first case?
|
# rewrite the method in the first case?
|
||||||
# right now there's no way to detect a timeout..
|
# right now there's no way to detect a timeout..
|
||||||
return [], np.empty(0), dt_duration
|
return [], np.empty(0), dt_duration
|
||||||
|
|
||||||
log.info(query_info)
|
log.info(query_info)
|
||||||
|
|
||||||
|
# ------ GAP-DETECTION ------
|
||||||
# NOTE XXX: ensure minimum duration in bars?
|
# NOTE XXX: ensure minimum duration in bars?
|
||||||
# => recursively call this method until we get at least as
|
# => recursively call this method until we get at least as
|
||||||
# many bars such that they sum in aggregate to the the
|
# many bars such that they sum in aggregate to the the
|
||||||
# desired total time (duration) at most.
|
# desired total time (duration) at most.
|
||||||
# - if you query over a gap and get no data
|
# - if you query over a gap and get no data
|
||||||
# that may short circuit the history
|
# that may short circuit the history
|
||||||
if (
|
if end_dt:
|
||||||
# XXX XXX XXX
|
|
||||||
# => WHY DID WE EVEN NEED THIS ORIGINALLY!? <=
|
|
||||||
# XXX XXX XXX
|
|
||||||
False
|
|
||||||
and end_dt
|
|
||||||
):
|
|
||||||
nparr: np.ndarray = bars_to_np(bars)
|
nparr: np.ndarray = bars_to_np(bars)
|
||||||
times: np.ndarray = nparr['time']
|
times: np.ndarray = nparr['time']
|
||||||
first: float = times[0]
|
first: float = times[0]
|
||||||
tdiff: float = times[-1] - first
|
last: float = times[-1]
|
||||||
|
# frame_dur: float = times[-1] - first
|
||||||
|
|
||||||
|
first_dt: DateTime = from_timestamp(first)
|
||||||
|
last_dt: DateTime = from_timestamp(last)
|
||||||
|
tdiff: int = (
|
||||||
|
last_dt
|
||||||
|
-
|
||||||
|
first_dt
|
||||||
|
).in_seconds() + sample_period_s
|
||||||
|
|
||||||
|
# XXX, do gap detections.
|
||||||
|
if (
|
||||||
|
last_dt.add(seconds=sample_period_s)
|
||||||
|
<
|
||||||
|
end_dt
|
||||||
|
):
|
||||||
|
details: ContractDetails = (
|
||||||
|
await self.ib.reqContractDetailsAsync(contract)
|
||||||
|
)[0]
|
||||||
|
from .venues import (
|
||||||
|
is_venue_open,
|
||||||
|
has_weekend,
|
||||||
|
sesh_times,
|
||||||
|
is_venue_closure,
|
||||||
|
)
|
||||||
|
_open_now: bool = is_venue_open(
|
||||||
|
con_deats=details,
|
||||||
|
)
|
||||||
|
open_time, close_time = sesh_times(details)
|
||||||
|
# XXX, always calc gap in mkt-venue-local timezone
|
||||||
|
tz: str = details.timeZoneId
|
||||||
|
gap: Interval = (
|
||||||
|
end_dt.in_tz(tz)
|
||||||
|
-
|
||||||
|
last_dt.in_tz(tz)
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
not has_weekend(gap)
|
||||||
|
and
|
||||||
|
# XXX NOT outside venue closures.
|
||||||
|
# !TODO, replace with,
|
||||||
|
# `not is_venue_closure()`
|
||||||
|
# per below assert on inverse case!
|
||||||
|
gap.end.time() != open_time
|
||||||
|
and
|
||||||
|
gap.start.time() != close_time
|
||||||
|
):
|
||||||
|
breakpoint()
|
||||||
|
log.warning(
|
||||||
|
f'Invalid non-closure gap for {fqme!r} ?!?\n'
|
||||||
|
f'is-open-now: {_open_now}\n'
|
||||||
|
f'\n'
|
||||||
|
f'{gap}\n'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert is_venue_closure(
|
||||||
|
gap=gap,
|
||||||
|
con_deats=details,
|
||||||
|
)
|
||||||
|
log.debug(
|
||||||
|
f'Detected venue closure gap (weekend),\n'
|
||||||
|
f'{gap}\n'
|
||||||
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
|
start_dt is None
|
||||||
|
and
|
||||||
|
tdiff
|
||||||
|
<
|
||||||
|
dt_duration.in_seconds()
|
||||||
|
# and
|
||||||
# len(bars) * sample_period_s) < dt_duration.in_seconds()
|
# len(bars) * sample_period_s) < dt_duration.in_seconds()
|
||||||
tdiff < dt_duration.in_seconds()
|
|
||||||
# and False
|
|
||||||
):
|
):
|
||||||
end_dt: DateTime = from_timestamp(first)
|
end_dt: DateTime = from_timestamp(first)
|
||||||
log.warning(
|
log.error(
|
||||||
f'Frame result was shorter then {dt_duration}!?\n'
|
f'Frame result was shorter then {dt_duration}!?\n'
|
||||||
'Recursing for more bars:\n'
|
|
||||||
f'end_dt: {end_dt}\n'
|
f'end_dt: {end_dt}\n'
|
||||||
f'dt_duration: {dt_duration}\n'
|
f'dt_duration: {dt_duration}\n'
|
||||||
|
# f'\n'
|
||||||
|
# f'Recursing for more bars:\n'
|
||||||
)
|
)
|
||||||
(
|
breakpoint()
|
||||||
r_bars,
|
# XXX ? TODO? recursively try to re-request?
|
||||||
r_arr,
|
# => i think *NO* right?
|
||||||
r_duration,
|
#
|
||||||
) = await self.bars(
|
# (
|
||||||
fqme,
|
# r_bars,
|
||||||
start_dt=start_dt,
|
# r_arr,
|
||||||
end_dt=end_dt,
|
# r_duration,
|
||||||
sample_period_s=sample_period_s,
|
# ) = await self.bars(
|
||||||
|
# fqme,
|
||||||
|
# start_dt=start_dt,
|
||||||
|
# end_dt=end_dt,
|
||||||
|
# sample_period_s=sample_period_s,
|
||||||
|
|
||||||
# TODO: make a table for Duration to
|
# # TODO: make a table for Duration to
|
||||||
# the ib str values in order to use this?
|
# # the ib str values in order to use this?
|
||||||
# duration=duration,
|
# # duration=duration,
|
||||||
)
|
# )
|
||||||
r_bars.extend(bars)
|
# r_bars.extend(bars)
|
||||||
bars = r_bars
|
# bars = r_bars
|
||||||
|
|
||||||
nparr: np.ndarray = bars_to_np(bars)
|
nparr: np.ndarray = bars_to_np(bars)
|
||||||
|
|
||||||
|
|
@ -784,9 +880,16 @@ class Client:
|
||||||
# crypto$
|
# crypto$
|
||||||
elif exch == 'PAXOS': # btc.paxos
|
elif exch == 'PAXOS': # btc.paxos
|
||||||
con = Crypto(
|
con = Crypto(
|
||||||
symbol=symbol,
|
symbol=symbol.upper(),
|
||||||
currency=currency,
|
currency='USD',
|
||||||
|
exchange='PAXOS',
|
||||||
)
|
)
|
||||||
|
# XXX, on `ib_insync` when first tried this,
|
||||||
|
# > Error 10299, reqId 141: Expected what to show is
|
||||||
|
# > AGGTRADES, please use that instead of TRADES.,
|
||||||
|
# > contract: Crypto(conId=479624278, symbol='BTC',
|
||||||
|
# > exchange='PAXOS', currency='USD',
|
||||||
|
# > localSymbol='BTC.USD', tradingClass='BTC')
|
||||||
|
|
||||||
# stonks
|
# stonks
|
||||||
else:
|
else:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue