Add datetime col de-duplicator
parent
b9af6176c5
commit
b6d2550f33
|
@ -263,6 +263,19 @@ def with_dts(
|
||||||
# )
|
# )
|
||||||
|
|
||||||
|
|
||||||
|
def dedup_dt(
|
||||||
|
df: pl.DataFrame,
|
||||||
|
) -> pl.DataFrame:
|
||||||
|
'''
|
||||||
|
Drop duplicate date-time rows (normally from an OHLC frame).
|
||||||
|
|
||||||
|
'''
|
||||||
|
return df.unique(
|
||||||
|
subset=['dt'],
|
||||||
|
maintain_order=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def detect_time_gaps(
|
def detect_time_gaps(
|
||||||
df: pl.DataFrame,
|
df: pl.DataFrame,
|
||||||
|
|
||||||
|
@ -294,10 +307,12 @@ def detect_time_gaps(
|
||||||
'''
|
'''
|
||||||
return (
|
return (
|
||||||
with_dts(df)
|
with_dts(df)
|
||||||
|
# First by a seconds unit step size
|
||||||
.filter(
|
.filter(
|
||||||
pl.col('s_diff').abs() > expect_period
|
pl.col('s_diff').abs() > expect_period
|
||||||
)
|
)
|
||||||
.filter(
|
.filter(
|
||||||
|
# Second by an arbitrary dt-unit step size
|
||||||
getattr(
|
getattr(
|
||||||
pl.col('dt_diff').dt,
|
pl.col('dt_diff').dt,
|
||||||
gap_dt_unit,
|
gap_dt_unit,
|
||||||
|
|
Loading…
Reference in New Issue