piker/piker/storage/nativedb.py

678 lines
18 KiB
Python

# piker: trading gear for hackers
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
`nativedb`: a lulzy Apache-parquet file manager (that some might
call a poor man's tsdb).
AKA a `piker`-native file-system native "time series database"
without needing an extra process and no standard TSDB features,
YET!
'''
# TODO: like there's soo much..
# - better name like "parkdb" or "nativedb" (lel)? bundle this lib with
# others to make full system:
# - tractor for failover and reliablity?
# - borg for replication and sync?
#
# - use `fastparquet` for appends:
# https://fastparquet.readthedocs.io/en/latest/api.html#fastparquet.write
# (presuming it's actually faster then overwrites and
# makes sense in terms of impl?)
#
# - use `polars` support for lazy scanning, processing and schema
# validation?
# - https://pola-rs.github.io/polars-book/user-guide/io/parquet/#scan
# - https://pola-rs.github.io/polars-book/user-guide/concepts/lazy-vs-eager/
# - consider delta writes for appends?
# - https://github.com/pola-rs/polars/blob/main/py-polars/polars/dataframe/frame.py#L3232
# - consider multi-file appends with appropriate time-range naming?
# - https://pola-rs.github.io/polars-book/user-guide/io/multiple/
#
# - use `borg` for replication?
# - https://borgbackup.readthedocs.io/en/stable/quickstart.html#remote-repositories
# - https://github.com/borgbackup/borg
# - https://borgbackup.readthedocs.io/en/stable/faq.html#usage-limitations
# - https://github.com/borgbackup/community
# - https://github.com/spslater/borgapi
# - https://nixos.wiki/wiki/ZFS
from __future__ import annotations
from contextlib import asynccontextmanager as acm
from datetime import datetime
import os
from pathlib import Path
from tempfile import NamedTemporaryFile
import time
from bidict import bidict
import numpy as np
import polars as pl
from pendulum import (
from_timestamp,
)
from piker import config
from piker import tsp
from tractor.ipc._shm import ShmArray
from piker.data import def_iohlcv_fields
from piker.log import get_logger
from . import TimeseriesNotFound
log = get_logger('storage.nativedb')
_price_fields: tuple[str, ...] = (
'open',
'high',
'low',
'close',
)
def detect_period(shm: ShmArray) -> float:
'''
Attempt to detect the series time step sampling period
in seconds.
'''
# TODO: detect sample rate helper?
# calc ohlc sample period for naming
ohlcv: np.ndarray = shm.array
times: np.ndarray = ohlcv['time']
period: float = times[-1] - times[-2]
if period == 0:
# maybe just last sample is borked?
period: float = times[-2] - times[-3]
return period
def mk_ohlcv_shm_keyed_filepath(
fqme: str,
period: float | int, # ow known as the "timeframe"
datadir: Path,
) -> Path:
if period < 1.:
raise ValueError('Sample period should be >= 1.!?')
path: Path = (
datadir
/
f'{fqme}.ohlcv{int(period)}s.parquet'
)
return path
def unpack_fqme_from_parquet_filepath(path: Path) -> str:
filename: str = str(path.name)
fqme, fmt_descr, suffix = filename.split('.')
assert suffix == 'parquet'
return fqme
# Only hydrate fields shared by canonical storage and provider SHM.
# Provider-only fields remain at their buffer defaults on restart.
ohlc_key_map = bidict({
name: name
for name, _ in def_iohlcv_fields
if name != 'index'
})
class NativeStorageClient:
'''
High level storage api for OHLCV time series stored in
a (modern) filesystem as apache parquet files B)
Part of a grander scheme to use arrow and parquet as our main
lowlevel data framework: https://arrow.apache.org/faq/.
'''
name: str = 'nativedb'
def __init__(
self,
datadir: Path,
) -> None:
self._datadir = datadir
self._index: dict[str, dict] = {}
# series' cache from tsdb reads
self._dfs: dict[str, dict[str, pl.DataFrame]] = {}
@property
def address(self) -> str:
return self._datadir.as_uri()
@property
def cardinality(self) -> int:
return len(self._index)
# @property
# def compression(self) -> str:
# ...
async def list_keys(self) -> list[str]:
return list(self._index)
def index_files(self):
for path in self._datadir.iterdir():
if (
path.is_dir()
or
path.suffix != '.parquet'
# or
# path.name in {'borked', 'expired',}
):
continue
key: str = path.name.removesuffix('.parquet')
fqme, _, descr = key.rpartition('.')
prefix, _, suffix = descr.partition('ohlcv')
period: int = int(suffix.strip('s'))
# cache description data
self._index[fqme] = {
'path': path,
'period': period,
}
return self._index
# async def search_keys(self, pattern: str) -> list[str]:
# '''
# Search for time series key in the storage backend.
# '''
# ...
# async def write_ticks(self, ticks: list) -> None:
# ...
async def load(
self,
fqme: str,
timeframe: int,
) -> tuple[
np.ndarray, # timeframe sampled array-series
datetime | None, # first dt
datetime | None, # last dt
] | None:
try:
array: np.ndarray = await self.read_ohlcv(
fqme,
timeframe,
)
except FileNotFoundError as fnfe:
bs_fqme, _, *_ = fqme.rpartition('.')
possible_matches: list[str] = []
for tskey in self._index:
if bs_fqme in tskey:
possible_matches.append(tskey)
match_str: str = '\n'.join(sorted(possible_matches))
raise TimeseriesNotFound(
f'No entry for `{fqme}`?\n'
f'Maybe you need a more specific fqme-key like:\n\n'
f'{match_str}'
) from fnfe
times = array['time']
if array.size == 0:
return None
return (
array,
from_timestamp(times[0]),
from_timestamp(times[-1]),
)
def mk_path(
self,
fqme: str,
period: float,
) -> Path:
return mk_ohlcv_shm_keyed_filepath(
fqme=fqme,
period=period,
datadir=self._datadir,
)
def _cache_df(
self,
fqme: str,
df: pl.DataFrame,
timeframe: float,
) -> None:
# cache df for later usage since we (currently) need to
# convert to np.ndarrays to push to our `ShmArray` rt
# buffers subsys but later we may operate entirely on
# pyarrow arrays/buffers so keeping the dfs around for
# a variety of purposes is handy.
self._dfs.setdefault(
timeframe,
{},
)[fqme] = df
def _canonicalize_ohlcv(
self,
df: pl.DataFrame,
) -> pl.DataFrame:
'''
Normalize a provider frame to the durable OHLCV schema.
Provider frames may omit the derived ``index`` field and add
provider-only fields. Durable frames contain only canonical
fields in their declared order.
'''
required: tuple[str, ...] = tuple(
name
for name, _ in def_iohlcv_fields
)
schema = {
name: pl.Int64 if field_type is int else pl.Float64
for name, field_type in def_iohlcv_fields
}
missing: set[str] = set(required).difference(df.columns)
missing.discard('index')
if missing:
raise ValueError(
f'OHLCV frame is missing columns: {sorted(missing)}'
)
times: np.ndarray = df['time'].to_numpy()
if (
not np.issubdtype(times.dtype, np.number)
or
not np.all(np.isfinite(times))
):
raise ValueError(
"OHLCV column 'time' must contain finite numeric data"
)
if np.any(times != np.floor(times)):
raise ValueError(
'OHLCV timestamps must use whole-second values'
)
return (
df
.with_columns(
pl.Series('index', np.arange(df.height))
)
.select(required)
.cast(schema)
)
def _repair_stored_ohlcv(
self,
df: pl.DataFrame,
timeframe: int,
) -> pl.DataFrame:
'''
Drop known legacy sentinels before hydration or merge.
'''
stored: pl.DataFrame = self._canonicalize_ohlcv(df)
zero_prices: pl.Expr = pl.all_horizontal([
pl.col(field) == 0
for field in _price_fields
]).fill_null(False)
invalid: pl.Expr = (
(pl.col('time') <= 0)
|
(
(pl.col('time') <= timeframe)
&
zero_prices
)
)
stored_len: int = stored.height
stored = stored.filter(~invalid)
dropped: int = stored_len - stored.height
if dropped:
log.warning(
f'Dropping {dropped} invalid persisted OHLCV '
f'row(s) during read repair'
)
stored = self._canonicalize_ohlcv(stored)
return stored
async def read_ohlcv(
self,
fqme: str,
timeframe: int | str,
end: float | None = None, # epoch or none
# limit: int = int(200e3),
) -> np.ndarray:
path: Path = self.mk_path(
fqme,
period=int(timeframe),
)
df: pl.DataFrame = self._repair_stored_ohlcv(
pl.read_parquet(path),
timeframe=int(timeframe),
)
self._cache_df(
fqme=fqme,
df=df,
timeframe=timeframe,
)
# TODO: filter by end and limit inputs
# times: pl.Series = df['time']
array: np.ndarray = tsp.pl2np(
df,
dtype=np.dtype(def_iohlcv_fields),
)
return array
async def as_df(
self,
fqme: str,
period: int = 60,
load_from_offline: bool = True,
) -> pl.DataFrame:
try:
return self._dfs[period][fqme]
except KeyError:
if not load_from_offline:
raise
await self.read_ohlcv(fqme, period)
return self._dfs[period][fqme]
def _write_ohlcv(
self,
fqme: str,
ohlcv: np.ndarray | pl.DataFrame,
timeframe: int,
) -> Path:
'''
Sync version of the public interface meth, since we don't
currently actually need or support an async impl.
'''
path: Path = mk_ohlcv_shm_keyed_filepath(
fqme=fqme,
period=timeframe,
datadir=self._datadir,
)
if isinstance(ohlcv, np.ndarray):
df: pl.DataFrame = tsp.np2pl(ohlcv)
else:
df = ohlcv
df = self._canonicalize_ohlcv(df)
self._validate_ohlcv(df, timeframe)
# TODO: in terms of managing the ultra long term data
# -[ ] use a proper profiler to measure all this IO and
# roundtripping!
# -[ ] implement parquet append!? see issue:
# https://github.com/pikers/piker/issues/536
# -[ ] try out ``fastparquet``'s append writing:
# https://fastparquet.readthedocs.io/en/latest/api.html#fastparquet.write
start = time.time()
with NamedTemporaryFile(
dir=path.parent,
prefix=f'.{path.name}.',
suffix='.tmp',
delete=False,
) as tmp_file:
tmp_path = Path(tmp_file.name)
try:
df.write_parquet(tmp_path)
committed: pl.DataFrame = pl.read_parquet(tmp_path)
self._validate_ohlcv(committed, timeframe)
if not committed.equals(df):
raise IOError(
'Temporary parquet differs from input frame'
)
with tmp_path.open('rb') as tmp_file:
os.fsync(tmp_file.fileno())
tmp_path.replace(path)
self._cache_df(
fqme=fqme,
df=committed,
timeframe=timeframe,
)
dir_fd: int = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(dir_fd)
finally:
os.close(dir_fd)
except BaseException:
tmp_path.unlink(missing_ok=True)
raise
delay: float = round(
time.time() - start,
ndigits=6,
)
log.info(
f'parquet write took {delay} secs\n'
f'file path: {path}'
)
return path
def _validate_ohlcv(
self,
df: pl.DataFrame,
timeframe: int,
) -> None:
'''
Validate timestamp invariants before publishing a series.
'''
if df.is_empty():
raise ValueError('Can not write an empty OHLCV series')
required: tuple[str, ...] = tuple(
name
for name, _ in def_iohlcv_fields
)
missing: set[str] = set(required).difference(df.columns)
if missing:
raise ValueError(
f'OHLCV frame is missing columns: {sorted(missing)}'
)
for field in required:
values: np.ndarray = df[field].to_numpy()
if (
not np.issubdtype(values.dtype, np.number)
or
not np.all(np.isfinite(values))
):
raise ValueError(
f'OHLCV column {field!r} must contain '
f'finite numeric data'
)
times: np.ndarray = df['time'].to_numpy()
if (
not np.all(np.isfinite(times))
or
np.any(times <= 0)
):
raise ValueError(
'OHLCV timestamps must be finite and positive'
)
prices: np.ndarray = df.select(_price_fields).to_numpy()
epoch_sentinel: np.ndarray = (
(times <= timeframe)
&
np.all(prices == 0, axis=1)
)
if np.any(epoch_sentinel):
raise ValueError(
'OHLCV frame contains a zero-initialized epoch sentinel'
)
if np.any(np.diff(times) <= 0):
raise ValueError(
'OHLCV timestamps must be strictly increasing'
)
async def update_ohlcv(
self,
fqme: str,
ohlcv: np.ndarray | pl.DataFrame,
timeframe: int,
) -> Path:
'''
Merge an incremental frame into durable OHLCV history.
Incoming rows replace stored rows at matching timestamps.
Writer ownership belongs to the persistent ``datad`` feed task.
'''
if isinstance(ohlcv, np.ndarray):
incoming: pl.DataFrame = tsp.np2pl(ohlcv)
else:
incoming = ohlcv
incoming = self._canonicalize_ohlcv(incoming)
self._validate_ohlcv(incoming, timeframe)
path: Path = self.mk_path(fqme, timeframe)
if path.exists():
stored: pl.DataFrame = self._repair_stored_ohlcv(
pl.read_parquet(path),
timeframe=timeframe,
)
merged: pl.DataFrame = pl.concat(
[stored, incoming],
how='diagonal_relaxed',
)
merged = (
merged
.unique(
subset='time',
keep='last',
)
.sort('time')
)
else:
merged = incoming
self._validate_ohlcv(merged, timeframe)
return self._write_ohlcv(
fqme,
merged,
timeframe,
)
async def write_ohlcv(
self,
fqme: str,
ohlcv: np.ndarray | pl.DataFrame,
timeframe: int,
) -> Path:
'''
Write input ohlcv time series for fqme and sampling period
to (local) disk.
'''
return self._write_ohlcv(
fqme,
ohlcv,
timeframe,
)
async def delete_ts(
self,
key: str,
timeframe: int | None = None,
) -> bool:
path: Path = mk_ohlcv_shm_keyed_filepath(
fqme=key,
period=timeframe,
datadir=self._datadir,
)
if path.is_file():
path.unlink()
log.warning(f'Deleting parquet entry:\n{path}')
else:
log.error(f'No path exists:\n{path}')
return path
# TODO: allow wiping and refetching a segment of the OHLCV timeseries
# data.
# def clear_range(
# self,
# key: str,
# start_dt: datetime,
# end_dt: datetime,
# timeframe: int | None = None,
# ) -> pl.DataFrame:
# '''
# Clear and re-fetch a range of datums for the OHLCV time series.
# Useful for series editing from a chart B)
# '''
# ...
# TODO: does this need to be async on average?
# I guess for any IPC connected backend yes?
@acm
async def get_client(
# TODO: eventually support something something apache arrow
# transport over ssh something..?
# host: str | None = None,
**kwargs,
) -> NativeStorageClient:
'''
Load a ``anyio_marketstore`` grpc client connected
to an existing ``marketstore`` server.
'''
datadir: Path = config.get_conf_dir() / 'nativedb'
if not datadir.is_dir():
log.info(f'Creating `nativedb` dir: {datadir}')
datadir.mkdir()
client = NativeStorageClient(datadir)
client.index_files()
yield client