From c1546eb043248952db84f56329707d7d937232fd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 8 Jun 2023 18:47:22 -0400 Subject: [PATCH] Add note about appending parquet files on write --- piker/storage/nativedb.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piker/storage/nativedb.py b/piker/storage/nativedb.py index e96856d1..9561d4e9 100644 --- a/piker/storage/nativedb.py +++ b/piker/storage/nativedb.py @@ -296,7 +296,11 @@ class NativeStorageClient: else: df = ohlcv - # TODO: use a proper profiler + # TODO: in terms of managing the ultra long term data + # - use a proper profiler to measure all this IO and + # roundtripping! + # - try out ``fastparquet``'s append writing: + # https://fastparquet.readthedocs.io/en/latest/api.html#fastparquet.write start = time.time() df.write_parquet(path) delay: float = round(