Deribit's feed fix

- `FeedInit` for init_msgs in `stream_quotes`. - new cache is `client_pairs` so is replacing the old `client.cache_symbols`. - `get_mkt_info` added - `get_ohlc` fixed to comply the new ways of the feed.
Deribit's api fix
2025-02-03 18:57:01 -03:00 · 2025-02-03 18:57:01 -03:00 · 2025-02-03 18:57:01 -03:00 · 2025-02-03 18:55:47 -03:00 · 2025-02-03 18:55:47 -03:00 · 2025-02-03 18:55:47 -03:00
2 changed files with 121 additions and 109 deletions
--- a/default.nix
+++ b/default.nix
@ -1,6 +1,7 @@
 with (import <nixpkgs> {});
 let
  glibStorePath = lib.getLib glib;
+  zlibStorePath = lib.getLib zlib;
  zstdStorePath = lib.getLib zstd;
  dbusStorePath = lib.getLib dbus;
  libGLStorePath = lib.getLib libGL;
@ -29,6 +30,7 @@ stdenv.mkDerivation {
  buildInputs = [
    # System requirements.
    glib
+    zlib
    dbus
    zstd
    libGL
@ -70,6 +72,7 @@ stdenv.mkDerivation {
    LIB_GCC_PATH="${libgcc.lib}/lib"
    GLIB_PATH="${glibStorePath}/lib"
    ZSTD_PATH="${zstdStorePath}/lib"
+    ZLIB_PATH="${zlibStorePath}/lib"
    DBUS_PATH="${dbusStorePath}/lib"
    LIBGL_PATH="${libGLStorePath}/lib"
    FREETYPE_PATH="${freetypeStorePath}/lib"
@ -92,6 +95,7 @@ stdenv.mkDerivation {
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$LIB_GCC_PATH"
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DBUS_PATH"
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GLIB_PATH"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$ZLIB_PATH"
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$ZSTD_PATH"
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$LIBGL_PATH"
    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$FONTCONFIG_PATH"
@ -121,7 +125,7 @@ stdenv.mkDerivation {
    PATCH="$PATCH:$PYQT6_PATH"
    PATCH="$PATCH:$PYQT6_SIP_PATH"

-    export PATCH 
+    export PATCH

    # Install deps
    uv lock
--- a/pyproject.toml
+++ b/pyproject.toml
@ -15,8 +15,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 [build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

 # ------ - ------

@ -34,114 +34,122 @@ ignore = []

 # ------ - ------

-[project]
+[tool.poetry]
 name = "piker"
-version = "0.1.0a0dev0"
+version = "0.1.0.alpha0.dev0"
 description = "trading gear for hackers"
-authors = [{ name = "Tyler Goodlet", email = "goodboy_foss@protonmail.com" }]
-requires-python = ">=3.12, <3.13"
-license = "AGPL-3.0-or-later"
+authors = ["Tyler Goodlet <goodboy_foss@protonmail.com>"]
+license = "AGPLv3"
 readme = "README.rst"
-keywords = [
-    "async",
-    "trading",
-    "finance",
-    "quant",
-    "charting",
+
+# ------ - ------
+
+[tool.poetry.dependencies]
+async-generator = "^1.10"
+attrs = "^23.1.0"
+bidict = "^0.22.1"
+colorama = "^0.4.6"
+colorlog = "^6.7.0"
+cython = "^3.0.0"
+greenback = "^1.1.1"
+ib-insync = "^0.9.86"
+msgspec = "^0.18.0"
+numba = "^0.59.0"
+numpy = "^1.25"
+polars = "^0.18.13"
+pygments = "^2.16.1"
+python = ">=3.11, <3.13"
+rich = "^13.5.2"
+# setuptools = "^68.0.0"
+tomli = "^2.0.1"
+tomli-w = "^1.0.0"
+trio-util = "^0.7.0"
+trio-websocket = "^0.10.3"
+typer = "^0.9.0"
+rapidfuzz = "^3.5.2"
+pdbp = "^1.5.0"
+trio = "^0.24"
+pendulum = "^3.0.0"
+httpx = "^0.27.0"
+
+[tool.poetry.dependencies.tractor]
+develop = true
+git = 'https://github.com/goodboy/tractor.git'
+branch = 'asyncio_debugger_support'
+# path = "../tractor"
+
+[tool.poetry.dependencies.asyncvnc]
+git = 'https://github.com/pikers/asyncvnc.git'
+branch = 'main'
+
+[tool.poetry.dependencies.tomlkit]
+develop = true
+git = 'https://github.com/pikers/tomlkit.git'
+branch = 'piker_pin'
+# path = "../tomlkit/"
+
+[tool.poetry.group.uis]
+optional = true
+[tool.poetry.group.uis.dependencies]
+# https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+# TODO: make sure the levenshtein shit compiles on nix..
+# rapidfuzz = {extras = ["speedup"], version = "^0.18.0"}
+rapidfuzz = "^3.2.0"
+qdarkstyle = ">=3.0.2"
+pyqtgraph = { git = 'https://github.com/pikers/pyqtgraph.git' }
+
+# ------ - ------
+pyqt6 = "^6.7.0"
+
+[tool.poetry.group.dev]
+optional = true
+[tool.poetry.group.dev.dependencies]
+# testing / CI
+pytest = "^6.0.0"
+elasticsearch = "^8.9.0"
+xonsh = "^0.14.2"
+prompt-toolkit = "3.0.40"
+
+# console ehancements and eventually remote debugging
+# extras/helpers.
+# TODO: add a toolset that makes debugging a `pikerd` service
+# (tree) easy to hack on directly using more or less the local env:
+# - xonsh + xxh
+# - rsyscall + pdbp
+# - actor runtime control console like BEAM/OTP
+
+# ------ - ------
+
+# TODO: add an `--only daemon` group for running non-ui / pikerd
+# service tree in distributed mode B)
+# https://python-poetry.org/docs/managing-dependencies/#installing-group-dependencies
+# [tool.poetry.group.daemon.dependencies]
+
+[tool.poetry.scripts]
+piker = 'piker.cli:cli'
+pikerd = 'piker.cli:pikerd'
+ledger = 'piker.accounting.cli:ledger'
+
+
+[project]
+name="piker"
+keywords=[
+  "async",
+  "trading",
+  "finance",
+  "quant",
+  "charting",
 ]
-classifiers = [
-    "Development Status :: 3 - Alpha",
-    "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
-    "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
-    "Intended Audience :: Financial and Insurance Industry",
-    "Intended Audience :: Science/Research",
-    "Intended Audience :: Developers",
-    "Intended Audience :: Education",
+classifiers=[
+  'Development Status :: 3 - Alpha',
+  "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+  'Operating System :: POSIX :: Linux',
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: 3 :: Only",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  'Intended Audience :: Financial and Insurance Industry',
+  'Intended Audience :: Science/Research',
+  'Intended Audience :: Developers',
+  'Intended Audience :: Education',
 ]
-dependencies = [
-    "async-generator >=1.10, <2.0.0",
-    "attrs >=23.1.0, <24.0.0",
-    "bidict >=0.22.1, <0.23.0",
-    "colorama >=0.4.6, <0.5.0",
-    "colorlog >=6.7.0, <7.0.0",
-    "ib-insync >=0.9.86, <0.10.0",
-    "numba >=0.59.0, <0.60.0",
-    "numpy >=1.25, <2.0",
-    "polars >=0.18.13, <0.19.0",
-    "pygments >=2.16.1, <3.0.0",
-    "rich >=13.5.2, <14.0.0",
-    "tomli >=2.0.1, <3.0.0",
-    "tomli-w >=1.0.0, <2.0.0",
-    "trio-util >=0.7.0, <0.8.0",
-    "trio-websocket >=0.10.3, <0.11.0",
-    "typer >=0.9.0, <1.0.0",
-    "rapidfuzz >=3.5.2, <4.0.0",
-    "pdbp >=1.5.0, <2.0.0",
-    "trio >=0.24, <0.25",
-    "pendulum >=3.0.0, <4.0.0",
-    "httpx >=0.27.0, <0.28.0",
-    "cryptofeed >=2.4.0, <3.0.0",
-    "pyarrow >=17.0.0, <18.0.0",
-    "websockets ==12.0",
-    "msgspec",
-    "tractor",
-    "asyncvnc",
-    "tomlkit",
-]
-
-[project.optional-dependencies]
-uis = [
-    # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
-    # TODO: make sure the levenshtein shit compiles on nix..
-    # rapidfuzz = {extras = ["speedup"], version = "^0.18.0"}
-    "rapidfuzz >=3.2.0, <4.0.0",
-    "qdarkstyle >=3.0.2, <4.0.0",
-    "pyqt6 >=6.7.0, <7.0.0",
-    "pyqtgraph",
-
-    # ------ - ------
-
-    # TODO: add an `--only daemon` group for running non-ui / pikerd
-    # service tree in distributed mode B)
-    # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
-    # [project.optional-dependencies]
-]
-
-[dependency-groups]
-dev = [
-    "pytest >=6.0.0, <7.0.0",
-    "elasticsearch >=8.9.0, <9.0.0",
-    "xonsh >=0.14.2, <0.15.0",
-    "prompt-toolkit ==3.0.40",
-    "cython >=3.0.0, <4.0.0",
-    "greenback >=1.1.1, <2.0.0",
-    # console ehancements and eventually remote debugging
-    # extras/helpers.
-    # TODO: add a toolset that makes debugging a `pikerd` service
-    # (tree) easy to hack on directly using more or less the local env:
-    # - xonsh + xxh
-    # - rsyscall + pdbp
-    # - actor runtime control console like BEAM/OTP
-]
-
-[project.scripts]
-piker = "piker.cli:cli"
-pikerd = "piker.cli:pikerd"
-ledger = "piker.accounting.cli:ledger"
-
-[tool.hatch.build.targets.sdist]
-include = ["piker"]
-
-[tool.hatch.build.targets.wheel]
-include = ["piker"]
-
-[tool.uv.sources]
-pyqtgraph = { git = "https://github.com/pikers/pyqtgraph.git" }
-asyncvnc = { git = "https://github.com/pikers/asyncvnc.git", branch = "main" }
-tomlkit = { git = "https://github.com/pikers/tomlkit.git", branch ="piker_pin" }
-msgspec = { git = "https://github.com/jcrist/msgspec.git" }
-tractor = { path = "../tractor" }
Author	SHA1	Message	Date
Nelson Torres	6b48bbca3c	Deribit's feed fix - `FeedInit` for init_msgs in `stream_quotes`. - new cache is `client_pairs` so is replacing the old `client.cache_symbols`. - `get_mkt_info` added - `get_ohlc` fixed to comply the new ways of the feed.	2025-02-03 18:57:01 -03:00
Nelson Torres	6b30b17afb	Deribit's api fix key changes: - Resolved the issue with the expiration dates from deribits, now we int instead of the crazy custom deribits format. - The client now has a new `_json_rpc_auth_wrapper` that adquires a first access token and then will refresh the access token when this expires. - `get_assets` fixed, now we use the public endpoint to check the availables assets, in the future probably this will change, but for now is working just fine. - `get_mkt_pairs` added. - `exch_info` added. - `cache_symbols` fixed. - Also a lot of reformat made in api.	2025-02-03 18:57:01 -03:00
Nelson Torres	0ec2c76c6e	Venues Moved from api to venues all the msgspecs structs, also added critical imports in api, feed and __init__ mods.	2025-02-03 18:57:01 -03:00
Tyler Goodlet	fe59ffc63f	Allow ledger passes to ignore (symcache) unknown fqmes For example in the paper-eng, if you have a backend that doesn't fully support a symcache (yet) it's handy to be able to ignore processing other paper-eng txns when all you care about at the moment is the simulated symbol. NOTE, that currently this will still result in a key-error when you load more then one mkt with the paper engine (for which the backend does not have the symcache implemented) since no fqme ad-hoc query was made for the 2nd symbol (and i'm not sure we should support that kinda hackery over just encouraging the sym-cache being added?). Def needs a little more thought depending on how many backends are never going to be able to (easily) support caching..	2025-02-03 18:55:47 -03:00
Tyler Goodlet	01081e1d80	.clearing._ems: Don't require `first_quote['last']` Instead just check for the field (which i'm not huge on the key-name for anyway) and if not found get the "last price" from the real-time shm buffer's latest 'close' sample. Unrelatedly, use a `subs.copy()` in the `Router.client_broadcast()` loop such that if a `client_stream` is popped on connection failure, we don't RTE for the "size changed on iteration".	2025-02-03 18:55:47 -03:00
Tyler Goodlet	a30c6d0ea1	`kucoin`: repair live quotes streaming.. This must have broke at some point during the new `MktPair` and thus `.fqme: str` updates; mas-o-menos the symbol key in the quote-msg-`dict` was NOT set to the `MktPair.bs_fqme: str` value and thus wasn't being processed by the downstream sampling and feed subsys. So fix that as well as a few other refinements, - set the `topic: mkt.bs_fqme` in quote msgs obvi. - drop the "wait for first clearing vlm" quote poll loop; going to fix the sampler to handle a `first_quote` without a `'last'` key. - add some typing around calls to `get_mkt_info()`. - rename `stream_messages()` -> `iter_normed_quotes()`.	2025-02-03 18:55:47 -03:00
Nelson Torres	7091a80a1d	Deleted settlePlan field from binance FutesPair.	2025-02-03 18:55:47 -03:00
Nelson Torres	5db98ef531	Added missing fields for kucoin. feeCategory, makerFeeCoefficient, takerFeeCoefficient and st.	2025-02-03 18:55:47 -03:00
Nelson Torres	3e999b164a	Default.nix fix: Since the last `poetry` update the command `poetry shell` have been moved to a plugin, this affects the entire instalation, so now we need to add more deps to the `buildInputs` and added to the `LD_LIBRARY_PATH` To run a command now you need to do something like this: `poetry run piker ...`	2025-02-03 18:55:47 -03:00
Tyler Goodlet	e87939e1f2	data._web_bs: try to raise jsonrpc errors in parent task	2025-02-03 18:52:11 -03:00
Nelson Torres	3630c02f48	Updated tractor method name.	2025-02-03 18:52:11 -03:00
Tyler Goodlet	fdf34e51bb	Drop `.cancel_actor()` from `maybe_spawn_daemon()` Since `tractor`'s new and improved inter-actor cancellation semantics are much more pedantic, AND bc we use the `ServiceMngr` for spawning service actors on-demand, the caller of `maybe_spawn_daemon()` should NEVER conduct a so called "out of band" `Actor`-runtime cancel request since this is precisely the job of our `ServiceMngr` XD Add a super in depth note explaining the underlying issue and adding a todo list of how we should prolly augment `tractor` to make such cases easier to grok and fix in the future!	2025-02-03 18:52:11 -03:00
Tyler Goodlet	a0b540022e	Prep service mngr for move to `tractor.hilevel` Given it's a fairly simple yet useful abstraction, it makes sense to offer this sub-sys alongside the core `tractor` runtime lib. Without going into extreme detail on the impl changes (it'll come in the commit that moves to the other repo) here is the high level summary: ------ - ------ - rename `Services` -> `ServiceMngr` and use an factory `@acm` to guarantee a single-instance-per-actor using a niche approach for a singleton object using a default keyword-arg B) - the mod level `open_service_mngr()` and `get_service_mngr()` are the new allocation/access API. - add a `ServiceMngr.start_service()` method which does the work of both spawning a new subactor (for the daemon) and uses its portal to start the mngr side supervision task. - open actor/task nurseries inside the `@acm` allocator. Adjust other dependent subsystems to match: ------ - ------ - use `open_service_mngr()` when first allocated in `open_pikerd()`. - use `get_service_mngr()` instead of importing the class ref inside `.service.maybe_spawn_daemon()`, `.brokers._daemon.spawn_brokerd()` and `.data._sampling.spawn_samplerd()` using a `partial` to pack in the endpoint ctx kwargs (unpacked inside `.start_service()` XD).	2025-02-03 18:52:11 -03:00
Tyler Goodlet	20f6343be2	Lel, forgot to add a `SPOT` venue for `binance`..	2025-02-03 18:52:11 -03:00
Tyler Goodlet	2f30fc4fb8	Mask no-data pause-point, add perps to sig. Was orig for debugging an issue with `kucoin` i think but definitely shouldn't be left in XD Also add `'perpetual_future'` to `start_backfill()` input literal set.	2025-02-03 18:52:11 -03:00
Nelson Torres	76b5547c2d	uv migration	2025-02-03 18:49:07 -03:00
Nelson Torres	08811f9a61	Default.nix fix: Since the last `poetry` update the command `poetry shell` have been moved to a plugin, this affects the entire instalation, so now we need to add more deps to the `buildInputs` and added to the `LD_LIBRARY_PATH` To run a command now you need to do something like this: `poetry run piker ...`	2025-02-03 18:46:20 -03:00