Mk jsronrpc's underlying ws timeout `float('inf')`

Since currently we're only using this IPC subsys for `deribit`, and generally speaking we're primarly supporting options markets (which are fairly "slow moving"), flip to a default of NOT resetting the `NoBsWs` on timeout since doing so normally breaks the jsron-rpc IPC session. Without a proper `fixture` passed to `open_autorecon_ws()` (which we should eventually implement!!) relying on a timeout-to-reset more or less will just cause breakage issues - a proper reconnect sequence must be implemented before using that feature. Deats, - expose and proxy through the `msg_recv_timeout` from `open_jsonrpc_session()` into the underlying `open_autorecon_ws()` call.
Refine history gap/termination signalling
2025-02-17 14:56:26 -05:00 · 2025-02-17 14:56:26 -05:00 · 2025-02-17 14:56:26 -05:00 · 2025-02-17 14:56:26 -05:00 · 2025-02-17 14:56:26 -05:00 · 2025-02-17 14:56:26 -05:00
3 changed files with 133 additions and 27 deletions
--- a/pyproject.toml
+++ b/pyproject.toml
@ -18,22 +18,6 @@
 requires = ["hatchling"]
 build-backend = "hatchling.build"

-# ------ - ------
-
-[tool.ruff.lint]
-# https://docs.astral.sh/ruff/settings/#lint_ignore
-ignore = []
-
-# https://docs.astral.sh/ruff/settings/#lint_per-file-ignores
-"piker/ui/qt.py" = [
-  "E402",
-  'F401',  # unused imports (without __all__ or blah as blah)
-  # "F841", # unused variable rules
-]
-# ignore-init-module-imports = false
-
-# ------ - ------
-
 [project]
 name = "piker"
 version = "0.1.0a0dev0"
@ -103,15 +87,23 @@ uis = [
    "pyqt6 >=6.7.0, <7.0.0",
    "pyqtgraph",

-    # ------ - ------
+    # for consideration,
+    # - 'visidata'

    # TODO: add an `--only daemon` group for running non-ui / pikerd
    # service tree in distributed mode B)
    # https://docs.astral.sh/uv/concepts/projects/dependencies/#optional-dependencies
-    # [project.optional-dependencies]
 ]

 [dependency-groups]
+# TODO: a toolset that makes debugging a `pikerd` service (tree) easy
+# to hack on directly using more or less the local env:
+# - xonsh + xxh
+# - rsyscall + pdbp
+# - actor runtime control console like BEAM/OTP
+#
+# console ehancements and eventually remote debugging extras/helpers.
+# use `uv --dev` to enable
 dev = [
    "pytest >=6.0.0, <7.0.0",
    "elasticsearch >=8.9.0, <9.0.0",
@ -119,13 +111,7 @@ dev = [
    "prompt-toolkit ==3.0.40",
    "cython >=3.0.0, <4.0.0",
    "greenback >=1.1.1, <2.0.0",
-    # console ehancements and eventually remote debugging
-    # extras/helpers.
-    # TODO: add a toolset that makes debugging a `pikerd` service
-    # (tree) easy to hack on directly using more or less the local env:
-    # - xonsh + xxh
-    # - rsyscall + pdbp
-    # - actor runtime control console like BEAM/OTP
+    "ruff>=0.9.6",
 ]

 [project.scripts]
--- a/ruff.toml
+++ b/ruff.toml
@ -0,0 +1,93 @@
+# from default `ruff.toml` @
+# https://docs.astral.sh/ruff/configuration/
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".ipynb_checkpoints",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pyenv",
+    ".pytest_cache",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    ".vscode",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "site-packages",
+    "venv",
+]
+
+# Same as Black.
+line-length = 88
+indent-width = 4
+
+# Assume Python 3.9
+target-version = "py312"
+
+# ------ - ------
+# TODO, stop warnings around `anext()` builtin use?
+# tool.ruff.target-version = "py310"
+
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+ignore-init-module-imports = false
+
+[lint.per-file-ignores]
+"piker/ui/qt.py" = [
+  "E402",
+  'F401',  # unused imports (without __all__ or blah as blah)
+  # "F841", # unused variable rules
+]
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[format]
+# Use single quotes in `ruff format`.
+quote-style = "single"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"
+
+# Enable auto-formatting of code examples in docstrings. Markdown,
+# reStructuredText code/literal blocks and doctests are all supported.
+#
+# This is currently disabled by default, but it is planned for this
+# to be opt-out in the future.
+docstring-code-format = false
+
+# Set the line length limit used when formatting code snippets in
+# docstrings.
+#
+# This only has an effect when the `docstring-code-format` setting is
+# enabled.
+docstring-code-line-length = "dynamic"
--- a/uv.lock
+++ b/uv.lock
@ -708,6 +708,7 @@ dev = [
    { name = "greenback" },
    { name = "prompt-toolkit" },
    { name = "pytest" },
+    { name = "ruff" },
    { name = "xonsh" },
 ]

@ -739,7 +740,7 @@ requires-dist = [
    { name = "tomli", specifier = ">=2.0.1,<3.0.0" },
    { name = "tomli-w", specifier = ">=1.0.0,<2.0.0" },
    { name = "tomlkit", git = "https://github.com/pikers/tomlkit.git?branch=piker_pin" },
-    { name = "tractor", directory = "../tractor" },
+    { name = "tractor", editable = "../tractor" },
    { name = "trio", specifier = ">=0.24,<0.25" },
    { name = "trio-util", specifier = ">=0.7.0,<0.8.0" },
    { name = "trio-websocket", specifier = ">=0.10.3,<0.11.0" },
@ -754,6 +755,7 @@ dev = [
    { name = "greenback", specifier = ">=1.1.1,<2.0.0" },
    { name = "prompt-toolkit", specifier = "==3.0.40" },
    { name = "pytest", specifier = ">=6.0.0,<7.0.0" },
+    { name = "ruff", specifier = ">=0.9.6" },
    { name = "xonsh", specifier = ">=0.14.2,<0.15.0" },
 ]

@ -1073,6 +1075,31 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424 },
 ]

+[[package]]
+name = "ruff"
+version = "0.9.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/2a/e1/e265aba384343dd8ddd3083f5e33536cd17e1566c41453a5517b5dd443be/ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9", size = 3639454 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/e3/3d2c022e687e18cf5d93d6bfa2722d46afc64eaa438c7fbbdd603b3597be/ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba", size = 11714128 },
+    { url = "https://files.pythonhosted.org/packages/e1/22/aff073b70f95c052e5c58153cba735748c9e70107a77d03420d7850710a0/ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504", size = 11682539 },
+    { url = "https://files.pythonhosted.org/packages/75/a7/f5b7390afd98a7918582a3d256cd3e78ba0a26165a467c1820084587cbf9/ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83", size = 11132512 },
+    { url = "https://files.pythonhosted.org/packages/a6/e3/45de13ef65047fea2e33f7e573d848206e15c715e5cd56095589a7733d04/ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc", size = 11929275 },
+    { url = "https://files.pythonhosted.org/packages/7d/f2/23d04cd6c43b2e641ab961ade8d0b5edb212ecebd112506188c91f2a6e6c/ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b", size = 11466502 },
+    { url = "https://files.pythonhosted.org/packages/b5/6f/3a8cf166f2d7f1627dd2201e6cbc4cb81f8b7d58099348f0c1ff7b733792/ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e", size = 12676364 },
+    { url = "https://files.pythonhosted.org/packages/f5/c4/db52e2189983c70114ff2b7e3997e48c8318af44fe83e1ce9517570a50c6/ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666", size = 13335518 },
+    { url = "https://files.pythonhosted.org/packages/66/44/545f8a4d136830f08f4d24324e7db957c5374bf3a3f7a6c0bc7be4623a37/ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5", size = 12823287 },
+    { url = "https://files.pythonhosted.org/packages/c5/26/8208ef9ee7431032c143649a9967c3ae1aae4257d95e6f8519f07309aa66/ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5", size = 14592374 },
+    { url = "https://files.pythonhosted.org/packages/31/70/e917781e55ff39c5b5208bda384fd397ffd76605e68544d71a7e40944945/ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217", size = 12500173 },
+    { url = "https://files.pythonhosted.org/packages/84/f5/e4ddee07660f5a9622a9c2b639afd8f3104988dc4f6ba0b73ffacffa9a8c/ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6", size = 11906555 },
+    { url = "https://files.pythonhosted.org/packages/f1/2b/6ff2fe383667075eef8656b9892e73dd9b119b5e3add51298628b87f6429/ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897", size = 11538958 },
+    { url = "https://files.pythonhosted.org/packages/3c/db/98e59e90de45d1eb46649151c10a062d5707b5b7f76f64eb1e29edf6ebb1/ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08", size = 12117247 },
+    { url = "https://files.pythonhosted.org/packages/ec/bc/54e38f6d219013a9204a5a2015c09e7a8c36cedcd50a4b01ac69a550b9d9/ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656", size = 12554647 },
+    { url = "https://files.pythonhosted.org/packages/a5/7d/7b461ab0e2404293c0627125bb70ac642c2e8d55bf590f6fce85f508f1b2/ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d", size = 9949214 },
+    { url = "https://files.pythonhosted.org/packages/ee/30/c3cee10f915ed75a5c29c1e57311282d1a15855551a64795c1b2bbe5cf37/ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa", size = 10999914 },
+    { url = "https://files.pythonhosted.org/packages/e8/a8/d71f44b93e3aa86ae232af1f2126ca7b95c0f515ec135462b3e1f351441c/ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a", size = 10177499 },
+]
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@ -1188,7 +1215,7 @@ source = { git = "https://github.com/pikers/tomlkit.git?branch=piker_pin#8e0239a
 [[package]]
 name = "tractor"
 version = "0.1.0a6.dev0"
-source = { directory = "../tractor" }
+source = { editable = "../tractor" }
 dependencies = [
    { name = "colorlog" },
    { name = "msgspec" },
Author	SHA1	Message	Date
Tyler Goodlet	d98fd54627	Mk jsronrpc's underlying ws timeout `float('inf')` Since currently we're only using this IPC subsys for `deribit`, and generally speaking we're primarly supporting options markets (which are fairly "slow moving"), flip to a default of NOT resetting the `NoBsWs` on timeout since doing so normally breaks the jsron-rpc IPC session. Without a proper `fixture` passed to `open_autorecon_ws()` (which we should eventually implement!!) relying on a timeout-to-reset more or less will just cause breakage issues - a proper reconnect sequence must be implemented before using that feature. Deats, - expose and proxy through the `msg_recv_timeout` from `open_jsonrpc_session()` into the underlying `open_autorecon_ws()` call.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	8892408039	Refine history gap/termination signalling Namely handling backends which do not provide a default "frame size-duration" in their init-config by making the backfiller guess the value based on the first frame received. Deats, - adjust `start_backfill()` to take a more explicit `def_frame_duration: Duration` expected to be unpacked from any backend hist init-config by the `tsdb_backfill()` caller which now also computes a value from the first received frame when the config section isn't provided. - in `start_backfill()` we now always expect the `def_frame_duration` input and always decrement the query range by this value whenever a `NoData` is raised by the provider-backend paired with an explicit `log.warning()` about the handling. - also relay any `DataUnavailable.args[0]` message from the provider in the handler. - repair "gap reporting" which checks for expected frame duration vs. that received with much better humanized logging on the missing segment using `pendulum.Interval/Duration.in_words()` output.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	5989810fbd	Only use `frame_types` if delivered during enter The `open_history_client()` provider endpoint can optionally deliver a `frame_types: dict[int, pendulum.Duration]` subsection in its `config: dict[str, dict]` (as was implemented with the `ib` backend). This allows the `tsp` backfilling machinery to use this "recommended frame duration" to subtract from the `last_start_dt` any time a `NoData` gap is signalled by the `get_hist()` call allowing gaps to be ignored safely without missing history by knowing the next earliest dt we can query from using the `end_dt`. However, currently all crypto$ providers haven't implemented this feat yet.. As such only try to use the `frame_types` feature if provided when handling `NoData` conditions inside `tsp.start_backfill()` and otherwise raise as normal.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	7a70ff857c	TOCHERRY service_mng_to_tractor: type fix to `.hilevel.ServicecMngr`	2025-02-17 14:56:26 -05:00
Tyler Goodlet	7a912deb4a	Enable `greenback` for `.pause_from_sync()` by default?	2025-02-17 14:56:26 -05:00
Tyler Goodlet	f6625b5929	`.brokers.cli`: module type and todo for `--pdb` flag to NOT src from sub-cmd	2025-02-17 14:56:26 -05:00
Tyler Goodlet	992d5f5af8	Catch using `Sampler.bcast_errors` where possible In all other possible IPC disconnect handling blocks. Also more comprehensive typing throughout `uniform_rate_send()`.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	04f5a79d36	Group bcast errors as `Sampler.bcast_errors` A new class var `tuple[Exception]` such that the err set can be reffed externally as needed for catching other similar pub-sub/IPC failures in other (related) real-time sub-systems. Also added some now-masked logging for debugging live-feed stream reading issues that should ONLY be used for debugging since they'll greatly degrade HFT perf. Used the new `log.mk_repr()` stuff (that one day we should prolly pull from `modden` as a dep) for pretty console emissions.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	e4044e4547	Suppress `trio.EndOfChannel`s raised by remote peer Since now `tractor` will raise this native `trio`-exc translated from a `Stop` msg when the peer gracefully terminates a `tractor.MsgStream`. Just `info()` log in such cases versus continuing to warn for the others.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	d91697c917	`.tsp._anal`: add (unused) `detect_vlm_gaps()`	2025-02-17 14:56:26 -05:00
Tyler Goodlet	70f77d67cd	`.storage.cli`: collect gap-markup-aids into `tf2aids: dict` prior to pause for introspection	2025-02-17 14:56:26 -05:00
Tyler Goodlet	18bf3484e0	Delegate to `tractor.msg.pretty_struct` since it was factored from here!	2025-02-17 14:56:26 -05:00
Tyler Goodlet	2ab7038d48	Teensie `piker.data` styling tweaks - use more compact optional value style with `\|`-union - fix `.flows` typing-only import since we need `MktPair` to be immediately defined for use on a `msgspec.Struct` field. - more "tree-like" warning msg in `.validate()` reporting.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	046e7dcec5	Invert `getattr()` check for `get_mkt_pairs()` ep Such that we `return` early when not defined by the provider backend to reduce an indent level in `SymbologyCache.load()`.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	320e8ddc57	Various `.clearing` todos/notes on potential issues with loglevel settings..	2025-02-17 14:56:26 -05:00
Tyler Goodlet	a4ffbb7927	Type loaded backend modules	2025-02-17 14:56:26 -05:00
Tyler Goodlet	aa506d124a	Bump various `.brokers.core` doc string content/style	2025-02-17 14:56:26 -05:00
Tyler Goodlet	de05b4ae17	Ignore any non-`.parquet` files under `.config/piker/nativedb/` subdir	2025-02-17 14:56:26 -05:00
Tyler Goodlet	c6d6ccd702	Doc-n-clean `.data._web_bs.open_jsonrpc_session()` Add a doc-string reflecting recent refinements, drop all the old hook params, rename `n: trio.Nursery` -> `tn` for "task nursery" fitting with code base's naming style.	2025-02-17 14:56:26 -05:00
Tyler Goodlet	b02a03a0a8	Allow ledger passes to ignore (symcache) unknown fqmes For example in the paper-eng, if you have a backend that doesn't fully support a symcache (yet) it's handy to be able to ignore processing other paper-eng txns when all you care about at the moment is the simulated symbol. NOTE, that currently this will still result in a key-error when you load more then one mkt with the paper engine (for which the backend does not have the symcache implemented) since no fqme ad-hoc query was made for the 2nd symbol (and i'm not sure we should support that kinda hackery over just encouraging the sym-cache being added?). Def needs a little more thought depending on how many backends are never going to be able to (easily) support caching..	2025-02-17 14:56:26 -05:00
Tyler Goodlet	dce73af6cc	.clearing._ems: Don't require `first_quote['last']` Instead just check for the field (which i'm not huge on the key-name for anyway) and if not found get the "last price" from the real-time shm buffer's latest 'close' sample. Unrelatedly, use a `subs.copy()` in the `Router.client_broadcast()` loop such that if a `client_stream` is popped on connection failure, we don't RTE for the "size changed on iteration".	2025-02-17 14:56:26 -05:00
Tyler Goodlet	b990a66e8c	data._web_bs: try to raise jsonrpc errors in parent task	2025-02-17 14:56:26 -05:00
Tyler Goodlet	f427956a68	Mask no-data pause, add perps to no-`/src`-in-fqme asset set Was orig for debugging an issue with `kucoin` i think but definitely shouldn't be left in XD Also add `'perpetual_future'` to the `.start_backfill()` input literal set since we don't expect the 'btc/usd.perp.binance' for now.	2025-02-17 14:56:26 -05:00
goodboy	1e3942fdc2	Merge pull request 'Add `ruff` to deps, bump `uv.lock`' (#32 ) from add_ruff_linter into gitea_feats Reviewed-on: #32	2025-02-17 19:55:32 +00:00
Nelson Torres	49ea380503	Add new `ruff.toml` config file Based on the default provided in their [docs](https://docs.astral.sh/ruff/configuration/) and migrating previous config from the prior `poetry`-verion of our `pyproject.toml`	2025-02-17 14:48:10 -05:00
Tyler Goodlet	933f169938	Add/reorg back some content from `poetry` old config	2025-02-14 13:47:02 -05:00
Nelson Torres	51337052a4	Remove legacy `poetry` config content from pyproject.toml	2025-02-14 15:01:29 -03:00
Tyler Goodlet	8abe55dcc6	Add `ruff` to deps, bump `uv.lock` Such that we start encouraging devs to lint code they touch and hopefully we include a pass as part of our tests/CI eventually B) Also, mk local `tractor` install `--editable` since without it being a locally hackable repo it's kinda pointless to install from the local fs Xp	2025-02-13 21:20:11 -05:00