Add `.ui` and `.dev` deps groups via `poetry` Bo

Since we eventually want to allow users to minimally deploy `pikerd`
service-tree (aka distributed cross host) installs, we need to offer
a "headless" deps group. Really this is just the core dep set minus Qt
and some aux search related libs (for now).

The new `.dev` group is for adding hacking and testing tools including
`xonsh` since that will eventually be our REPL of choice more then
likely B)

Oh, and fix the namespace path (was a typo) for the `ledger` CLI and
of course bump the lock file.

flake.lock

@@ -0,0 +1,138 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "poetry2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1688870561,
+        "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1692174805,
+        "narHash": "sha256-xmNPFDi/AUMIxwgOH/IVom55Dks34u1g7sFKKebxUm0=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "caac0eb6bdcad0b32cb2522e03e4002c8975c62e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "poetry2nix": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1692048894,
+        "narHash": "sha256-cDw03rso2V4CDc3Mll0cHN+ztzysAvdI8pJ7ybbz714=",
+        "ref": "refs/heads/pyqt6",
+        "rev": "b059ad4c3051f45d6c912e17747aae37a9ec1544",
+        "revCount": 2276,
+        "type": "git",
+        "url": "file:///home/lord_fomo/repos/poetry2nix"
+      },
+      "original": {
+        "type": "git",
+        "url": "file:///home/lord_fomo/repos/poetry2nix"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs",
+        "poetry2nix": "poetry2nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,178 @@
+# NOTE: to convert to a poetry2nix env like this here are the
+# steps:
+# - install poetry in your system nix config
+# - convert the repo to use poetry using `poetry init`:
+#   https://python-poetry.org/docs/basic-usage/#initialising-a-pre-existing-project
+# - then manually ensuring all deps are converted over:
+# - add this file to the repo and commit it
+# - 
+{
+  description = "piker: trading gear for hackers (pkged with poetry2nix)";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+
+  # see https://github.com/nix-community/poetry2nix/tree/master#api
+  inputs.poetry2nix = {
+    # url = "github:nix-community/poetry2nix";
+    # url = "github:K900/poetry2nix/qt5-explicit-deps";
+    url = "/home/lord_fomo/repos/poetry2nix";
+
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+    poetry2nix,
+  }:
+    # TODO: build cross-OS and use the `${system}` var thingy..
+    flake-utils.lib.eachDefaultSystem (system:
+      let
+        # use PWD as sources
+        projectDir = ./.;
+        pyproject = ./pyproject.toml;
+        poetrylock = ./poetry.lock;
+
+        # TODO: port to 3.11 and support both versions?
+        python = "python3.10";
+
+        # for more functions and examples.
+        # inherit
+        # (poetry2nix.legacyPackages.${system})
+        # mkPoetryApplication;
+        # pkgs = nixpkgs.legacyPackages.${system};
+
+        pkgs = nixpkgs.legacyPackages.x86_64-linux;
+        lib = pkgs.lib;
+        p2npkgs = poetry2nix.legacyPackages.x86_64-linux;
+
+        # define all pkg overrides per dep, see edgecases.md:
+        # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md
+        # TODO: add these into the json file:
+        # https://github.com/nix-community/poetry2nix/blob/master/overrides/build-systems.json
+        pypkgs-build-requirements = {
+          asyncvnc = [ "setuptools" ];
+          eventkit = [ "setuptools" ];
+          ib-insync = [ "setuptools" "flake8" ];
+          msgspec = [ "setuptools"];
+          pdbp = [ "setuptools" ];
+          pyqt6-sip = [ "setuptools" ];
+          tabcompleter = [ "setuptools" ];
+          tractor = [ "setuptools" ];
+          tricycle = [ "setuptools" ];
+          trio-typing = [ "setuptools" ];
+          trio-util = [ "setuptools" ];
+          xonsh = [ "setuptools" ];
+        };
+
+        # auto-generate override entries
+        p2n-overrides = p2npkgs.defaultPoetryOverrides.extend (self: super:
+          builtins.mapAttrs (package: build-requirements:
+            (builtins.getAttr package super).overridePythonAttrs (old: {
+              buildInputs = (
+                old.buildInputs or [ ]
+              ) ++ (
+                builtins.map (
+                  pkg: if builtins.isString pkg then builtins.getAttr pkg super else pkg
+                  ) build-requirements
+              );
+            })
+          ) pypkgs-build-requirements
+        );
+
+        # override some ahead-of-time compiled extensions
+        # to be built with their wheels.
+        ahot_overrides = p2n-overrides.extend(
+          final: prev: {
+
+            # llvmlite = prev.llvmlite.override {
+            #   preferWheel = false;
+            # };
+
+            # TODO: get this workin with p2n and nixpkgs..
+            # pyqt6 = prev.pyqt6.override {
+            #   preferWheel = true;
+            # };
+
+            # NOTE: this DOESN'T work atm but after a fix
+            # to poetry2nix, it will and actually this line
+            # won't be needed - thanks @k900:
+            # https://github.com/nix-community/poetry2nix/pull/1257
+            pyqt5 = prev.pyqt5.override {
+              withWebkit = false;
+              preferWheel = true;
+            };
+
+            # see PR from @k900:
+            # https://github.com/nix-community/poetry2nix/pull/1257
+            # pyqt5-qt5 = prev.pyqt5-qt5.override {
+            #   withWebkit = false;
+            #   preferWheel = true;
+            # };
+
+            # TODO: patch in an override for polars to build
+            # from src! See the details likely needed from
+            # the cryptography entry:
+            # https://github.com/nix-community/poetry2nix/blob/master/overrides/default.nix#L426-L435
+            polars = prev.polars.override {
+              preferWheel = true;
+            };
+          }
+      );
+
+      # WHY!? -> output-attrs that `nix develop` scans for:
+      # https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-develop.html#flake-output-attributes
+      in {
+          packages = {
+            # piker = poetry2nix.legacyPackages.x86_64-linux.mkPoetryEditablePackage {
+            #   editablePackageSources = { piker = ./piker; };
+
+            piker = p2npkgs.mkPoetryApplication {
+              projectDir = projectDir;
+
+              # SEE ABOVE for auto-genned input set, override
+              # buncha deps with extras.. like `setuptools` mostly.
+              # TODO: maybe propose a patch to p2n to show that you
+              # can even do this in the edgecases docs?
+              overrides = ahot_overrides;
+
+              # XXX: won't work on llvmlite..
+              # preferWheels = true;
+            };
+          };
+
+          devShells.default = pkgs.mkShell {
+            # packages = [ poetry2nix.packages.${system}.poetry ];
+            packages = [ poetry2nix.packages.x86_64-linux.poetry ];
+            inputsFrom = [ self.packages.x86_64-linux.piker ];
+
+            # TODO: boot xonsh inside the poetry virtualenv when
+            # defined via a custom entry point?
+            # NOTE XXX: apparently DON'T do these..?
+            # shellHook = "poetry run xonsh";
+            # shellHook = "poetry shell";
+          };
+
+
+          # TODO: grok the difference here..
+          # - avoid re-cloning git repos on every develop entry..
+          # - ideally allow hacking on the src code of some deps
+          #   (tractor, pyqtgraph, tomlkit, etc.) WITHOUT having to
+          #   re-install them every time a change is made.
+
+          # devShells.default = (p2npkgs.mkPoetryEnv {
+          # # let {
+          # #   devEnv = p2npkgs.mkPoetryEnv {
+          #     projectDir = projectDir;
+          #     overrides = ahot_overrides;
+          #     inputsFrom = [ self.packages.x86_64-linux.piker ];
+          #   }).env.overrideAttrs (old: {
+          #       buildInputs = [ packages.piker ];
+          #     }
+          #   );
+
+        }
+    );  # end of .outputs scope
+}

piker/cli/__init__.py

@@ -226,8 +226,6 @@ def services(config, tl, ports):
 
 
 def _load_clis() -> None:
-    from ..service import marketstore  # noqa
-    from ..service import elastic
     from ..data import cli  # noqa
     from ..brokers import cli  # noqa
     from ..ui import cli  # noqa

piker/data/cli.py

@@ -22,14 +22,14 @@ import trio
 import tractor
 import click
 
-from ..service.marketstore import (
+# from ..service.marketstore import (
-    # get_client,
+#     # get_client,
-    # stream_quotes,
+#     # stream_quotes,
-    ingest_quote_stream,
+#     ingest_quote_stream,
-    # _url,
+#     # _url,
-    # _tick_tbk_ids,
+#     # _tick_tbk_ids,
-    # mk_tbk,
+#     # mk_tbk,
-)
+# )
 from ..cli import cli
 from .. import watchlists as wl
 from ._util import (
@@ -212,44 +212,44 @@ def storage(
     trio.run(main)
 
 
-@cli.command()
+# @cli.command()
-@click.option('--test-file', '-t', help='Test quote stream file')
+# @click.option('--test-file', '-t', help='Test quote stream file')
-@click.option('--tl', is_flag=True, help='Enable tractor logging')
+# @click.option('--tl', is_flag=True, help='Enable tractor logging')
-@click.argument('name', nargs=1, required=True)
+# @click.argument('name', nargs=1, required=True)
-@click.pass_obj
+# @click.pass_obj
-def ingest(config, name, test_file, tl):
+# def ingest(config, name, test_file, tl):
-    '''
+#     '''
-    Ingest real-time broker quotes and ticks to a marketstore instance.
+#     Ingest real-time broker quotes and ticks to a marketstore instance.
 
-    '''
+#     '''
-    # global opts
+#     # global opts
-    loglevel = config['loglevel']
+#     loglevel = config['loglevel']
-    tractorloglevel = config['tractorloglevel']
+#     tractorloglevel = config['tractorloglevel']
-    # log = config['log']
+#     # log = config['log']
 
-    watchlist_from_file = wl.ensure_watchlists(config['wl_path'])
+#     watchlist_from_file = wl.ensure_watchlists(config['wl_path'])
-    watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins)
+#     watchlists = wl.merge_watchlist(watchlist_from_file, wl._builtins)
-    symbols = watchlists[name]
+#     symbols = watchlists[name]
 
-    grouped_syms = {}
+#     grouped_syms = {}
-    for sym in symbols:
+#     for sym in symbols:
-        symbol, _, provider = sym.rpartition('.')
+#         symbol, _, provider = sym.rpartition('.')
-        if provider not in grouped_syms:
+#         if provider not in grouped_syms:
-            grouped_syms[provider] = []
+#             grouped_syms[provider] = []
 
-        grouped_syms[provider].append(symbol)
+#         grouped_syms[provider].append(symbol)
 
-    async def entry_point():
+#     async def entry_point():
-        async with tractor.open_nursery() as n:
+#         async with tractor.open_nursery() as n:
-            for provider, symbols in grouped_syms.items():
+#             for provider, symbols in grouped_syms.items():
-                await n.run_in_actor(
+#                 await n.run_in_actor(
-                    ingest_quote_stream,
+#                     ingest_quote_stream,
-                    name='ingest_marketstore',
+#                     name='ingest_marketstore',
-                    symbols=symbols,
+#                     symbols=symbols,
-                    brokername=provider,
+#                     brokername=provider,
-                    tries=1,
+#                     tries=1,
-                    actorloglevel=loglevel,
+#                     actorloglevel=loglevel,
-                    loglevel=tractorloglevel
+#                     loglevel=tractorloglevel
-                )
+#                 )
 
-    tractor.run(entry_point)
+#     tractor.run(entry_point)

pyproject.toml

@@ -0,0 +1,110 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# ------ - ------
+
+[tool.poetry]
+name = "piker"
+version = "0.1.0.alpha0.dev0"
+description = "trading gear for hackers"
+authors = ["Tyler Goodlet <jgbt@protonmail.com>"]
+license = "AGPLv3"
+readme = "README.rst"
+
+# ------ - ------
+
+# NOTE: the default install is like an `--only daemon` deps group
+# for running a non-ui / headless / pikerd-service-actor-tree in
+# disti mode:
+# https://python-poetry.org/docs/managing-dependencies/#installing-group-dependencies
+# <SEE BELOW for installing the `--with uis` group>
+[tool.poetry.dependencies]
+asks = "^3.0.0"
+async-generator = "^1.10"
+attrs = "^23.1.0"
+bidict = "^0.22.1"
+colorama = "^0.4.6"
+colorlog = "^6.7.0"
+cython = "^3.0.0"
+greenback = "^1.1.1"
+ib-insync = "^0.9.86"
+msgspec = "^0.18.0"
+numba = "^0.57.1"
+numpy = "1.24"
+pendulum = "^2.1.2"
+polars = "^0.18.13"
+pygments = "^2.16.1"
+python = "^3.10"
+rich = "^13.5.2"
+# setuptools = "^68.0.0"
+tomli = "^2.0.1"
+tomli-w = "^1.0.0"
+trio = "^0.22.2"
+trio-util = "^0.7.0"
+trio-websocket = "^0.10.3"
+typer = "^0.9.0"
+
+
+[tool.poetry.dependencies.asyncvnc]
+git = 'https://github.com/pikers/asyncvnc.git'
+branch = 'main'
+
+[tool.poetry.dependencies.tomlkit]
+# git = 'https://github.com/pikers/tomlkit.git',
+# branch = 'piker_pin'
+path = "../tomlkit/"
+develop = true
+
+# ------ - ------
+
+[tool.poetry.dependencies.tractor]
+# path = '../tractor/'
+# branch = 'asyncio_debugger_support'
+git = 'https://github.com/goodboy/tractor.git'
+branch = 'piker_pin'
+develop = true
+
+# ------ - ------
+
+[tool.poetry.group.uis]
+optional=true
+[tool.poetry.group.uis.dependencies]
+# https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+# TODO: outstanding pkging issues on nixos.. no wheel and missing
+# overlay code in the flake to get AOTC workin.
+fuzzywuzzy = {extras = ["speedup"], version = "^0.18.0"}
+# fuzzywuzzy = "^0.18.0"
+qdarkstyle = ">=3.0.2"
+pyqt5 = "^5.15.9"
+pyqtgraph = { git = 'https://github.com/pikers/pyqtgraph.git' }
+# pyqt6 = "^6.5.2"
+
+# ------ - ------
+
+[tool.poetry.group.dev]
+optional = true
+[tool.poetry.group.dev.dependencies]
+# testing / CI
+pytest = "^6.0.0"
+elasticsearch = "^8.9.0"
+
+# console ehancements and eventually remote debugging
+# extras/helpers.
+# TODO: add a toolset that makes debugging a `pikerd` service
+# (tree) easy to hack on directly using more or less the local env:
+# - xonsh + xxh
+# - rsyscall + pdbp
+# - actor runtime control console like BEAM/OTP
+xonsh = "^0.14.0"  # XXX: explicit env install for shell use w nix
+prompt-toolkit = "^3.0.39"
+
+# ------ - ------
+
+[tool.poetry.scripts]
+# poetry = "poetry.console.application:main"
+piker = 'piker.cli:cli'
+pikerd = 'piker.cli:pikerd'
+ledger = 'piker.accounting.cli:ledger'
+
+# ------ - ------

setup.py

@@ -76,7 +76,7 @@ setup(
         'pendulum', # easier datetimes
         'bidict',  # 2 way map
         'cython',
-        'numpy',
+        'numpy==1.24',
         'numba',
 
         # UI