From 6b90e2e3ee9879092ab557f707c7b98cc72700e1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 15 Aug 2023 11:39:03 -0400 Subject: [PATCH] Factor and gen per-dep overrides via "fancy" `.extend()` As per the hot tip from the edgecases.md, https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md#modulenotfounderror-no-module-named-packagename Factor all the (mostly `setuptools`) overrides into a `pypkgs-build-requirements` set and `.extend()` in any `preferWheel` additions (`polars`, `pyqt`, etc.) before passing to to `mkPoetryApplication(overrides=)`. Add a buncha todos for improving the poetry2nix pkging including: - adding the override requirements to the json file for all our deps in the `pypkgs-build-requirement` set. - maybe propose docs for the edgecases.md to show how to do the auto-gen set (via func) AND extend with further overrides like `preferWheel`? - task to support `polars` build from src (by copying `cryptography` stuff) instead of only from a wheel? - get pyqt5 building from wheel since it seems to be taking forever from src.. - get pyqt6 working in general - going to require taking stuff from nixpkgs and applying it in the overrides of p2n. --- flake.nix | 115 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 26bdcd02..7442781d 100644 --- a/flake.nix +++ b/flake.nix @@ -4,13 +4,20 @@ # - convert the repo to use poetry using `poetry init`: # https://python-poetry.org/docs/basic-usage/#initialising-a-pre-existing-project # - then manually ensuring all deps are converted over: +# - add this file to the repo and commit it +# - { description = "piker: trading gear for hackers (pkged with poetry2nix)"; inputs.flake-utils.url = "github:numtide/flake-utils"; inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + # see https://github.com/nix-community/poetry2nix/tree/master#api inputs.poetry2nix = { url = "github:nix-community/poetry2nix"; + # url = "github:K900/poetry2nix/qt5-explicit-deps"; + # url = "/home/lord_fomo/repos/poetry2nix"; + inputs.nixpkgs.follows = "nixpkgs"; }; @@ -20,6 +27,7 @@ flake-utils, poetry2nix, }: + # TODO: build cross-OS and use the `${system}` var thingy.. flake-utils.lib.eachDefaultSystem (system: let # use PWD as sources @@ -30,13 +38,52 @@ # TODO: port to 3.11 and support both versions? python = "python3.10"; - # see https://github.com/nix-community/poetry2nix/tree/master#api # for more functions and examples. # inherit # (poetry2nix.legacyPackages.${system}) # mkPoetryApplication; # pkgs = nixpkgs.legacyPackages.${system}; + pkgs = nixpkgs.legacyPackages.x86_64-linux; + lib = pkgs.lib; + p2npkgs = poetry2nix.legacyPackages.x86_64-linux; + + # define all pkg overrides per dep, see edgecases.md: + # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md + # TODO: add these into the json file: + # https://github.com/nix-community/poetry2nix/blob/master/overrides/build-systems.json + pypkgs-build-requirements = { + asyncvnc = [ "setuptools" ]; + eventkit = [ "setuptools" ]; + ib-insync = [ "setuptools" "flake8" ]; + msgspec = [ "setuptools"]; + tabcompleter = [ "setuptools" ]; + pdbp = [ "setuptools" ]; + xonsh = [ "setuptools" ]; + trio-typing = [ "setuptools" ]; + trio-util = [ "setuptools" ]; + tricycle = [ "setuptools" ]; + tractor = [ "setuptools" ]; + pyqt6-sip = [ "setuptools" ]; + + # don't need these right? + # tomlkit = [ "setuptools" ]; + }; + + # auto-generate override entries + p2n-overrides = p2npkgs.defaultPoetryOverrides.extend (self: super: + builtins.mapAttrs (package: build-requirements: + (builtins.getAttr package super).overridePythonAttrs (old: { + buildInputs = ( + old.buildInputs or [ ] + ) ++ ( + builtins.map ( + pkg: if builtins.isString pkg then builtins.getAttr pkg super else pkg + ) build-requirements + ); + }) + ) pypkgs-build-requirements + ); in { @@ -46,38 +93,62 @@ # }; packages = { - piker = poetry2nix.mkPoetryEditablePackage { - # env = poetry2nix.mkPoetryEnv { + # piker = poetry2nix.legacyPackages.x86_64-linux.mkPoetryEditablePackage { + # editablePackageSources = { piker = ./piker; }; + piker = p2npkgs.mkPoetryApplication { + projectDir = projectDir; - # NOTE: taken from surrounding inputs - # projectDir = projectDir; - editablePackageSources = { piker = ./piker; }; + # SEE ABOVE for auto-genned input set, override + # buncha deps with extras.. like `setuptools` mostly. + # TODO: maybe propose a patch to p2n to show that you + # can even do this in the edgecases docs? + overrides = p2n-overrides.extend( + final: prev: { - # override msgspec to include setuptools as input - # https://github.com/nix-community/poetry2nix/blob/master/docs/edgecases.md#modulenotfounderror-no-module-named-packagenamed - overrides = poetry2nix.defaultPoetryOverrides.extend - (self: super: { - msgspec = super.msgspec.overridePythonAttrs - ( - old: { - buildInputs = (old.buildInputs or [ ]) ++ [ super.setuptools ]; - } - ); - } - ); + # TODO: get this workin with p2n and nixpkgs.. + # pyqt6 = prev.pyqt6.override { + # preferWheel = true; + # }; + # NOTE: this DOESN'T work atm but after a fix + # to poetry2nix, it will and actually this line + # won't be needed - thanks @k900: + # https://github.com/nix-community/poetry2nix/pull/1257 + pyqt5 = prev.pyqt5.override { + withWebkit = false; + preferWheel = true; + }; + + # see PR from @k900: + # https://github.com/nix-community/poetry2nix/pull/1257 + pyqt5-qt5 = prev.pyqt5-qt5.override { + withWebkit = false; + preferWheel = true; + }; + + # TODO: patch in an override for polars to build + # from src! See the details likely needed from + # the cryptography entry: + # https://github.com/nix-community/poetry2nix/blob/master/overrides/default.nix#L426-L435 + polars = prev.polars.override { + preferWheel = true; + }; + } + ); }; }; - - # boot xonsh inside the poetry virtualenv when - # define the custom entry point via an expected # output-attr that `nix-develop` scans for: # https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-develop.html#flake-output-attributes devShells.default = pkgs.mkShell { # packages = [ poetry2nix.packages.${system}.poetry ]; packages = [ poetry2nix.packages.x86_64-linux.poetry ]; - shellHook = "poetry run xonsh"; + inputsFrom = [ self.packages.x86_64-linux.piker ]; + + # TODO: boot xonsh inside the poetry virtualenv when + # defined via a custom entry point? + # NOTE XXX: apparently DON'T do these..? + # shellHook = "poetry run xonsh"; # shellHook = "poetry shell"; }; }