diff --git a/docs/reference/options.md b/docs/reference/options.md index 99a0d25c2..638c9a3d4 100644 --- a/docs/reference/options.md +++ b/docs/reference/options.md @@ -2713,17 +2713,16 @@ boolean - [https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix](https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix) - ## languages.python.venv.requirements -Contents of pip requirements.txt file. +Path to pip requirements.txt file as a string. Must be relative to devenv root. This is passed to `pip install -r` during `devenv shell` initialisation. *Type:* -null or strings concatenated with ā€œ\\nā€ or path +null or string @@ -2734,7 +2733,6 @@ null or strings concatenated with ā€œ\\nā€ or path - [https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix](https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix) - ## languages.python.version The Python version to use. diff --git a/examples/python/.test.sh b/examples/python/.test.sh index 0ad7cfa28..12c4d124a 100755 --- a/examples/python/.test.sh +++ b/examples/python/.test.sh @@ -1,4 +1,5 @@ -#!/usr/bin/env bash +#!/bin/sh set -ex python --version | grep "3.11.3" -python -c "import requests;print(requests)" \ No newline at end of file +python -c "import requests;print(requests)" +python -c "import arrow;print(arrow.__version__)" | grep "1.2.3" diff --git a/examples/python/devenv.nix b/examples/python/devenv.nix index dd34e1b7d..dcb01bc1a 100644 --- a/examples/python/devenv.nix +++ b/examples/python/devenv.nix @@ -6,6 +6,6 @@ version = "3.11.3"; venv.enable = true; - venv.requirements = ./requirements.txt; + venv.requirements = "./requirements.txt"; }; } diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt index f2293605c..becae46f3 100644 --- a/examples/python/requirements.txt +++ b/examples/python/requirements.txt @@ -1 +1,2 @@ requests +-r requirements2.txt diff --git a/examples/python/requirements2.txt b/examples/python/requirements2.txt new file mode 100644 index 000000000..ec8058b23 --- /dev/null +++ b/examples/python/requirements2.txt @@ -0,0 +1 @@ +-r subrequirements/requirements3.txt \ No newline at end of file diff --git a/examples/python/subrequirements/constraints.txt b/examples/python/subrequirements/constraints.txt new file mode 100644 index 000000000..57889784a --- /dev/null +++ b/examples/python/subrequirements/constraints.txt @@ -0,0 +1 @@ +arrow==1.2.3 \ No newline at end of file diff --git a/examples/python/subrequirements/requirements3.txt b/examples/python/subrequirements/requirements3.txt new file mode 100644 index 000000000..4447788e1 --- /dev/null +++ b/examples/python/subrequirements/requirements3.txt @@ -0,0 +1,2 @@ +arrow +-c constraints.txt \ No newline at end of file diff --git a/src/devenv/cli.py b/src/devenv/cli.py index 6c949be97..dedae7327 100644 --- a/src/devenv/cli.py +++ b/src/devenv/cli.py @@ -179,12 +179,10 @@ def cli(ctx, offline, system, debugger, nix_flags, verbose): ctx.obj["gc_root"] = DEVENV_HOME_GC ctx.obj["gc_project"] = DEVENV_HOME_GC / str(int(time.time() * 1000)) - @cli.group() def processes(): pass - os.environ["DEVENV_DIR"] = str(DEVENV_DIR) DEVENV_GC = DEVENV_DIR / "gc" os.environ["DEVENV_GC"] = str(DEVENV_GC) diff --git a/src/modules/languages/python.nix b/src/modules/languages/python.nix index e391d195b..dee1f95a5 100644 --- a/src/modules/languages/python.nix +++ b/src/modules/languages/python.nix @@ -2,6 +2,7 @@ let cfg = config.languages.python; + flattenreq = pkgs.writers.writePython3 "flattenreq" { flakeIgnore = [ "E501" ]; } (builtins.readFile ../support/flattenreq.py); libraries = lib.makeLibraryPath ( cfg.libraries ++ (lib.optional cfg.manylinux.enable pkgs.pythonManylinuxPackages.manylinux2014Package) @@ -26,12 +27,6 @@ let ]; }; - requirements = pkgs.writeText "requirements.txt" ( - if lib.isPath cfg.venv.requirements - then builtins.readFile cfg.venv.requirements - else cfg.venv.requirements - ); - nixpkgs-python = config.lib.getInput { name = "nixpkgs-python"; url = "github:cachix/nixpkgs-python"; @@ -45,10 +40,16 @@ let VENV_PATH="${config.env.DEVENV_STATE}/venv" + function recreate_venv () { + ${pkgs.coreutils}/bin/rm -rf "$VENV_PATH" + ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH" + echo "${package.interpreter}" > "$VENV_PATH/.devenv_interpreter" + } + profile_python="$(${readlink} ${package.interpreter})" devenv_interpreter_path="$(${pkgs.coreutils}/bin/cat "$VENV_PATH/.devenv_interpreter" 2> /dev/null|| false )" venv_python="$(${readlink} "$devenv_interpreter_path")" - requirements="${lib.optionalString (cfg.venv.requirements != null) ''${requirements}''}" + requirements="${lib.optionalString (cfg.venv.requirements != null) ''$DEVENV_ROOT/"${cfg.venv.requirements}"''}" # recreate venv if necessary if [ -z $venv_python ] || [ $profile_python != $venv_python ] @@ -58,24 +59,37 @@ let ${lib.optionalString cfg.poetry.enable '' [ -f "${config.env.DEVENV_STATE}/poetry.lock.checksum" ] && rm ${config.env.DEVENV_STATE}/poetry.lock.checksum ''} - echo ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH" - ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH" - echo "${package.interpreter}" > "$VENV_PATH/.devenv_interpreter" + recreate_venv + venv_recreated=1 fi source "$VENV_PATH"/bin/activate # reinstall requirements if necessary + # -n means nonempty if [ -n "$requirements" ] then - devenv_requirements_path="$(${pkgs.coreutils}/bin/cat "$VENV_PATH/.devenv_requirements" 2> /dev/null|| false )" - devenv_requirements="$(${readlink} "$devenv_requirements_path")" - if [ -z $devenv_requirements ] || [ $devenv_requirements != $requirements ] + tmp="$(${pkgs.coreutils}/bin/mktemp -d)" + ${flattenreq} "$requirements" "$tmp" + + existing_requirements="$VENV_PATH/.devenv_requirements" + [ -f $existing_requirements ] || existing_requirements="/dev/null" + existing_constraints="$VENV_PATH/.devenv_constraints" + [ -f $existing_constraints ] || existing_constraints="/dev/null" + + if ! ${pkgs.diffutils}/bin/cmp --silent "$tmp/requirements.txt" "$existing_requirements" || ! ${pkgs.diffutils}/bin/cmp --silent "$tmp/constraints.txt" "$existing_constraints"; then - echo "${requirements}" > "$VENV_PATH/.devenv_requirements" - echo "Requirements changed, running pip install -r ${requirements}..." - "$VENV_PATH"/bin/pip install -r ${requirements} + if [ -z "$venv_recreated" ] + then + echo "Requirements changed, rebuilding Python venv..." + recreate_venv + fi + echo "Installing requirements..." + ${pkgs.coreutils}/bin/install "$tmp/requirements.txt" "$VENV_PATH/.devenv_requirements" + ${pkgs.coreutils}/bin/install "$tmp/constraints.txt" "$VENV_PATH/.devenv_constraints" + "$VENV_PATH"/bin/pip install -r "$VENV_PATH/.devenv_requirements" -c "$VENV_PATH/.devenv_constraints" fi + ${pkgs.coreutils}/bin/rm -rf "$tmp" fi ''; @@ -176,10 +190,10 @@ in venv.enable = lib.mkEnableOption "Python virtual environment"; venv.requirements = lib.mkOption { - type = lib.types.nullOr (lib.types.either lib.types.lines lib.types.path); + type = lib.types.nullOr lib.types.str; default = null; description = '' - Contents of pip requirements.txt file. + Path to pip requirements.txt file as a string. Must be relative to devenv root. This is passed to `pip install -r` during `devenv shell` initialisation. ''; }; diff --git a/src/modules/support/flattenreq.py b/src/modules/support/flattenreq.py new file mode 100644 index 000000000..cf86b1cd3 --- /dev/null +++ b/src/modules/support/flattenreq.py @@ -0,0 +1,74 @@ +import os +import re +import sys + +envvar_pattern = r"\$\{([^\}]+)\}" + + +def replace_envvars(text, file_path): + def replace(match): + env_var = match.group(1) + val = os.environ.get(env_var, None) + if val is None: + raise ValueError( + f"No such environment variable {env_var} in {text} within " + f"{file_path}" + ) + return val + + return re.sub(envvar_pattern, replace, text) + + +def flatten_requirements(file_path, outdir): + requirements = set() + constraints = set() + + def process_file(file_path): + if not file_path.startswith(os.path.sep): + prefix = os.getcwd() + file_path = os.path.join(prefix, file_path) + with open(file_path, "r") as file: + for line in file: + prefix = os.path.dirname(file_path) + line = line.strip() + if line.startswith("-r"): + line = replace_envvars(line, file_path) + nested_file_path = re.match(r"-r\s+(.+)", line).group(1) + if not nested_file_path.startswith(os.path.sep): + nested_file_path = os.path.join(prefix, nested_file_path) + process_file(nested_file_path) + elif line.startswith("-c"): + line = replace_envvars(line, file_path) + constraint_file_path = re.match(r"-c\s+(.+)", line).group(1) + if not constraint_file_path.startswith(os.path.sep): + constraint_file_path = os.path.join( + prefix, constraint_file_path + ) + process_constraints(constraint_file_path) + elif not line.startswith("#") and line: + requirements.add(line) + + def process_constraints(constraint_file_path): + with open(constraint_file_path, "r") as file: + for line in file: + line = line.strip() + if not line.startswith("#") and line: + constraints.add(line) + + process_file(file_path) + + with open(os.path.join(outdir, "requirements.txt"), "w") as output_file: + for req in sorted(requirements): + output_file.write(req + "\n") + + with open(os.path.join(outdir, "constraints.txt"), "w") as con_output_file: + for con in sorted(constraints): + con_output_file.write(con + "\n") + + +if __name__ == "__main__": + requirements_file_path, outdir = sys.argv[1], sys.argv[2] + if os.path.isfile(outdir): + raise OSError(f"{outdir} is an existing file") + os.makedirs(outdir, exist_ok=True) + flatten_requirements(requirements_file_path, outdir)