release-haskell.nix

/*
  This is the Hydra jobset for the `haskell-updates` branch in Nixpkgs.
  You can see the status of this jobset at
  https://hydra.nixos.org/jobset/nixpkgs/haskell-updates.

  To debug this expression you can use `hydra-eval-jobs` from
  `pkgs.hydra` which prints the jobset description
  to `stdout`:

  $ hydra-eval-jobs -I . pkgs/top-level/release-haskell.nix
*/
{
  supportedSystems ? import ../../ci/supportedSystems.nix,
}:

let

  releaseLib = import ./release-lib.nix {
    inherit supportedSystems;
  };

  inherit (releaseLib)
    lib
    mapTestOn
    packagePlatforms
    pkgs
    ;

  # Helper function which traverses a (nested) set
  # of derivations produced by mapTestOn and flattens
  # it to a list of derivations suitable to be passed
  # to `releaseTools.aggregate` as constituents.
  # Removes all non derivations from the input jobList.
  #
  # accumulateDerivations :: [ Either Derivation AttrSet ] -> [ Derivation ]
  #
  # > accumulateDerivations [ drv1 "string" { foo = drv2; bar = { baz = drv3; }; } ]
  # [ drv1 drv2 drv3 ]
  accumulateDerivations =
    jobList:
    lib.concatMap (
      attrs:
      if lib.isDerivation attrs then
        [ attrs ]
      else
        lib.optionals (lib.isAttrs attrs) (accumulateDerivations (lib.attrValues attrs))
    ) jobList;

  # names of all subsets of `pkgs.haskell.packages`
  #
  # compilerNames looks like the following:
  #
  # ```
  # {
  #   ghc810 = "ghc810";
  #   ghc8107Binary = "ghc8107Binary";
  #   ghc8107 = "ghc8107";
  #   ghc924 = "ghc924";
  #   ...
  # }
  # ```
  compilerNames = lib.mapAttrs (name: _: name) pkgs.haskell.packages;

  # list of all compilers to test specific packages on
  released = with compilerNames; [
    ghc8107
    ghc902
    ghc925
    ghc926
    ghc927
    ghc928
    ghc945
    ghc946
    ghc947
    ghc948
    ghc963
    ghc964
    ghc965
    ghc966
    ghc981
    ghc982
    ghc983
    ghc984
    ghc9101
  ];

  # packagePlatforms applied to `haskell.packages.*`
  #
  # This returns an attr set that looks like the following, where each Haskell
  # package in the compiler attr set has its list of supported platforms as its
  # value.
  #
  # ```
  # {
  #   ghc810 = {
  #     conduit = [ ... ];
  #     lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ]
  #     ...
  #   };
  #   ghc902 = { ... };
  #   ...
  # }
  # ```
  compilerPlatforms = lib.mapAttrs (_: v: packagePlatforms v) pkgs.haskell.packages;

  # This function lets you specify specific packages
  # which are to be tested on a list of specific GHC
  # versions and returns a job set for all specified
  # combinations.
  #
  # You can call versionedCompilerJobs like the following:
  #
  # ```
  # versionedCompilerJobs {
  #   ghc-tags = ["ghc902" "ghc924"];
  # }
  # ```
  #
  # This would produce an output like the following:
  #
  # ```
  # {
  #   haskell.packages = {
  #     ghc884 = {};
  #     ghc810 = {};
  #     ghc902 = {
  #       ghc-tags = {
  #         aarch64-darwin = <derivation...>;
  #         aarch64-linux = <derivation...>;
  #         ...
  #       };
  #     };
  #     ghc924 = {
  #       ghc-tags = { ... };
  #     };
  #     ...
  #   };
  # }
  # ```
  versionedCompilerJobs =
    config:
    mapTestOn {
      haskell.packages =
        let
          # Mapping function that takes an attrset of jobs, and
          # removes all jobs that are not specified in config.
          #
          # For example, imagine a call to onlyConfigJobs like:
          #
          # ```
          # onlyConfigJobs
          #   "ghc902"
          #   {
          #     conduit = [ ... ];
          #     lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ];
          #   }
          # ```
          #
          # onlyConfigJobs pulls out only those jobs that are specified in config.
          #
          # For instance, if config is `{ lens = [ "ghc902" ]; }`, then the above
          # example call to onlyConfigJobs will return:
          #
          # ```
          # { lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ]; }
          # ```
          #
          # If config is `{ lens = [ "ghc8107" ]; }`, then the above example call
          # to onlyConfigJobs returns `{}`.
          #
          # onlyConfigJobs will also remove all platforms from a job that are not
          # supported by the GHC it is compiled with.
          onlyConfigJobs =
            ghc: jobs:
            let
              configFilteredJobset = lib.filterAttrs (
                jobName: platforms: lib.elem ghc (config."${jobName}" or [ ])
              ) jobs;

              # Remove platforms from each job that are not supported by GHC.
              # This is important so that we don't build jobs for platforms
              # where GHC can't be compiled.
              jobsetWithGHCPlatforms = lib.mapAttrs (
                _: platforms: lib.intersectLists jobs.ghc platforms
              ) configFilteredJobset;
            in
            jobsetWithGHCPlatforms;
        in
        lib.mapAttrs onlyConfigJobs compilerPlatforms;
    };

  # hydra jobs for `pkgs` of which we import a subset of
  pkgsPlatforms = packagePlatforms pkgs;

  # names of packages in an attribute set that are maintained
  maintainedPkgNames =
    set:
    builtins.attrNames (lib.filterAttrs (_: v: builtins.length (v.meta.maintainers or [ ]) > 0) set);

  recursiveUpdateMany = builtins.foldl' lib.recursiveUpdate { };

  # Remove multiple elements from a list at once.
  #
  # removeMany
  #   :: [a]  -- list of elements to remove
  #   -> [a]  -- list of elements from which to remove
  #   -> [a]
  #
  # > removeMany ["aarch64-linux" "x86_64-darwin"] ["aarch64-linux" "x86_64-darwin" "x86_64-linux"]
  # ["x86_64-linux"]
  removeMany = itemsToRemove: list: lib.foldr lib.remove list itemsToRemove;

  # Recursively remove platforms from the values in an attribute set.
  #
  # removePlatforms
  #   :: [String]
  #   -> AttrSet
  #   -> AttrSet
  #
  # > attrSet = {
  #     foo = ["aarch64-linux" "x86_64-darwin" "x86_64-linux"];
  #     bar.baz = ["aarch64-linux" "x86_64-linux"];
  #     bar.quux = ["aarch64-linux" "x86_64-darwin"];
  #   }
  # > removePlatforms ["aarch64-linux" "x86_64-darwin"] attrSet
  # {
  #   foo = ["x86_64-linux"];
  #   bar = {
  #     baz = ["x86_64-linux"];
  #     quux = [];
  #   };
  # }
  removePlatforms =
    platformsToRemove: packageSet:
    lib.mapAttrsRecursive (
      _: val: if lib.isList val then removeMany platformsToRemove val else val
    ) packageSet;

  jobs = recursiveUpdateMany [
    (mapTestOn {
      haskellPackages = packagePlatforms pkgs.haskellPackages;
      haskell.compiler =
        packagePlatforms pkgs.haskell.compiler
        // (lib.genAttrs
          [
            "ghcjs"
            "ghcjs810"
          ]
          (ghcjsName: {
            # We can't build ghcjs itself, since it exceeds 3GB (Hydra's output limit) due
            # to the size of its bundled libs. We can however save users a bit of compile
            # time by building the bootstrap ghcjs on Hydra. For this reason, we overwrite
            # the ghcjs attributes in haskell.compiler with a reference to the bootstrap
            # ghcjs attribute in their bootstrap package set (exposed via passthru) which
            # would otherwise be ignored by Hydra.
            bootGhcjs = (packagePlatforms pkgs.haskell.compiler.${ghcjsName}.passthru).bootGhcjs;
          })
        );

      tests.haskell = packagePlatforms pkgs.tests.haskell;

      nixosTests = {
        inherit (packagePlatforms pkgs.nixosTests)
          agda
          xmonad
          xmonad-xdg-autostart
          ;
      };

      agdaPackages = packagePlatforms pkgs.agdaPackages;

      # top-level packages that depend on haskellPackages
      inherit (pkgsPlatforms)
        agda
        alex
        arion
        bench
        blucontrol
        cabal-install
        cabal2nix
        cachix
        # carp broken on 2024-04-09
        changelog-d
        cornelis
        cedille
        client-ip-echo
        darcs
        dconf2nix
        dhall
        dhall-bash
        dhall-docs
        dhall-lsp-server
        dhall-json
        dhall-nix
        dhall-nixpkgs
        dhall-yaml
        diagrams-builder
        echidna
        elm2nix
        emanote
        fffuu
        futhark
        ghcid
        git-annex
        git-brunch
        gitit
        glirc
        hadolint
        happy
        haskell-ci
        haskell-language-server
        hasura-graphql-engine
        hci
        hercules-ci-agent
        hinit
        hedgewars
        hledger
        hledger-check-fancyassertions
        hledger-iadd
        hledger-interest
        hledger-ui
        hledger-web
        hlint
        hpack
        hscolour
        icepeak
        ihaskell
        jacinda
        jl
        json2yaml
        koka
        krank
        lambdabot
        lhs2tex
        madlang
        matterhorn
        mkjson
        mueval
        naproche
        niv
        nix-delegate
        nix-deploy
        nix-diff
        nix-linter
        nix-output-monitor
        nix-script
        nix-tree
        nixfmt
        nixfmt-classic
        nixfmt-rfc-style
        nota
        nvfetcher
        oama
        ormolu
        pakcs
        pandoc
        place-cursor-at
        pinboard-notes-backup
        pretty-simple
        purenix
        shake
        shellcheck
        shellcheck-minimal
        sourceAndTags
        spacecookie
        spago
        specup
        splot
        stack
        stack2nix
        stutter
        stylish-haskell
        taffybar
        tamarin-prover
        termonad
        tldr-hs
        tweet-hs
        update-nix-fetchgit
        uusi
        uqm
        uuagc
        # vaultenv: broken by connection on 2024-03-16
        wstunnel
        xmobar
        xmonadctl
        xmonad-with-packages
        yi
        zsh-git-prompt
        ;

      # Members of the elmPackages set that are Haskell derivations
      elmPackages = {
        inherit (pkgsPlatforms.elmPackages)
          elm
          elm-format
          elm-instrument
          elmi-to-json
          ;
      };

      # GHCs linked to musl.
      pkgsMusl =
        removePlatforms
          [
            # pkgsMusl is compiled natively with musl.  It is not
            # cross-compiled (unlike pkgsStatic).  We can only
            # natively bootstrap GHC with musl on x86_64-linux because
            # upstream doesn't provide a musl bindist for aarch64.
            "aarch64-linux"

            # musl only supports linux, not darwin.
            "x86_64-darwin"
            "aarch64-darwin"
          ]
          {
            haskell.compiler = lib.recursiveUpdate (packagePlatforms pkgs.pkgsMusl.haskell.compiler) {
              # remove musl ghc865Binary since it is known to be broken and
              # causes an evaluation error on darwin.
              ghc865Binary = { };

              ghcjs = { };
              ghcjs810 = { };
            };

            # Get some cache going for MUSL-enabled GHC.
            haskellPackages = {
              inherit (packagePlatforms pkgs.pkgsMusl.haskellPackages)
                hello
                lens
                random
                ;
            };
          };

      # Test some statically linked packages to catch regressions
      # and get some cache going for static compilation with GHC.
      # Use native-bignum to avoid GMP linking problems (LGPL)
      pkgsStatic =
        removePlatforms
          [
            "aarch64-linux" # times out on Hydra

            # Static doesn't work on darwin
            "x86_64-darwin"
            "aarch64-darwin"
          ]
          {
            haskellPackages = {
              inherit (packagePlatforms pkgs.pkgsStatic.haskellPackages)
                hello
                lens
                random
                QuickCheck
                cabal2nix
                terminfo # isn't bundled for cross
                xhtml # isn't bundled for cross
                ;
            };

            haskell.packages.native-bignum.ghc948 = {
              inherit (packagePlatforms pkgs.pkgsStatic.haskell.packages.native-bignum.ghc948)
                hello
                lens
                random
                QuickCheck
                cabal2nix
                terminfo # isn't bundled for cross
                xhtml # isn't bundled for cross
                ;
            };

            haskell.packages.native-bignum.ghc984 = {
              inherit (packagePlatforms pkgs.pkgsStatic.haskell.packages.native-bignum.ghc984)
                hello
                random
                QuickCheck
                terminfo # isn't bundled for cross
                ;
            };
          };

      pkgsCross = {
        ghcjs =
          removePlatforms
            [
              # Hydra output size of 3GB is exceeded
              "aarch64-linux"
            ]
            {
              haskellPackages = {
                inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskellPackages)
                  ghc
                  hello
                  microlens
                  ;
              };

              haskell.packages.ghc98 = {
                inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskell.packages.ghc98)
                  ghc
                  hello
                  microlens
                  ;
              };

              haskell.packages.ghcHEAD = {
                inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskell.packages.ghcHEAD)
                  ghc
                  hello
                  microlens
                  ;
              };
            };

        riscv64 = {
          # Cross compilation of GHC
          haskell.compiler = {
            inherit (packagePlatforms pkgs.pkgsCross.riscv64.haskell.compiler)
              # Our oldest GHC which still uses its own expression. 8.10.7 can
              # theoretically be used to chain bootstrap all GHCs on riscv64
              # which doesn't have official bindists.
              ghc8107
              # Latest GHC we are able to cross-compile.
              ghc948
              ;
          };
        };

        aarch64-multiplatform = {
          # Cross compilation of GHC
          haskell.compiler = {
            inherit (packagePlatforms pkgs.pkgsCross.aarch64-multiplatform.haskell.compiler)
              # Uses a separate expression and LLVM backend for aarch64.
              ghc8107
              # Latest GHC we are able to cross-compile. Uses NCG backend.
              ghc948
              ;
          };
        };
      };
    })
    (versionedCompilerJobs {
      # Packages which should be checked on more than the
      # default GHC version. This list can be used to test
      # the state of the package set with newer compilers
      # and to confirm that critical packages for the
      # package sets (like Cabal, jailbreak-cabal) are
      # working as expected.
      cabal-install = lib.subtractLists [
        # It is recommended to use pkgs.cabal-install instead of cabal-install
        # from the package sets. Due to (transitively) requiring recent versions
        # of core packages, it is not always reasonable to get cabal-install to
        # work with older compilers.
        compilerNames.ghc8107
        compilerNames.ghc902
        compilerNames.ghc925
        compilerNames.ghc926
        compilerNames.ghc927
        compilerNames.ghc928
        compilerNames.ghc945
        compilerNames.ghc946
        compilerNames.ghc947
        compilerNames.ghc948
        compilerNames.ghc9101
      ] released;
      Cabal_3_10_3_0 = released;
      Cabal_3_12_1_0 = released;
      Cabal_3_14_0_0 = released;
      cabal2nix = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      cabal2nix-unstable = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      funcmp = released;
      haskell-language-server = lib.subtractLists [
        # Support ceased as of 2.3.0.0
        compilerNames.ghc8107
        # Support ceased as of 2.5.0.0
        compilerNames.ghc902
      ] released;
      hoogle = lib.subtractLists [
      ] released;
      hlint = lib.subtractLists [
        compilerNames.ghc902
        compilerNames.ghc9101
      ] released;
      hpack = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      hsdns = released;
      jailbreak-cabal = released;
      language-nix = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      nix-paths = released;
      titlecase = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      ghc-api-compat = [
        compilerNames.ghc8107
        compilerNames.ghc902
      ];
      ghc-bignum = [
        compilerNames.ghc8107
      ];
      ghc-lib = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      ghc-lib-parser = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      ghc-lib-parser-ex = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      ghc-source-gen = [
        # Feel free to remove these as they break,
        compilerNames.ghc8107
        compilerNames.ghc902
        compilerNames.ghc928
      ];
      ghc-tags = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      hashable = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      primitive = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
      weeder = lib.subtractLists [
        compilerNames.ghc9101
      ] released;
    })
    {
      mergeable = pkgs.releaseTools.aggregate {
        name = "haskell-updates-mergeable";
        meta = {
          description = ''
            Critical haskell packages that should work at all times,
            serves as minimum requirement for an update merge
          '';
          maintainers = lib.teams.haskell.members;
        };
        constituents = accumulateDerivations [
          # haskell specific tests
          jobs.tests.haskell
          # important top-level packages
          jobs.cabal-install
          jobs.cabal2nix
          jobs.cachix
          jobs.darcs
          jobs.haskell-language-server
          jobs.hledger
          jobs.hledger-ui
          jobs.hpack
          jobs.niv
          jobs.pandoc
          jobs.stack
          jobs.stylish-haskell
          jobs.shellcheck
          # important haskell (library) packages
          jobs.haskellPackages.cabal-plan
          jobs.haskellPackages.distribution-nixpkgs
          jobs.haskellPackages.hackage-db
          jobs.haskellPackages.xmonad
          jobs.haskellPackages.xmonad-contrib
          # haskell packages maintained by @peti
          # imported from the old hydra jobset
          jobs.haskellPackages.hopenssl
          jobs.haskellPackages.hsemail
          jobs.haskellPackages.hsyslog
        ];
      };
      maintained = pkgs.releaseTools.aggregate {
        name = "maintained-haskell-packages";
        meta = {
          description = "Aggregate jobset of all haskell packages with a maintainer";
          maintainers = lib.teams.haskell.members;
        };
        constituents = accumulateDerivations (
          builtins.map (name: jobs.haskellPackages."${name}") (maintainedPkgNames pkgs.haskellPackages)
        );
      };

      muslGHCs = pkgs.releaseTools.aggregate {
        name = "haskell-pkgsMusl-ghcs";
        meta = {
          description = "GHCs built with musl";
          maintainers = with lib.maintainers; [
            nh2
          ];
        };
        constituents = accumulateDerivations [
          jobs.pkgsMusl.haskell.compiler.ghc8107Binary
          jobs.pkgsMusl.haskell.compiler.ghc8107
          jobs.pkgsMusl.haskell.compiler.ghc902
          jobs.pkgsMusl.haskell.compiler.ghc925
          jobs.pkgsMusl.haskell.compiler.ghc926
          jobs.pkgsMusl.haskell.compiler.ghc927
          jobs.pkgsMusl.haskell.compiler.ghc928
          jobs.pkgsMusl.haskell.compiler.ghcHEAD
          jobs.pkgsMusl.haskell.compiler.integer-simple.ghc8107
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghc902
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghc925
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghc926
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghc927
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghc928
          jobs.pkgsMusl.haskell.compiler.native-bignum.ghcHEAD
        ];
      };

      staticHaskellPackages = pkgs.releaseTools.aggregate {
        name = "static-haskell-packages";
        meta = {
          description = "Static haskell builds using the pkgsStatic infrastructure";
          maintainers = [
            lib.maintainers.sternenseemann
            lib.maintainers.rnhmjoj
          ];
        };
        constituents = accumulateDerivations [
          jobs.pkgsStatic.haskell.packages.native-bignum.ghc948 # non-hadrian
          jobs.pkgsStatic.haskellPackages
          jobs.pkgsStatic.haskell.packages.native-bignum.ghc984
        ];
      };
    }
  ];

in
jobs