pkgs/top-level/release.nix

   1 /* This file defines the builds that constitute the Nixpkgs.
   2    Everything defined here ends up in the Nixpkgs channel.  Individual
   3    jobs can be tested by running:
   4
   5    $ nix-build pkgs/top-level/release.nix -A <jobname>.<system>
   6
   7    e.g.
   8
   9    $ nix-build pkgs/top-level/release.nix -A coreutils.x86_64-linux
  10 */
  11 { nixpkgs ? { outPath = (import ../../lib).cleanSource ../..; revCount = 1234; shortRev = "abcdef"; revision = "0000000000000000000000000000000000000000"; }
  12 , system ? builtins.currentSystem
  13 , officialRelease ? false
  14   # The platform doubles for which we build Nixpkgs.
  15 , supportedSystems ? [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin" ]
  16   # The platform triples for which we build bootstrap tools.
  17 , bootstrapConfigs ? [
  18     "aarch64-apple-darwin"
  19     "aarch64-unknown-linux-gnu"
  20     "aarch64-unknown-linux-musl"
  21     "i686-unknown-linux-gnu"
  22     "x86_64-apple-darwin"
  23     "x86_64-unknown-linux-gnu"
  24     "x86_64-unknown-linux-musl"
  25     # we can uncomment that once our bootstrap tarballs are fixed
  26     #"x86_64-unknown-freebsd"
  27   ]
  28   # Strip most of attributes when evaluating to spare memory usage
  29 , scrubJobs ? true
  30   # Attributes passed to nixpkgs. Don't build packages marked as unfree.
  31 , nixpkgsArgs ? { config = {
  32     allowUnfree = false;
  33     inHydra = true;
  34     # Exceptional unsafe packages that we still build and distribute,
  35     # so users choosing to allow don't have to rebuild them every time.
  36     permittedInsecurePackages = [
  37       "olm-3.2.16" # see PR #347899
  38     ];
  39   }; }
  40
  41   # This flag, if set to true, will inhibit the use of `mapTestOn`
  42   # and `release-lib.packagePlatforms`.  Generally, it causes the
  43   # resulting tree of attributes to *not* have a ".${system}"
  44   # suffixed upon every job name like Hydra expects.
  45   #
  46   # This flag exists mainly for use by
  47   # pkgs/top-level/release-attrnames-superset.nix; see that file for
  48   # full details.  The exact behavior of this flag may change; it
  49   # should be considered an internal implementation detail of
  50   # pkgs/top-level/.
  51   #
  52 , attrNamesOnly ? false
  53 }:
  54
  55 let
  56   release-lib = import ./release-lib.nix {
  57     inherit supportedSystems scrubJobs nixpkgsArgs system;
  58   };
  59
  60   inherit (release-lib) mapTestOn pkgs;
  61
  62   inherit (release-lib.lib)
  63     collect
  64     elem
  65     genAttrs
  66     hasInfix
  67     hasSuffix
  68     id
  69     isDerivation
  70     optionals
  71     ;
  72
  73   inherit (release-lib.lib.attrsets) unionOfDisjoint;
  74
  75   supportDarwin = genAttrs [
  76     "x86_64"
  77     "aarch64"
  78   ] (arch: elem "${arch}-darwin" supportedSystems);
  79
  80   nonPackageJobs =
  81     { tarball = import ./make-tarball.nix { inherit pkgs nixpkgs officialRelease; };
  82
  83       release-checks = import ./nixpkgs-basic-release-checks.nix { inherit pkgs nixpkgs supportedSystems; };
  84
  85       manual = pkgs.nixpkgs-manual.override { inherit nixpkgs; };
  86       metrics = import ./metrics.nix { inherit pkgs nixpkgs; };
  87       lib-tests = import ../../lib/tests/release.nix { inherit pkgs; };
  88       pkgs-lib-tests = import ../pkgs-lib/tests { inherit pkgs; };
  89
  90       darwin-tested = if supportDarwin.x86_64 || supportDarwin.aarch64 then pkgs.releaseTools.aggregate
  91         { name = "nixpkgs-darwin-${jobs.tarball.version}";
  92           meta.description = "Release-critical builds for the Nixpkgs darwin channel";
  93           constituents = [
  94             jobs.tarball
  95             jobs.release-checks
  96           ]
  97           ++ optionals supportDarwin.x86_64 [
  98             jobs.cabal2nix.x86_64-darwin
  99             jobs.ghc.x86_64-darwin
 100             jobs.git.x86_64-darwin
 101             jobs.go.x86_64-darwin
 102             jobs.mariadb.x86_64-darwin
 103             jobs.nix.x86_64-darwin
 104             jobs.nixpkgs-review.x86_64-darwin
 105             jobs.nix-info.x86_64-darwin
 106             jobs.nix-info-tested.x86_64-darwin
 107             jobs.openssh.x86_64-darwin
 108             jobs.openssl.x86_64-darwin
 109             jobs.pandoc.x86_64-darwin
 110             jobs.postgresql.x86_64-darwin
 111             jobs.python3.x86_64-darwin
 112             jobs.ruby.x86_64-darwin
 113             jobs.rustc.x86_64-darwin
 114             # blocking ofBorg CI 2020-02-28
 115             # jobs.stack.x86_64-darwin
 116             jobs.stdenv.x86_64-darwin
 117             jobs.vim.x86_64-darwin
 118             jobs.cachix.x86_64-darwin
 119             jobs.darwin.linux-builder.x86_64-darwin
 120
 121             # UI apps
 122             # jobs.firefox-unwrapped.x86_64-darwin
 123             jobs.qt5.qtmultimedia.x86_64-darwin
 124             jobs.inkscape.x86_64-darwin
 125             jobs.gimp.x86_64-darwin
 126             jobs.emacs.x86_64-darwin
 127             jobs.wireshark.x86_64-darwin
 128             jobs.transmission_3-gtk.x86_64-darwin
 129             jobs.transmission_4-gtk.x86_64-darwin
 130
 131             # Tests
 132             /*
 133             jobs.tests.cc-wrapper.default.x86_64-darwin
 134             jobs.tests.cc-wrapper.llvmPackages.clang.x86_64-darwin
 135             jobs.tests.cc-wrapper.llvmPackages.libcxx.x86_64-darwin
 136             jobs.tests.stdenv-inputs.x86_64-darwin
 137             jobs.tests.macOSSierraShared.x86_64-darwin
 138             jobs.tests.stdenv.hooks.patch-shebangs.x86_64-darwin
 139             */
 140           ]
 141           ++ optionals supportDarwin.aarch64 [
 142             jobs.cabal2nix.aarch64-darwin
 143             jobs.ghc.aarch64-darwin
 144             jobs.git.aarch64-darwin
 145             jobs.go.aarch64-darwin
 146             jobs.mariadb.aarch64-darwin
 147             jobs.nix.aarch64-darwin
 148             jobs.nixpkgs-review.aarch64-darwin
 149             jobs.nix-info.aarch64-darwin
 150             jobs.nix-info-tested.aarch64-darwin
 151             jobs.openssh.aarch64-darwin
 152             jobs.openssl.aarch64-darwin
 153             jobs.pandoc.aarch64-darwin
 154             jobs.postgresql.aarch64-darwin
 155             jobs.python3.aarch64-darwin
 156             jobs.ruby.aarch64-darwin
 157             jobs.rustc.aarch64-darwin
 158             # blocking ofBorg CI 2020-02-28
 159             # jobs.stack.aarch64-darwin
 160             jobs.stdenv.aarch64-darwin
 161             jobs.vim.aarch64-darwin
 162             jobs.cachix.aarch64-darwin
 163             jobs.darwin.linux-builder.aarch64-darwin
 164
 165             # UI apps
 166             # jobs.firefox-unwrapped.aarch64-darwin
 167             jobs.qt5.qtmultimedia.aarch64-darwin
 168             jobs.inkscape.aarch64-darwin
 169             jobs.gimp.aarch64-darwin
 170             jobs.emacs.aarch64-darwin
 171             jobs.wireshark.aarch64-darwin
 172             jobs.transmission_3-gtk.aarch64-darwin
 173             jobs.transmission_4-gtk.aarch64-darwin
 174
 175             # Tests
 176             /*
 177             jobs.tests.cc-wrapper.default.aarch64-darwin
 178             jobs.tests.cc-wrapper.llvmPackages.clang.aarch64-darwin
 179             jobs.tests.cc-wrapper.llvmPackages.libcxx.aarch64-darwin
 180             jobs.tests.stdenv-inputs.aarch64-darwin
 181             jobs.tests.macOSSierraShared.aarch64-darwin
 182             jobs.tests.stdenv.hooks.patch-shebangs.aarch64-darwin
 183             */
 184           ];
 185         } else null;
 186
 187       unstable = pkgs.releaseTools.aggregate
 188         { name = "nixpkgs-${jobs.tarball.version}";
 189           meta.description = "Release-critical builds for the Nixpkgs unstable channel";
 190           constituents =
 191             [ jobs.tarball
 192               jobs.release-checks
 193               jobs.metrics
 194               jobs.manual
 195               jobs.lib-tests
 196               jobs.pkgs-lib-tests
 197               jobs.stdenv.x86_64-linux
 198               jobs.cargo.x86_64-linux
 199               jobs.go.x86_64-linux
 200               jobs.linux.x86_64-linux
 201               jobs.nix.x86_64-linux
 202               jobs.pandoc.x86_64-linux
 203               jobs.python3.x86_64-linux
 204               # Needed by contributors to test PRs (by inclusion of the PR template)
 205               jobs.nixpkgs-review.x86_64-linux
 206               # Needed for support
 207               jobs.nix-info.x86_64-linux
 208               jobs.nix-info-tested.x86_64-linux
 209               # Ensure that X11/GTK are in order.
 210               jobs.firefox-unwrapped.x86_64-linux
 211               jobs.cachix.x86_64-linux
 212               jobs.devenv.x86_64-linux
 213
 214               /*
 215               TODO: re-add tests; context: https://github.com/NixOS/nixpkgs/commit/36587a587ab191eddd868179d63c82cdd5dee21b
 216
 217               jobs.tests.cc-wrapper.default.x86_64-linux
 218               jobs.tests.cc-wrapper.gcc7Stdenv.x86_64-linux
 219               jobs.tests.cc-wrapper.gcc8Stdenv.x86_64-linux
 220
 221               # broken see issue #40038
 222
 223               jobs.tests.cc-wrapper.llvmPackages.clang.x86_64-linux
 224               jobs.tests.cc-wrapper.llvmPackages.libcxx.x86_64-linux
 225               jobs.tests.cc-multilib-gcc.x86_64-linux
 226               jobs.tests.cc-multilib-clang.x86_64-linux
 227               jobs.tests.stdenv-inputs.x86_64-linux
 228               jobs.tests.stdenv.hooks.patch-shebangs.x86_64-linux
 229               */
 230             ]
 231             ++ collect isDerivation jobs.stdenvBootstrapTools
 232             ++ optionals supportDarwin.x86_64 [
 233               jobs.stdenv.x86_64-darwin
 234               jobs.cargo.x86_64-darwin
 235               jobs.cachix.x86_64-darwin
 236               jobs.devenv.x86_64-darwin
 237               jobs.go.x86_64-darwin
 238               jobs.python3.x86_64-darwin
 239               jobs.nixpkgs-review.x86_64-darwin
 240               jobs.nix.x86_64-darwin
 241               jobs.nix-info.x86_64-darwin
 242               jobs.nix-info-tested.x86_64-darwin
 243               jobs.git.x86_64-darwin
 244               jobs.mariadb.x86_64-darwin
 245               jobs.vim.x86_64-darwin
 246               jobs.inkscape.x86_64-darwin
 247               jobs.qt5.qtmultimedia.x86_64-darwin
 248               jobs.darwin.linux-builder.x86_64-darwin
 249               /*
 250               jobs.tests.cc-wrapper.default.x86_64-darwin
 251               jobs.tests.cc-wrapper.gcc7Stdenv.x86_64-darwin
 252               jobs.tests.cc-wrapper.gcc8Stdenv.x86_64-darwin
 253               jobs.tests.cc-wrapper.llvmPackages.clang.x86_64-darwin
 254               jobs.tests.cc-wrapper.llvmPackages.libcxx.x86_64-darwin
 255               jobs.tests.stdenv-inputs.x86_64-darwin
 256               jobs.tests.macOSSierraShared.x86_64-darwin
 257               jobs.tests.stdenv.hooks.patch-shebangs.x86_64-darwin
 258               */
 259             ]
 260             ++ optionals supportDarwin.aarch64 [
 261               jobs.stdenv.aarch64-darwin
 262               jobs.cargo.aarch64-darwin
 263               jobs.cachix.aarch64-darwin
 264               jobs.devenv.aarch64-darwin
 265               jobs.go.aarch64-darwin
 266               jobs.python3.aarch64-darwin
 267               jobs.nixpkgs-review.aarch64-darwin
 268               jobs.nix.aarch64-darwin
 269               jobs.nix-info.aarch64-darwin
 270               jobs.nix-info-tested.aarch64-darwin
 271               jobs.git.aarch64-darwin
 272               jobs.mariadb.aarch64-darwin
 273               jobs.vim.aarch64-darwin
 274               jobs.inkscape.aarch64-darwin
 275               jobs.qt5.qtmultimedia.aarch64-darwin
 276               jobs.darwin.linux-builder.aarch64-darwin
 277               /* consider adding tests, as suggested above for x86_64-darwin */
 278             ];
 279         };
 280
 281       stdenvBootstrapTools = genAttrs bootstrapConfigs (config:
 282           if hasInfix "-linux-" config then
 283             let
 284               bootstrap = import ../stdenv/linux/make-bootstrap-tools.nix {
 285                 pkgs = import ../.. {
 286                   localSystem = { inherit config; };
 287                 };
 288               };
 289             in {
 290               inherit (bootstrap) build test;
 291             }
 292           else if hasSuffix "-darwin" config then
 293             let
 294               bootstrap = import ../stdenv/darwin/make-bootstrap-tools.nix {
 295                 localSystem = { inherit config; };
 296               };
 297             in {
 298               # Lightweight distribution and test
 299               inherit (bootstrap) build test;
 300               # Test a full stdenv bootstrap from the bootstrap tools definition
 301               # TODO: Re-enable once the new bootstrap-tools are in place.
 302               #inherit (bootstrap.test-pkgs) stdenv;
 303             }
 304           else if hasSuffix "-freebsd" config then
 305             let
 306               bootstrap = import ../stdenv/freebsd/make-bootstrap-tools.nix {
 307                 pkgs = import ../.. {
 308                   localSystem = { inherit config; };
 309                 };
 310               };
 311             in {
 312               inherit (bootstrap) build;  # test does't exist yet
 313             }
 314           else
 315             abort "No bootstrap implementation for system: ${config}"
 316         );
 317     };
 318
 319   # Do not allow attribute collision between jobs inserted in
 320   # 'nonPackageAttrs' and jobs pulled in from 'pkgs'.
 321   # Conflicts usually cause silent job drops like in
 322   #   https://github.com/NixOS/nixpkgs/pull/182058
 323   jobs = let
 324     packagePlatforms = if attrNamesOnly then id else release-lib.packagePlatforms;
 325     packageJobs = {
 326       haskell.compiler = packagePlatforms pkgs.haskell.compiler;
 327       haskellPackages = packagePlatforms pkgs.haskellPackages;
 328       # Build selected packages (HLS) for multiple Haskell compilers to rebuild
 329       # the cache after a staging merge
 330       haskell.packages = genAttrs [
 331         # TODO: share this list between release.nix and release-haskell.nix
 332         "ghc90"
 333         "ghc92"
 334         "ghc94"
 335         "ghc96"
 336         "ghc98"
 337       ] (compilerName: {
 338         inherit (packagePlatforms pkgs.haskell.packages.${compilerName})
 339           haskell-language-server;
 340       });
 341       idrisPackages = packagePlatforms pkgs.idrisPackages;
 342       agdaPackages = packagePlatforms pkgs.agdaPackages;
 343
 344       pkgsLLVM.stdenv = [ "x86_64-linux" "aarch64-linux" ];
 345       pkgsArocc.stdenv = [ "x86_64-linux" "aarch64-linux" ];
 346       pkgsZig.stdenv = [ "x86_64-linux" "aarch64-linux" ];
 347       pkgsMusl.stdenv = [ "x86_64-linux" "aarch64-linux" ];
 348       pkgsStatic.stdenv = [ "x86_64-linux" "aarch64-linux" ];
 349
 350       tests = packagePlatforms pkgs.tests;
 351
 352       # Language packages disabled in https://github.com/NixOS/nixpkgs/commit/ccd1029f58a3bb9eca32d81bf3f33cb4be25cc66
 353
 354       #emacsPackages = packagePlatforms pkgs.emacsPackages;
 355       #rPackages = packagePlatforms pkgs.rPackages;
 356       ocamlPackages = { };
 357       perlPackages = { };
 358
 359       darwin = packagePlatforms pkgs.darwin // {
 360         xcode = {};
 361       };
 362     };
 363     mapTestOn-packages =
 364       if attrNamesOnly
 365       then pkgs // packageJobs
 366       else mapTestOn ((packagePlatforms pkgs) // packageJobs);
 367   in
 368     unionOfDisjoint nonPackageJobs mapTestOn-packages;
 369
 370 in jobs