nixpkgs/pkgs/top-level/release-attrpaths-superset.nix

199 lines
6.4 KiB
Nix

# This expression will, as efficiently as possible, dump a
# *superset* of all attrpaths of derivations which might be
# part of a release on *any* platform.
#
# Both this expression and what ofborg uses (release-outpaths.nix)
# are essentially single-threaded (under the current cppnix
# implementation).
#
# This expression runs much, much, much faster and uses much, much
# less memory than the ofborg script by skipping the
# platform-relevance checks. The ofborg outpaths.nix script takes
# half an hour on a 3ghz core and peaks at 60gbytes of memory; this
# expression runs on the same machine in 44 seconds with peak memory
# usage of 5gbytes.
#
# Once you have the list of attrnames you can split it up into
# $NUM_CORES batches and run the platform checks separately for each
# batch, in parallel.
#
# To dump the attrnames:
#
# nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names
#
{ lib ? import (path + "/lib")
, trace ? false
, enableWarnings ? true
, checkMeta ? true
, path ? ./../..
}:
let
# No release package attrpath may have any of these attrnames as
# its initial component.
#
# If you can find a way to remove any of these entries without
# causing CI to fail, please do so.
#
excluded-toplevel-attrs = {
# spliced packagesets
__splicedPackages = true;
pkgsBuildBuild = true;
pkgsBuildHost = true;
pkgsBuildTarget = true;
pkgsHostHost = true;
pkgsHostTarget = true;
pkgsTargetTarget = true;
buildPackages = true;
targetPackages = true;
# cross packagesets
pkgsLLVM = true;
pkgsMusl = true;
pkgsStatic = true;
pkgsCross = true;
pkgsx86_64Darwin = true;
pkgsi686Linux = true;
pkgsLinux = true;
pkgsExtraHardening = true;
};
# No release package attrname may have any of these at a component
# anywhere in its attrpath. These are the names of gigantic
# top-level attrsets that have leaked into so many sub-packagesets
# that it's easier to simply exclude them entirely.
#
# If you can find a way to remove any of these entries without
# causing CI to fail, please do so.
#
excluded-attrnames-at-any-depth = {
lib = true;
override = true;
__functor = true;
__functionArgs = true;
__splicedPackages = true;
newScope = true;
scope = true;
pkgs = true;
test-pkgs = true;
buildHaskellPackages = true;
buildPackages = true;
buildFreebsd = true;
generateOptparseApplicativeCompletions = true;
callPackage = true;
mkDerivation = true;
overrideDerivation = true;
overrideScope = true;
overrideScope' = true;
# Special case: lib/types.nix leaks into a lot of nixos-related
# derivations, and does not eval deeply.
type = true;
};
# __attrsFailEvaluation is a temporary workaround to get top-level
# eval to succeed (under builtins.tryEval) for the entire
# packageset, without deep invasve changes into individual
# packages.
#
# Now that CI has been added, ensuring that top-level eval will
# not be broken by any new commits, you should not add any new
# occurrences of __attrsFailEvaluation, and should remove them
# wherever you are able to (doing so will likely require deep
# adjustments within packages). Once all of the uses of
# __attrsFailEvaluation are removed, it will be deleted from the
# routine below. In the meantime,
#
# The intended semantics are that an attrpath rooted at pkgs is
# part of the (unfiltered) release jobset iff all of the following
# are true:
#
# 1. The first component of the attrpath is not in
# `excluded-toplevel-attrs`
#
# 2. No attrname in the attrpath belongs to the list of forbidden
# attrnames `excluded-attrnames-at-any-depth`
#
# 3. The attrpath leads to a value for which lib.isDerivation is true
#
# 4. No proper prefix of the attrpath has __attrsFailEvaluation=true
#
# 5. Any proper prefix of the attrpath at which lib.isDerivation
# is true also has __recurseIntoDerivationForReleaseJobs=true.
#
# The last condition is unfortunately necessary because there are
# Hydra release jobnames which have proper prefixes which are
# attrnames of derivations (!). We should probably restructure
# the job tree so that this is not the case.
#
justAttrNames = path: value:
let
attempt =
if lib.isDerivation value &&
# in some places we have *derivations* with jobsets as subattributes, ugh
!(value.__recurseIntoDerivationForReleaseJobs or false) then
[ path ]
# Even wackier case: we have meta.broken==true jobs with
# !meta.broken jobs as subattributes with license=unfree, and
# check-meta.nix won't throw an "unfree" failure because the
# enclosing derivation is marked broken. Yeah. Bonkers.
# We should just forbid jobsets enclosed by derivations.
else if lib.isDerivation value &&
!value.meta.available then []
else if !(lib.isAttrs value) then []
else if (value.__attrsFailEvaluation or false) then []
else lib.pipe value [
(builtins.mapAttrs
(name: value:
if excluded-attrnames-at-any-depth.${name} or false then [] else
(justAttrNames (path ++ [name]) value)))
builtins.attrValues
builtins.concatLists
];
seq = builtins.deepSeq attempt attempt;
tried = builtins.tryEval seq;
result =
if tried.success
then tried.value
else if enableWarnings && path != [ "AAAAAASomeThingsFailToEvaluate" ]
then lib.warn "tryEval failed at: ${lib.concatStringsSep "." path}" []
else [];
in
if !trace
then result
else lib.trace "** ${lib.concatStringsSep "." path}" result;
unfiltered = import ./release-outpaths.nix {
inherit checkMeta;
attrNamesOnly = true;
inherit path;
};
filtered = lib.pipe unfiltered [
(pkgs: builtins.removeAttrs pkgs (builtins.attrNames excluded-toplevel-attrs))
];
paths =
[
# I am not entirely sure why these three packages end up in
# the Hydra jobset. But they do, and they don't meet the
# criteria above, so at the moment they are special-cased.
[ "pkgsLLVM" "stdenv" ]
[ "pkgsStatic" "stdenv" ]
[ "pkgsMusl" "stdenv" ]
] ++ justAttrNames [] filtered;
names =
map (path: (lib.concatStringsSep "." path)) paths;
in
{
inherit paths names;
}