feat(v1): port cargo-lock translator to drv-parts

This commit is contained in:
DavHau 2023-05-29 22:33:20 +02:00
parent 69560f5128
commit 8843ecf03d
8 changed files with 867 additions and 0 deletions

View File

@ -0,0 +1,12 @@
{lib, ...}: let
l = builtins // lib;
sanitizePath = path: let
absolute = (l.substring 0 1 path) == "/";
sanitizedRelPath = l.removePrefix "/" (l.toString (l.toPath "/${path}"));
in
if absolute
then "/${sanitizedRelPath}"
else sanitizedRelPath;
in
sanitizePath

View File

@ -0,0 +1,7 @@
{lib, ...}: let
l = builtins // lib;
sanitizeRelativePath = path:
l.removePrefix "/" (l.toString (l.toPath "/${path}"));
in
sanitizeRelativePath

View File

@ -0,0 +1,296 @@
{lib, ...}: let
l = builtins // lib;
nameVersionPair = name: version: {inherit name version;};
expectedFields = [
"name"
"version"
"sourceSpec"
"dependencies"
];
mkFinalObjects = rawObjects: extractors:
l.map
(rawObj: let
finalObj =
{inherit rawObj;}
// l.mapAttrs
(key: extractFunc: extractFunc rawObj finalObj)
extractors;
in
finalObj)
rawObjects;
# checks validity of all objects by iterating over them
mkValidatedFinalObjects = finalObjects: translatorName: extraObjects:
l.map
(finalObj:
if l.any (field: ! finalObj ? "${field}") expectedFields
then
throw
''
Translator ${translatorName} failed.
The following object does not contain all required fields:
Object:
${l.toJSON finalObj}
Missing fields:
${l.subtractLists expectedFields (l.attrNames finalObj)}
''
# TODO: validate sourceSpec as well
else finalObj)
(finalObjects ++ extraObjects);
mkExportedFinalObjects = finalObjects: exportedPackages:
l.filter
(finalObj:
exportedPackages.${finalObj.name} or null == finalObj.version)
finalObjects;
mkRelevantFinalObjects = exportedFinalObjects: allDependencies:
l.genericClosure {
startSet =
l.map
(finalObj:
finalObj
// {key = "${finalObj.name}#${finalObj.version}";})
exportedFinalObjects;
operator = finalObj:
l.map
(c:
allDependencies.${c.name}.${c.version}
// {key = "${c.name}#${c.version}";})
finalObj.dependencies;
};
/*
format:
{
foo = {
"1.0.0" = finalObj
}
}
*/
makeDependencies = finalObjects:
l.foldl'
(result: finalObj:
l.recursiveUpdate
result
{
"${finalObj.name}" = {
"${finalObj.version}" = finalObj;
};
})
{}
finalObjects;
translate = func: let
final =
func
{
inherit objectsByKey;
};
rawObjects = final.serializedRawObjects;
finalObjects' = mkFinalObjects rawObjects final.extractors;
objectsByKey =
l.mapAttrs
(key: keyFunc:
l.foldl'
(merged: finalObj:
merged
// {"${keyFunc finalObj.rawObj finalObj}" = finalObj;})
{}
finalObjects')
final.keys;
dreamLockData = magic final;
magic = {
defaultPackage,
exportedPackages,
extractors,
extraObjects ? [],
keys ? {},
location ? "",
serializedRawObjects,
subsystemName,
subsystemAttrs ? {},
translatorName,
}: let
inputs = {
inherit
defaultPackage
exportedPackages
extractors
extraObjects
keys
location
serializedRawObjects
subsystemName
subsystemAttrs
translatorName
;
};
finalObjects =
mkValidatedFinalObjects
finalObjects'
translatorName
(final.extraObjects or []);
allDependencies = makeDependencies finalObjects;
exportedFinalObjects =
mkExportedFinalObjects finalObjects exportedPackages;
relevantFinalObjects =
mkRelevantFinalObjects exportedFinalObjects allDependencies;
relevantDependencies = makeDependencies relevantFinalObjects;
sources =
l.mapAttrs
(name: versions: let
# Filter out all `path` sources which link to store paths.
# The removed sources can be added back via source override later
filteredObjects =
l.filterAttrs
(version: finalObj:
(finalObj.sourceSpec.type != "path")
|| ! l.isStorePath (l.removeSuffix "/" finalObj.sourceSpec.path))
versions;
in
l.mapAttrs
(version: finalObj: finalObj.sourceSpec)
filteredObjects)
relevantDependencies;
dependencyGraph =
l.mapAttrs
(name: versions:
l.mapAttrs
(version: finalObj: finalObj.dependencies)
versions)
relevantDependencies;
cyclicDependencies =
if dependencyGraph == {}
then {}
else cyclicDependencies';
cyclicDependencies' =
# TODO: inefficient! Implement some kind of early cutoff
let
depGraphWithFakeRoot =
l.recursiveUpdate
dependencyGraph
{
__fake-entry.__fake-version =
l.mapAttrsToList
nameVersionPair
exportedPackages;
};
findCycles = node: prevNodes: cycles: let
children =
depGraphWithFakeRoot."${node.name}"."${node.version}";
cyclicChildren =
l.filter
(child: prevNodes ? "${child.name}#${child.version}")
children;
nonCyclicChildren =
l.filter
(child: ! prevNodes ? "${child.name}#${child.version}")
children;
cycles' =
cycles
++ (l.map (child: {
from = node;
to = child;
})
cyclicChildren);
# use set for efficient lookups
prevNodes' =
prevNodes
// {"${node.name}#${node.version}" = null;};
in
if nonCyclicChildren == []
then cycles'
else
l.flatten
(l.map
(child: findCycles child prevNodes' cycles')
nonCyclicChildren);
cyclesList =
findCycles
(
nameVersionPair
"__fake-entry"
"__fake-version"
)
{}
[];
in
l.foldl'
(cycles: cycle: (
let
existing =
cycles."${cycle.from.name}"."${cycle.from.version}"
or [];
reverse =
cycles."${cycle.to.name}"."${cycle.to.version}"
or [];
in
# if edge or reverse edge already in cycles, do nothing
if
l.elem cycle.from reverse
|| l.elem cycle.to existing
then cycles
else
l.recursiveUpdate
cycles
{
"${cycle.from.name}"."${cycle.from.version}" =
existing ++ [cycle.to];
}
))
{}
cyclesList;
data =
{
decompressed = true;
_generic = {
inherit
defaultPackage
location
;
packages = exportedPackages;
subsystem = subsystemName;
sourcesAggregatedHash = null;
};
# build system specific attributes
_subsystem = subsystemAttrs;
inherit cyclicDependencies sources;
}
// {dependencies = dependencyGraph;};
in {
inherit data;
inherit inputs;
};
in
dreamLockData.data;
in
translate

View File

@ -0,0 +1,31 @@
{
config,
lib,
dream2nix,
...
}: let
l = lib // builtins;
cfg = config.rust-cargo-lock;
parseSpdxId = import ../../../lib/internal/parseSpdxId.nix {inherit lib;};
sanitizePath = import ../../../lib/internal/sanitizePath.nix {inherit lib;};
sanitizeRelativePath = import ../../../lib/internal/sanitizeRelativePath.nix {inherit lib;};
prepareSourceTree = import ../../../lib/internal/prepareSourceTree.nix {inherit lib;};
simpleTranslate2 = import ../../../lib/internal/simpleTranslate2.nix {inherit lib;};
translate = import ./translate.nix {
inherit lib parseSpdxId sanitizePath sanitizeRelativePath simpleTranslate2;
};
dreamLock = translate {
projectRelPath = "";
tree = prepareSourceTree {source = cfg.source;};
};
in {
imports = [
./interface.nix
dream2nix.modules.drv-parts.mkDerivation
];
rust-cargo-lock = {
inherit dreamLock;
};
}

View File

@ -0,0 +1,22 @@
{lib, ...}: let
discoverCrates = {tree}: let
cargoToml = tree.files."Cargo.toml".tomlContent or {};
subdirCrates =
lib.flatten
(lib.mapAttrsToList
(dirName: dir: discoverCrates {tree = dir;})
(tree.directories or {}));
in
if cargoToml ? package.name
then
[
{
inherit (cargoToml.package) name version;
inherit (tree) relPath;
}
]
++ subdirCrates
else subdirCrates;
in
discoverCrates

View File

@ -0,0 +1,21 @@
{
config,
lib,
...
}: let
l = lib // builtins;
t = l.types;
in {
options.rust-cargo-lock = l.mapAttrs (_: l.mkOption) {
dreamLock = {
type = t.attrs;
internal = true;
description = "The content of the dream2nix generated lock file";
};
source = {
type = t.either t.path t.package;
description = "Source of the package";
default = config.mkDerivation.src;
};
};
}

View File

@ -0,0 +1,450 @@
{
lib,
parseSpdxId,
sanitizePath,
sanitizeRelativePath,
simpleTranslate2,
...
}: let
l = lib // builtins;
translate = {
# project,
projectRelPath,
tree,
...
}: let
# get the root source and project source
rootTree = tree;
projectTree = rootTree.getNodeFromPath projectRelPath;
rootSource = rootTree.fullPath;
projectSource = sanitizePath "${rootSource}/${projectRelPath}";
# pull all crates from subsystemInfo, if not find all of them
# this is mainly helpful when `projects` is defined manually in which case
# crates won't be available, so we will reduce burden on the user here.
allCrates =
(import ./findAllCrates.nix {inherit lib;})
{tree = rootTree;};
# Get the root toml
rootToml = {
relPath = projectRelPath;
value = projectTree.files."Cargo.toml".tomlContent;
};
# use workspace members from discover phase
# or discover them again ourselves
workspaceMembers =
l.flatten
(
l.map
(
memberName: let
components = l.splitString "/" memberName;
in
# Resolve globs if there are any
if l.last components == "*"
then let
parentDirRel = l.concatStringsSep "/" (l.init components);
dirs = (rootTree.getNodeFromPath parentDirRel).directories;
in
l.mapAttrsToList
(name: _: "${parentDirRel}/${name}")
dirs
else memberName
)
(rootToml.value.workspace.members or [])
);
# Get cargo packages (for workspace members)
workspaceCargoPackages =
l.map
(relPath: {
inherit relPath;
value = (projectTree.getNodeFromPath "${relPath}/Cargo.toml").tomlContent;
})
# Filter root referencing member, we already parsed this (rootToml)
(l.filter (relPath: relPath != ".") workspaceMembers);
# All cargo packages that we will output
cargoPackages =
if l.hasAttrByPath ["package" "name"] rootToml.value
# Note: the ordering is important here, since packageToml assumes
# the rootToml to be at 0 index (if it is a package)
then [rootToml] ++ workspaceCargoPackages
else workspaceCargoPackages;
# Get a "main" package toml
packageToml = l.elemAt cargoPackages 0;
# Parse Cargo.lock and extract dependencies
parsedLock = projectTree.files."Cargo.lock".tomlContent;
parsedDeps = parsedLock.package;
makeDepId = dep: "${dep.name} ${dep.version} (${dep.source or ""})";
# Gets a checksum from the [metadata] table of the lockfile
getChecksum = dep: parsedLock.metadata."checksum ${makeDepId dep}";
# map of dependency names to a list of the possible versions
depNamesToVersions =
l.foldl'
(
all: el:
if l.hasAttr el.name all
then all // {${el.name} = all.${el.name} ++ [el];}
else all // {${el.name} = [el];}
)
{}
parsedDeps;
# takes a parsed dependency entry and finds the original dependency from `Cargo.lock`
findOriginalDep = let
# checks dep against another dep to see if they are the same
# this is only for checking entries from a `dependencies` list
# against a dependency entry under `package` from Cargo.lock
isSameDependency = dep: againstDep:
l.foldl'
(previousResult: result: previousResult && result)
true
(
l.mapAttrsToList
(
name: value:
if l.hasAttr name againstDep
then
# if git source, we need to get rid of the revision part
if name == "source" && l.hasPrefix "git+" againstDep.source
then l.concatStringsSep "#" (l.init (l.splitString "#" againstDep.source)) == value
else againstDep.${name} == value
else false
)
dep
);
in
dep: let
notFoundError = "no dependency found with name ${dep.name} in Cargo.lock";
foundCount = l.length depNamesToVersions.${dep.name};
found =
# if found one version, then that's the dependency we are looking for
if foundCount == 1
then l.head depNamesToVersions.${dep.name}
# if found multiple, then we need to check which dependency we are looking for
else if foundCount > 1
then
l.findFirst
(otherDep: isSameDependency dep otherDep)
(throw notFoundError)
depNamesToVersions.${dep.name}
else throw notFoundError;
in
found;
# This parses a "package-name version (source)" entry in the "dependencies"
# field of a dependency in Cargo.lock
parseDepEntryImpl = entry: let
parsed = l.splitString " " entry;
# name is always at the beginning
name = l.head parsed;
# parse the version if it exists
maybeVersion =
if l.length parsed > 1
then l.elemAt parsed 1
else null;
# parse the source if it exists
source =
if l.length parsed > 2
then l.removePrefix "(" (l.removeSuffix ")" (l.elemAt parsed 2))
else null;
# find the original dependency from the information we have
foundDep = findOriginalDep (
{inherit name;}
// l.optionalAttrs (source != null) {inherit source;}
// l.optionalAttrs (maybeVersion != null) {version = maybeVersion;}
);
in
foundDep;
# dependency entries mapped to their original dependency
entryToDependencyAttrs = let
makePair = entry: l.nameValuePair entry (parseDepEntryImpl entry);
depEntries = l.flatten (l.map (dep: dep.dependencies or []) parsedDeps);
in
l.listToAttrs (l.map makePair (l.unique depEntries));
parseDepEntry = entry: entryToDependencyAttrs.${entry};
# Parses a git source, taken straight from nixpkgs.
parseSourceImpl = src: let
parts = l.match ''git\+([^?]+)(\?(rev|tag|branch)=(.*))?#(.*)'' src;
type = l.elemAt parts 2; # rev, tag or branch
value = l.elemAt parts 3;
checkType = type: l.hasPrefix "${type}+" src;
in
if checkType "registry"
then
if src == "registry+https://github.com/rust-lang/crates.io-index"
then {
type = "crates-io";
value = null;
}
else throw "registries other than crates.io are not supported yet"
else if parts != null
then {
type = "git";
value =
{
url = l.elemAt parts 0;
sha = l.elemAt parts 4;
}
// (lib.optionalAttrs (type != null) {inherit type value;});
}
else throw "unknown or unsupported source type: ${src}";
parsedSources = l.listToAttrs (
l.map
(dep: l.nameValuePair dep.source (parseSourceImpl dep.source))
(l.filter (dep: dep ? source) parsedDeps)
);
parseSource = dep:
if dep ? source
then parsedSources.${dep.source}
else {
type = "path";
value = null;
};
package = rec {
toml = packageToml.value;
name = toml.package.name;
version =
toml.package.version
or (l.warn "no version found in Cargo.toml for ${name}, defaulting to unknown" "unknown");
};
extractVersionFromDep = rawObj: let
source = parseSource rawObj;
duplicateVersions =
l.filter
(dep: dep.version == rawObj.version)
depNamesToVersions.${rawObj.name};
in
if l.length duplicateVersions > 1 && source.type != "path"
then rawObj.version + "$" + source.type
else rawObj.version;
in
simpleTranslate2
({...}: {
translatorName = "cargo-lock";
# relative path of the project within the source tree.
location = projectRelPath;
# the name of the subsystem
subsystemName = "rust";
# Extract subsystem specific attributes.
# The structure of this should be defined in:
# ./src/specifications/{subsystem}
subsystemAttrs = {
relPathReplacements = let
# function to find path replacements for one package
findReplacements = package: let
# Extract dependencies from the Cargo.toml of the package
tomlDeps =
l.flatten
(
l.map
(
target:
(l.attrValues (target.dependencies or {}))
++ (l.attrValues (target.buildDependencies or {}))
)
([package.value] ++ (l.attrValues (package.value.target or {})))
);
# We only need to patch path dependencies
pathDeps = l.filter (dep: dep ? path) tomlDeps;
# filter out path dependencies whose path are same as in workspace members.
# this is because otherwise workspace.members paths will also get replaced in the build.
# and there is no reason to replace these anyways since they are in the source.
outsideDeps =
l.filter
(
dep:
!(l.any (memberPath: dep.path == memberPath) workspaceMembers)
)
pathDeps;
makeReplacement = dep: {
name = dep.path;
value = sanitizeRelativePath "${package.relPath}/${dep.path}";
};
replacements = l.listToAttrs (l.map makeReplacement outsideDeps);
# filter out replacements which won't replace anything
# this means that the path doesn't need to be replaced because it's
# already in the source that we are building
filtered =
l.filterAttrs
(
n: v: ! l.pathExists (sanitizePath "${projectSource}/${v}")
)
replacements;
in
filtered;
# find replacements for all packages we export
allPackageReplacements =
l.map
(
package: let
pkg = package.value.package;
replacements = findReplacements package;
in {${pkg.name}.${pkg.version} = replacements;}
)
cargoPackages;
in
l.foldl' l.recursiveUpdate {} allPackageReplacements;
gitSources = l.unique (
l.map (src: src.value) (
l.filter
(src: src.type == "git")
(l.map parseSource parsedDeps)
)
);
meta = l.foldl' l.recursiveUpdate {} (
l.map
(
package: let
pkg = package.value.package;
in {
${pkg.name}.${pkg.version} =
{license = parseSpdxId (pkg.license or "");}
// (
l.filterAttrs
(n: v: l.any (on: n == on) ["description" "homepage"])
pkg
);
}
)
cargoPackages
);
};
defaultPackage = package.name;
/*
List the package candidates which should be exposed to the user.
Only top-level packages should be listed here.
Users will not be interested in all individual dependencies.
*/
exportedPackages = let
makePair = p: let
pkg = p.value.package;
in
l.nameValuePair pkg.name pkg.version;
in
l.listToAttrs (l.map makePair cargoPackages);
/*
a list of raw package objects
If the upstream format is a deep attrset, this list should contain
a flattened representation of all entries.
*/
serializedRawObjects = parsedDeps;
/*
Define extractor functions which each extract one property from
a given raw object.
(Each rawObj comes from serializedRawObjects).
Extractors can access the fields extracted by other extractors
by accessing finalObj.
*/
extractors = {
name = rawObj: finalObj: rawObj.name;
version = rawObj: finalObj: extractVersionFromDep rawObj;
dependencies = rawObj: finalObj:
l.map
(dep: {
name = dep.name;
version = extractVersionFromDep dep;
})
(l.map parseDepEntry (rawObj.dependencies or []));
sourceSpec = rawObj: finalObj: let
source = parseSource rawObj;
depNameVersion = {
pname = rawObj.name;
version = l.removeSuffix ("$" + source.type) rawObj.version;
};
sourceConstructors = {
path = dependencyObject: let
findCrate =
l.findFirst
(
crate:
(crate.name == dependencyObject.name)
&& (crate.version == dependencyObject.version)
)
null;
workspaceCrates =
l.map
(
pkg: {
inherit (pkg.value.package) name version;
inherit (pkg) relPath;
}
)
cargoPackages;
workspaceCrate = findCrate workspaceCrates;
nonWorkspaceCrate = findCrate allCrates;
final =
if
(package.name == dependencyObject.name)
&& (package.version == dependencyObject.version)
then {
type = "path";
path = projectRelPath;
rootName = null;
rootVersion = null;
}
else if workspaceCrate != null
then {
type = "path";
path = workspaceCrate.relPath;
rootName = package.name;
rootVersion = package.version;
}
else if nonWorkspaceCrate != null
then {
type = "path";
path = nonWorkspaceCrate.relPath;
rootName = null;
rootVersion = null;
}
else throw "could not find crate '${dependencyObject.name}-${dependencyObject.version}'";
in
final;
git = dependencyObject: let
parsed = source.value;
maybeRef =
if parsed.type or null == "branch"
then {ref = "refs/heads/${parsed.value}";}
else if parsed.type or null == "tag"
then {ref = "refs/tags/${parsed.value}";}
else {};
in
maybeRef
// {
type = "git";
url = parsed.url;
rev = parsed.sha;
};
crates-io = dependencyObject:
depNameVersion
// {
type = "crates-io";
hash = dependencyObject.checksum or (getChecksum dependencyObject);
};
};
in
sourceConstructors."${source.type}" rawObj;
};
});
in
translate

View File

@ -0,0 +1,28 @@
{
lib,
config,
...
}: let
l = lib // builtins;
in {
imports = [
../../drv-parts/rust-cargo-lock
];
deps = {nixpkgs, ...}: {
inherit (nixpkgs) fetchFromGitHub;
inherit (nixpkgs) stdenv;
};
name = l.mkForce "ripgrep";
version = l.mkForce "13.0.0";
mkDerivation = {
src = config.deps.fetchFromGitHub {
owner = "BurntSushi";
repo = "ripgrep";
rev = config.version;
sha256 = "sha256-udEh+Re2PeO3DnX4fQThsaT1Y3MBHFfrX5Q5EN2XrF0=";
};
};
}