@@ -87,32 +87,37 @@ then
87 gitModulesStr = origSrcSubDir + "/.gitmodules";
87 gitModulesStr = origSrcSubDir + "/.gitmodules";
88 gitModules = builtins.path { name = "gitmodules"; path = gitModulesStr; };
88 gitModules = builtins.path { name = "gitmodules"; path = gitModulesStr; };
90+ # Files from the git repository related knownSubmoduleDirs.
91+ gitSubmoduleFiles = knownSubmoduleDirs:
92+ let pathsNeeded = map (p: toString (src.origSrcSubDir or src) + "/${p}") (
93+ lib.concatMap (x: all_paren_dirs (x + "/.git")) knownSubmoduleDirs);
90− gitSubmoduleFiles = cleanSourceWith {
94+ in cleanSourceWith {
91− caller = "cleanGit";
95+ caller = "cleanGit";
92− name = (if name == null then "" else name + "-") + "gitSubmoduleFiles";
96+ name = (if name == null then "" else name + "-") + "gitSubmoduleFiles";
93− inherit src;
97+ inherit src;
94− filter = path: type:
98+ filter = path: type:
95− type == "directory" # TODO get sudmodule directories from `.gitmodules`
99+ elem path pathsNeeded
96− # and use that to filter directory tree here
100+ ||
98− lib.any (i: (lib.hasSuffix i path)) [
99− "/.git" ".gitmodules" ".git/config" ".git/index" ".git/HEAD" ".git/objects" ".git/refs" ] ||
100− (lib.strings.hasInfix ".git/modules/" path &&
101 lib.any (i: (lib.hasSuffix i path)) [
101 lib.any (i: (lib.hasSuffix i path)) [
102+ ".gitmodules" ".git/config" ".git/index" ".git/HEAD" ".git/objects" ".git/refs" ]
104+ (lib.strings.hasInfix ".git/modules/" path &&
105+ lib.any (i: (lib.hasSuffix i path)) [
102− "config" "index" "HEAD" "objects" "refs" ]);
106+ "config" "index" "HEAD" "objects" "refs" ]);
109+ hasSubmodules = !isWorktree && builtins.pathExists gitModulesStr;
105 # Make a temporary dir that looks enough like the real thing for
111 # Make a temporary dir that looks enough like the real thing for
106 # `git ls-files --recurse-submodules` to give us an accurate list
112 # `git ls-files --recurse-submodules` to give us an accurate list
107 # of all the files in the index.
113 # of all the files in the index.
108− whitelist_file =
114+ whitelist_files = knownSubmoduleDirs:
109− runCommand "git-ls-files" {envVariable = true;} ''
115+ runCommand "git-ls-files" {} ''
110 tmp=$(mktemp -d)
116 tmp=$(mktemp -d)
112− ${ lib.optionalString (!isWorktree && builtins.pathExists gitModulesStr) ''
118+ ${ lib.optionalString hasSubmodules ''
113− cp -ra ${gitSubmoduleFiles}/. $tmp
119+ cp -ra ${gitSubmoduleFiles knownSubmoduleDirs}/. $tmp
114 chmod +w -R $tmp
120 chmod +w -R $tmp
117 cp -r ${gitDir} .git
122 cp -r ${gitDir} .git
118 chmod +w -R .git
123 chmod +w -R .git
@@ -123,10 +128,37 @@ then
123 ${ lib.optionalString (isWorktree && builtins.pathExists gitModulesStr) ''
128 ${ lib.optionalString (isWorktree && builtins.pathExists gitModulesStr) ''
124 cp ${gitModules} ./.gitmodules
129 cp ${gitModules} ./.gitmodules
132+ ${ lib.optionalString hasSubmodules ''
133+ ${git}/bin/git submodule status --recursive | awk '{ print $2 }' > $out/submoduleDirs
126− ${git}/bin/git ls-files --recurse-submodules > $out
135+ ${git}/bin/git ls-files --recurse-submodules > $out/files
129− whitelist = lines (readFile (whitelist_file.out));
138+ # Find the submodules
139+ whitelist_recursive = knownSubmoduleDirs:
141+ # Get the new list of submoduleDirs and files
142+ # (useing the submoduleDirs we already know about)
143+ files = whitelist_files knownSubmoduleDirs;
145+ submoduleDirs = lines (readFile (files + "/submoduleDirs"));
146+ files = lines (readFile (files + "/files"));
149+ # If we are not expecting submodules or if the new list does not
150+ # contain any additional submodules we are done.
151+ if !hasSubmodules || lib.all (x: elem x knownSubmoduleDirs) new.submoduleDirs
154+ # Look again using what we know now about the submodules
155+ whitelist_recursive new.submoduleDirs;
157+ # Use empty as a starting point for looking for submodules.
158+ # We could allow a list to be passed into cleanGit, but when testing
159+ # on the `ghcjs` repo (one with a lot of submodules) it did not
160+ # make much of a difference to the speed of `cleanGit`.
161+ whitelist = whitelist_recursive [];
131 all_paren_dirs = p:
163 all_paren_dirs = p:
132 if p == "." || p == "/"
164 if p == "." || p == "/"
@@ -138,9 +170,9 @@ then
138 concatMap (p:
170 concatMap (p:
139 # Using `origSrcSubDir` (if present) makes it possible to cleanGit src that
171 # Using `origSrcSubDir` (if present) makes it possible to cleanGit src that
140 # has already been cleaned with cleanSrcWith.
172 # has already been cleaned with cleanSrcWith.
141− let full_path = src.origSrcSubDir or (toString src) + "/${p}"; in
173+ let full_path = toString (src.origSrcSubDir or src) + "/${p}"; in
142 map (p': { name = p'; value = true; }) (all_paren_dirs full_path)
174 map (p': { name = p'; value = true; }) (all_paren_dirs full_path)
143− ) whitelist
175+ ) whitelist.files
146 # Identify files in the `.git` dir
178 # Identify files in the `.git` dir
@@ -155,6 +187,9 @@ then
155 cleanSourceWith {
187 cleanSourceWith {
156 caller = "cleanGit";
188 caller = "cleanGit";
157 inherit name src subDir includeSiblings filter;
189 inherit name src subDir includeSiblings filter;
192+ # inherit whitelist whitelist_set;