git.iko.soy/typeable/nixpkgs
on master help sign in
squashfs: use -no-hardlinks for reproducible squashfs images (#114454)

the nix store may contain hardlinks: derivations may output them directly, or users may be using store optimization which automatically hardlinks identical files in the nix store. The presence of these links are intended to be a 'transparent' optimization. However, when creating a squashfs image, the image will be different depending on whether hard links were present on the filesystem, leading to reproducibility problems. By passing '-no-hardlinks' to mksquashfs the files are stored as duplicates in the squashfs image. Since squashfs has support for duplicate files this does not lead to a larger image. For more details see https://github.com/NixOS/nixpkgs/issues/114331

commit0aeba64fb26e4defa0842a942757144659c6e29f authorArnout Engelen <arnout@bzzt.net> dateSun 28 Feb 2021, 19:03 (6 years ago) parentc6898f8
3 files changed+81−1
nixos/lib/make-squashfs.nix+1−1 pkgs/tools/filesystems/squashfs/0001-Mksquashfs-add-no-hardlinks-option.patch+76 pkgs/tools/filesystems/squashfs/default.nix+4
nixos/lib/make-squashfs.nix+1−1
@@ -23,6 +23,6 @@ stdenv.mkDerivation {
2323
2424 # Generate the squashfs image.
2525 mksquashfs nix-path-registration $(cat $closureInfo/store-paths) $out \
26− -keep-as-directory -all-root -b 1048576 -comp ${comp}
26+ -no-hardlinks -keep-as-directory -all-root -b 1048576 -comp ${comp}
2727 '';
2828}
pkgs/tools/filesystems/squashfs/0001-Mksquashfs-add-no-hardlinks-option.patchnew file+76
@@ -0,0 +1,76 @@
1+From d925c9a11ee2e88ac8aac03f51892746f2bcf8cd Mon Sep 17 00:00:00 2001
2+From: Phillip Lougher <phillip@squashfs.org.uk>
3+Date: Thu, 25 Feb 2021 23:12:10 +0000
4+Subject: [PATCH] Mksquashfs: add -no-hardlinks option
5+
6+Normally Mksquashfs will detect hardlinks (multiple files with the
7+same inode) and hardlink them in the Squashfs image.
8+
9+But often hardlinks are used in the original filesystem
10+to save space, when files are discovered to be duplicate.
11+In this special case the only reason the files are
12+hardlinked is to save space, and where the filesystem
13+doesn't handle duplicate files (different inode, same
14+data).
15+
16+Squashfs does handle duplicate files, and so add
17+an option to ignore hardlinks and instead
18+store them as duplicates.
19+
20+Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
21+---
22+ squashfs-tools/mksquashfs.c | 16 +++++++++++-----
23+ 1 file changed, 11 insertions(+), 5 deletions(-)
24+
25+diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
26+index a45b77f..d4dc359 100644
27+--- a/squashfs-tools/mksquashfs.c
28++++ b/squashfs-tools/mksquashfs.c
29+@@ -312,6 +312,9 @@ struct dir_info *root_dir;
30+ FILE *log_fd;
31+ int logging=FALSE;
32+
33++/* Should Mksquashfs detect hardlinked files? */
34++int no_hardlinks = FALSE;
35++
36+ static char *read_from_disk(long long start, unsigned int avail_bytes);
37+ void add_old_root_entry(char *name, squashfs_inode inode, int inode_number,
38+ int type);
39+@@ -3093,11 +3096,11 @@ struct inode_info *lookup_inode3(struct stat *buf, int pseudo, int id,
40+
41+ /*
42+ * Look-up inode in hash table, if it already exists we have a
43+- * hard-link, so increment the nlink count and return it.
44+- * Don't do the look-up for directories because we don't hard-link
45+- * directories.
46++ * hardlink, so increment the nlink count and return it.
47++ * Don't do the look-up for directories because Unix/Linux doesn't
48++ * allow hard-links to directories.
49+ */
50+- if ((buf->st_mode & S_IFMT) != S_IFDIR) {
51++ if ((buf->st_mode & S_IFMT) != S_IFDIR && !no_hardlinks) {
52+ for(inode = inode_info[ino_hash]; inode; inode = inode->next) {
53+ if(memcmp(buf, &inode->buf, sizeof(struct stat)) == 0) {
54+ inode->nlink ++;
55+@@ -5447,7 +5450,9 @@ int main(int argc, char *argv[])
56+ comp = lookup_compressor(COMP_DEFAULT);
57+
58+ for(i = source + 2; i < argc; i++) {
59+- if(strcmp(argv[i], "-mkfs-time") == 0 ||
60++ if(strcmp(argv[i], "-no-hardlinks") == 0)
61++ no_hardlinks = TRUE;
62++ else if(strcmp(argv[i], "-mkfs-time") == 0 ||
63+ strcmp(argv[i], "-fstime") == 0) {
64+ if((++i == argc) || !parse_num_unsigned(argv[i], &mkfs_time)) {
65+ ERROR("%s: %s missing or invalid time value\n", argv[0], argv[i - 1]);
66+@@ -5893,6 +5898,7 @@ printOptions:
67+ "files larger than block size\n");
68+ ERROR("-no-duplicates\t\tdo not perform duplicate "
69+ "checking\n");
70++ ERROR("-no-hardlinks\t\tdo not hardlink files, instead store duplicates\n");
71+ ERROR("-all-root\t\tmake all files owned by root\n");
72+ ERROR("-root-mode <mode>\tset root directory permissions to octal <mode>\n");
73+ ERROR("-force-uid <uid>\tset all file uids to <uid>\n");
74+--
75+2.30.0
76+
pkgs/tools/filesystems/squashfs/default.nix+4
@@ -19,6 +19,10 @@ stdenv.mkDerivation {
1919 # This patch adds an option to pad filesystems (increasing size) in
2020 # exchange for better chunking / binary diff calculation.
2121 ./4k-align.patch
22+ # Add -no-hardlinks option. This is a rebased version of
23+ # c37bb4da4a5fa8c1cf114237ba364692dd522262, can be removed
24+ # when upgrading to the next version after 4.4
25+ ./0001-Mksquashfs-add-no-hardlinks-option.patch
2226 ] ++ lib.optional stdenv.isDarwin ./darwin.patch;
2327
2428 buildInputs = [ zlib xz zstd lz4 ];