git.iko.soy/typeable/nixpkgs
on master help sign in
squashfsTools: 4.4dev_20180612 -> 4.4

A new release has been made upstream. Reproducibility issues were fixed in that release, so we no longer need those patches. For a full overview of the changes, see the 4.4-specific readme at [1]. The alignment patch no longer applies cleanly; I disabled it for now, and I will try to restore it in a follow-up commit. [1]: https://github.com/plougher/squashfs-tools/blob/52eb4c279cd283ed9802dd1ceb686560b22ffb67/README-4.4

commit442cf4f7834077c38320fa7c8aee6ff6d576bfde authorRuud van Asseldonk <dev@veniogames.com> dateSat 7 Sep 2019, 16:12 (7 years ago) parentd3983ca
4 files changed+6−404
pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch−90 pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch−83 pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patch−220 pkgs/tools/filesystems/squashfs/default.nix+6−11
pkgs/tools/filesystems/squashfs/0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patchdeleted−90
@@ -1,90 +0,0 @@
1−From 0ab12a8585373be2de5129e14d979c62e7a90d82 Mon Sep 17 00:00:00 2001
2−From: Chris Lamb <lamby@debian.org>
3−Date: Mon, 21 Nov 2016 09:33:05 +0100
4−Subject: [PATCH] If SOURCE_DATE_EPOCH is set, override timestamps with that
5− value.
6−
7−See https://reproducible-builds.org/specs/source-date-epoch/ for more
8−information about this environment variable.
9−
10−Based on a patch by Alexander Couzens <lynxis@fe...> posted on
11−https://sourceforge.net/p/squashfs/mailman/message/34673610/
12−
13−Signed-off-by: Chris Lamb <lamby@debian.org>
14−---
15− squashfs-tools/mksquashfs.c | 38 ++++++++++++++++++++++++++++++++++++-
16− 1 file changed, 37 insertions(+), 1 deletion(-)
17−
18−diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
19−index c2098bd..b49e956 100644
20−--- a/squashfs-tools/mksquashfs.c
21−+++ b/squashfs-tools/mksquashfs.c
22−@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
23− /* inode lookup table */
24− squashfs_inode *inode_lookup_table = NULL;
25−
26−+/* override filesystem creation time */
27−+time_t mkfs_fixed_time = -1;
28−+
29− /* in memory directory data */
30− #define I_COUNT_SIZE 128
31− #define DIR_ENTRIES 32
32−@@ -5104,6 +5107,9 @@ int main(int argc, char *argv[])
33− int total_mem = get_default_phys_mem();
34− int progress = TRUE;
35− int force_progress = FALSE;
36−+ char *source_date_epoch, *endptr;
37−+ unsigned long long epoch;
38−+
39− struct file_buffer **fragment = NULL;
40−
41− if(argc > 1 && strcmp(argv[1], "-version") == 0) {
42−@@ -5641,6 +5647,36 @@ printOptions:
43− }
44− }
45−
46−+ /* if SOURCE_DATE_EPOCH is set, use that timestamp for the mkfs time */
47−+ source_date_epoch = getenv("SOURCE_DATE_EPOCH");
48−+ if(source_date_epoch) {
49−+ errno = 0;
50−+ epoch = strtoull(source_date_epoch, &endptr, 10);
51−+ if((errno == ERANGE && (epoch == ULLONG_MAX || epoch == 0))
52−+ || (errno != 0 && epoch == 0)) {
53−+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
54−+ "strtoull: %s\n", strerror(errno));
55−+ EXIT_MKSQUASHFS();
56−+ }
57−+ if(endptr == source_date_epoch) {
58−+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
59−+ "No digits were found: %s\n", endptr);
60−+ EXIT_MKSQUASHFS();
61−+ }
62−+ if(*endptr != '\0') {
63−+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
64−+ "Trailing garbage: %s\n", endptr);
65−+ EXIT_MKSQUASHFS();
66−+ }
67−+ if(epoch > ULONG_MAX) {
68−+ ERROR("Environment variable $SOURCE_DATE_EPOCH: "
69−+ "value must be smaller than or equal to "
70−+ "%lu but was found to be: %llu \n", ULONG_MAX, epoch);
71−+ EXIT_MKSQUASHFS();
72−+ }
73−+ mkfs_fixed_time = (time_t)epoch;
74−+ }
75−+
76− /*
77− * Some compressors may need the options to be checked for validity
78− * once all the options have been processed
79−@@ -5993,7 +6029,7 @@ printOptions:
80− sBlk.flags = SQUASHFS_MKFLAGS(noI, noD, noF, noX, no_fragments,
81− always_use_fragments, duplicate_checking, exportable,
82− no_xattrs, comp_opts);
83−- sBlk.mkfs_time = time(NULL);
84−+ sBlk.mkfs_time = mkfs_fixed_time != -1 ? mkfs_fixed_time : time(NULL);
85−
86− disable_info();
87−
88−--
89−2.17.0
90−
pkgs/tools/filesystems/squashfs/0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patchdeleted−83
@@ -1,83 +0,0 @@
1−From 32a07d4156a281084c90a4b78affc8b0b32a26fc Mon Sep 17 00:00:00 2001
2−From: intrigeri <intrigeri@boum.org>
3−Date: Mon, 21 Nov 2016 11:41:28 +0000
4−Subject: [PATCH] If SOURCE_DATE_EPOCH is set, also clamp content timestamps
5− with that value.
6−
7−Based on a patch by Alexander Couzens <lynxis@fe...> posted on
8−https://sourceforge.net/p/squashfs/mailman/message/34673610/
9−---
10− squashfs-tools/mksquashfs.c | 15 ++++++++++++---
11− 1 file changed, 12 insertions(+), 3 deletions(-)
12−
13−diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
14−index b49e956..9f020bf 100644
15−--- a/squashfs-tools/mksquashfs.c
16−+++ b/squashfs-tools/mksquashfs.c
17−@@ -137,6 +137,9 @@ unsigned int cache_bytes = 0, cache_size = 0, inode_count = 0;
18− /* inode lookup table */
19− squashfs_inode *inode_lookup_table = NULL;
20−
21−+/* clamp all timestamps to SOURCE_DATE_EPOCH */
22−+time_t content_clamp_time = -1;
23−+
24− /* override filesystem creation time */
25− time_t mkfs_fixed_time = -1;
26−
27−@@ -2246,6 +2249,8 @@ restat:
28− pathname_reader(dir_ent), strerror(errno));
29− goto read_err;
30− }
31−+ if(content_clamp_time != -1 && buf2.st_mtime >= content_clamp_time)
32−+ buf2.st_mtime = content_clamp_time;
33−
34− if(read_size != buf2.st_size) {
35− close(file);
36−@@ -3101,7 +3106,7 @@ void dir_scan(squashfs_inode *inode, char *pathname,
37− buf.st_mode = S_IRWXU | S_IRWXG | S_IRWXO | S_IFDIR;
38− buf.st_uid = getuid();
39− buf.st_gid = getgid();
40−- buf.st_mtime = time(NULL);
41−+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
42− buf.st_dev = 0;
43− buf.st_ino = 0;
44− dir_ent->inode = lookup_inode2(&buf, PSEUDO_FILE_OTHER, 0);
45−@@ -3127,6 +3115,8 @@ void dir_scan(squashfs_inode *inode, char *pathname,
46− /* source directory has disappeared? */
47− BAD_ERROR("Cannot stat source directory %s because %s\n",
48− pathname, strerror(errno));
49−+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
50−+ buf.st_mtime = content_clamp_time;
51− dir_ent->inode = lookup_inode(&buf);
52− }
53−
54−@@ -3365,6 +3372,8 @@ struct dir_info *dir_scan1(char *filename, char *subpath,
55− free_dir_entry(dir_ent);
56− continue;
57− }
58−+ if(content_clamp_time != -1 && buf.st_mtime >= content_clamp_time)
59−+ buf.st_mtime = content_clamp_time;
60−
61− if((buf.st_mode & S_IFMT) != S_IFREG &&
62− (buf.st_mode & S_IFMT) != S_IFDIR &&
63−@@ -3544,7 +3553,7 @@ void dir_scan2(struct dir_info *dir, struct pseudo *pseudo)
64− buf.st_gid = pseudo_ent->dev->gid;
65− buf.st_rdev = makedev(pseudo_ent->dev->major,
66− pseudo_ent->dev->minor);
67−- buf.st_mtime = time(NULL);
68−+ buf.st_mtime = content_clamp_time != -1 ? content_clamp_time : time(NULL);
69− buf.st_ino = pseudo_ino ++;
70−
71− if(pseudo_ent->dev->type == 'd') {
72−@@ -5674,7 +5683,7 @@ printOptions:
73− "%lu but was found to be: %llu \n", ULONG_MAX, epoch);
74− EXIT_MKSQUASHFS();
75− }
76−- mkfs_fixed_time = (time_t)epoch;
77−+ mkfs_fixed_time = content_clamp_time = (time_t)epoch;
78− }
79−
80− /*
81−--
82−2.17.0
83−
pkgs/tools/filesystems/squashfs/0003-remove-frag-deflator-thread.patchdeleted−220
@@ -1,220 +0,0 @@
1−From afc0c76a170bd17cbd29bbec6ae6d2227e398570 Mon Sep 17 00:00:00 2001
2−From: Alexander Couzens <lynxis@fe80.eu>
3−Date: Fri, 13 Jan 2017 22:00:37 +0100
4−Subject: [PATCH] remove frag_deflator_thread
5−
6−frag_deflator_thread compress fragments.
7−Replace the deflator_thread with a function and
8−use the function instead of the to_frag queue.
9−---
10− squashfs-tools/info.c | 5 ---
11− squashfs-tools/mksquashfs.c | 76 +++++++++++++------------------------
12− squashfs-tools/mksquashfs.h | 2 +-
13− squashfs-tools/restore.c | 15 +-------
14− 4 files changed, 30 insertions(+), 68 deletions(-)
15−
16−diff --git a/squashfs-tools/info.c b/squashfs-tools/info.c
17−index 7968c77..028d578 100644
18−--- a/squashfs-tools/info.c
19−+++ b/squashfs-tools/info.c
20−@@ -96,11 +96,6 @@ void dump_state()
21− printf("compressed block queue (deflate thread(s) -> main thread)\n");
22− dump_seq_queue(to_main, 0);
23−
24−- printf("uncompressed packed fragment queue (main thread -> fragment"
25−- " deflate thread(s))\n");
26−- dump_queue(to_frag);
27−-
28−-
29− printf("locked frag queue (compressed frags waiting while multi-block"
30− " file is written)\n");
31− dump_queue(locked_fragment);
32−diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
33−index cf48e40..cacf14c 100644
34−--- a/squashfs-tools/mksquashfs.c
35−+++ b/squashfs-tools/mksquashfs.c
36−@@ -270,10 +270,10 @@ unsigned int sid_count = 0, suid_count = 0, sguid_count = 0;
37− struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
38− struct cache *bwriter_buffer, *fwriter_buffer;
39− struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
40−- *to_frag, *locked_fragment, *to_process_frag;
41−+ *locked_fragment, *to_process_frag;
42− struct seq_queue *to_main;
43− pthread_t reader_thread, writer_thread, main_thread;
44−-pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
45−+pthread_t *deflator_thread, *frag_thread;
46− pthread_t *restore_thread = NULL;
47− pthread_mutex_t fragment_mutex = PTHREAD_MUTEX_INITIALIZER;
48− pthread_mutex_t pos_mutex = PTHREAD_MUTEX_INITIALIZER;
49−@@ -323,7 +323,7 @@ struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
50− void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
51− unsigned short get_checksum_mem(char *buff, int bytes);
52− void check_usable_phys_mem(int total_mem);
53−-
54−+void frag_deflator(struct file_buffer *file_buffer);
55−
56− void prep_exit()
57− {
58−@@ -1540,7 +1540,7 @@ void write_fragment(struct file_buffer *fragment)
59− pthread_mutex_lock(&fragment_mutex);
60− fragment_table[fragment->block].unused = 0;
61− fragments_outstanding ++;
62−- queue_put(to_frag, fragment);
63−+ frag_deflator(fragment);
64− pthread_cleanup_pop(1);
65− }
66−
67−@@ -2412,51 +2412,34 @@ void *deflator(void *arg)
68− }
69−
70−
71−-void *frag_deflator(void *arg)
72−+void frag_deflator(struct file_buffer *file_buffer)
73− {
74−- void *stream = NULL;
75−- int res;
76−
77−- res = compressor_init(comp, &stream, block_size, 1);
78−- if(res)
79−- BAD_ERROR("frag_deflator:: compressor_init failed\n");
80−-
81−- pthread_cleanup_push((void *) pthread_mutex_unlock, &fragment_mutex);
82−-
83−- while(1) {
84−- int c_byte, compressed_size;
85−- struct file_buffer *file_buffer = queue_get(to_frag);
86−- struct file_buffer *write_buffer =
87−+ int c_byte, compressed_size;
88−+ struct file_buffer *write_buffer =
89− cache_get(fwriter_buffer, file_buffer->block);
90−
91−- c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
92−- file_buffer->size, block_size, noF, 1);
93−- compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
94−- write_buffer->size = compressed_size;
95−- pthread_mutex_lock(&fragment_mutex);
96−- if(fragments_locked == FALSE) {
97−- fragment_table[file_buffer->block].size = c_byte;
98−- fragment_table[file_buffer->block].start_block = bytes;
99−- write_buffer->block = bytes;
100−- bytes += compressed_size;
101−- fragments_outstanding --;
102−- queue_put(to_writer, write_buffer);
103−- pthread_mutex_unlock(&fragment_mutex);
104−- TRACE("Writing fragment %lld, uncompressed size %d, "
105−- "compressed size %d\n", file_buffer->block,
106−- file_buffer->size, compressed_size);
107−- } else {
108−- add_pending_fragment(write_buffer, c_byte,
109−- file_buffer->block);
110−- pthread_mutex_unlock(&fragment_mutex);
111−- }
112−- cache_block_put(file_buffer);
113−+ c_byte = mangle2(stream, write_buffer->data, file_buffer->data,
114−+ file_buffer->size, block_size, noF, 1);
115−+ compressed_size = SQUASHFS_COMPRESSED_SIZE_BLOCK(c_byte);
116−+ write_buffer->size = compressed_size;
117−+ if(fragments_locked == FALSE) {
118−+ fragment_table[file_buffer->block].size = c_byte;
119−+ fragment_table[file_buffer->block].start_block = bytes;
120−+ write_buffer->block = bytes;
121−+ bytes += compressed_size;
122−+ fragments_outstanding --;
123−+ queue_put(to_writer, write_buffer);
124−+ TRACE("Writing fragment %lld, uncompressed size %d, "
125−+ "compressed size %d\n", file_buffer->block,
126−+ file_buffer->size, compressed_size);
127−+ } else {
128−+ add_pending_fragment(write_buffer, c_byte,
129−+ file_buffer->block);
130− }
131−-
132−- pthread_cleanup_pop(0);
133−+ cache_block_put(file_buffer);
134− }
135−
136−-
137− struct file_buffer *get_file_buffer()
138− {
139− struct file_buffer *file_buffer = seq_queue_get(to_main);
140−@@ -4257,19 +4240,17 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
141− multiply_overflow(processors * 3, sizeof(pthread_t)))
142− BAD_ERROR("Processors too large\n");
143−
144−- deflator_thread = malloc(processors * 3 * sizeof(pthread_t));
145−+ deflator_thread = malloc(processors * 2 * sizeof(pthread_t));
146− if(deflator_thread == NULL)
147− MEM_ERROR();
148−
149−- frag_deflator_thread = &deflator_thread[processors];
150−- frag_thread = &frag_deflator_thread[processors];
151−+ frag_thread = &deflator_thread[processors];
152−
153− to_reader = queue_init(1);
154− to_deflate = queue_init(reader_size);
155− to_process_frag = queue_init(reader_size);
156− to_writer = queue_init(bwriter_size + fwriter_size);
157− from_writer = queue_init(1);
158−- to_frag = queue_init(fragment_size);
159− locked_fragment = queue_init(fragment_size);
160− to_main = seq_queue_init();
161− reader_buffer = cache_init(block_size, reader_size, 0, 0);
162−@@ -4285,9 +4266,6 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
163− for(i = 0; i < processors; i++) {
164− if(pthread_create(&deflator_thread[i], NULL, deflator, NULL))
165− BAD_ERROR("Failed to create thread\n");
166−- if(pthread_create(&frag_deflator_thread[i], NULL, frag_deflator,
167−- NULL) != 0)
168−- BAD_ERROR("Failed to create thread\n");
169− if(pthread_create(&frag_thread[i], NULL, frag_thrd,
170− (void *) destination_file) != 0)
171− BAD_ERROR("Failed to create thread\n");
172−diff --git a/squashfs-tools/mksquashfs.h b/squashfs-tools/mksquashfs.h
173−index 55708a3..dc5bde4 100644
174−--- a/squashfs-tools/mksquashfs.h
175−+++ b/squashfs-tools/mksquashfs.h
176−@@ -135,7 +135,7 @@ struct append_file {
177− extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
178− struct cache *bwriter_buffer, *fwriter_buffer;
179− extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
180−- *to_frag, *locked_fragment, *to_process_frag;
181−+ *locked_fragment, *to_process_frag;
182− extern struct append_file **file_mapping;
183− extern struct seq_queue *to_main;
184− extern pthread_mutex_t fragment_mutex, dup_mutex;
185−diff --git a/squashfs-tools/restore.c b/squashfs-tools/restore.c
186−index 5e336b3..a7aaf2e 100644
187−--- a/squashfs-tools/restore.c
188−+++ b/squashfs-tools/restore.c
189−@@ -47,8 +47,8 @@
190− #define TRUE 1
191−
192− extern pthread_t reader_thread, writer_thread, main_thread;
193−-extern pthread_t *deflator_thread, *frag_deflator_thread, *frag_thread;
194−-extern struct queue *to_deflate, *to_writer, *to_frag, *to_process_frag;
195−+extern pthread_t *deflator_thread, *frag_thread;
196−+extern struct queue *to_deflate, *to_writer, *to_process_frag;
197− extern struct seq_queue *to_main;
198− extern void restorefs();
199− extern int processors;
200−@@ -120,17 +120,6 @@ void *restore_thrd(void *arg)
201− pthread_cancel(main_thread);
202− pthread_join(main_thread, NULL);
203−
204−- /* then flush the main thread to fragment deflator thread(s)
205−- * queue. The fragment deflator thread(s) will idle
206−- */
207−- queue_flush(to_frag);
208−-
209−- /* now kill the fragment deflator thread(s) */
210−- for(i = 0; i < processors; i++)
211−- pthread_cancel(frag_deflator_thread[i]);
212−- for(i = 0; i < processors; i++)
213−- pthread_join(frag_deflator_thread[i], NULL);
214−-
215− /*
216− * then flush the main thread/fragment deflator thread(s)
217− * to writer thread queue. The writer thread will idle
218−--
219−2.17.0
220−
pkgs/tools/filesystems/squashfs/default.nix+6−11
@@ -8,26 +8,21 @@ assert lz4Support -> (lz4 != null);
88
99stdenv.mkDerivation {
1010 pname = "squashfs";
11− version = "4.4dev_20180612";
11+ version = "4.4";
1212
1313 src = fetchFromGitHub {
1414 owner = "plougher";
1515 repo = "squashfs-tools";
16− sha256 = "1y53z8dkph3khdyhkmkmy0sg9p1n8czv3vj4l324nj8kxyih3l2c";
17− rev = "6e242dc95485ada8d1d0b3dd9346c5243d4a517f";
16+ sha256 = "0697fv8n6739mcyn57jclzwwbbqwpvjdfkv1qh9s56lvyqnplwaw";
17+ # Tag "4.4" points to this commit.
18+ rev = "52eb4c279cd283ed9802dd1ceb686560b22ffb67";
1819 };
1920
2021 patches = [
21− # These patches ensures that mksquashfs output is reproducible.
22− # See also https://reproducible-builds.org/docs/system-images/
23− # and https://github.com/NixOS/nixpkgs/issues/40144.
24− ./0001-If-SOURCE_DATE_EPOCH-is-set-override-timestamps-with.patch
25− ./0002-If-SOURCE_DATE_EPOCH-is-set-also-clamp-content-times.patch
26− ./0003-remove-frag-deflator-thread.patch
27−
2822 # This patch adds an option to pad filesystems (increasing size) in
2923 # exchange for better chunking / binary diff calculation.
30− ./squashfs-tools-4.4-4k-align.patch
24+ # TODO(ruuda): make this patch apply to the proper 4.4 release.
25+ # ./squashfs-tools-4.4-4k-align.patch
3126 ] ++ stdenv.lib.optional stdenv.isDarwin ./darwin.patch;
3227
3328 buildInputs = [ zlib xz zstd ]