wok-current view fusecloop/stuff/fusecloop.u @ rev 24519
updated etherape (0.9.18 -> 0.9.20)
| author | Hans-G?nter Theisgen |
|---|---|
| date | Tue Feb 22 14:48:21 2022 +0100 (2022-02-22) |
| parents | c6796c6304f6 |
| children |
line source
1 === Add v3 support
2 --- compressed_loop.h
3 +++ compressed_loop.h
4 @@ -41,6 +41,80 @@
5 /* data_index (num_blocks 64bit pointers, network order)... */
6 /* compressed data (gzip block compressed format)... */
8 +struct cloop_tail
9 +{
10 + u_int32_t table_size;
11 + u_int32_t index_size;
12 + u_int32_t num_blocks;
13 +};
14 +
15 +struct block_info
16 +{
17 + loff_t offset; /* 64-bit offsets of compressed block */
18 + u_int32_t size; /* 32-bit compressed block size */
19 + u_int32_t optidx; /* 32-bit index number */
20 +};
21 +
22 +static inline char *build_index(struct block_info *offsets, unsigned long n)
23 +{
24 + u_int32_t *ofs32 = (u_int32_t *) offsets;
25 + loff_t *ofs64 = (loff_t *) offsets;
26 +
27 + if (ofs32[0] == 0) {
28 + if (ofs32[2]) { /* ACCELERATED KNOPPIX V1.0 */
29 + while (n--) {
30 + offsets[n].offset = __be64_to_cpu(offsets[n].offset);
31 + offsets[n].size = ntohl(offsets[n].size);
32 + }
33 + return (char *) "128BE accelerated knoppix 1.0";
34 + }
35 + else { /* V2.0 */
36 + loff_t last = __be64_to_cpu(ofs64[n]);
37 + while (n--) {
38 + offsets[n].size = last -
39 + (offsets[n].offset = __be64_to_cpu(ofs64[n]));
40 + last = offsets[n].offset;
41 + }
42 + return (char *) "64BE v2.0";
43 + }
44 + }
45 + else if (ofs32[1] == 0) { /* V1.0 */
46 + loff_t last = __le64_to_cpu(ofs64[n]);
47 + while (n--) {
48 + offsets[n].size = last -
49 + (offsets[n].offset = __le64_to_cpu(ofs64[n]));
50 + last = offsets[n].offset;
51 + }
52 + return (char *) "64LE v1.0";
53 + }
54 + else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
55 + loff_t last = ntohl(ofs32[n]);
56 + while (n--) {
57 + offsets[n].size = last -
58 + (offsets[n].offset = ntohl(ofs32[n]));
59 + last = offsets[n].offset;
60 + }
61 + return (char *) "32BE v0.68";
62 + }
63 + else { /* V3.0 */
64 + unsigned long i;
65 + loff_t j;
66 +
67 + for (i = n; i-- != 0; )
68 + offsets[i].size = ntohl(ofs32[i]);
69 + for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
70 + offsets[i].offset = j;
71 + if (offsets[i].size & 0x80000000) {
72 + unsigned long k = offsets[i].size & 0x7FFFFFFF;
73 + offsets[i].offset = offsets[k].offset;
74 + offsets[i].size = offsets[k].size;
75 + }
76 + else j += offsets[i].size;
77 + }
78 + return (char *) "32BE v3.0";
79 + }
80 +}
81 +
82 /* Cloop suspend IOCTL */
83 #define CLOOP_SUSPEND 0x4C07
86 --- cloopreader.h
87 +++ cloopreader.h
88 @@ -33,7 +33,7 @@
89 int numblocks;
90 ulong blocksize;
92 - loff_t* toc; /* Data index */
93 + struct block_info *toc; /* Data index */
94 size_t tocsize;
96 unsigned char* cblock; /* Compressed block */
98 --- cloopreader.c
99 +++ cloopreader.c
100 @@ -59,10 +59,32 @@
102 ALLOC(c->pblock,c->blocksize);
104 - c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */
105 - ALLOC(c->toc,c->tocsize);
106 + if (c->numblocks + 1 == 0) {
107 + struct cloop_tail tail;
108 + loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
109 + void *p;
110 + ulong toclen, len;
112 - OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
113 + OP(lseek(c->fh, end - sizeof(tail), SEEK_SET));
114 + OP(read_all(c->fh, &tail, sizeof(tail)));
115 + c->numblocks = ntohl(tail.num_blocks);
116 + c->tocsize = sizeof(*c->toc) * c->numblocks;
117 + len = ntohl(tail.table_size);
118 + toclen = (ntohl(tail.index_size) & 255) * c->numblocks;
119 + OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
120 + ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
121 + ALLOC(p,len);
122 + OP(read_all(c->fh,p,len)); /* read Data Index */
123 + if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK)
124 + exit(1);
125 + free(p);
126 + }
127 + else {
128 + c->tocsize = sizeof(*c->toc) * c->numblocks;
129 + ALLOC(c->toc,c->tocsize);
130 + OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
131 + }
132 + build_index(c->toc, c->numblocks);
133 c->cblocksizecur=0;
134 c->curblock=-1;
135 return 0;
136 @@ -79,10 +101,10 @@
137 if(page>=c->numblocks){errno=EFAULT;return -1;}
138 c->curblock=page;
140 - bprintf("Seeking to 0x%Lx\n",btc(c->toc[page]));
141 - OP(lseek(c->fh,btc(c->toc[page]), SEEK_SET));
142 + bprintf("Seeking to 0x%Lx\n",c->toc[page].offset);
143 + OP(lseek(c->fh,c->toc[page].offset, SEEK_SET));
145 - c->cblocksize=btc(c->toc[page+1]) - btc(c->toc[page]);
146 + c->cblocksize=c->toc[page].size;
147 bprintf("Compressed size=%lu\n",c->cblocksize);
148 if(c->cblocksize > c->cblocksizecur){
149 if(c->cblocksizecur)free(c->cblock);
151 --- extract_compressed_fs.c
152 +++ extract_compressed_fs.c
153 @@ -1,19 +1,23 @@
154 /* Extracts a filesystem back from a compressed fs file */
155 +#define _LARGEFILE64_SOURCE
156 #include "common_header.h"
157 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
159 int main(int argc, char *argv[])
160 {
161 int handle;
162 struct cloop_head head;
163 unsigned int i;
164 + unsigned long num_blocks, block_size, zblock_maxsize, lastlen = 0;
165 unsigned char *buffer, *clear_buffer;
166 + struct block_info *offsets;
168 - if (argc != 2) {
169 - fprintf(stderr, "Need filename\n");
170 + if (argc < 2 || argv[1][0] == '-') {
171 + fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
172 exit(1);
173 }
175 - handle = open(argv[1], O_RDONLY);
176 + handle = open(argv[1], O_RDONLY|O_LARGEFILE);
177 if (handle < 0) {
178 perror("Opening compressed file\n");
179 exit(1);
180 @@ -24,66 +28,100 @@
181 exit(1);
182 }
184 - buffer = malloc(ntohl(head.block_size) + ntohl(head.block_size)/1000
185 - + 12 + 4);
186 - clear_buffer = malloc(ntohl(head.block_size));
187 - fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n",
188 - ntohl(head.num_blocks), ntohl(head.block_size), head.preamble);
189 + num_blocks = ntohl(head.num_blocks);
190 + block_size = ntohl(head.block_size);
191 + zblock_maxsize = block_size + block_size/1000 + 12 + 4;
192 + buffer = malloc(zblock_maxsize);
193 + clear_buffer = malloc(block_size);
195 - for (i = 0; i < ntohl(head.num_blocks); i++) {
196 - int currpos;
197 - unsigned long destlen = ntohl(head.block_size);
198 - loff_t offset[2];
199 - unsigned int size;
200 + if (num_blocks == 0xFFFFFFFF) {
201 + void *table;
202 + struct cloop_tail tail;
203 + unsigned long len, table_size;
204 + loff_t end = lseek64(handle, 0, SEEK_END);
205 +
206 + if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
207 + read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
208 + lseek64(handle, end - sizeof(tail) -
209 + ntohl(tail.table_size), SEEK_SET) < 0) {
210 + perror("Reading tail\n");
211 + exit(1);
212 + }
213 + head.num_blocks = tail.num_blocks;
214 + num_blocks = ntohl(head.num_blocks);
215 + table_size = ntohl(tail.table_size);
216 + table = malloc(table_size);
217 + len = i = num_blocks * (ntohl(tail.index_size) & 255);
218 + lastlen = ntohl(tail.index_size) / 256;
219 + offsets = malloc(num_blocks * sizeof(*offsets));
220 + if (!table || !offsets ||
221 + read(handle, table, table_size) != table_size ||
222 + uncompress((void *)offsets, &len, table, table_size) != Z_OK ||
223 + len != i) {
224 + perror("Reading index\n");
225 + exit(1);
226 + }
227 + free(table);
228 + }
229 + else {
230 + offsets = malloc(i = num_blocks * sizeof(*offsets));
231 + if (!offsets || read(handle, offsets, i) != i) {
232 + perror("Reading index\n");
233 + exit(1);
234 + }
235 + }
236 +
237 + fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
238 + num_blocks, block_size, head.preamble);
239 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
240 +
241 + if (argc > 2) {
242 + unsigned n;
243 + loff_t data, offset = ((num_blocks + 1) * sizeof(offset)) + sizeof(head);
244 +
245 + strcpy(head.preamble, CLOOP_PREAMBLE);
246 + write(STDOUT_FILENO, &head, n = sizeof(head));
247 + for (i = 0; i < num_blocks; i++) {
248 + data = __be64_to_cpu(offset);
249 + write(STDOUT_FILENO, &data, sizeof(data));
250 + n += sizeof(data);
251 + offset += offsets[i].size;
252 + }
253 + data = __be64_to_cpu(offset);
254 + write(STDOUT_FILENO, &data, sizeof(data));
255 + for (i = 0; i < num_blocks && lseek64(handle, offsets[i].offset, SEEK_SET) >= 0; i++) {
256 + read(handle, buffer, offsets[i].size);
257 + write(STDOUT_FILENO, buffer, offsets[i].size);
258 + n += offsets[i].size;
259 + }
260 + n &= 0x1FF;
261 + if (n) {
262 + memset(buffer, 0, 512);
263 + write(STDOUT_FILENO, buffer, 512 - n);
264 + }
265 + return 0;
266 + }
267 +
268 + for (i = 0; i < num_blocks; i++) {
269 + unsigned long destlen = block_size;
270 + unsigned int size = offsets[i].size;
272 - read(handle, &offset, 2*sizeof(loff_t));
273 - lseek(handle, -sizeof(loff_t), SEEK_CUR);
274 -
275 - currpos = lseek(handle, 0, SEEK_CUR);
276 - if (lseek(handle, __be64_to_cpu(offset[0]), SEEK_SET) < 0) {
277 + if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
278 fprintf(stderr, "lseek to %Lu: %s\n",
279 - __be64_to_cpu(offset[0]), strerror(errno));
280 + offsets[i].offset, strerror(errno));
281 exit(1);
282 }
284 - size=__be64_to_cpu(offset[1])-__be64_to_cpu(offset[0]);
285 - if (size > ntohl(head.block_size) + ntohl(head.block_size)/1000
286 - + 12 + 4) {
287 + if (size > zblock_maxsize) {
288 fprintf(stderr,
289 "Size %u for block %u (offset %Lu) too big\n",
290 - size, i, __be64_to_cpu(offset[0]));
291 + size, i, offsets[i].offset);
292 exit(1);
293 }
294 read(handle, buffer, size);
295 - if (lseek(handle, currpos, SEEK_SET) < 0) {
296 - perror("seeking");
297 - exit(1);
298 - }
300 - fprintf(stderr, "Block %u length %u => %lu\n",
301 - i, size, destlen);
302 - if (i == 3) {
303 - fprintf(stderr,
304 - "Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n",
305 - buffer[0],
306 - buffer[1],
307 - buffer[2],
308 - buffer[3],
309 - buffer[4],
310 - buffer[5],
311 - buffer[6],
312 - buffer[7]);
313 - fprintf(stderr,
314 - "Block tail:%02X%02X%02X%02X%02X%02X%02X%02X\n",
315 - buffer[3063],
316 - buffer[3064],
317 - buffer[3065],
318 - buffer[3066],
319 - buffer[3067],
320 - buffer[3068],
321 - buffer[3069],
322 - buffer[3070]);
323 - }
324 + fprintf(stderr, "Block %u at %llu length %u",
325 + i, offsets[i].offset, size);
326 switch (uncompress(clear_buffer, &destlen,
327 buffer, size)) {
328 case Z_OK:
329 @@ -105,12 +143,13 @@
330 fprintf(stderr, "Uncomp: unknown error %u\n", i);
331 exit(1);
332 }
333 - if (destlen != ntohl(head.block_size)) {
334 - fprintf(stderr, "Uncomp: bad len %u (%lu not %u)\n", i,
335 - destlen, ntohl(head.block_size));
336 + fprintf(stderr, " => %lu\n", destlen);
337 + if (destlen != block_size && i != num_blocks - 1) {
338 + fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
339 + destlen, block_size);
340 exit(1);
341 }
342 - write(STDOUT_FILENO, clear_buffer, ntohl(head.block_size));
343 + write(STDOUT_FILENO, clear_buffer, (lastlen != 0 && (i+1) == num_blocks) ? lastlen : block_size);
344 }
345 return 0;
346 }
347 --- Makefile
348 +++ Makefile
349 @@ -1,16 +1,19 @@
350 PROGNAME=fusecloop
351 ARCFILES=*.c *.h *.pl Makefile configure README VERSION HELP INSTALL typescript *.cloop COPYING
352 -PROGS=fusecloop cloopreaderdemo extract_compressed_fs
353 +PROGS=fusecloop cloopreaderdemo extract_compressed_fs create_compressed_fs
354 FUSECFLAGS=`pkg-config fuse --cflags`
355 FUSELDFLAGS=`pkg-config fuse --libs`
357 CFLAGS= -Wall
359 -all: fusecloop extract_compressed_fs
360 +all: fusecloop extract_compressed_fs create_compressed_fs
362 extract_compressed_fs: extract_compressed_fs.c
363 ${CC} ${CFLAGS} ${LDFLAGS} -lz extract_compressed_fs.c -o extract_compressed_fs
365 +create_compressed_fs: create_compressed_fs.c md5sum.c
366 + ${CC} ${CFLAGS} ${LDFLAGS} -lz create_compressed_fs.c -o create_compressed_fs
367 +
368 fusecloop: fusecloop.c cloopreader.o strver debug.o
369 ${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop
373 --- md5sum.c
374 +++ md5sum.c
375 @@ -0,0 +1,246 @@
376 +/*
377 + * Based on busybox code.
378 + *
379 + * Compute MD5 checksum of strings according to the
380 + * definition of MD5 in RFC 1321 from April 1992.
381 + *
382 + * Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995.
383 + *
384 + * Copyright (C) 1995-1999 Free Software Foundation, Inc.
385 + * Copyright (C) 2001 Manuel Novoa III
386 + * Copyright (C) 2003 Glenn L. McGrath
387 + * Copyright (C) 2003 Erik Andersen
388 + * Copyright (C) 2010 Denys Vlasenko
389 + * Copyright (C) 2012 Pascal Bellard
390 + *
391 + * Licensed under GPLv2 or later
392 + */
393 +
394 +#define ALIGN1
395 +
396 +static uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */
397 +static uint64_t total64; /* must be directly before hash[] */
398 +static uint32_t hash[8]; /* 4 elements for md5, 5 for sha1, 8 for sha256 */
399 +
400 +/* Emit a string of hex representation of bytes */
401 +/* static char* bin2hex(char *p)
402 +{
403 + static const char bb_hexdigits_upcase[] ALIGN1 = "0123456789abcdef";
404 + int count = 16;
405 + const char *cp = (const char *) hash;
406 + while (count) {
407 + unsigned char c = *cp++;
408 + // put lowercase hex digits
409 + *p++ = bb_hexdigits_upcase[c >> 4];
410 + *p++ = bb_hexdigits_upcase[c & 0xf];
411 + count--;
412 + }
413 + return p;
414 +} */
415 +
416 +//#define rotl32(x,n) (((x) << (n)) | ((x) >> (32 - (n))))
417 +static uint32_t rotl32(uint32_t x, unsigned n)
418 +{
419 + return (x << n) | (x >> (32 - n));
420 +}
421 +
422 +static void md5_process_block64(void);
423 +
424 +/* Feed data through a temporary buffer.
425 + * The internal buffer remembers previous data until it has 64
426 + * bytes worth to pass on.
427 + */
428 +static void common64_hash(const void *buffer, size_t len)
429 +{
430 + unsigned bufpos = total64 & 63;
431 +
432 + total64 += len;
433 +
434 + while (1) {
435 + unsigned remaining = 64 - bufpos;
436 + if (remaining > len)
437 + remaining = len;
438 + /* Copy data into aligned buffer */
439 + memcpy(wbuffer + bufpos, buffer, remaining);
440 + len -= remaining;
441 + buffer = (const char *)buffer + remaining;
442 + bufpos += remaining;
443 + /* clever way to do "if (bufpos != 64) break; ... ; bufpos = 0;" */
444 + bufpos -= 64;
445 + if (bufpos != 0)
446 + break;
447 + /* Buffer is filled up, process it */
448 + md5_process_block64();
449 + /*bufpos = 0; - already is */
450 + }
451 +}
452 +
453 +/* Process the remaining bytes in the buffer */
454 +static void common64_end(void)
455 +{
456 + unsigned bufpos = total64 & 63;
457 + /* Pad the buffer to the next 64-byte boundary with 0x80,0,0,0... */
458 + wbuffer[bufpos++] = 0x80;
459 +
460 + /* This loop iterates either once or twice, no more, no less */
461 + while (1) {
462 + unsigned remaining = 64 - bufpos;
463 + memset(wbuffer + bufpos, 0, remaining);
464 + /* Do we have enough space for the length count? */
465 + if (remaining >= 8) {
466 + /* Store the 64-bit counter of bits in the buffer */
467 + uint64_t t = total64 << 3;
468 + /* wbuffer is suitably aligned for this */
469 + *(uint64_t *) (&wbuffer[64 - 8]) = t;
470 + }
471 + md5_process_block64();
472 + if (remaining >= 8)
473 + break;
474 + bufpos = 0;
475 + }
476 +}
477 +
478 +/* These are the four functions used in the four steps of the MD5 algorithm
479 + * and defined in the RFC 1321. The first function is a little bit optimized
480 + * (as found in Colin Plumbs public domain implementation).
481 + * #define FF(b, c, d) ((b & c) | (~b & d))
482 + */
483 +#undef FF
484 +#undef FG
485 +#undef FH
486 +#undef FI
487 +#define FF(b, c, d) (d ^ (b & (c ^ d)))
488 +#define FG(b, c, d) FF(d, b, c)
489 +#define FH(b, c, d) (b ^ c ^ d)
490 +#define FI(b, c, d) (c ^ (b | ~d))
491 +
492 +/* Hash a single block, 64 bytes long and 4-byte aligned */
493 +static void md5_process_block64(void)
494 +{
495 + /* Before we start, one word to the strange constants.
496 + They are defined in RFC 1321 as
497 + T[i] = (int)(4294967296.0 * fabs(sin(i))), i=1..64
498 + */
499 + static const uint32_t C_array[] = {
500 + /* round 1 */
501 + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
502 + 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
503 + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
504 + 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
505 + /* round 2 */
506 + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
507 + 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
508 + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
509 + 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
510 + /* round 3 */
511 + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
512 + 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
513 + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x4881d05,
514 + 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
515 + /* round 4 */
516 + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
517 + 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
518 + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
519 + 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391
520 + };
521 + static const char P_array[] ALIGN1 = {
522 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 1 */
523 + 1, 6, 11, 0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, /* 2 */
524 + 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, /* 3 */
525 + 0, 7, 14, 5, 12, 3, 10, 1, 8, 15, 6, 13, 4, 11, 2, 9 /* 4 */
526 + };
527 + uint32_t *words = (uint32_t *) wbuffer;
528 + uint32_t A = hash[0];
529 + uint32_t B = hash[1];
530 + uint32_t C = hash[2];
531 + uint32_t D = hash[3];
532 +
533 + static const char S_array[] ALIGN1 = {
534 + 7, 12, 17, 22,
535 + 5, 9, 14, 20,
536 + 4, 11, 16, 23,
537 + 6, 10, 15, 21
538 + };
539 + const uint32_t *pc;
540 + const char *pp;
541 + const char *ps;
542 + int i;
543 + uint32_t temp;
544 +
545 +
546 + pc = C_array;
547 + pp = P_array;
548 + ps = S_array - 4;
549 +
550 + for (i = 0; i < 64; i++) {
551 + if ((i & 0x0f) == 0)
552 + ps += 4;
553 + temp = A;
554 + switch (i >> 4) {
555 + case 0:
556 + temp += FF(B, C, D);
557 + break;
558 + case 1:
559 + temp += FG(B, C, D);
560 + break;
561 + case 2:
562 + temp += FH(B, C, D);
563 + break;
564 + case 3:
565 + temp += FI(B, C, D);
566 + }
567 + temp += words[(int) (*pp++)] + *pc++;
568 + temp = rotl32(temp, ps[i & 3]);
569 + temp += B;
570 + A = D;
571 + D = C;
572 + C = B;
573 + B = temp;
574 + }
575 + /* Add checksum to the starting values */
576 + hash[0] += A;
577 + hash[1] += B;
578 + hash[2] += C;
579 + hash[3] += D;
580 +
581 +}
582 +#undef FF
583 +#undef FG
584 +#undef FH
585 +#undef FI
586 +
587 +/* Initialize structure containing state of computation.
588 + * (RFC 1321, 3.3: Step 3)
589 + */
590 +static void md5_begin(void)
591 +{
592 + hash[0] = 0x67452301;
593 + hash[1] = 0xefcdab89;
594 + hash[2] = 0x98badcfe;
595 + hash[3] = 0x10325476;
596 + total64 = 0;
597 +}
598 +
599 +/* Used also for sha1 and sha256 */
600 +#define md5_hash common64_hash
601 +
602 +/* Process the remaining bytes in the buffer and put result from CTX
603 + * in first 16 bytes following RESBUF. The result is always in little
604 + * endian byte order, so that a byte-wise output yields to the wanted
605 + * ASCII representation of the message digest.
606 + */
607 +#define md5_end common64_end
608 +
609 +typedef struct { char hash[16]; } md5hash;
610 +
611 +static md5hash md5sum(uint8_t *buffer, int len)
612 +{
613 + md5hash val;
614 +
615 + md5_begin();
616 + md5_hash(buffer, len);
617 + md5_end();
618 + memcpy(&val, hash, 16);
619 +
620 + return val;
621 +}
622 --- create_compressed_fs.c
623 +++ create_compressed_fs.c
624 @@ -0,0 +1,226 @@
625 +#ifdef FIND_BEST_COMPRESSION
626 +#include <compress.h>
627 +extern "C" {
628 +#include <stdlib.h>
629 +#include <string.h>
630 +
631 +#define ZMAX 9
632 +static shrink_t level;
633 +static int pass, iter;
634 +static int best_compress(unsigned char *compressed,
635 + unsigned long *compressed_len,
636 + unsigned char *uncompressed,
637 + unsigned long uncompressed_len)
638 +{
639 + int i, j, err;
640 + unsigned char *buf[2];
641 + unsigned len;
642 + unsigned long llen, best = *compressed_len * 2;
643 + static unsigned char *buffer;
644 + static unsigned long buffersz;
645 +
646 + if (buffersz < *compressed_len) {
647 + if (buffer) free(buffer);
648 + buffer = (unsigned char *) malloc(buffersz = *compressed_len);
649 + if (!buffer) return Z_MEM_ERROR;
650 + }
651 + buf[0] = compressed;
652 + buf[1] = buffer;
653 + for (i = j = 0; i <= ZMAX+3 && (pass == 0 || i < pass); i++) {
654 + llen = len = *compressed_len;
655 + if (i >= ZMAX+1) {
656 + level.level = (i == ZMAX+1) ? shrink_normal :
657 + (i == ZMAX+2) ? shrink_extra : shrink_insane;
658 + err = (compress_zlib(level, buf[j],
659 + len, uncompressed,
660 + uncompressed_len)) ? Z_OK : Z_DATA_ERROR;
661 + } else {
662 + err = compress2(buf[j], &llen, uncompressed,
663 + uncompressed_len, i);
664 + len = llen;
665 + }
666 + if (err != Z_OK) return err;
667 + if (len < best) {
668 + best = len;
669 + j = 1 - j;
670 + }
671 + }
672 + *compressed_len = best;
673 + if (j == 0)
674 + memcpy(compressed, buffer, best);
675 + return err;
676 +}
677 +#define compress2(a,b,c,d,e) best_compress(a,b,c,d)
678 +#endif
679 +
680 +#include <signal.h>
681 +
682 +/* Creates a compressed file */
683 +#include "common_header.h"
684 +
685 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
686 +#define CHUNK 65536
687 +#define DEFAULT_BLOCKSIZE 65536
688 +
689 +static void quit(const char *s)
690 +{
691 + fprintf(stderr, "%s\n", s);
692 + exit(1);
693 +}
694 +
695 +static int readblock(unsigned char *buffer, int n)
696 +{
697 + int i;
698 +
699 + memset(buffer, 0, n);
700 + for (i = 0 ; i < n;) {
701 + int j = read(STDIN_FILENO, buffer + i, n - i);
702 + if (j < 0 && errno == EINTR) continue;
703 + if (j <= 0) break;
704 + i += j;
705 + }
706 + return i;
707 +}
708 +
709 +#ifdef FIND_BEST_COMPRESSION
710 +#include "md5sum.c"
711 +#endif
712 +
713 +static unsigned n;
714 +static unsigned long lastlen, pos;
715 +static u_int32_t *block_index;
716 +static unsigned char *compressed;
717 +static unsigned long block_size = 0;
718 +static void flush_index(int sig)
719 +{
720 + static char padding[512];
721 + struct cloop_tail tail;
722 + unsigned long len;
723 +
724 + fprintf(stderr, "Write index for %lu blocks\n", n);
725 + if (block_size >= 0x1000000) lastlen = 0;
726 + tail.index_size = ntohl(sizeof(*block_index) + 256*(lastlen % 0xFFffFF));
727 + tail.num_blocks = ntohl(n);
728 + n *= sizeof(*block_index);
729 + len = n + n/1000 + 12;
730 + compressed = (unsigned char *) realloc(compressed, len);
731 + if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
732 + n, Z_BEST_SPEED) != Z_OK)
733 + quit("Index compression failed");
734 + tail.table_size = ntohl(len);
735 + pos += len + sizeof(tail);
736 + n = pos & 511;
737 + // if (n) write(STDOUT_FILENO, padding, 512 - n);
738 + write(STDOUT_FILENO, compressed, len);
739 + write(STDOUT_FILENO, &tail, sizeof(tail));
740 + exit(sig != 0);
741 +}
742 +
743 +int main(int argc, char *argv[])
744 +{
745 + struct cloop_head head;
746 + unsigned char *uncompressed;
747 + unsigned long len;
748 + unsigned indexmax, zlenmax;
749 +#ifdef FIND_BEST_COMPRESSION
750 + unsigned i, j, hashmax;
751 + md5hash *hash;
752 +#endif
753 +
754 +#ifdef FIND_BEST_COMPRESSION
755 + while (argc > 1) {
756 + if (argv[1][0] == '-') {
757 + int *p = &pass;
758 + switch (argv[1][1]) {
759 + case 'i' : p = &iter;
760 + case 'n' : *p = atoi(argv[2]);
761 + argc -= 2;
762 + argv += 2;
763 + continue;
764 + }
765 + }
766 + argc--;
767 + if (argv[1][0] < '0' || argv[1][0] > '9')
768 + quit("Usage : create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
769 +#else
770 + if (argc > 1) {
771 + if (argv[1][0] < '0' || argv[1][0] > '9')
772 + quit("Usage : create_compressed_fs [block size] < input > output");
773 +#endif
774 + block_size = atoi(argv[1]);
775 + }
776 + if (block_size < 4096)
777 + block_size = DEFAULT_BLOCKSIZE;
778 + fprintf(stderr, "Block size is %lu\n", block_size);
779 + zlenmax = block_size + block_size/1000 + 12;
780 +
781 + memset(&head, 0, sizeof(head));
782 + strcpy(head.preamble, CLOOP_PREAMBLE);
783 + head.num_blocks = -1;
784 + head.block_size = htonl(block_size);
785 + write(STDOUT_FILENO, &head, sizeof(head));
786 + pos = sizeof(head);
787 +
788 + compressed = (unsigned char *) malloc(zlenmax);
789 + uncompressed = (unsigned char *) malloc(block_size);
790 + block_index = (u_int32_t *) malloc(indexmax = CHUNK);
791 +#ifdef FIND_BEST_COMPRESSION
792 + hash = (md5hash *) malloc(hashmax = CHUNK);
793 + if (!compressed || !uncompressed || !block_index || !hash)
794 +#else
795 + if (!compressed || !uncompressed || !block_index)
796 +#endif
797 + quit("Malloc failed");
798 +
799 + signal(SIGINT,flush_index);
800 + signal(SIGQUIT,flush_index);
801 + signal(SIGTERM,flush_index);
802 +
803 + for (n = 0; (len = readblock(uncompressed, block_size)) != 0; n++) {
804 + lastlen = len;
805 + if (n * sizeof(*block_index) >= indexmax) {
806 + block_index = (u_int32_t *) realloc(block_index,
807 + indexmax += CHUNK);
808 + if (!block_index)
809 + quit("Realloc");
810 + }
811 +#ifdef FIND_BEST_COMPRESSION
812 + if (n * sizeof(*hash) >= hashmax) {
813 + hash = (md5hash *) realloc(hash, hashmax += CHUNK);
814 + if (!hash)
815 + quit("Realloc hash");
816 + }
817 + hash[n] = md5sum(uncompressed, len);
818 + j = 0x7FFFFFFF;
819 + if (n < j)
820 + j = n;
821 + for (i = 0; i < j; i++) {
822 + if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
823 + && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
824 + break;
825 + }
826 + if (i != j) {
827 + block_index[n] = ntohl(0x80000000 | i);
828 + fprintf(stderr, "Block %u length %lu => duplicate %lu\n",
829 + n, block_size, i);
830 + }
831 + else
832 +#endif
833 + {
834 + len = zlenmax;
835 + if (compress2(compressed, &len, uncompressed, lastlen,
836 + Z_BEST_SPEED) != Z_OK)
837 + quit("Compression failed");
838 + fprintf(stderr, "Block %u length %lu => %lu\n",
839 + n, block_size, len);
840 + write(STDOUT_FILENO, compressed, len);
841 + pos += len;
842 + block_index[n] = ntohl(len);
843 + }
844 + }
845 + flush_index(0);
846 + return 0;
847 +}
848 +#ifdef FIND_BEST_COMPRESSION
849 +}
850 +#endif
851 --- fusecloop.c
852 +++ fusecloop.c
853 @@ -65,7 +65,7 @@
855 memcpy(stbuf,&stb,sizeof stb);
856 stbuf->st_mode&=~0222;
857 - stbuf->st_size = cd.blocksize * cd.numblocks;
858 + stbuf->st_size = (loff_t) cd.blocksize * cd.numblocks;
859 /*
860 stbuf->st_mode = S_IFREG | 0444;
861 stbuf->st_nlink = 1;
863 === Add v4 support
864 --- compressed_loop.h
865 +++ compressed_loop.h
866 @@ -30,6 +30,54 @@
867 /* ...padding up to CLOOP_HEADROOM... */
868 /* block_size (32bit number, network order) */
869 /* num_blocks (32bit number, network order) */
870 +/*
871 +* Starting with Format V4.0 (cloop version 4.x), cloop can now have two *
872 +* alternative structures: *
873 +* 1. Header first: *
874 +* +---------------------------- FIXED SIZE ---------------------------+ *
875 +* |Signature (128 bytes) | *
876 +* |block_size (32bit number, network order) | *
877 +* |num_blocks (32bit number, network order) | *
878 +* +--------------------------- VARIABLE SIZE -------------------------+ *
879 +* |num_blocks * FlagsOffset (upper 4 bits flags, lower 60 bits offset)| *
880 +* |compressed data blocks of variable size ... | *
881 +* +-------------------------------------------------------------------+ *
882 +* *
883 +* 2. Footer (header last): *
884 +* +--------------------------- VARIABLE SIZE -------------------------+ *
885 +* |compressed data blocks of variable size ... | *
886 +* |num_blocks * FlagsOffset (upper 4 bits flags, lower 60 bits offset)| *
887 +* +---------------------------- FIXED SIZE ---------------------------+ *
888 +* |Signature (128 bytes) | *
889 +* |block_size (32bit number, network order) | *
890 +* |num_blocks (32bit number, network order) | *
891 +* +-------------------------------------------------------------------+ *
892 +* *
893 +* Offsets are always relative to beginning of file, in all formats. *
894 +* The block index contains num_blocks+1 offsets, followed (1) or *
895 +* preceded (2) by the compressed blocks. *
896 +* *
897 +* CLOOP4 flags for each compressed block *
898 +* Value Meaning *
899 +* 0 GZIP/7ZIP compression (compatible with V2.0 Format) *
900 +* 1 no compression (incompressible data) *
901 +* 2 xz compression (currently best space saver) *
902 +* 3 lz4 compression *
903 +* 4 lzo compression (fastest) *
904 +* 15 block link *
905 +*/
906 +/* Get value of first 4 bits */
907 +#define CLOOP_BLOCK_FLAGS(x) ((unsigned int)(((x) & 0xf000000000000000LLU) >> 60))
908 +/* Get value of last 60 bits */
909 +#define CLOOP_BLOCK_OFFSET(x) ((x) & 0x0fffffffffffffffLLU)
910 +
911 +#define CLOOP_COMPRESSOR_ZLIB 0x0
912 +#define CLOOP_COMPRESSOR_NONE 0x1
913 +#define CLOOP_COMPRESSOR_XZ 0x2
914 +#define CLOOP_COMPRESSOR_LZ4 0x3
915 +#define CLOOP_COMPRESSOR_LZO 0x4
916 +#define CLOOP_COMPRESSOR_LINK 0xF
917 +
919 struct cloop_head
920 {
921 @@ -43,47 +91,86 @@
923 struct cloop_tail
924 {
925 - u_int32_t table_size;
926 - u_int32_t index_size;
927 + u_int32_t table_size;
928 + u_int32_t index_size; /* size:4 comp:3 ctrl-c:1 lastlen:24 */
929 +#define CLOOP3_INDEX_SIZE(x) ((unsigned int)((x) & 0xF))
930 +#define CLOOP3_BLOCKS_FLAGS(x) ((unsigned int)((x) & 0x70) >> 4)
931 +#define CLOOP3_TRUNCATED(x) ((unsigned int)((x) & 0x80) >> 7)
932 +#define CLOOP3_LASTLEN(x) (unsigned int)((x) >> 8)
933 u_int32_t num_blocks;
934 };
936 +#define GZIP_MAX_BUFFER(n) ((n) + (n)/1000 + 12)
937 +
938 struct block_info
939 {
940 loff_t offset; /* 64-bit offsets of compressed block */
941 u_int32_t size; /* 32-bit compressed block size */
942 - u_int32_t optidx; /* 32-bit index number */
943 + u_int32_t flags; /* 32-bit compression flags */
944 +
945 };
947 -static inline char *build_index(struct block_info *offsets, unsigned long n)
948 +static inline char *build_index(struct block_info *offsets, unsigned long n,
949 + unsigned long block_size, unsigned global_flags)
950 {
951 u_int32_t *ofs32 = (u_int32_t *) offsets;
952 loff_t *ofs64 = (loff_t *) offsets;
953 -
954 +
955 + /* v3 64bits bug: v1 assumed */
956 + unsigned long v3_64;
957 + loff_t prev;
958 +
959 + if (ofs32[0] != 0 && ofs32[1] == 0) {
960 + for (v3_64=(n+1)/2, prev=__le64_to_cpu(ofs64[v3_64]);
961 + v3_64 > 0 && __le64_to_cpu(ofs64[--v3_64]) < prev;
962 + prev=__le64_to_cpu(ofs64[v3_64]));
963 + }
964 +
965 if (ofs32[0] == 0) {
966 if (ofs32[2]) { /* ACCELERATED KNOPPIX V1.0 */
967 while (n--) {
968 offsets[n].offset = __be64_to_cpu(offsets[n].offset);
969 offsets[n].size = ntohl(offsets[n].size);
970 + offsets[n].flags = 0;
971 }
972 return (char *) "128BE accelerated knoppix 1.0";
973 }
974 - else { /* V2.0 */
975 - loff_t last = __be64_to_cpu(ofs64[n]);
976 - while (n--) {
977 + else { /* V2.0/V4.0 */
978 + loff_t last = CLOOP_BLOCK_OFFSET(__be64_to_cpu(ofs64[n]));
979 + u_int32_t flags;
980 + static char v4[11];
981 + unsigned long i = n;
982 +
983 + for (flags = 0; n-- ;) {
984 + loff_t data = __be64_to_cpu(ofs64[n]);
985 +
986 offsets[n].size = last -
987 - (offsets[n].offset = __be64_to_cpu(ofs64[n]));
988 + (offsets[n].offset = CLOOP_BLOCK_OFFSET(data));
989 last = offsets[n].offset;
990 + offsets[n].flags = CLOOP_BLOCK_FLAGS(data);
991 + flags |= 1 << offsets[n].flags;
992 + }
993 + if (flags < 2) return (char *) "64BE v2.0";
994 + while (i--) {
995 + if (offsets[i].flags == CLOOP_COMPRESSOR_LINK) {
996 + offsets[i] = offsets[offsets[i].offset];
997 + }
998 + }
999 + strcpy(v4, (char *) "64BE v4.0a");
1000 + v4[10] = 'a' + ((flags-1) & 0xF); // compressors used
1001 + if (flags > 0x10) { // with links ?
1002 + v4[10] += 'A' - 'a';
1003 }
1004 - return (char *) "64BE v2.0";
1005 + return v4;
1006 }
1007 }
1008 - else if (ofs32[1] == 0) { /* V1.0 */
1009 + else if (ofs32[1] == 0 && v3_64 == 0) { /* V1.0 */
1010 loff_t last = __le64_to_cpu(ofs64[n]);
1011 while (n--) {
1012 offsets[n].size = last -
1013 (offsets[n].offset = __le64_to_cpu(ofs64[n]));
1014 last = offsets[n].offset;
1015 + offsets[n].flags = 0;
1016 }
1017 return (char *) "64LE v1.0";
1018 }
1019 @@ -93,25 +180,37 @@
1020 offsets[n].size = last -
1021 (offsets[n].offset = ntohl(ofs32[n]));
1022 last = offsets[n].offset;
1023 + offsets[n].flags = 0;
1024 }
1025 return (char *) "32BE v0.68";
1026 }
1027 else { /* V3.0 */
1028 unsigned long i;
1029 loff_t j;
1030 + static char v3[11];
1032 + v3_64 = (ofs32[1] == 0) ? 2 : 1;
1033 for (i = n; i-- != 0; )
1034 - offsets[i].size = ntohl(ofs32[i]);
1035 + offsets[i].size = ntohl(ofs32[i*v3_64]);
1036 for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
1037 offsets[i].offset = j;
1038 + offsets[i].flags = global_flags;
1039 + if ((offsets[i].size & 0x80000000) == 0) {
1040 + j += offsets[i].size;
1041 + }
1042 + else if (offsets[i].size == 0xFFFFFFFF) {
1043 + offsets[i].flags = CLOOP_COMPRESSOR_NONE;
1044 + j += offsets[i].size = block_size;
1045 + }
1046 + }
1047 + for (i = 0; i < n; i++) {
1048 if (offsets[i].size & 0x80000000) {
1049 - unsigned long k = offsets[i].size & 0x7FFFFFFF;
1050 - offsets[i].offset = offsets[k].offset;
1051 - offsets[i].size = offsets[k].size;
1052 + offsets[i] = offsets[offsets[i].size & 0x7FFFFFFF];
1053 }
1054 - else j += offsets[i].size;
1055 }
1056 - return (char *) "32BE v3.0";
1057 + strcpy(v3, (char *) (--v3_64) ? "64BE v3.0a" : "32BE v3.0a");
1058 + v3[10] += global_flags;
1059 + return v3;
1060 }
1061 }
1063 --- cloopreader.c
1064 +++ cloopreader.c
1065 @@ -25,6 +25,8 @@
1066 #include "debug.h"
1067 #include "cloopreader.h"
1069 +#include "cloopunpack.c"
1070 +
1071 int read_all(int fh, void* block, size_t size){
1072 bfuncinfo("fh=%d block=0x%lx size=0x%lx",
1073 fh,(ulong)block,(ulong)size);
1074 @@ -50,18 +52,29 @@
1075 bfuncinfo("fh=%d",fh);
1076 c->fh=fh;
1077 struct cloop_head head;
1078 - OP(read_all(c->fh,&head,sizeof head)); /* read Header */
1079 + int v4_header_last, flags;
1080 + loff_t end;
1081 +
1082 + for (v4_header_last=0;;v4_header_last++) {
1083 + OP(read_all(c->fh,&head,sizeof head)); /* read Header */
1084 + if (!memcmp(&head,"#!/bin/sh",9)) break;
1085 + if (v4_header_last) exit(1);
1086 + end = lseek(c->fh, 0, SEEK_END);
1087 + OP(lseek(c->fh, end - sizeof(head), SEEK_SET));
1088 + }
1090 c->numblocks=ntohl(head.num_blocks);
1091 c->blocksize=ntohl(head.block_size);
1093 bprintf("block_size=%lx num_blocks=%x\n", c->blocksize, c->numblocks);
1094 + if (v4_header_last)
1095 + OP(lseek(c->fh, end - sizeof(head) - (sizeof(*c->toc) * c->numblocks), SEEK_SET));
1097 ALLOC(c->pblock,c->blocksize);
1099 if (c->numblocks + 1 == 0) {
1100 struct cloop_tail tail;
1101 - loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
1102 + end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
1103 void *p;
1104 ulong toclen, len;
1106 @@ -70,21 +83,23 @@
1107 c->numblocks = ntohl(tail.num_blocks);
1108 c->tocsize = sizeof(*c->toc) * c->numblocks;
1109 len = ntohl(tail.table_size);
1110 - toclen = (ntohl(tail.index_size) & 255) * c->numblocks;
1111 + flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
1112 + toclen = CLOOP3_INDEX_SIZE(ntohl(tail.index_size)) * c->numblocks;
1113 OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
1114 ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
1115 ALLOC(p,len);
1116 OP(read_all(c->fh,p,len)); /* read Data Index */
1117 - if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK)
1118 + if (unpack[flags]((void *)c->toc,&toclen,p,len) != Z_OK)
1119 exit(1);
1120 free(p);
1121 }
1122 else {
1123 + flags = 0;
1124 c->tocsize = sizeof(*c->toc) * c->numblocks;
1125 ALLOC(c->toc,c->tocsize);
1126 OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
1127 }
1128 - build_index(c->toc, c->numblocks);
1129 + build_index(c->toc, c->numblocks, c->blocksize, flags);
1130 c->cblocksizecur=0;
1131 c->curblock=-1;
1132 return 0;
1133 @@ -121,7 +136,8 @@
1134 "pblock=0x%lx &destlen=0x%lx cblock=0x%lx cblocksize=%lu\n",
1135 (ulong)c->pblock,(ulong)&destlen,(ulong)c->cblock,c->cblocksize
1136 );
1137 - switch(uncompress(c->pblock,&destlen,c->cblock,c->cblocksize)){
1138 + if(c->toc[page].flags <= CLOOP_COMPRESSOR_MAX){
1139 + switch(unpack[c->toc[page].flags](c->pblock,&destlen,c->cblock,c->cblocksize)){
1140 case Z_OK: break;
1141 #define entry(x)\
1142 case x: bprintf( #x"\n"); break;
1143 @@ -130,7 +146,9 @@
1144 entry(Z_DATA_ERROR)
1145 #undef entry
1146 default: bprintf("Z_UNKNOWN_ERROR\n");
1147 + }
1148 }
1149 + else bprintf("Unsuppoted compression type\n");
1150 if(destlen!=c->blocksize)bprintf("Size mismatch\n");
1151 return 0;
1152 }
1153 --- /dev/null
1154 +++ cloopunpack.c
1155 @@ -0,0 +1,74 @@
1156 +
1157 +static int none_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
1158 +{
1159 + memcpy(dest,source,*destLen = sourceLen);
1160 + return Z_OK;
1161 +}
1162 +
1163 +#include <lzma.h>
1164 +static int xz_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
1165 +{
1166 + size_t src_pos = 0;
1167 + size_t dest_pos = 0;
1168 + uint64_t memlimit = 32*1024*1024;
1169 +
1170 + lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL,
1171 + source, &src_pos, sourceLen, dest, &dest_pos, *destLen);
1172 +
1173 + if(res == LZMA_OK && sourceLen == (int) src_pos) {
1174 + *destLen = dest_pos;
1175 + return Z_OK;
1176 + }
1177 + else return Z_ERRNO;
1178 +}
1179 +
1180 +#include <lz4.h>
1181 +static int lz4_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
1182 +{
1183 + long size = LZ4_decompress_safe((const char *) source, (char *) dest, sourceLen, *destLen);
1184 +
1185 + if (size < 0) return Z_ERRNO;
1186 + *destLen = size;
1187 + return Z_OK;
1188 +}
1189 +
1190 +#include <lzo/lzo1x.h>
1191 +static int lzo_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
1192 +{
1193 + lzo_uint outlen = *destLen;
1194 +
1195 + if (lzo1x_decompress_safe(source, sourceLen, dest, &outlen, NULL) == LZO_E_OK) {
1196 + *destLen = outlen;
1197 + return Z_OK;
1198 + }
1199 + else return Z_ERRNO;
1200 +}
1201 +
1202 +#include <zstd.h>
1203 +static int zstd_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
1204 +{
1205 + const size_t res = ZSTD_decompress(dest, *destLen, source, sourceLen);
1206 +
1207 + if (ZSTD_isError(res)) {
1208 + return Z_ERRNO;
1209 + }
1210 +
1211 + *destLen = res;
1212 + return Z_OK;
1213 +}
1214 +
1215 +#define CLOOP_COMPRESSOR_ZSTD 0x5
1216 +
1217 +#define CLOOP_COMPRESSOR_MAX CLOOP_COMPRESSOR_ZSTD
1218 +
1219 +#define CLOOP_COMPRESSOR_NAMES "gzip","copy","xz","lz4","lzo","zstd"
1220 +
1221 +static int (*unpack[CLOOP_COMPRESSOR_MAX+1])(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen) = {
1222 + uncompress,
1223 + none_uncompress,
1224 + xz_uncompress,
1225 + lz4_uncompress,
1226 + lzo_uncompress,
1227 + zstd_uncompress
1228 +};
1229 +
1230 --- extract_compressed_fs.c
1231 +++ extract_compressed_fs.c
1232 @@ -3,14 +3,78 @@
1233 #include "common_header.h"
1234 #define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
1236 +#include "cloopunpack.c"
1237 +static char *packnames[CLOOP_COMPRESSOR_MAX+1] = { CLOOP_COMPRESSOR_NAMES };
1238 +
1239 +static unsigned char *out;
1240 +static unsigned long zblock_maxsize, block_size, num_blocks;
1241 +static int handle, err;
1242 +static unsigned char *buffer, *clear_buffer;
1243 +static struct block_info *offsets;
1244 +
1245 +static uLongf get_block(unsigned int i, int argc)
1246 +{
1247 + unsigned long len;
1248 + uLongf ulen;
1249 + int flags = offsets[i].flags;
1250 +
1251 + if (flags > CLOOP_COMPRESSOR_MAX) {
1252 + fprintf(stderr, "Block %u: unsupported compression %d \n",
1253 + i, flags);
1254 + exit(1);
1255 + }
1256 +
1257 + len = offsets[i].size;
1258 + if (len > zblock_maxsize) {
1259 + fprintf(stderr,
1260 + "Size %lu for block %u (offset %Lu) too big\n",
1261 + len, i, offsets[i].offset);
1262 + exit(1);
1263 + }
1264 +
1265 + if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
1266 + fprintf(stderr, "lseek to %Lu: %s\n",
1267 + offsets[i].offset, strerror(errno));
1268 + exit(1);
1269 + }
1270 +
1271 + read(handle, out = buffer, ulen = len);
1272 +
1273 + if (flags != CLOOP_COMPRESSOR_ZLIB || argc <= 2) {
1274 + ulen = block_size;
1275 + err = unpack[flags](out = clear_buffer, &ulen, buffer, len);
1276 + if (err != Z_OK) {
1277 + fprintf(stderr, "Unpack %s block %u error %d \n",
1278 + packnames[flags], i, err);
1279 + exit(1);
1280 + }
1281 + if (ulen != block_size && i != num_blocks - 1) {
1282 + fprintf(stderr, "Uncomp %s: bad len %u (%lu not %lu)\n",
1283 + packnames[flags], i, ulen, block_size);
1284 + exit(1);
1285 + }
1286 + if (argc > 2) {
1287 + len = ulen;
1288 + ulen = zblock_maxsize;
1289 + err = compress2(out = buffer, &ulen, clear_buffer, len, Z_BEST_SPEED);
1290 + if (err != Z_OK) {
1291 + fprintf(stderr, "Compress %s block %u error %d \n",
1292 + packnames[flags], i, err);
1293 + exit(1);
1294 + }
1295 + }
1296 + }
1297 + return ulen;
1298 +}
1299 +
1300 int main(int argc, char *argv[])
1301 {
1302 - int handle;
1303 + int flags;
1304 struct cloop_head head;
1305 - unsigned int i;
1306 - unsigned long num_blocks, block_size, zblock_maxsize, lastlen = 0;
1307 - unsigned char *buffer, *clear_buffer;
1308 - struct block_info *offsets;
1309 + unsigned int i, v4_header_last, global_flags;
1310 + unsigned long n, len;
1311 + uLongf ulen;
1312 + loff_t end;
1314 if (argc < 2 || argv[1][0] == '-') {
1315 fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
1316 @@ -23,23 +87,35 @@
1317 exit(1);
1318 }
1320 - if (read(handle, &head, sizeof(head)) != sizeof(head)) {
1321 - perror("Reading compressed file header\n");
1322 + for (v4_header_last=0;; v4_header_last++) {
1323 + if (read(handle, &head, sizeof(head)) != sizeof(head)) {
1324 + perror("Reading compressed file header\n");
1325 + exit(1);
1326 + }
1327 + if (!memcmp(&head,"#!/bin/sh",9)) break;
1328 + end = lseek64(handle, 0, SEEK_END);
1329 + lseek64(handle, end - sizeof(head), SEEK_SET);
1330 + if (v4_header_last == 0) continue;
1331 + perror("Not a cloop file\n");
1332 exit(1);
1333 }
1335 num_blocks = ntohl(head.num_blocks);
1336 block_size = ntohl(head.block_size);
1337 - zblock_maxsize = block_size + block_size/1000 + 12 + 4;
1338 + zblock_maxsize = GZIP_MAX_BUFFER(block_size);
1339 buffer = malloc(zblock_maxsize);
1340 clear_buffer = malloc(block_size);
1342 + if (v4_header_last) {
1343 + lseek64(handle, end - ((num_blocks+1) * sizeof(loff_t)) - sizeof(head), SEEK_SET);
1344 + }
1345 +
1346 if (num_blocks == 0xFFFFFFFF) {
1347 void *table;
1348 struct cloop_tail tail;
1349 - unsigned long len, table_size;
1350 - loff_t end = lseek64(handle, 0, SEEK_END);
1351 + unsigned long table_size;
1353 + end = lseek64(handle, 0, SEEK_END);
1354 if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
1355 read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
1356 lseek64(handle, end - sizeof(tail) -
1357 @@ -51,105 +127,108 @@
1358 num_blocks = ntohl(head.num_blocks);
1359 table_size = ntohl(tail.table_size);
1360 table = malloc(table_size);
1361 - len = i = num_blocks * (ntohl(tail.index_size) & 255);
1362 - lastlen = ntohl(tail.index_size) / 256;
1363 - offsets = malloc(num_blocks * sizeof(*offsets));
1364 - if (!table || !offsets ||
1365 - read(handle, table, table_size) != table_size ||
1366 - uncompress((void *)offsets, &len, table, table_size) != Z_OK ||
1367 - len != i) {
1368 + len = num_blocks * CLOOP3_INDEX_SIZE(ntohl(tail.index_size));
1369 + global_flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
1370 + if (global_flags > CLOOP_COMPRESSOR_MAX) {
1371 + fprintf(stderr, "Unsupported compression %d\n",
1372 + global_flags);
1373 + exit(1);
1374 + }
1375 + ulen = num_blocks * sizeof(*offsets);
1376 + offsets = malloc(ulen);
1377 + if (!table || !offsets || !buffer || !clear_buffer) {
1378 + fprintf(stderr,"Out of memory\n");
1379 + exit(1);
1380 + }
1381 + if (read(handle, table, table_size) != table_size) {
1382 perror("Reading index\n");
1383 exit(1);
1384 }
1385 + err = unpack[global_flags]((void *) offsets, &ulen, table, table_size);
1386 + if (err != Z_OK) {
1387 + fprintf(stderr, "Unpack %s index error %d\n",
1388 + packnames[global_flags],err);
1389 + exit(1);
1390 + }
1391 free(table);
1392 }
1393 else {
1394 - offsets = malloc(i = num_blocks * sizeof(*offsets));
1395 - if (!offsets || read(handle, offsets, i) != i) {
1396 + global_flags = 0;
1397 + len = num_blocks * sizeof(*offsets);
1398 + offsets = malloc(len);
1399 + if (v4_header_last) {
1400 + len = (num_blocks+1) * sizeof(loff_t);
1401 + }
1402 + if (!offsets || !buffer || !clear_buffer) {
1403 + fprintf(stderr,"Out of memory\n");
1404 + exit(1);
1405 + }
1406 + if (read(handle, offsets, len) != len) {
1407 perror("Reading index\n");
1408 exit(1);
1409 }
1410 }
1412 + if (v4_header_last) {
1413 + lseek64(handle, 0, SEEK_SET);
1414 + }
1415 +
1416 fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
1417 num_blocks, block_size, head.preamble);
1418 - fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
1419 +#if 1
1420 + if (getenv("CLOOP_INDEX") != NULL) {
1421 + fprintf(stderr, "CLOOP_INDEX: binary\n");
1422 + write(STDOUT_FILENO, offsets, len);
1423 + exit(0);
1424 + }
1425 +#endif
1426 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size, global_flags));
1428 +#if 1
1429 + if (getenv("CLOOP_TABLE") != NULL) {
1430 + fprintf(stderr, "CLOOP_TABLE ascii: offset, size, flags\n");
1431 + for (i = 0; i < num_blocks; i++) {
1432 + printf("%llu %u %u\n",
1433 + offsets[i].offset,
1434 + offsets[i].size,
1435 + offsets[i].flags);
1436 + }
1437 + exit(0);
1438 + }
1439 +#endif
1440 +
1441 if (argc > 2) {
1442 - unsigned n;
1443 - loff_t data, offset = ((num_blocks + 1) * sizeof(offset)) + sizeof(head);
1444 + loff_t data, ofs = ((num_blocks + 1) * sizeof(ofs)) + sizeof(head);
1446 strcpy(head.preamble, CLOOP_PREAMBLE);
1447 write(STDOUT_FILENO, &head, n = sizeof(head));
1448 for (i = 0; i < num_blocks; i++) {
1449 - data = __be64_to_cpu(offset);
1450 + data = __be64_to_cpu(ofs);
1451 write(STDOUT_FILENO, &data, sizeof(data));
1452 n += sizeof(data);
1453 - offset += offsets[i].size;
1454 + if (offsets[i].flags == CLOOP_COMPRESSOR_ZLIB)
1455 + ofs += offsets[i].size;
1456 + else
1457 + ofs += get_block(i,argc);
1458 }
1459 - data = __be64_to_cpu(offset);
1460 + data = __be64_to_cpu(ofs);
1461 write(STDOUT_FILENO, &data, sizeof(data));
1462 - for (i = 0; i < num_blocks && lseek64(handle, offsets[i].offset, SEEK_SET) >= 0; i++) {
1463 - read(handle, buffer, offsets[i].size);
1464 - write(STDOUT_FILENO, buffer, offsets[i].size);
1465 - n += offsets[i].size;
1466 - }
1467 + }
1468 + for (i = 0; i < num_blocks; i++) {
1469 +
1470 + fprintf(stderr, "Block %u %s at %llu length %lu ",
1471 + i, packnames[offsets[i].flags], offsets[i].offset, offsets[i].size);
1472 + ulen = get_block(i, argc);
1473 + fprintf(stderr, " => %lu\n", ulen);
1474 + write(STDOUT_FILENO, out, ulen);
1475 + n += ulen;
1476 + }
1477 + if (argc > 2) {
1478 n &= 0x1FF;
1479 if (n) {
1480 memset(buffer, 0, 512);
1481 write(STDOUT_FILENO, buffer, 512 - n);
1482 }
1483 - return 0;
1484 - }
1485 -
1486 - for (i = 0; i < num_blocks; i++) {
1487 - unsigned long destlen = block_size;
1488 - unsigned int size = offsets[i].size;
1489 -
1490 - if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
1491 - fprintf(stderr, "lseek to %Lu: %s\n",
1492 - offsets[i].offset, strerror(errno));
1493 - exit(1);
1494 - }
1495 -
1496 - if (size > zblock_maxsize) {
1497 - fprintf(stderr,
1498 - "Size %u for block %u (offset %Lu) too big\n",
1499 - size, i, offsets[i].offset);
1500 - exit(1);
1501 - }
1502 - read(handle, buffer, size);
1503 -
1504 - fprintf(stderr, "Block %u at %llu length %u",
1505 - i, offsets[i].offset, size);
1506 - switch (uncompress(clear_buffer, &destlen,
1507 - buffer, size)) {
1508 - case Z_OK:
1509 - break;
1510 -
1511 - case Z_MEM_ERROR:
1512 - fprintf(stderr, "Uncomp: oom block %u\n", i);
1513 - exit(1);
1514 -
1515 - case Z_BUF_ERROR:
1516 - fprintf(stderr, "Uncomp: not enough out room %u\n", i);
1517 - exit(1);
1518 -
1519 - case Z_DATA_ERROR:
1520 - fprintf(stderr, "Uncomp: input corrupt %u\n", i);
1521 - exit(1);
1522 -
1523 - default:
1524 - fprintf(stderr, "Uncomp: unknown error %u\n", i);
1525 - exit(1);
1526 - }
1527 - fprintf(stderr, " => %lu\n", destlen);
1528 - if (destlen != block_size && i != num_blocks - 1) {
1529 - fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
1530 - destlen, block_size);
1531 - exit(1);
1532 - }
1533 - write(STDOUT_FILENO, clear_buffer, (lastlen != 0 && (i+1) == num_blocks) ? lastlen : block_size);
1534 }
1535 return 0;
1536 }
1537 --- create_compressed_fs.c
1538 +++ create_compressed_fs.c
1539 @@ -7,10 +7,11 @@
1540 #define ZMAX 9
1541 static shrink_t level;
1542 static int pass, iter;
1543 -static int best_compress(unsigned char *compressed,
1544 - unsigned long *compressed_len,
1545 - unsigned char *uncompressed,
1546 - unsigned long uncompressed_len)
1547 +static int best_compress(Bytef *compressed,
1548 + uLongf *compressed_len,
1549 + const Bytef *uncompressed,
1550 + uLong uncompressed_len,
1551 + int dummy)
1552 {
1553 int i, j, err;
1554 unsigned char *buf[2];
1555 @@ -19,6 +20,7 @@
1556 static unsigned char *buffer;
1557 static unsigned long buffersz;
1559 + (void) dummy;
1560 if (buffersz < *compressed_len) {
1561 if (buffer) free(buffer);
1562 buffer = (unsigned char *) malloc(buffersz = *compressed_len);
1563 @@ -50,9 +52,95 @@
1564 memcpy(compressed, buffer, best);
1565 return err;
1566 }
1567 -#define compress2(a,b,c,d,e) best_compress(a,b,c,d)
1568 +#else
1569 +#include <stdlib.h>
1570 +#include <string.h>
1571 +#include <zlib.h>
1572 #endif
1573 +
1574 +#include <lzma.h>
1575 +static int xz_compress(Bytef *compressed,
1576 + uLongf *compressed_len,
1577 + const Bytef *uncompressed,
1578 + uLong uncompressed_len,
1579 + int level)
1580 +{
1581 + int res = Z_ERRNO;
1582 + lzma_stream strm = LZMA_STREAM_INIT;
1583 +
1584 + if (lzma_easy_encoder(&strm, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC32) == LZMA_OK) {
1585 +
1586 + strm.next_in = uncompressed;
1587 + strm.avail_in = uncompressed_len;
1588 + strm.next_out = compressed;
1589 + strm.avail_out = *compressed_len;
1590 +
1591 + if (lzma_code(&strm, LZMA_FINISH) == LZMA_STREAM_END) {
1592 +
1593 + *compressed_len -= strm.avail_out;
1594 + res = Z_OK;
1595 + }
1596 + }
1597 + lzma_end(&strm);
1598 + return res;
1599 +}
1600 +
1601 +#include <lz4.h>
1602 +#include <lz4hc.h>
1603 +#ifndef LZ4HC_CLEVEL_MAX
1604 +#define LZ4HC_CLEVEL_MAX 12
1605 +#endif
1606 +static int lz4_compress(Bytef *compressed,
1607 + uLongf *compressed_len,
1608 + const Bytef *uncompressed,
1609 + uLong uncompressed_len,
1610 + int level)
1611 +{
1612 + int res = LZ4_compress_HC((const char *) uncompressed, (char *) compressed,
1613 + uncompressed_len, *compressed_len, LZ4HC_CLEVEL_MAX);
1614 + (void) level;
1615 + if (res <= 0) return Z_ERRNO;
1616 + *compressed_len = res;
1617 + return Z_OK;
1618 +}
1619 +
1620 +static int setup_compress(Bytef *compressed,
1621 + uLongf *compressed_len,
1622 + const Bytef *uncompressed,
1623 + uLong uncompressed_len,
1624 + int level);
1625 +
1626 +static int (*compress3)(Bytef *compressed,
1627 + uLongf *compressed_len,
1628 + const Bytef *uncompressed,
1629 + uLong uncompressed_len,
1630 + int level) = setup_compress;
1632 +static int setup_compress(Bytef *compressed,
1633 + uLongf *compressed_len,
1634 + const Bytef *uncompressed,
1635 + uLong uncompressed_len,
1636 + int level)
1637 +{
1638 + char *s = getenv("CLOOP_COMP");
1639 + if (s && !strcmp(s,"XZ")) {
1640 + compress3 = xz_compress;
1641 + }
1642 + else if (s && !strcmp(s,"LZ4")) {
1643 + compress3 = lz4_compress;
1644 + }
1645 + else
1646 +#ifdef FIND_BEST_COMPRESSION
1647 + if (s && !strcmp(s,"GZIP")) {
1648 + compress3 = compress2;
1649 + }
1650 + else compress3 = best_compress;
1651 +#else
1652 + compress3 = compress2;
1653 +#endif
1654 + return compress3(compressed,compressed_len,uncompressed,uncompressed_len,level);
1655 +}
1656 +
1657 #include <signal.h>
1659 /* Creates a compressed file */
1660 @@ -82,9 +170,7 @@
1661 return i;
1662 }
1664 -#ifdef FIND_BEST_COMPRESSION
1665 #include "md5sum.c"
1666 -#endif
1668 static unsigned n;
1669 static unsigned long lastlen, pos;
1670 @@ -96,15 +182,23 @@
1671 static char padding[512];
1672 struct cloop_tail tail;
1673 unsigned long len;
1674 + int flags = 0;
1676 - fprintf(stderr, "Write index for %lu blocks\n", n);
1677 + fprintf(stderr, "Write index for %u blocks\n", n);
1678 if (block_size >= 0x1000000) lastlen = 0;
1679 - tail.index_size = ntohl(sizeof(*block_index) + 256*(lastlen % 0xFFffFF));
1680 + if (sig) flags = 0x80;
1681 + if (compress3 == xz_compress) {
1682 + flags |= (CLOOP_COMPRESSOR_XZ << 4);
1683 + }
1684 + if (compress3 == lz4_compress) {
1685 + flags |= (CLOOP_COMPRESSOR_LZ4 << 4);
1686 + }
1687 + tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
1688 tail.num_blocks = ntohl(n);
1689 n *= sizeof(*block_index);
1690 - len = n + n/1000 + 12;
1691 + len = GZIP_MAX_BUFFER(n);
1692 compressed = (unsigned char *) realloc(compressed, len);
1693 - if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
1694 + if (!compressed || compress3(compressed, &len, (unsigned char *) block_index,
1695 n, Z_BEST_SPEED) != Z_OK)
1696 quit("Index compression failed");
1697 tail.table_size = ntohl(len);
1698 @@ -122,11 +216,10 @@
1699 unsigned char *uncompressed;
1700 unsigned long len;
1701 unsigned indexmax, zlenmax;
1702 -#ifdef FIND_BEST_COMPRESSION
1703 - unsigned i, j, hashmax;
1704 + unsigned i, j, hashmax, domd5;
1705 md5hash *hash;
1706 -#endif
1708 + domd5 = getenv("CLOOP_NOMD5") == NULL;
1709 #ifdef FIND_BEST_COMPRESSION
1710 while (argc > 1) {
1711 if (argv[1][0] == '-') {
1712 @@ -141,11 +234,11 @@
1713 }
1714 argc--;
1715 if (argv[1][0] < '0' || argv[1][0] > '9')
1716 - quit("Usage : create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
1717 + quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5] create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
1718 #else
1719 if (argc > 1) {
1720 if (argv[1][0] < '0' || argv[1][0] > '9')
1721 - quit("Usage : create_compressed_fs [block size] < input > output");
1722 + quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] create_compressed_fs [block size] < input > output");
1723 #endif
1724 block_size = atoi(argv[1]);
1725 }
1726 @@ -164,12 +257,8 @@
1727 compressed = (unsigned char *) malloc(zlenmax);
1728 uncompressed = (unsigned char *) malloc(block_size);
1729 block_index = (u_int32_t *) malloc(indexmax = CHUNK);
1730 -#ifdef FIND_BEST_COMPRESSION
1731 hash = (md5hash *) malloc(hashmax = CHUNK);
1732 if (!compressed || !uncompressed || !block_index || !hash)
1733 -#else
1734 - if (!compressed || !uncompressed || !block_index)
1735 -#endif
1736 quit("Malloc failed");
1738 signal(SIGINT,flush_index);
1739 @@ -184,39 +273,42 @@
1740 if (!block_index)
1741 quit("Realloc");
1742 }
1743 -#ifdef FIND_BEST_COMPRESSION
1744 - if (n * sizeof(*hash) >= hashmax) {
1745 - hash = (md5hash *) realloc(hash, hashmax += CHUNK);
1746 - if (!hash)
1747 - quit("Realloc hash");
1748 - }
1749 - hash[n] = md5sum(uncompressed, len);
1750 - j = 0x7FFFFFFF;
1751 - if (n < j)
1752 - j = n;
1753 - for (i = 0; i < j; i++) {
1754 - if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
1755 - && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
1756 - break;
1757 - }
1758 - if (i != j) {
1759 - block_index[n] = ntohl(0x80000000 | i);
1760 - fprintf(stderr, "Block %u length %lu => duplicate %lu\n",
1761 - n, block_size, i);
1762 + if (domd5) {
1763 + if (n * sizeof(*hash) >= hashmax) {
1764 + hash = (md5hash *) realloc(hash, hashmax += CHUNK);
1765 + if (!hash)
1766 + quit("Realloc hash");
1767 + }
1768 + hash[n] = md5sum(uncompressed, len);
1769 + j = 0x7FFFFFFF;
1770 + if (n < j)
1771 + j = n;
1772 + for (i = 0; i < j; i++) {
1773 + if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
1774 + && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
1775 + break;
1776 + }
1777 + if (i != j) {
1778 + block_index[n] = ntohl(0x80000000 | i);
1779 + fprintf(stderr, "Block %u length %lu => duplicate %u\n",
1780 + n, block_size, i);
1781 + continue;
1782 + }
1783 }
1784 - else
1785 -#endif
1786 - {
1787 - len = zlenmax;
1788 - if (compress2(compressed, &len, uncompressed, lastlen,
1789 - Z_BEST_SPEED) != Z_OK)
1790 - quit("Compression failed");
1791 - fprintf(stderr, "Block %u length %lu => %lu\n",
1792 - n, block_size, len);
1793 - write(STDOUT_FILENO, compressed, len);
1794 - pos += len;
1795 + len = zlenmax;
1796 + if (compress3(compressed, &len, uncompressed, lastlen,
1797 + Z_BEST_SPEED) != Z_OK || len >= lastlen) {
1798 + len = lastlen;
1799 + block_index[n] = ntohl(0xFFFFFFFF);
1800 + write(STDOUT_FILENO, uncompressed, len);
1801 + }
1802 + else {
1803 block_index[n] = ntohl(len);
1804 + write(STDOUT_FILENO, compressed, len);
1805 }
1806 + fprintf(stderr, "Block %u length %lu => %lu\n",
1807 + n, block_size, len);
1808 + pos += len;
1809 }
1810 flush_index(0);
1811 return 0;
1812 --- compressed_loop.h
1813 +++ compressed_loop.h
1814 @@ -92,9 +92,8 @@
1815 struct cloop_tail
1816 {
1817 u_int32_t table_size;
1818 - u_int32_t index_size; /* size:4 comp:3 ctrl-c:1 lastlen:24 */
1819 + u_int32_t index_size; /* size:4 unused:3 ctrl-c:1 lastlen:24 */
1820 #define CLOOP3_INDEX_SIZE(x) ((unsigned int)((x) & 0xF))
1821 -#define CLOOP3_BLOCKS_FLAGS(x) ((unsigned int)((x) & 0x70) >> 4)
1822 #define CLOOP3_TRUNCATED(x) ((unsigned int)((x) & 0x80) >> 7)
1823 #define CLOOP3_LASTLEN(x) (unsigned int)((x) >> 8)
1824 u_int32_t num_blocks;
1825 @@ -107,12 +106,12 @@
1826 loff_t offset; /* 64-bit offsets of compressed block */
1827 u_int32_t size; /* 32-bit compressed block size */
1828 u_int32_t flags; /* 32-bit compression flags */
1829 -
1830 };
1832 -static inline char *build_index(struct block_info *offsets, unsigned long n,
1833 - unsigned long block_size, unsigned global_flags)
1834 +static inline char *build_index(struct block_info *offsets, unsigned long n, unsigned long block_size)
1835 {
1836 + static char v[sizeof("64BE v4.0a")];
1837 + u_int32_t flags;
1838 u_int32_t *ofs32 = (u_int32_t *) offsets;
1839 loff_t *ofs64 = (loff_t *) offsets;
1841 @@ -137,8 +136,6 @@
1842 }
1843 else { /* V2.0/V4.0 */
1844 loff_t last = CLOOP_BLOCK_OFFSET(__be64_to_cpu(ofs64[n]));
1845 - u_int32_t flags;
1846 - static char v4[11];
1847 unsigned long i = n;
1849 for (flags = 0; n-- ;) {
1850 @@ -156,12 +153,7 @@
1851 offsets[i] = offsets[offsets[i].offset];
1852 }
1853 }
1854 - strcpy(v4, (char *) "64BE v4.0a");
1855 - v4[10] = 'a' + ((flags-1) & 0xF); // compressors used
1856 - if (flags > 0x10) { // with links ?
1857 - v4[10] += 'A' - 'a';
1858 - }
1859 - return v4;
1860 + strcpy(v, (char *) "64BE v4.0a");
1861 }
1862 }
1863 else if (ofs32[1] == 0 && v3_64 == 0) { /* V1.0 */
1864 @@ -174,44 +166,50 @@
1865 }
1866 return (char *) "64LE v1.0";
1867 }
1868 - else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
1869 - loff_t last = ntohl(ofs32[n]);
1870 - while (n--) {
1871 - offsets[n].size = last -
1872 - (offsets[n].offset = ntohl(ofs32[n]));
1873 - last = offsets[n].offset;
1874 - offsets[n].flags = 0;
1875 - }
1876 - return (char *) "32BE v0.68";
1877 - }
1878 - else { /* V3.0 */
1879 + else { /* V3.0 or V0.68 */
1880 unsigned long i;
1881 loff_t j;
1882 - static char v3[11];
1884 - v3_64 = (ofs32[1] == 0) ? 2 : 1;
1885 - for (i = n; i-- != 0; )
1886 - offsets[i].size = ntohl(ofs32[i*v3_64]);
1887 + for (i = 0; i < n && ntohl(ofs32[i]) < ntohl(ofs32[i+1]); i++);
1888 + if (i == n && ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
1889 + loff_t last = ntohl(ofs32[n]);
1890 + while (n--) {
1891 + offsets[n].size = last -
1892 + (offsets[n].offset = ntohl(ofs32[n]));
1893 + last = offsets[n].offset;
1894 + offsets[n].flags = 0;
1895 + }
1896 + return (char *) "32BE v0.68";
1897 + }
1898 +
1899 + v3_64 = (ofs32[1] == 0);
1900 + for (i = n; i-- != 0; ) {
1901 + offsets[i].size = ntohl(ofs32[i << v3_64]);
1902 + if (offsets[i].size == 0xFFFFFFFF) {
1903 + offsets[i].size = 0x10000000 | block_size;
1904 + }
1905 + offsets[i].flags = (offsets[i].size >> 28);
1906 + offsets[i].size &= 0x0FFFFFFF;
1907 + }
1908 for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
1909 offsets[i].offset = j;
1910 - offsets[i].flags = global_flags;
1911 - if ((offsets[i].size & 0x80000000) == 0) {
1912 + if (offsets[i].flags < 8) {
1913 j += offsets[i].size;
1914 }
1915 - else if (offsets[i].size == 0xFFFFFFFF) {
1916 - offsets[i].flags = CLOOP_COMPRESSOR_NONE;
1917 - j += offsets[i].size = block_size;
1918 - }
1919 }
1920 for (i = 0; i < n; i++) {
1921 - if (offsets[i].size & 0x80000000) {
1922 - offsets[i] = offsets[offsets[i].size & 0x7FFFFFFF];
1923 + flags |= 1 << offsets[i].flags;
1924 + if (offsets[i].flags >= 8) {
1925 + offsets[i] = offsets[offsets[i].size];
1926 }
1927 }
1928 - strcpy(v3, (char *) (--v3_64) ? "64BE v3.0a" : "32BE v3.0a");
1929 - v3[10] += global_flags;
1930 - return v3;
1931 + strcpy(v, (char *) (v3_64) ? "64BE v3.0a" : "32BE v3.0a");
1932 + }
1933 + v[9] = 'a' + ((flags-1) & 0xF); // compressors used
1934 + if (flags > 0x10) { // with links ?
1935 + v[9] += 'A' - 'a';
1936 }
1937 + return v;
1938 }
1940 /* Cloop suspend IOCTL */
1941 --- cloopreader.c
1942 +++ cloopreader.c
1943 @@ -52,7 +52,7 @@
1944 bfuncinfo("fh=%d",fh);
1945 c->fh=fh;
1946 struct cloop_head head;
1947 - int v4_header_last, flags;
1948 + int v4_header_last;
1949 loff_t end;
1951 for (v4_header_last=0;;v4_header_last++) {
1952 @@ -83,23 +83,21 @@
1953 c->numblocks = ntohl(tail.num_blocks);
1954 c->tocsize = sizeof(*c->toc) * c->numblocks;
1955 len = ntohl(tail.table_size);
1956 - flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
1957 toclen = CLOOP3_INDEX_SIZE(ntohl(tail.index_size)) * c->numblocks;
1958 OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
1959 ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
1960 ALLOC(p,len);
1961 OP(read_all(c->fh,p,len)); /* read Data Index */
1962 - if (unpack[flags]((void *)c->toc,&toclen,p,len) != Z_OK)
1963 + if (unpack[CLOOP_COMPRESSOR_ZLIB]((void *)c->toc,&toclen,p,len) != Z_OK)
1964 exit(1);
1965 free(p);
1966 }
1967 else {
1968 - flags = 0;
1969 c->tocsize = sizeof(*c->toc) * c->numblocks;
1970 ALLOC(c->toc,c->tocsize);
1971 OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
1972 }
1973 - build_index(c->toc, c->numblocks, c->blocksize, flags);
1974 + build_index(c->toc, c->numblocks, c->blocksize);
1975 c->cblocksizecur=0;
1976 c->curblock=-1;
1977 return 0;
1978 --- create_compressed_fs.c
1979 +++ create_compressed_fs.c
1980 @@ -58,6 +58,9 @@
1981 #include <zlib.h>
1982 #endif
1984 +/* Creates a compressed file */
1985 +#include "common_header.h"
1986 +
1987 #include <lzma.h>
1988 static int xz_compress(Bytef *compressed,
1989 uLongf *compressed_len,
1990 @@ -78,6 +81,7 @@
1991 if (lzma_code(&strm, LZMA_FINISH) == LZMA_STREAM_END) {
1993 *compressed_len -= strm.avail_out;
1994 + *compressed_len |= (CLOOP_COMPRESSOR_XZ << 28);
1995 res = Z_OK;
1996 }
1997 }
1998 @@ -101,6 +105,7 @@
1999 (void) level;
2000 if (res <= 0) return Z_ERRNO;
2001 *compressed_len = res;
2002 + *compressed_len |= (CLOOP_COMPRESSOR_LZ4 << 28);
2003 return Z_OK;
2004 }
2006 @@ -143,9 +148,6 @@
2008 #include <signal.h>
2010 -/* Creates a compressed file */
2011 -#include "common_header.h"
2012 -
2013 #define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
2014 #define CHUNK 65536
2015 #define DEFAULT_BLOCKSIZE 65536
2016 @@ -179,7 +181,6 @@
2017 static unsigned long block_size = 0;
2018 static void flush_index(int sig)
2019 {
2020 - static char padding[512];
2021 struct cloop_tail tail;
2022 unsigned long len;
2023 int flags = 0;
2024 @@ -187,24 +188,21 @@
2025 fprintf(stderr, "Write index for %u blocks\n", n);
2026 if (block_size >= 0x1000000) lastlen = 0;
2027 if (sig) flags = 0x80;
2028 - if (compress3 == xz_compress) {
2029 - flags |= (CLOOP_COMPRESSOR_XZ << 4);
2030 - }
2031 - if (compress3 == lz4_compress) {
2032 - flags |= (CLOOP_COMPRESSOR_LZ4 << 4);
2033 - }
2034 tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
2035 tail.num_blocks = ntohl(n);
2036 n *= sizeof(*block_index);
2037 len = GZIP_MAX_BUFFER(n);
2038 compressed = (unsigned char *) realloc(compressed, len);
2039 - if (!compressed || compress3(compressed, &len, (unsigned char *) block_index,
2040 +#ifdef FIND_BEST_COMPRESSION
2041 + if (!compressed || best_compress(compressed, &len, (unsigned char *) block_index,
2042 +#else
2043 + if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
2044 +#endif
2045 n, Z_BEST_SPEED) != Z_OK)
2046 quit("Index compression failed");
2047 tail.table_size = ntohl(len);
2048 pos += len + sizeof(tail);
2049 n = pos & 511;
2050 - // if (n) write(STDOUT_FILENO, padding, 512 - n);
2051 write(STDOUT_FILENO, compressed, len);
2052 write(STDOUT_FILENO, &tail, sizeof(tail));
2053 exit(sig != 0);
2054 @@ -242,7 +240,7 @@
2055 #endif
2056 block_size = atoi(argv[1]);
2057 }
2058 - if (block_size < 4096)
2059 + if (block_size < 4096 || block_size > 0x0FFFFFFF)
2060 block_size = DEFAULT_BLOCKSIZE;
2061 fprintf(stderr, "Block size is %lu\n", block_size);
2062 zlenmax = block_size + block_size/1000 + 12;
2063 @@ -280,7 +278,7 @@
2064 quit("Realloc hash");
2065 }
2066 hash[n] = md5sum(uncompressed, len);
2067 - j = 0x7FFFFFFF;
2068 + j = 0x0FFFFFFF;
2069 if (n < j)
2070 j = n;
2071 for (i = 0; i < j; i++) {
2072 @@ -289,7 +287,7 @@
2073 break;
2074 }
2075 if (i != j) {
2076 - block_index[n] = ntohl(0x80000000 | i);
2077 + block_index[n] = ntohl((CLOOP_COMPRESSOR_LINK << 28) | i);
2078 fprintf(stderr, "Block %u length %lu => duplicate %u\n",
2079 n, block_size, i);
2080 continue;
2081 @@ -297,14 +295,14 @@
2082 }
2083 len = zlenmax;
2084 if (compress3(compressed, &len, uncompressed, lastlen,
2085 - Z_BEST_SPEED) != Z_OK || len >= lastlen) {
2086 + Z_BEST_SPEED) != Z_OK || (len & 0x0FFFFFFF) >= lastlen) {
2087 len = lastlen;
2088 - block_index[n] = ntohl(0xFFFFFFFF);
2089 + block_index[n] = ntohl((CLOOP_COMPRESSOR_NONE << 28) | lastlen);
2090 write(STDOUT_FILENO, uncompressed, len);
2091 }
2092 else {
2093 block_index[n] = ntohl(len);
2094 - write(STDOUT_FILENO, compressed, len);
2095 + write(STDOUT_FILENO, compressed, len &= 0x0FFFFFFF);
2096 }
2097 fprintf(stderr, "Block %u length %lu => %lu\n",
2098 n, block_size, len);
2099 --- extract_compressed_fs.c
2100 +++ extract_compressed_fs.c
2101 @@ -69,9 +69,8 @@
2103 int main(int argc, char *argv[])
2104 {
2105 - int flags;
2106 struct cloop_head head;
2107 - unsigned int i, v4_header_last, global_flags;
2108 + unsigned int i, v4_header_last;
2109 unsigned long n, len;
2110 uLongf ulen;
2111 loff_t end;
2112 @@ -128,12 +127,6 @@
2113 table_size = ntohl(tail.table_size);
2114 table = malloc(table_size);
2115 len = num_blocks * CLOOP3_INDEX_SIZE(ntohl(tail.index_size));
2116 - global_flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
2117 - if (global_flags > CLOOP_COMPRESSOR_MAX) {
2118 - fprintf(stderr, "Unsupported compression %d\n",
2119 - global_flags);
2120 - exit(1);
2121 - }
2122 ulen = num_blocks * sizeof(*offsets);
2123 offsets = malloc(ulen);
2124 if (!table || !offsets || !buffer || !clear_buffer) {
2125 @@ -144,16 +137,15 @@
2126 perror("Reading index\n");
2127 exit(1);
2128 }
2129 - err = unpack[global_flags]((void *) offsets, &ulen, table, table_size);
2130 + err = unpack[CLOOP_COMPRESSOR_ZLIB]((void *) offsets, &ulen, table, table_size);
2131 if (err != Z_OK) {
2132 fprintf(stderr, "Unpack %s index error %d\n",
2133 - packnames[global_flags],err);
2134 + packnames[CLOOP_COMPRESSOR_ZLIB],err);
2135 exit(1);
2136 }
2137 free(table);
2138 }
2139 else {
2140 - global_flags = 0;
2141 len = num_blocks * sizeof(*offsets);
2142 offsets = malloc(len);
2143 if (v4_header_last) {
2144 @@ -182,7 +174,7 @@
2145 exit(0);
2146 }
2147 #endif
2148 - fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size, global_flags));
2149 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size));
2151 #if 1
2152 if (getenv("CLOOP_TABLE") != NULL) {
2153 === Add v4 support
2154 --- compressed_loop.h
2155 +++ compressed_loop.h
2156 @@ -64,6 +64,7 @@
2157 * 2 xz compression (currently best space saver) *
2158 * 3 lz4 compression *
2159 * 4 lzo compression (fastest) *
2160 +* 5 zstd compression *
2161 * 15 block link *
2162 */
2163 /* Get value of first 4 bits */
2164 @@ -76,6 +77,7 @@
2165 #define CLOOP_COMPRESSOR_XZ 0x2
2166 #define CLOOP_COMPRESSOR_LZ4 0x3
2167 #define CLOOP_COMPRESSOR_LZO 0x4
2168 +#define CLOOP_COMPRESSOR_ZSTD 0x5
2169 #define CLOOP_COMPRESSOR_LINK 0xF
2172 --- create_compressed_fs.c
2173 +++ create_compressed_fs.c
2174 @@ -6,7 +6,9 @@
2176 #define ZMAX 9
2177 static shrink_t level;
2178 -static int pass, iter;
2179 +static struct {
2180 + int pass, iter, v4;
2181 +} conf;
2182 static int best_compress(Bytef *compressed,
2183 uLongf *compressed_len,
2184 const Bytef *uncompressed,
2185 @@ -28,7 +30,7 @@
2186 }
2187 buf[0] = compressed;
2188 buf[1] = buffer;
2189 - for (i = j = 0; i <= ZMAX+3 && (pass == 0 || i < pass); i++) {
2190 + for (i = j = 0; i <= ZMAX+3 && (conf.pass == 0 || i < conf.pass); i++) {
2191 llen = len = *compressed_len;
2192 if (i >= ZMAX+1) {
2193 level.level = (i == ZMAX+1) ? shrink_normal :
2194 @@ -56,6 +58,9 @@
2195 #include <stdlib.h>
2196 #include <string.h>
2197 #include <zlib.h>
2198 +static struct {
2199 + int v4;
2200 +} conf;
2201 #endif
2203 /* Creates a compressed file */
2204 @@ -148,7 +153,8 @@
2206 #include <signal.h>
2208 -#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
2209 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
2210 +#define CLOOP_PREAMBLE_VERSION 12
2211 #define CHUNK 65536
2212 #define DEFAULT_BLOCKSIZE 65536
2214 @@ -174,11 +180,22 @@
2216 #include "md5sum.c"
2218 +//#define CLOOP_DEBUG
2219 +#ifdef CLOOP_DEBUG
2220 +static void write_file(const char *s, void *data, unsigned n)
2221 +{
2222 + int fd = open(s,O_WRONLY|O_CREAT|O_TRUNC,0600);
2223 + if (fd == 0) return;
2224 + write(fd,data,n);
2225 + close(fd);
2226 +}
2227 +#endif
2228 static unsigned n;
2229 -static unsigned long lastlen, pos;
2230 +static unsigned long lastlen, pos = 0;
2231 static u_int32_t *block_index;
2232 static unsigned char *compressed;
2233 static unsigned long block_size = 0;
2234 +static struct cloop_head head;
2235 static void flush_index(int sig)
2236 {
2237 struct cloop_tail tail;
2238 @@ -187,11 +204,26 @@
2240 fprintf(stderr, "Write index for %u blocks\n", n);
2241 if (block_size >= 0x1000000) lastlen = 0;
2242 + if (conf.v4 == 1) {
2243 + unsigned i;
2244 + loff_t ofs64;
2245 + for (i = 0, ofs64 = 0; i < n; ofs64 += (loff_t) ntohl(block_index[i++] & 0xFFffFFf0)) {
2246 + loff_t index64 = ((block_index[i] & 0xF) == CLOOP_COMPRESSOR_NONE) ? ((loff_t) CLOOP_COMPRESSOR_NONE << 60) : 0;
2247 + index64 = __le64_to_cpu(index64 + ofs64);
2248 + write(STDOUT_FILENO, &index64, sizeof(index64));
2249 + }
2250 + head.num_blocks = htonl(n);
2251 + write(STDOUT_FILENO, &head, sizeof(head));
2252 + exit(sig != 0);
2253 + }
2254 if (sig) flags = 0x80;
2255 tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
2256 tail.num_blocks = ntohl(n);
2257 n *= sizeof(*block_index);
2258 len = GZIP_MAX_BUFFER(n);
2259 +#ifdef CLOOP_DEBUG
2260 + write_file("cloop.index",block_index,n);
2261 +#endif
2262 compressed = (unsigned char *) realloc(compressed, len);
2263 #ifdef FIND_BEST_COMPRESSION
2264 if (!compressed || best_compress(compressed, &len, (unsigned char *) block_index,
2265 @@ -201,8 +233,10 @@
2266 n, Z_BEST_SPEED) != Z_OK)
2267 quit("Index compression failed");
2268 tail.table_size = ntohl(len);
2269 - pos += len + sizeof(tail);
2270 - n = pos & 511;
2271 +#ifdef CLOOP_DEBUG
2272 + write_file("cloop.zindex",compressed,len);
2273 + write_file("cloop.tail",&tail,sizeof(tail));
2274 +#endif
2275 write(STDOUT_FILENO, compressed, len);
2276 write(STDOUT_FILENO, &tail, sizeof(tail));
2277 exit(sig != 0);
2278 @@ -210,7 +244,6 @@
2280 int main(int argc, char *argv[])
2281 {
2282 - struct cloop_head head;
2283 unsigned char *uncompressed;
2284 unsigned long len;
2285 unsigned indexmax, zlenmax;
2286 @@ -221,9 +254,12 @@
2287 #ifdef FIND_BEST_COMPRESSION
2288 while (argc > 1) {
2289 if (argv[1][0] == '-') {
2290 - int *p = &pass;
2291 + int *p = &conf.pass;
2292 switch (argv[1][1]) {
2293 - case 'i' : p = &iter;
2294 + case 'v' : conf.v4++;
2295 + argv++; argc--;
2296 + continue;
2297 + case 'i' : p = &conf.iter;
2298 case 'n' : *p = atoi(argv[2]);
2299 argc -= 2;
2300 argv += 2;
2301 @@ -232,11 +268,23 @@
2302 }
2303 argc--;
2304 if (argv[1][0] < '0' || argv[1][0] > '9')
2305 - quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5] create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
2306 + quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5=1] "
2307 +# ifdef CLOOP_DEBUG
2308 + "[CLOOP_DEBUG=1] "
2309 +# endif
2310 + "create_compressed_fs [-v4] [-n <pass>][ -i <iter>] [block size] < input > output");
2311 #else
2312 if (argc > 1) {
2313 + if (strcmp(argv[1],"-v4") == 0) {
2314 + conf.v4++;
2315 + argv++; argc--;
2316 + }
2317 if (argv[1][0] < '0' || argv[1][0] > '9')
2318 - quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] create_compressed_fs [block size] < input > output");
2319 + quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] "
2320 +# ifdef CLOOP_DEBUG
2321 + "[CLOOP_DEBUG=1] "
2322 +# endif
2323 + "create_compressed_fs [-v4] [block size] < input > output");
2324 #endif
2325 block_size = atoi(argv[1]);
2326 }
2327 @@ -247,10 +295,13 @@
2329 memset(&head, 0, sizeof(head));
2330 strcpy(head.preamble, CLOOP_PREAMBLE);
2331 - head.num_blocks = -1;
2332 head.block_size = htonl(block_size);
2333 - write(STDOUT_FILENO, &head, sizeof(head));
2334 - pos = sizeof(head);
2335 + if (conf.v4 == 0) {
2336 + head.num_blocks = -1;
2337 + write(STDOUT_FILENO, &head, sizeof(head));
2338 + pos = sizeof(head);
2339 + }
2340 + else head.preamble[CLOOP_PREAMBLE_VERSION] = '4';
2342 compressed = (unsigned char *) malloc(zlenmax);
2343 uncompressed = (unsigned char *) malloc(block_size);
2344 @@ -277,14 +328,20 @@
2345 if (!hash)
2346 quit("Realloc hash");
2347 }
2348 - hash[n] = md5sum(uncompressed, len);
2349 + hash[n] = md5sum(uncompressed, lastlen);
2350 j = 0x0FFFFFFF;
2351 if (n < j)
2352 j = n;
2353 for (i = 0; i < j; i++) {
2354 - if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
2355 - && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
2356 - break;
2357 + if (* (uint32_t *) &hash[i] != * (uint32_t *) &hash[n]
2358 + || memcmp(&hash[i],&hash[n],sizeof(*hash)))
2359 + continue;
2360 + len = zlenmax;
2361 + if (compress3(compressed, &len, uncompressed, lastlen,
2362 + Z_BEST_SPEED) != Z_OK || (len & 0x0FFFFFFF) >= lastlen) {
2363 + len = (CLOOP_COMPRESSOR_NONE << 28) | lastlen;
2364 + }
2365 + if (block_index[i] == ntohl(len)) break;
2366 }
2367 if (i != j) {
2368 block_index[n] = ntohl((CLOOP_COMPRESSOR_LINK << 28) | i);
2369 --- extract_compressed_fs.c
2370 +++ extract_compressed_fs.c
2371 @@ -67,16 +67,21 @@
2372 return ulen;
2373 }
2375 +//#define CLOOP_DEBUG
2376 int main(int argc, char *argv[])
2377 {
2378 struct cloop_head head;
2379 unsigned int i, v4_header_last;
2380 unsigned long n, len;
2381 uLongf ulen;
2382 - loff_t end;
2383 + off64_t end;
2385 if (argc < 2 || argv[1][0] == '-') {
2386 - fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
2387 + fprintf(stderr, "Usage: "
2388 +#ifdef CLOOP_DEBUG
2389 + "[CLOOP_TAIL=1|CLOOP_ZINDEX=1|CLOOP_INDEX=1|CLOOP_TABLE=1] "
2390 +#endif
2391 + "extract_compressed_fs file [--convert-to-v2] > output\n");
2392 exit(1);
2393 }
2395 @@ -122,6 +127,17 @@
2396 perror("Reading tail\n");
2397 exit(1);
2398 }
2399 +#ifdef CLOOP_DEBUG
2400 + if (getenv("CLOOP_TAIL") != NULL) {
2401 + fprintf(stderr, "Table size:32 %u, index_size:4 %u, unused:3\n",
2402 + tail.table_size,CLOOP3_INDEX_SIZE(tail.index_size));
2403 + fprintf(stderr, "Truncated:1 %u, lastlen:24 %u, num_blocks:32 %u end=%llu\n",
2404 + CLOOP3_TRUNCATED(tail.index_size),CLOOP3_LASTLEN(tail.index_size),tail.num_blocks,end);
2405 + fprintf(stderr, "CLOOP_TAIL: binary\n");
2406 + write(STDOUT_FILENO, &tail, sizeof(tail));
2407 + exit(0);
2408 + }
2409 +#endif
2410 head.num_blocks = tail.num_blocks;
2411 num_blocks = ntohl(head.num_blocks);
2412 table_size = ntohl(tail.table_size);
2413 @@ -137,6 +153,13 @@
2414 perror("Reading index\n");
2415 exit(1);
2416 }
2417 +#ifdef CLOOP_DEBUG
2418 + if (getenv("CLOOP_ZINDEX") != NULL) {
2419 + fprintf(stderr, "CLOOP_ZINDEX: binary\n");
2420 + write(STDOUT_FILENO, table, table_size);
2421 + exit(0);
2422 + }
2423 +#endif
2424 err = unpack[CLOOP_COMPRESSOR_ZLIB]((void *) offsets, &ulen, table, table_size);
2425 if (err != Z_OK) {
2426 fprintf(stderr, "Unpack %s index error %d\n",
2427 @@ -167,7 +190,7 @@
2429 fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
2430 num_blocks, block_size, head.preamble);
2431 -#if 1
2432 +#ifdef CLOOP_DEBUG
2433 if (getenv("CLOOP_INDEX") != NULL) {
2434 fprintf(stderr, "CLOOP_INDEX: binary\n");
2435 write(STDOUT_FILENO, offsets, len);
2436 @@ -175,8 +198,8 @@
2437 }
2438 #endif
2439 fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size));
2440 -
2441 -#if 1
2442 +
2443 +#ifdef CLOOP_DEBUG
2444 if (getenv("CLOOP_TABLE") != NULL) {
2445 fprintf(stderr, "CLOOP_TABLE ascii: offset, size, flags\n");
2446 for (i = 0; i < num_blocks; i++) {