wok-6.x annotate fusecloop/stuff/fusecloop.u @ rev 25176

updated python-formalchemy (1.5.5 -> 1.5.6)
author Hans-G?nter Theisgen
date Fri Jul 01 17:52:21 2022 +0100 (2022-07-01)
parents c6796c6304f6
children
rev   line source
pascal@23696 1 === Add v3 support
pascal@10932 2 --- compressed_loop.h
pascal@10932 3 +++ compressed_loop.h
pascal@18828 4 @@ -41,6 +41,80 @@
pascal@10932 5 /* data_index (num_blocks 64bit pointers, network order)... */
pascal@10932 6 /* compressed data (gzip block compressed format)... */
pascal@10932 7
pascal@10933 8 +struct cloop_tail
pascal@10933 9 +{
pascal@12214 10 + u_int32_t table_size;
pascal@10933 11 + u_int32_t index_size;
pascal@10933 12 + u_int32_t num_blocks;
pascal@10933 13 +};
pascal@10933 14 +
pascal@10932 15 +struct block_info
pascal@10932 16 +{
pascal@10932 17 + loff_t offset; /* 64-bit offsets of compressed block */
pascal@10932 18 + u_int32_t size; /* 32-bit compressed block size */
pascal@10932 19 + u_int32_t optidx; /* 32-bit index number */
pascal@10932 20 +};
pascal@10932 21 +
pascal@10935 22 +static inline char *build_index(struct block_info *offsets, unsigned long n)
pascal@10932 23 +{
pascal@10932 24 + u_int32_t *ofs32 = (u_int32_t *) offsets;
pascal@10932 25 + loff_t *ofs64 = (loff_t *) offsets;
pascal@10933 26 +
pascal@10932 27 + if (ofs32[0] == 0) {
pascal@10932 28 + if (ofs32[2]) { /* ACCELERATED KNOPPIX V1.0 */
pascal@10933 29 + while (n--) {
pascal@10932 30 + offsets[n].offset = __be64_to_cpu(offsets[n].offset);
pascal@10932 31 + offsets[n].size = ntohl(offsets[n].size);
pascal@10933 32 + }
pascal@12214 33 + return (char *) "128BE accelerated knoppix 1.0";
pascal@10932 34 + }
pascal@10932 35 + else { /* V2.0 */
pascal@10933 36 + loff_t last = __be64_to_cpu(ofs64[n]);
pascal@10933 37 + while (n--) {
pascal@10932 38 + offsets[n].size = last -
pascal@10932 39 + (offsets[n].offset = __be64_to_cpu(ofs64[n]));
pascal@10932 40 + last = offsets[n].offset;
pascal@10933 41 + }
pascal@12214 42 + return (char *) "64BE v2.0";
pascal@10932 43 + }
pascal@10932 44 + }
pascal@10932 45 + else if (ofs32[1] == 0) { /* V1.0 */
pascal@11113 46 + loff_t last = __le64_to_cpu(ofs64[n]);
pascal@10933 47 + while (n--) {
pascal@10932 48 + offsets[n].size = last -
pascal@10932 49 + (offsets[n].offset = __le64_to_cpu(ofs64[n]));
pascal@10932 50 + last = offsets[n].offset;
pascal@10933 51 + }
pascal@12214 52 + return (char *) "64LE v1.0";
pascal@10932 53 + }
pascal@10933 54 + else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
pascal@10933 55 + loff_t last = ntohl(ofs32[n]);
pascal@10933 56 + while (n--) {
pascal@10932 57 + offsets[n].size = last -
pascal@10932 58 + (offsets[n].offset = ntohl(ofs32[n]));
pascal@10932 59 + last = offsets[n].offset;
pascal@10933 60 + }
pascal@12214 61 + return (char *) "32BE v0.68";
pascal@10932 62 + }
pascal@10933 63 + else { /* V3.0 */
pascal@17169 64 + unsigned long i;
pascal@17168 65 + loff_t j;
pascal@10933 66 +
pascal@17169 67 + for (i = n; i-- != 0; )
pascal@10935 68 + offsets[i].size = ntohl(ofs32[i]);
pascal@10935 69 + for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
pascal@10933 70 + offsets[i].offset = j;
pascal@18828 71 + if (offsets[i].size & 0x80000000) {
pascal@18829 72 + unsigned long k = offsets[i].size & 0x7FFFFFFF;
pascal@18828 73 + offsets[i].offset = offsets[k].offset;
pascal@18828 74 + offsets[i].size = offsets[k].size;
pascal@18828 75 + }
pascal@18828 76 + else j += offsets[i].size;
pascal@10933 77 + }
pascal@12214 78 + return (char *) "32BE v3.0";
pascal@10933 79 + }
pascal@10932 80 +}
pascal@10932 81 +
pascal@10932 82 /* Cloop suspend IOCTL */
pascal@10932 83 #define CLOOP_SUSPEND 0x4C07
pascal@10932 84
pascal@10932 85
pascal@10932 86 --- cloopreader.h
pascal@10932 87 +++ cloopreader.h
pascal@10932 88 @@ -33,7 +33,7 @@
pascal@10932 89 int numblocks;
pascal@10932 90 ulong blocksize;
pascal@10932 91
pascal@10932 92 - loff_t* toc; /* Data index */
pascal@10932 93 + struct block_info *toc; /* Data index */
pascal@10932 94 size_t tocsize;
pascal@10932 95
pascal@10932 96 unsigned char* cblock; /* Compressed block */
pascal@10932 97
pascal@10932 98 --- cloopreader.c
pascal@10932 99 +++ cloopreader.c
pascal@12214 100 @@ -59,10 +59,32 @@
pascal@10932 101
pascal@10932 102 ALLOC(c->pblock,c->blocksize);
pascal@10932 103
pascal@10932 104 - c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */
pascal@12214 105 - ALLOC(c->toc,c->tocsize);
pascal@12214 106 + if (c->numblocks + 1 == 0) {
pascal@10933 107 + struct cloop_tail tail;
pascal@10935 108 + loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
pascal@12214 109 + void *p;
pascal@12214 110 + ulong toclen, len;
pascal@12214 111
pascal@12214 112 - OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
pascal@10935 113 + OP(lseek(c->fh, end - sizeof(tail), SEEK_SET));
pascal@10933 114 + OP(read_all(c->fh, &tail, sizeof(tail)));
pascal@10933 115 + c->numblocks = ntohl(tail.num_blocks);
pascal@12214 116 + c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@12214 117 + len = ntohl(tail.table_size);
pascal@12214 118 + toclen = (ntohl(tail.index_size) & 255) * c->numblocks;
pascal@12214 119 + OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
pascal@17168 120 + ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
pascal@12214 121 + ALLOC(p,len);
pascal@12214 122 + OP(read_all(c->fh,p,len)); /* read Data Index */
pascal@12214 123 + if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK)
pascal@12214 124 + exit(1);
pascal@12214 125 + free(p);
pascal@10933 126 + }
pascal@12214 127 + else {
pascal@12214 128 + c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@12214 129 + ALLOC(c->toc,c->tocsize);
pascal@12214 130 + OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
pascal@12214 131 + }
pascal@10935 132 + build_index(c->toc, c->numblocks);
pascal@10932 133 c->cblocksizecur=0;
pascal@10932 134 c->curblock=-1;
pascal@10932 135 return 0;
pascal@12214 136 @@ -79,10 +101,10 @@
pascal@10932 137 if(page>=c->numblocks){errno=EFAULT;return -1;}
pascal@10932 138 c->curblock=page;
pascal@10932 139
pascal@10932 140 - bprintf("Seeking to 0x%Lx\n",btc(c->toc[page]));
pascal@10932 141 - OP(lseek(c->fh,btc(c->toc[page]), SEEK_SET));
pascal@10932 142 + bprintf("Seeking to 0x%Lx\n",c->toc[page].offset);
pascal@10932 143 + OP(lseek(c->fh,c->toc[page].offset, SEEK_SET));
pascal@10932 144
pascal@10932 145 - c->cblocksize=btc(c->toc[page+1]) - btc(c->toc[page]);
pascal@10932 146 + c->cblocksize=c->toc[page].size;
pascal@10932 147 bprintf("Compressed size=%lu\n",c->cblocksize);
pascal@10932 148 if(c->cblocksize > c->cblocksizecur){
pascal@10932 149 if(c->cblocksizecur)free(c->cblock);
pascal@10933 150
pascal@10932 151 --- extract_compressed_fs.c
pascal@10932 152 +++ extract_compressed_fs.c
pascal@17885 153 @@ -1,19 +1,23 @@
pascal@10935 154 /* Extracts a filesystem back from a compressed fs file */
pascal@10937 155 +#define _LARGEFILE64_SOURCE
pascal@10935 156 #include "common_header.h"
pascal@12214 157 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@10935 158
pascal@10935 159 int main(int argc, char *argv[])
pascal@10935 160 {
pascal@10935 161 int handle;
pascal@10932 162 struct cloop_head head;
pascal@10932 163 unsigned int i;
pascal@17996 164 + unsigned long num_blocks, block_size, zblock_maxsize, lastlen = 0;
pascal@10932 165 unsigned char *buffer, *clear_buffer;
pascal@10932 166 + struct block_info *offsets;
pascal@10932 167
pascal@10935 168 - if (argc != 2) {
pascal@10935 169 - fprintf(stderr, "Need filename\n");
pascal@10935 170 + if (argc < 2 || argv[1][0] == '-') {
pascal@10935 171 + fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
pascal@10935 172 exit(1);
pascal@10935 173 }
pascal@10932 174
pascal@17885 175 - handle = open(argv[1], O_RDONLY);
pascal@17885 176 + handle = open(argv[1], O_RDONLY|O_LARGEFILE);
pascal@17885 177 if (handle < 0) {
pascal@17885 178 perror("Opening compressed file\n");
pascal@17885 179 exit(1);
pascal@18894 180 @@ -24,66 +28,100 @@
pascal@10935 181 exit(1);
pascal@10935 182 }
pascal@10935 183
pascal@10935 184 - buffer = malloc(ntohl(head.block_size) + ntohl(head.block_size)/1000
pascal@10935 185 - + 12 + 4);
pascal@10935 186 - clear_buffer = malloc(ntohl(head.block_size));
pascal@10935 187 - fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n",
pascal@10935 188 - ntohl(head.num_blocks), ntohl(head.block_size), head.preamble);
pascal@12214 189 + num_blocks = ntohl(head.num_blocks);
pascal@12214 190 + block_size = ntohl(head.block_size);
pascal@12214 191 + zblock_maxsize = block_size + block_size/1000 + 12 + 4;
pascal@12214 192 + buffer = malloc(zblock_maxsize);
pascal@12214 193 + clear_buffer = malloc(block_size);
pascal@12214 194
pascal@10935 195 - for (i = 0; i < ntohl(head.num_blocks); i++) {
pascal@10935 196 - int currpos;
pascal@10935 197 - unsigned long destlen = ntohl(head.block_size);
pascal@10935 198 - loff_t offset[2];
pascal@10935 199 - unsigned int size;
pascal@20947 200 + if (num_blocks == 0xFFFFFFFF) {
pascal@12214 201 + void *table;
pascal@10933 202 + struct cloop_tail tail;
pascal@12214 203 + unsigned long len, table_size;
pascal@10937 204 + loff_t end = lseek64(handle, 0, SEEK_END);
pascal@12214 205 +
pascal@10937 206 + if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
pascal@10933 207 + read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
pascal@10937 208 + lseek64(handle, end - sizeof(tail) -
pascal@12214 209 + ntohl(tail.table_size), SEEK_SET) < 0) {
pascal@10933 210 + perror("Reading tail\n");
pascal@10933 211 + exit(1);
pascal@10933 212 + }
pascal@10933 213 + head.num_blocks = tail.num_blocks;
pascal@10935 214 + num_blocks = ntohl(head.num_blocks);
pascal@12214 215 + table_size = ntohl(tail.table_size);
pascal@12214 216 + table = malloc(table_size);
pascal@12214 217 + len = i = num_blocks * (ntohl(tail.index_size) & 255);
pascal@18894 218 + lastlen = ntohl(tail.index_size) / 256;
pascal@12214 219 + offsets = malloc(num_blocks * sizeof(*offsets));
pascal@12214 220 + if (!table || !offsets ||
pascal@12214 221 + read(handle, table, table_size) != table_size ||
pascal@12214 222 + uncompress((void *)offsets, &len, table, table_size) != Z_OK ||
pascal@12214 223 + len != i) {
pascal@12214 224 + perror("Reading index\n");
pascal@12214 225 + exit(1);
pascal@12214 226 + }
pascal@12214 227 + free(table);
pascal@10933 228 + }
pascal@12214 229 + else {
pascal@12214 230 + offsets = malloc(i = num_blocks * sizeof(*offsets));
pascal@12214 231 + if (!offsets || read(handle, offsets, i) != i) {
pascal@12214 232 + perror("Reading index\n");
pascal@12214 233 + exit(1);
pascal@12214 234 + }
pascal@10932 235 + }
pascal@10932 236 +
pascal@12214 237 + fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
pascal@12214 238 + num_blocks, block_size, head.preamble);
pascal@10935 239 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
pascal@10932 240 +
pascal@10935 241 + if (argc > 2) {
pascal@14712 242 + unsigned n;
pascal@10936 243 + loff_t data, offset = ((num_blocks + 1) * sizeof(offset)) + sizeof(head);
pascal@10935 244 +
pascal@10935 245 + strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@14712 246 + write(STDOUT_FILENO, &head, n = sizeof(head));
pascal@10935 247 + for (i = 0; i < num_blocks; i++) {
pascal@10936 248 + data = __be64_to_cpu(offset);
pascal@10936 249 + write(STDOUT_FILENO, &data, sizeof(data));
pascal@14712 250 + n += sizeof(data);
pascal@10936 251 + offset += offsets[i].size;
pascal@10935 252 + }
pascal@10936 253 + data = __be64_to_cpu(offset);
pascal@10936 254 + write(STDOUT_FILENO, &data, sizeof(data));
pascal@10937 255 + for (i = 0; i < num_blocks && lseek64(handle, offsets[i].offset, SEEK_SET) >= 0; i++) {
pascal@10935 256 + read(handle, buffer, offsets[i].size);
pascal@10935 257 + write(STDOUT_FILENO, buffer, offsets[i].size);
pascal@14712 258 + n += offsets[i].size;
pascal@14712 259 + }
pascal@14712 260 + n &= 0x1FF;
pascal@14712 261 + if (n) {
pascal@14712 262 + memset(buffer, 0, 512);
pascal@14712 263 + write(STDOUT_FILENO, buffer, 512 - n);
pascal@10935 264 + }
pascal@10935 265 + return 0;
pascal@10935 266 + }
pascal@10935 267 +
pascal@10935 268 + for (i = 0; i < num_blocks; i++) {
pascal@10935 269 + unsigned long destlen = block_size;
pascal@10932 270 + unsigned int size = offsets[i].size;
pascal@10932 271
pascal@10932 272 - read(handle, &offset, 2*sizeof(loff_t));
pascal@10932 273 - lseek(handle, -sizeof(loff_t), SEEK_CUR);
pascal@10932 274 -
pascal@10932 275 - currpos = lseek(handle, 0, SEEK_CUR);
pascal@10932 276 - if (lseek(handle, __be64_to_cpu(offset[0]), SEEK_SET) < 0) {
pascal@10937 277 + if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
pascal@10932 278 fprintf(stderr, "lseek to %Lu: %s\n",
pascal@10932 279 - __be64_to_cpu(offset[0]), strerror(errno));
pascal@10932 280 + offsets[i].offset, strerror(errno));
pascal@10932 281 exit(1);
pascal@10932 282 }
pascal@10932 283
pascal@10932 284 - size=__be64_to_cpu(offset[1])-__be64_to_cpu(offset[0]);
pascal@10935 285 - if (size > ntohl(head.block_size) + ntohl(head.block_size)/1000
pascal@10935 286 - + 12 + 4) {
pascal@10935 287 + if (size > zblock_maxsize) {
pascal@10932 288 fprintf(stderr,
pascal@10932 289 "Size %u for block %u (offset %Lu) too big\n",
pascal@10932 290 - size, i, __be64_to_cpu(offset[0]));
pascal@10932 291 + size, i, offsets[i].offset);
pascal@10932 292 exit(1);
pascal@10932 293 }
pascal@10932 294 read(handle, buffer, size);
pascal@10932 295 - if (lseek(handle, currpos, SEEK_SET) < 0) {
pascal@10932 296 - perror("seeking");
pascal@10932 297 - exit(1);
pascal@10932 298 - }
pascal@10932 299
pascal@10935 300 - fprintf(stderr, "Block %u length %u => %lu\n",
pascal@10935 301 - i, size, destlen);
pascal@18894 302 - if (i == 3) {
pascal@18894 303 - fprintf(stderr,
pascal@18894 304 - "Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n",
pascal@18894 305 - buffer[0],
pascal@18894 306 - buffer[1],
pascal@18894 307 - buffer[2],
pascal@18894 308 - buffer[3],
pascal@18894 309 - buffer[4],
pascal@18894 310 - buffer[5],
pascal@18894 311 - buffer[6],
pascal@18894 312 - buffer[7]);
pascal@18894 313 - fprintf(stderr,
pascal@18894 314 - "Block tail:%02X%02X%02X%02X%02X%02X%02X%02X\n",
pascal@18894 315 - buffer[3063],
pascal@18894 316 - buffer[3064],
pascal@18894 317 - buffer[3065],
pascal@18894 318 - buffer[3066],
pascal@18894 319 - buffer[3067],
pascal@18894 320 - buffer[3068],
pascal@18894 321 - buffer[3069],
pascal@18894 322 - buffer[3070]);
pascal@18894 323 - }
pascal@18894 324 + fprintf(stderr, "Block %u at %llu length %u",
pascal@18894 325 + i, offsets[i].offset, size);
pascal@18894 326 switch (uncompress(clear_buffer, &destlen,
pascal@18894 327 buffer, size)) {
pascal@18894 328 case Z_OK:
pascal@18894 329 @@ -105,12 +143,13 @@
pascal@10937 330 fprintf(stderr, "Uncomp: unknown error %u\n", i);
pascal@10937 331 exit(1);
pascal@10937 332 }
pascal@10937 333 - if (destlen != ntohl(head.block_size)) {
pascal@10937 334 - fprintf(stderr, "Uncomp: bad len %u (%lu not %u)\n", i,
pascal@10937 335 - destlen, ntohl(head.block_size));
pascal@18894 336 + fprintf(stderr, " => %lu\n", destlen);
pascal@18894 337 + if (destlen != block_size && i != num_blocks - 1) {
pascal@10937 338 + fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
pascal@10937 339 + destlen, block_size);
pascal@10937 340 exit(1);
pascal@10937 341 }
pascal@10937 342 - write(STDOUT_FILENO, clear_buffer, ntohl(head.block_size));
pascal@17996 343 + write(STDOUT_FILENO, clear_buffer, (lastlen != 0 && (i+1) == num_blocks) ? lastlen : block_size);
pascal@10937 344 }
pascal@10937 345 return 0;
pascal@10937 346 }
pascal@10935 347 --- Makefile
pascal@10935 348 +++ Makefile
pascal@10935 349 @@ -1,16 +1,19 @@
pascal@10935 350 PROGNAME=fusecloop
pascal@10935 351 ARCFILES=*.c *.h *.pl Makefile configure README VERSION HELP INSTALL typescript *.cloop COPYING
pascal@10935 352 -PROGS=fusecloop cloopreaderdemo extract_compressed_fs
pascal@10935 353 +PROGS=fusecloop cloopreaderdemo extract_compressed_fs create_compressed_fs
pascal@10935 354 FUSECFLAGS=`pkg-config fuse --cflags`
pascal@10935 355 FUSELDFLAGS=`pkg-config fuse --libs`
pascal@10935 356
pascal@10935 357 CFLAGS= -Wall
pascal@10935 358
pascal@10935 359 -all: fusecloop extract_compressed_fs
pascal@10935 360 +all: fusecloop extract_compressed_fs create_compressed_fs
pascal@10935 361
pascal@10935 362 extract_compressed_fs: extract_compressed_fs.c
pascal@10935 363 ${CC} ${CFLAGS} ${LDFLAGS} -lz extract_compressed_fs.c -o extract_compressed_fs
pascal@10935 364
pascal@18894 365 +create_compressed_fs: create_compressed_fs.c md5sum.c
pascal@10935 366 + ${CC} ${CFLAGS} ${LDFLAGS} -lz create_compressed_fs.c -o create_compressed_fs
pascal@10935 367 +
pascal@10935 368 fusecloop: fusecloop.c cloopreader.o strver debug.o
pascal@10935 369 ${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop
pascal@10935 370
pascal@10935 371
pascal@12214 372
pascal@18894 373 --- md5sum.c
pascal@18894 374 +++ md5sum.c
pascal@18894 375 @@ -0,0 +1,246 @@
pascal@18894 376 +/*
pascal@18894 377 + * Based on busybox code.
pascal@18894 378 + *
pascal@18894 379 + * Compute MD5 checksum of strings according to the
pascal@18894 380 + * definition of MD5 in RFC 1321 from April 1992.
pascal@18894 381 + *
pascal@18894 382 + * Written by Ulrich Drepper <drepper@gnu.ai.mit.edu>, 1995.
pascal@18894 383 + *
pascal@18894 384 + * Copyright (C) 1995-1999 Free Software Foundation, Inc.
pascal@18894 385 + * Copyright (C) 2001 Manuel Novoa III
pascal@18894 386 + * Copyright (C) 2003 Glenn L. McGrath
pascal@18894 387 + * Copyright (C) 2003 Erik Andersen
pascal@18894 388 + * Copyright (C) 2010 Denys Vlasenko
pascal@18894 389 + * Copyright (C) 2012 Pascal Bellard
pascal@18894 390 + *
pascal@18894 391 + * Licensed under GPLv2 or later
pascal@18894 392 + */
pascal@18894 393 +
pascal@18894 394 +#define ALIGN1
pascal@18894 395 +
pascal@18894 396 +static uint8_t wbuffer[64]; /* always correctly aligned for uint64_t */
pascal@18894 397 +static uint64_t total64; /* must be directly before hash[] */
pascal@18894 398 +static uint32_t hash[8]; /* 4 elements for md5, 5 for sha1, 8 for sha256 */
pascal@18894 399 +
pascal@18894 400 +/* Emit a string of hex representation of bytes */
pascal@23702 401 +/* static char* bin2hex(char *p)
pascal@18894 402 +{
pascal@18894 403 + static const char bb_hexdigits_upcase[] ALIGN1 = "0123456789abcdef";
pascal@18894 404 + int count = 16;
pascal@18894 405 + const char *cp = (const char *) hash;
pascal@18894 406 + while (count) {
pascal@18894 407 + unsigned char c = *cp++;
pascal@23703 408 + // put lowercase hex digits
pascal@18894 409 + *p++ = bb_hexdigits_upcase[c >> 4];
pascal@18894 410 + *p++ = bb_hexdigits_upcase[c & 0xf];
pascal@18894 411 + count--;
pascal@18894 412 + }
pascal@18894 413 + return p;
pascal@23702 414 +} */
pascal@18894 415 +
pascal@18894 416 +//#define rotl32(x,n) (((x) << (n)) | ((x) >> (32 - (n))))
pascal@18894 417 +static uint32_t rotl32(uint32_t x, unsigned n)
pascal@18894 418 +{
pascal@18894 419 + return (x << n) | (x >> (32 - n));
pascal@18894 420 +}
pascal@18894 421 +
pascal@18894 422 +static void md5_process_block64(void);
pascal@18894 423 +
pascal@18894 424 +/* Feed data through a temporary buffer.
pascal@18894 425 + * The internal buffer remembers previous data until it has 64
pascal@18894 426 + * bytes worth to pass on.
pascal@18894 427 + */
pascal@18894 428 +static void common64_hash(const void *buffer, size_t len)
pascal@18894 429 +{
pascal@18894 430 + unsigned bufpos = total64 & 63;
pascal@18894 431 +
pascal@18894 432 + total64 += len;
pascal@18894 433 +
pascal@18894 434 + while (1) {
pascal@18894 435 + unsigned remaining = 64 - bufpos;
pascal@18894 436 + if (remaining > len)
pascal@18894 437 + remaining = len;
pascal@18894 438 + /* Copy data into aligned buffer */
pascal@18894 439 + memcpy(wbuffer + bufpos, buffer, remaining);
pascal@18894 440 + len -= remaining;
pascal@18894 441 + buffer = (const char *)buffer + remaining;
pascal@18894 442 + bufpos += remaining;
pascal@18894 443 + /* clever way to do "if (bufpos != 64) break; ... ; bufpos = 0;" */
pascal@18894 444 + bufpos -= 64;
pascal@18894 445 + if (bufpos != 0)
pascal@18894 446 + break;
pascal@18894 447 + /* Buffer is filled up, process it */
pascal@18894 448 + md5_process_block64();
pascal@18894 449 + /*bufpos = 0; - already is */
pascal@18894 450 + }
pascal@18894 451 +}
pascal@18894 452 +
pascal@18894 453 +/* Process the remaining bytes in the buffer */
pascal@18894 454 +static void common64_end(void)
pascal@18894 455 +{
pascal@18894 456 + unsigned bufpos = total64 & 63;
pascal@18894 457 + /* Pad the buffer to the next 64-byte boundary with 0x80,0,0,0... */
pascal@18894 458 + wbuffer[bufpos++] = 0x80;
pascal@18894 459 +
pascal@18894 460 + /* This loop iterates either once or twice, no more, no less */
pascal@18894 461 + while (1) {
pascal@18894 462 + unsigned remaining = 64 - bufpos;
pascal@18894 463 + memset(wbuffer + bufpos, 0, remaining);
pascal@18894 464 + /* Do we have enough space for the length count? */
pascal@18894 465 + if (remaining >= 8) {
pascal@18894 466 + /* Store the 64-bit counter of bits in the buffer */
pascal@18894 467 + uint64_t t = total64 << 3;
pascal@18894 468 + /* wbuffer is suitably aligned for this */
pascal@18894 469 + *(uint64_t *) (&wbuffer[64 - 8]) = t;
pascal@18894 470 + }
pascal@18894 471 + md5_process_block64();
pascal@18894 472 + if (remaining >= 8)
pascal@18894 473 + break;
pascal@18894 474 + bufpos = 0;
pascal@18894 475 + }
pascal@18894 476 +}
pascal@18894 477 +
pascal@18894 478 +/* These are the four functions used in the four steps of the MD5 algorithm
pascal@18894 479 + * and defined in the RFC 1321. The first function is a little bit optimized
pascal@18894 480 + * (as found in Colin Plumbs public domain implementation).
pascal@18894 481 + * #define FF(b, c, d) ((b & c) | (~b & d))
pascal@18894 482 + */
pascal@18894 483 +#undef FF
pascal@18894 484 +#undef FG
pascal@18894 485 +#undef FH
pascal@18894 486 +#undef FI
pascal@18894 487 +#define FF(b, c, d) (d ^ (b & (c ^ d)))
pascal@18894 488 +#define FG(b, c, d) FF(d, b, c)
pascal@18894 489 +#define FH(b, c, d) (b ^ c ^ d)
pascal@18894 490 +#define FI(b, c, d) (c ^ (b | ~d))
pascal@18894 491 +
pascal@18894 492 +/* Hash a single block, 64 bytes long and 4-byte aligned */
pascal@18894 493 +static void md5_process_block64(void)
pascal@18894 494 +{
pascal@18894 495 + /* Before we start, one word to the strange constants.
pascal@18894 496 + They are defined in RFC 1321 as
pascal@18894 497 + T[i] = (int)(4294967296.0 * fabs(sin(i))), i=1..64
pascal@18894 498 + */
pascal@18894 499 + static const uint32_t C_array[] = {
pascal@18894 500 + /* round 1 */
pascal@18894 501 + 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
pascal@18894 502 + 0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
pascal@18894 503 + 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
pascal@18894 504 + 0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
pascal@18894 505 + /* round 2 */
pascal@18894 506 + 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
pascal@18894 507 + 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
pascal@18894 508 + 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
pascal@18894 509 + 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
pascal@18894 510 + /* round 3 */
pascal@18894 511 + 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
pascal@18894 512 + 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
pascal@18894 513 + 0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x4881d05,
pascal@18894 514 + 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
pascal@18894 515 + /* round 4 */
pascal@18894 516 + 0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
pascal@18894 517 + 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
pascal@18894 518 + 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
pascal@18894 519 + 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391
pascal@18894 520 + };
pascal@18894 521 + static const char P_array[] ALIGN1 = {
pascal@18894 522 + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 1 */
pascal@18894 523 + 1, 6, 11, 0, 5, 10, 15, 4, 9, 14, 3, 8, 13, 2, 7, 12, /* 2 */
pascal@18894 524 + 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, /* 3 */
pascal@18894 525 + 0, 7, 14, 5, 12, 3, 10, 1, 8, 15, 6, 13, 4, 11, 2, 9 /* 4 */
pascal@18894 526 + };
pascal@18895 527 + uint32_t *words = (uint32_t *) wbuffer;
pascal@18894 528 + uint32_t A = hash[0];
pascal@18894 529 + uint32_t B = hash[1];
pascal@18894 530 + uint32_t C = hash[2];
pascal@18894 531 + uint32_t D = hash[3];
pascal@18894 532 +
pascal@18894 533 + static const char S_array[] ALIGN1 = {
pascal@18894 534 + 7, 12, 17, 22,
pascal@18894 535 + 5, 9, 14, 20,
pascal@18894 536 + 4, 11, 16, 23,
pascal@18894 537 + 6, 10, 15, 21
pascal@18894 538 + };
pascal@18894 539 + const uint32_t *pc;
pascal@18894 540 + const char *pp;
pascal@18894 541 + const char *ps;
pascal@18894 542 + int i;
pascal@18894 543 + uint32_t temp;
pascal@18894 544 +
pascal@18894 545 +
pascal@18894 546 + pc = C_array;
pascal@18894 547 + pp = P_array;
pascal@18894 548 + ps = S_array - 4;
pascal@18894 549 +
pascal@18894 550 + for (i = 0; i < 64; i++) {
pascal@18894 551 + if ((i & 0x0f) == 0)
pascal@18894 552 + ps += 4;
pascal@18894 553 + temp = A;
pascal@18894 554 + switch (i >> 4) {
pascal@18894 555 + case 0:
pascal@18894 556 + temp += FF(B, C, D);
pascal@18894 557 + break;
pascal@18894 558 + case 1:
pascal@18894 559 + temp += FG(B, C, D);
pascal@18894 560 + break;
pascal@18894 561 + case 2:
pascal@18894 562 + temp += FH(B, C, D);
pascal@18894 563 + break;
pascal@18894 564 + case 3:
pascal@18894 565 + temp += FI(B, C, D);
pascal@18894 566 + }
pascal@18894 567 + temp += words[(int) (*pp++)] + *pc++;
pascal@18894 568 + temp = rotl32(temp, ps[i & 3]);
pascal@18894 569 + temp += B;
pascal@18894 570 + A = D;
pascal@18894 571 + D = C;
pascal@18894 572 + C = B;
pascal@18894 573 + B = temp;
pascal@18894 574 + }
pascal@18894 575 + /* Add checksum to the starting values */
pascal@18894 576 + hash[0] += A;
pascal@18894 577 + hash[1] += B;
pascal@18894 578 + hash[2] += C;
pascal@18894 579 + hash[3] += D;
pascal@18894 580 +
pascal@18894 581 +}
pascal@18894 582 +#undef FF
pascal@18894 583 +#undef FG
pascal@18894 584 +#undef FH
pascal@18894 585 +#undef FI
pascal@18894 586 +
pascal@18894 587 +/* Initialize structure containing state of computation.
pascal@18894 588 + * (RFC 1321, 3.3: Step 3)
pascal@18894 589 + */
pascal@18894 590 +static void md5_begin(void)
pascal@18894 591 +{
pascal@18894 592 + hash[0] = 0x67452301;
pascal@18894 593 + hash[1] = 0xefcdab89;
pascal@18894 594 + hash[2] = 0x98badcfe;
pascal@18894 595 + hash[3] = 0x10325476;
pascal@18894 596 + total64 = 0;
pascal@18894 597 +}
pascal@18894 598 +
pascal@18894 599 +/* Used also for sha1 and sha256 */
pascal@18894 600 +#define md5_hash common64_hash
pascal@18894 601 +
pascal@18894 602 +/* Process the remaining bytes in the buffer and put result from CTX
pascal@18894 603 + * in first 16 bytes following RESBUF. The result is always in little
pascal@18894 604 + * endian byte order, so that a byte-wise output yields to the wanted
pascal@18894 605 + * ASCII representation of the message digest.
pascal@18894 606 + */
pascal@18894 607 +#define md5_end common64_end
pascal@18894 608 +
pascal@18894 609 +typedef struct { char hash[16]; } md5hash;
pascal@18894 610 +
pascal@18894 611 +static md5hash md5sum(uint8_t *buffer, int len)
pascal@18894 612 +{
pascal@18894 613 + md5hash val;
pascal@18894 614 +
pascal@18894 615 + md5_begin();
pascal@18894 616 + md5_hash(buffer, len);
pascal@18894 617 + md5_end();
pascal@18894 618 + memcpy(&val, hash, 16);
pascal@18894 619 +
pascal@18894 620 + return val;
pascal@18894 621 +}
pascal@10935 622 --- create_compressed_fs.c
pascal@10935 623 +++ create_compressed_fs.c
pascal@23425 624 @@ -0,0 +1,226 @@
pascal@12214 625 +#ifdef FIND_BEST_COMPRESSION
pascal@12214 626 +#include <compress.h>
pascal@12214 627 +extern "C" {
pascal@12214 628 +#include <stdlib.h>
pascal@12214 629 +#include <string.h>
pascal@12214 630 +
pascal@19561 631 +#define ZMAX 9
pascal@19408 632 +static shrink_t level;
pascal@19408 633 +static int pass, iter;
pascal@12214 634 +static int best_compress(unsigned char *compressed,
pascal@12214 635 + unsigned long *compressed_len,
pascal@12214 636 + unsigned char *uncompressed,
pascal@12214 637 + unsigned long uncompressed_len)
pascal@12214 638 +{
pascal@12214 639 + int i, j, err;
pascal@12214 640 + unsigned char *buf[2];
pascal@12214 641 + unsigned len;
pascal@12214 642 + unsigned long llen, best = *compressed_len * 2;
pascal@12214 643 + static unsigned char *buffer;
pascal@12214 644 + static unsigned long buffersz;
pascal@12214 645 +
pascal@12214 646 + if (buffersz < *compressed_len) {
pascal@12214 647 + if (buffer) free(buffer);
pascal@12214 648 + buffer = (unsigned char *) malloc(buffersz = *compressed_len);
pascal@12214 649 + if (!buffer) return Z_MEM_ERROR;
pascal@12214 650 + }
pascal@12214 651 + buf[0] = compressed;
pascal@12214 652 + buf[1] = buffer;
pascal@19561 653 + for (i = j = 0; i <= ZMAX+3 && (pass == 0 || i < pass); i++) {
pascal@12214 654 + llen = len = *compressed_len;
pascal@19560 655 + if (i >= ZMAX+1) {
pascal@19561 656 + level.level = (i == ZMAX+1) ? shrink_normal :
pascal@19561 657 + (i == ZMAX+2) ? shrink_extra : shrink_insane;
pascal@19408 658 + err = (compress_zlib(level, buf[j],
pascal@12214 659 + len, uncompressed,
pascal@12214 660 + uncompressed_len)) ? Z_OK : Z_DATA_ERROR;
pascal@19408 661 + } else {
pascal@12214 662 + err = compress2(buf[j], &llen, uncompressed,
pascal@12214 663 + uncompressed_len, i);
pascal@12214 664 + len = llen;
pascal@12214 665 + }
pascal@12214 666 + if (err != Z_OK) return err;
pascal@12214 667 + if (len < best) {
pascal@12214 668 + best = len;
pascal@12214 669 + j = 1 - j;
pascal@12214 670 + }
pascal@12214 671 + }
pascal@12214 672 + *compressed_len = best;
pascal@12214 673 + if (j == 0)
pascal@12214 674 + memcpy(compressed, buffer, best);
pascal@12214 675 + return err;
pascal@12214 676 +}
pascal@12214 677 +#define compress2(a,b,c,d,e) best_compress(a,b,c,d)
pascal@12214 678 +#endif
pascal@12214 679 +
pascal@18894 680 +#include <signal.h>
pascal@18894 681 +
pascal@10935 682 +/* Creates a compressed file */
pascal@10935 683 +#include "common_header.h"
pascal@10935 684 +
pascal@12214 685 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@10935 686 +#define CHUNK 65536
pascal@10935 687 +#define DEFAULT_BLOCKSIZE 65536
pascal@10935 688 +
pascal@12214 689 +static void quit(const char *s)
pascal@10935 690 +{
pascal@10935 691 + fprintf(stderr, "%s\n", s);
pascal@10935 692 + exit(1);
pascal@10935 693 +}
pascal@10935 694 +
pascal@10935 695 +static int readblock(unsigned char *buffer, int n)
pascal@10935 696 +{
pascal@10935 697 + int i;
pascal@10935 698 +
pascal@10935 699 + memset(buffer, 0, n);
pascal@10935 700 + for (i = 0 ; i < n;) {
pascal@10935 701 + int j = read(STDIN_FILENO, buffer + i, n - i);
pascal@10935 702 + if (j < 0 && errno == EINTR) continue;
pascal@10935 703 + if (j <= 0) break;
pascal@10935 704 + i += j;
pascal@10935 705 + }
pascal@10935 706 + return i;
pascal@10935 707 +}
pascal@10935 708 +
pascal@18894 709 +#ifdef FIND_BEST_COMPRESSION
pascal@18894 710 +#include "md5sum.c"
pascal@18894 711 +#endif
pascal@18894 712 +
pascal@18894 713 +static unsigned n;
pascal@23425 714 +static unsigned long lastlen, pos;
pascal@23425 715 +static u_int32_t *block_index;
pascal@18894 716 +static unsigned char *compressed;
pascal@18894 717 +static unsigned long block_size = 0;
pascal@18894 718 +static void flush_index(int sig)
pascal@18894 719 +{
pascal@18894 720 + static char padding[512];
pascal@18894 721 + struct cloop_tail tail;
pascal@18894 722 + unsigned long len;
pascal@18894 723 +
pascal@18894 724 + fprintf(stderr, "Write index for %lu blocks\n", n);
pascal@18894 725 + if (block_size >= 0x1000000) lastlen = 0;
pascal@18894 726 + tail.index_size = ntohl(sizeof(*block_index) + 256*(lastlen % 0xFFffFF));
pascal@18894 727 + tail.num_blocks = ntohl(n);
pascal@18894 728 + n *= sizeof(*block_index);
pascal@18894 729 + len = n + n/1000 + 12;
pascal@18894 730 + compressed = (unsigned char *) realloc(compressed, len);
pascal@18894 731 + if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
pascal@18894 732 + n, Z_BEST_SPEED) != Z_OK)
pascal@18894 733 + quit("Index compression failed");
pascal@18894 734 + tail.table_size = ntohl(len);
pascal@18894 735 + pos += len + sizeof(tail);
pascal@18894 736 + n = pos & 511;
pascal@23761 737 + // if (n) write(STDOUT_FILENO, padding, 512 - n);
pascal@18894 738 + write(STDOUT_FILENO, compressed, len);
pascal@18894 739 + write(STDOUT_FILENO, &tail, sizeof(tail));
pascal@18894 740 + exit(sig != 0);
pascal@18894 741 +}
pascal@18894 742 +
pascal@10935 743 +int main(int argc, char *argv[])
pascal@10935 744 +{
pascal@10935 745 + struct cloop_head head;
pascal@18894 746 + unsigned char *uncompressed;
pascal@18894 747 + unsigned long len;
pascal@18894 748 + unsigned indexmax, zlenmax;
pascal@18894 749 +#ifdef FIND_BEST_COMPRESSION
pascal@18894 750 + unsigned i, j, hashmax;
pascal@18894 751 + md5hash *hash;
pascal@18894 752 +#endif
pascal@10935 753 +
pascal@19408 754 +#ifdef FIND_BEST_COMPRESSION
pascal@19408 755 + while (argc > 1) {
pascal@19408 756 + if (argv[1][0] == '-') {
pascal@19408 757 + int *p = &pass;
pascal@19408 758 + switch (argv[1][1]) {
pascal@19408 759 + case 'i' : p = &iter;
pascal@19408 760 + case 'n' : *p = atoi(argv[2]);
pascal@19408 761 + argc -= 2;
pascal@19408 762 + argv += 2;
pascal@19408 763 + continue;
pascal@19408 764 + }
pascal@19408 765 + }
pascal@19408 766 + argc--;
pascal@19408 767 + if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@19408 768 + quit("Usage : create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
pascal@19408 769 +#else
pascal@10935 770 + if (argc > 1) {
pascal@10935 771 + if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@10935 772 + quit("Usage : create_compressed_fs [block size] < input > output");
pascal@19408 773 +#endif
pascal@10935 774 + block_size = atoi(argv[1]);
pascal@10935 775 + }
pascal@10935 776 + if (block_size < 4096)
pascal@10935 777 + block_size = DEFAULT_BLOCKSIZE;
pascal@10935 778 + fprintf(stderr, "Block size is %lu\n", block_size);
pascal@10935 779 + zlenmax = block_size + block_size/1000 + 12;
pascal@10935 780 +
pascal@10935 781 + memset(&head, 0, sizeof(head));
pascal@10935 782 + strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@10935 783 + head.num_blocks = -1;
pascal@10935 784 + head.block_size = htonl(block_size);
pascal@10935 785 + write(STDOUT_FILENO, &head, sizeof(head));
pascal@12214 786 + pos = sizeof(head);
pascal@10935 787 +
pascal@12214 788 + compressed = (unsigned char *) malloc(zlenmax);
pascal@12214 789 + uncompressed = (unsigned char *) malloc(block_size);
pascal@23425 790 + block_index = (u_int32_t *) malloc(indexmax = CHUNK);
pascal@18894 791 +#ifdef FIND_BEST_COMPRESSION
pascal@18894 792 + hash = (md5hash *) malloc(hashmax = CHUNK);
pascal@18894 793 + if (!compressed || !uncompressed || !block_index || !hash)
pascal@18894 794 +#else
pascal@18894 795 + if (!compressed || !uncompressed || !block_index)
pascal@18894 796 +#endif
pascal@10935 797 + quit("Malloc failed");
pascal@10935 798 +
pascal@18894 799 + signal(SIGINT,flush_index);
pascal@18894 800 + signal(SIGQUIT,flush_index);
pascal@18894 801 + signal(SIGTERM,flush_index);
pascal@18894 802 +
pascal@17996 803 + for (n = 0; (len = readblock(uncompressed, block_size)) != 0; n++) {
pascal@17997 804 + lastlen = len;
pascal@18894 805 + if (n * sizeof(*block_index) >= indexmax) {
pascal@23425 806 + block_index = (u_int32_t *) realloc(block_index,
pascal@12214 807 + indexmax += CHUNK);
pascal@18894 808 + if (!block_index)
pascal@10935 809 + quit("Realloc");
pascal@10935 810 + }
pascal@18894 811 +#ifdef FIND_BEST_COMPRESSION
pascal@18894 812 + if (n * sizeof(*hash) >= hashmax) {
pascal@18894 813 + hash = (md5hash *) realloc(hash, hashmax += CHUNK);
pascal@18895 814 + if (!hash)
pascal@18894 815 + quit("Realloc hash");
pascal@18894 816 + }
pascal@18894 817 + hash[n] = md5sum(uncompressed, len);
pascal@18894 818 + j = 0x7FFFFFFF;
pascal@18894 819 + if (n < j)
pascal@18894 820 + j = n;
pascal@18894 821 + for (i = 0; i < j; i++) {
pascal@18894 822 + if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
pascal@18894 823 + && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
pascal@18894 824 + break;
pascal@18894 825 + }
pascal@18894 826 + if (i != j) {
pascal@18894 827 + block_index[n] = ntohl(0x80000000 | i);
pascal@18894 828 + fprintf(stderr, "Block %u length %lu => duplicate %lu\n",
pascal@18894 829 + n, block_size, i);
pascal@18894 830 + }
pascal@18894 831 + else
pascal@18894 832 +#endif
pascal@18894 833 + {
pascal@18894 834 + len = zlenmax;
pascal@18894 835 + if (compress2(compressed, &len, uncompressed, lastlen,
pascal@18894 836 + Z_BEST_SPEED) != Z_OK)
pascal@18894 837 + quit("Compression failed");
pascal@18894 838 + fprintf(stderr, "Block %u length %lu => %lu\n",
pascal@18894 839 + n, block_size, len);
pascal@18894 840 + write(STDOUT_FILENO, compressed, len);
pascal@18894 841 + pos += len;
pascal@18894 842 + block_index[n] = ntohl(len);
pascal@18894 843 + }
pascal@10935 844 + }
pascal@18894 845 + flush_index(0);
pascal@10935 846 + return 0;
pascal@10935 847 +}
pascal@12214 848 +#ifdef FIND_BEST_COMPRESSION
pascal@12214 849 +}
pascal@12214 850 +#endif
pascal@10938 851 --- fusecloop.c
pascal@10938 852 +++ fusecloop.c
pascal@10938 853 @@ -65,7 +65,7 @@
pascal@10938 854
pascal@10938 855 memcpy(stbuf,&stb,sizeof stb);
pascal@10938 856 stbuf->st_mode&=~0222;
pascal@10938 857 - stbuf->st_size = cd.blocksize * cd.numblocks;
pascal@10938 858 + stbuf->st_size = (loff_t) cd.blocksize * cd.numblocks;
pascal@10938 859 /*
pascal@10938 860 stbuf->st_mode = S_IFREG | 0444;
pascal@10938 861 stbuf->st_nlink = 1;
pascal@23696 862
pascal@23696 863 === Add v4 support
pascal@23696 864 --- compressed_loop.h
pascal@23696 865 +++ compressed_loop.h
pascal@23696 866 @@ -30,6 +30,54 @@
pascal@23696 867 /* ...padding up to CLOOP_HEADROOM... */
pascal@23696 868 /* block_size (32bit number, network order) */
pascal@23696 869 /* num_blocks (32bit number, network order) */
pascal@23696 870 +/*
pascal@23696 871 +* Starting with Format V4.0 (cloop version 4.x), cloop can now have two *
pascal@23696 872 +* alternative structures: *
pascal@23696 873 +* 1. Header first: *
pascal@23696 874 +* +---------------------------- FIXED SIZE ---------------------------+ *
pascal@23696 875 +* |Signature (128 bytes) | *
pascal@23696 876 +* |block_size (32bit number, network order) | *
pascal@23696 877 +* |num_blocks (32bit number, network order) | *
pascal@23696 878 +* +--------------------------- VARIABLE SIZE -------------------------+ *
pascal@23696 879 +* |num_blocks * FlagsOffset (upper 4 bits flags, lower 60 bits offset)| *
pascal@23696 880 +* |compressed data blocks of variable size ... | *
pascal@23696 881 +* +-------------------------------------------------------------------+ *
pascal@23696 882 +* *
pascal@23696 883 +* 2. Footer (header last): *
pascal@23696 884 +* +--------------------------- VARIABLE SIZE -------------------------+ *
pascal@23696 885 +* |compressed data blocks of variable size ... | *
pascal@23696 886 +* |num_blocks * FlagsOffset (upper 4 bits flags, lower 60 bits offset)| *
pascal@23696 887 +* +---------------------------- FIXED SIZE ---------------------------+ *
pascal@23696 888 +* |Signature (128 bytes) | *
pascal@23696 889 +* |block_size (32bit number, network order) | *
pascal@23696 890 +* |num_blocks (32bit number, network order) | *
pascal@23696 891 +* +-------------------------------------------------------------------+ *
pascal@23696 892 +* *
pascal@23696 893 +* Offsets are always relative to beginning of file, in all formats. *
pascal@23696 894 +* The block index contains num_blocks+1 offsets, followed (1) or *
pascal@23696 895 +* preceded (2) by the compressed blocks. *
pascal@23696 896 +* *
pascal@23696 897 +* CLOOP4 flags for each compressed block *
pascal@23696 898 +* Value Meaning *
pascal@23696 899 +* 0 GZIP/7ZIP compression (compatible with V2.0 Format) *
pascal@23696 900 +* 1 no compression (incompressible data) *
pascal@23696 901 +* 2 xz compression (currently best space saver) *
pascal@23696 902 +* 3 lz4 compression *
pascal@23696 903 +* 4 lzo compression (fastest) *
pascal@23696 904 +* 15 block link *
pascal@23696 905 +*/
pascal@23696 906 +/* Get value of first 4 bits */
pascal@23696 907 +#define CLOOP_BLOCK_FLAGS(x) ((unsigned int)(((x) & 0xf000000000000000LLU) >> 60))
pascal@23696 908 +/* Get value of last 60 bits */
pascal@23696 909 +#define CLOOP_BLOCK_OFFSET(x) ((x) & 0x0fffffffffffffffLLU)
pascal@23696 910 +
pascal@23696 911 +#define CLOOP_COMPRESSOR_ZLIB 0x0
pascal@23696 912 +#define CLOOP_COMPRESSOR_NONE 0x1
pascal@23696 913 +#define CLOOP_COMPRESSOR_XZ 0x2
pascal@23696 914 +#define CLOOP_COMPRESSOR_LZ4 0x3
pascal@23696 915 +#define CLOOP_COMPRESSOR_LZO 0x4
pascal@23696 916 +#define CLOOP_COMPRESSOR_LINK 0xF
pascal@23696 917 +
pascal@23696 918
pascal@23696 919 struct cloop_head
pascal@23696 920 {
pascal@23696 921 @@ -43,47 +91,86 @@
pascal@23696 922
pascal@23696 923 struct cloop_tail
pascal@23696 924 {
pascal@23696 925 - u_int32_t table_size;
pascal@23696 926 - u_int32_t index_size;
pascal@23696 927 + u_int32_t table_size;
pascal@23696 928 + u_int32_t index_size; /* size:4 comp:3 ctrl-c:1 lastlen:24 */
pascal@23696 929 +#define CLOOP3_INDEX_SIZE(x) ((unsigned int)((x) & 0xF))
pascal@23696 930 +#define CLOOP3_BLOCKS_FLAGS(x) ((unsigned int)((x) & 0x70) >> 4)
pascal@23696 931 +#define CLOOP3_TRUNCATED(x) ((unsigned int)((x) & 0x80) >> 7)
pascal@23696 932 +#define CLOOP3_LASTLEN(x) (unsigned int)((x) >> 8)
pascal@23696 933 u_int32_t num_blocks;
pascal@23696 934 };
pascal@23696 935
pascal@23696 936 +#define GZIP_MAX_BUFFER(n) ((n) + (n)/1000 + 12)
pascal@23696 937 +
pascal@23696 938 struct block_info
pascal@23696 939 {
pascal@23696 940 loff_t offset; /* 64-bit offsets of compressed block */
pascal@23696 941 u_int32_t size; /* 32-bit compressed block size */
pascal@23696 942 - u_int32_t optidx; /* 32-bit index number */
pascal@23696 943 + u_int32_t flags; /* 32-bit compression flags */
pascal@23696 944 +
pascal@23696 945 };
pascal@23696 946
pascal@23696 947 -static inline char *build_index(struct block_info *offsets, unsigned long n)
pascal@23696 948 +static inline char *build_index(struct block_info *offsets, unsigned long n,
pascal@23696 949 + unsigned long block_size, unsigned global_flags)
pascal@23696 950 {
pascal@23696 951 u_int32_t *ofs32 = (u_int32_t *) offsets;
pascal@23696 952 loff_t *ofs64 = (loff_t *) offsets;
pascal@23696 953 -
pascal@23696 954 +
pascal@23696 955 + /* v3 64bits bug: v1 assumed */
pascal@23696 956 + unsigned long v3_64;
pascal@23696 957 + loff_t prev;
pascal@23696 958 +
pascal@23696 959 + if (ofs32[0] != 0 && ofs32[1] == 0) {
pascal@23696 960 + for (v3_64=(n+1)/2, prev=__le64_to_cpu(ofs64[v3_64]);
pascal@23696 961 + v3_64 > 0 && __le64_to_cpu(ofs64[--v3_64]) < prev;
pascal@23696 962 + prev=__le64_to_cpu(ofs64[v3_64]));
pascal@23696 963 + }
pascal@23696 964 +
pascal@23696 965 if (ofs32[0] == 0) {
pascal@23696 966 if (ofs32[2]) { /* ACCELERATED KNOPPIX V1.0 */
pascal@23696 967 while (n--) {
pascal@23696 968 offsets[n].offset = __be64_to_cpu(offsets[n].offset);
pascal@23696 969 offsets[n].size = ntohl(offsets[n].size);
pascal@23696 970 + offsets[n].flags = 0;
pascal@23696 971 }
pascal@23696 972 return (char *) "128BE accelerated knoppix 1.0";
pascal@23696 973 }
pascal@23696 974 - else { /* V2.0 */
pascal@23696 975 - loff_t last = __be64_to_cpu(ofs64[n]);
pascal@23696 976 - while (n--) {
pascal@23696 977 + else { /* V2.0/V4.0 */
pascal@23696 978 + loff_t last = CLOOP_BLOCK_OFFSET(__be64_to_cpu(ofs64[n]));
pascal@23696 979 + u_int32_t flags;
pascal@23696 980 + static char v4[11];
pascal@23696 981 + unsigned long i = n;
pascal@23696 982 +
pascal@23696 983 + for (flags = 0; n-- ;) {
pascal@23696 984 + loff_t data = __be64_to_cpu(ofs64[n]);
pascal@23696 985 +
pascal@23696 986 offsets[n].size = last -
pascal@23696 987 - (offsets[n].offset = __be64_to_cpu(ofs64[n]));
pascal@23696 988 + (offsets[n].offset = CLOOP_BLOCK_OFFSET(data));
pascal@23696 989 last = offsets[n].offset;
pascal@23696 990 + offsets[n].flags = CLOOP_BLOCK_FLAGS(data);
pascal@23696 991 + flags |= 1 << offsets[n].flags;
pascal@23696 992 + }
pascal@23696 993 + if (flags < 2) return (char *) "64BE v2.0";
pascal@23696 994 + while (i--) {
pascal@23696 995 + if (offsets[i].flags == CLOOP_COMPRESSOR_LINK) {
pascal@23696 996 + offsets[i] = offsets[offsets[i].offset];
pascal@23696 997 + }
pascal@23696 998 + }
pascal@23696 999 + strcpy(v4, (char *) "64BE v4.0a");
pascal@23696 1000 + v4[10] = 'a' + ((flags-1) & 0xF); // compressors used
pascal@23696 1001 + if (flags > 0x10) { // with links ?
pascal@23696 1002 + v4[10] += 'A' - 'a';
pascal@23696 1003 }
pascal@23696 1004 - return (char *) "64BE v2.0";
pascal@23696 1005 + return v4;
pascal@23696 1006 }
pascal@23696 1007 }
pascal@23696 1008 - else if (ofs32[1] == 0) { /* V1.0 */
pascal@23696 1009 + else if (ofs32[1] == 0 && v3_64 == 0) { /* V1.0 */
pascal@23696 1010 loff_t last = __le64_to_cpu(ofs64[n]);
pascal@23696 1011 while (n--) {
pascal@23696 1012 offsets[n].size = last -
pascal@23696 1013 (offsets[n].offset = __le64_to_cpu(ofs64[n]));
pascal@23696 1014 last = offsets[n].offset;
pascal@23696 1015 + offsets[n].flags = 0;
pascal@23696 1016 }
pascal@23696 1017 return (char *) "64LE v1.0";
pascal@23696 1018 }
pascal@23696 1019 @@ -93,25 +180,37 @@
pascal@23696 1020 offsets[n].size = last -
pascal@23696 1021 (offsets[n].offset = ntohl(ofs32[n]));
pascal@23696 1022 last = offsets[n].offset;
pascal@23696 1023 + offsets[n].flags = 0;
pascal@23696 1024 }
pascal@23696 1025 return (char *) "32BE v0.68";
pascal@23696 1026 }
pascal@23696 1027 else { /* V3.0 */
pascal@23696 1028 unsigned long i;
pascal@23696 1029 loff_t j;
pascal@23696 1030 + static char v3[11];
pascal@23696 1031
pascal@23696 1032 + v3_64 = (ofs32[1] == 0) ? 2 : 1;
pascal@23696 1033 for (i = n; i-- != 0; )
pascal@23696 1034 - offsets[i].size = ntohl(ofs32[i]);
pascal@23696 1035 + offsets[i].size = ntohl(ofs32[i*v3_64]);
pascal@23696 1036 for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
pascal@23696 1037 offsets[i].offset = j;
pascal@23696 1038 + offsets[i].flags = global_flags;
pascal@23696 1039 + if ((offsets[i].size & 0x80000000) == 0) {
pascal@23696 1040 + j += offsets[i].size;
pascal@23696 1041 + }
pascal@23696 1042 + else if (offsets[i].size == 0xFFFFFFFF) {
pascal@23696 1043 + offsets[i].flags = CLOOP_COMPRESSOR_NONE;
pascal@23696 1044 + j += offsets[i].size = block_size;
pascal@23696 1045 + }
pascal@23696 1046 + }
pascal@23696 1047 + for (i = 0; i < n; i++) {
pascal@23696 1048 if (offsets[i].size & 0x80000000) {
pascal@23696 1049 - unsigned long k = offsets[i].size & 0x7FFFFFFF;
pascal@23696 1050 - offsets[i].offset = offsets[k].offset;
pascal@23696 1051 - offsets[i].size = offsets[k].size;
pascal@23696 1052 + offsets[i] = offsets[offsets[i].size & 0x7FFFFFFF];
pascal@23696 1053 }
pascal@23696 1054 - else j += offsets[i].size;
pascal@23696 1055 }
pascal@23696 1056 - return (char *) "32BE v3.0";
pascal@23696 1057 + strcpy(v3, (char *) (--v3_64) ? "64BE v3.0a" : "32BE v3.0a");
pascal@23696 1058 + v3[10] += global_flags;
pascal@23696 1059 + return v3;
pascal@23696 1060 }
pascal@23696 1061 }
pascal@23696 1062
pascal@23696 1063 --- cloopreader.c
pascal@23696 1064 +++ cloopreader.c
pascal@23696 1065 @@ -25,6 +25,8 @@
pascal@23696 1066 #include "debug.h"
pascal@23696 1067 #include "cloopreader.h"
pascal@23696 1068
pascal@23696 1069 +#include "cloopunpack.c"
pascal@23696 1070 +
pascal@23696 1071 int read_all(int fh, void* block, size_t size){
pascal@23696 1072 bfuncinfo("fh=%d block=0x%lx size=0x%lx",
pascal@23696 1073 fh,(ulong)block,(ulong)size);
pascal@23696 1074 @@ -50,18 +52,29 @@
pascal@23696 1075 bfuncinfo("fh=%d",fh);
pascal@23696 1076 c->fh=fh;
pascal@23696 1077 struct cloop_head head;
pascal@23696 1078 - OP(read_all(c->fh,&head,sizeof head)); /* read Header */
pascal@23696 1079 + int v4_header_last, flags;
pascal@23696 1080 + loff_t end;
pascal@23696 1081 +
pascal@23696 1082 + for (v4_header_last=0;;v4_header_last++) {
pascal@23696 1083 + OP(read_all(c->fh,&head,sizeof head)); /* read Header */
pascal@23696 1084 + if (!memcmp(&head,"#!/bin/sh",9)) break;
pascal@23696 1085 + if (v4_header_last) exit(1);
pascal@23696 1086 + end = lseek(c->fh, 0, SEEK_END);
pascal@23696 1087 + OP(lseek(c->fh, end - sizeof(head), SEEK_SET));
pascal@23696 1088 + }
pascal@23696 1089
pascal@23696 1090 c->numblocks=ntohl(head.num_blocks);
pascal@23696 1091 c->blocksize=ntohl(head.block_size);
pascal@23696 1092
pascal@23696 1093 bprintf("block_size=%lx num_blocks=%x\n", c->blocksize, c->numblocks);
pascal@23696 1094 + if (v4_header_last)
pascal@23696 1095 + OP(lseek(c->fh, end - sizeof(head) - (sizeof(*c->toc) * c->numblocks), SEEK_SET));
pascal@23696 1096
pascal@23696 1097 ALLOC(c->pblock,c->blocksize);
pascal@23696 1098
pascal@23696 1099 if (c->numblocks + 1 == 0) {
pascal@23696 1100 struct cloop_tail tail;
pascal@23696 1101 - loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
pascal@23696 1102 + end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
pascal@23696 1103 void *p;
pascal@23696 1104 ulong toclen, len;
pascal@23696 1105
pascal@23696 1106 @@ -70,21 +83,23 @@
pascal@23696 1107 c->numblocks = ntohl(tail.num_blocks);
pascal@23696 1108 c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@23696 1109 len = ntohl(tail.table_size);
pascal@23696 1110 - toclen = (ntohl(tail.index_size) & 255) * c->numblocks;
pascal@23696 1111 + flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
pascal@23696 1112 + toclen = CLOOP3_INDEX_SIZE(ntohl(tail.index_size)) * c->numblocks;
pascal@23696 1113 OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
pascal@23696 1114 ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
pascal@23696 1115 ALLOC(p,len);
pascal@23696 1116 OP(read_all(c->fh,p,len)); /* read Data Index */
pascal@23696 1117 - if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK)
pascal@23696 1118 + if (unpack[flags]((void *)c->toc,&toclen,p,len) != Z_OK)
pascal@23696 1119 exit(1);
pascal@23696 1120 free(p);
pascal@23696 1121 }
pascal@23696 1122 else {
pascal@23696 1123 + flags = 0;
pascal@23696 1124 c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@23696 1125 ALLOC(c->toc,c->tocsize);
pascal@23696 1126 OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
pascal@23696 1127 }
pascal@23696 1128 - build_index(c->toc, c->numblocks);
pascal@23696 1129 + build_index(c->toc, c->numblocks, c->blocksize, flags);
pascal@23696 1130 c->cblocksizecur=0;
pascal@23696 1131 c->curblock=-1;
pascal@23696 1132 return 0;
pascal@23696 1133 @@ -121,7 +136,8 @@
pascal@23696 1134 "pblock=0x%lx &destlen=0x%lx cblock=0x%lx cblocksize=%lu\n",
pascal@23696 1135 (ulong)c->pblock,(ulong)&destlen,(ulong)c->cblock,c->cblocksize
pascal@23696 1136 );
pascal@23696 1137 - switch(uncompress(c->pblock,&destlen,c->cblock,c->cblocksize)){
pascal@23696 1138 + if(c->toc[page].flags <= CLOOP_COMPRESSOR_MAX){
pascal@23696 1139 + switch(unpack[c->toc[page].flags](c->pblock,&destlen,c->cblock,c->cblocksize)){
pascal@23696 1140 case Z_OK: break;
pascal@23696 1141 #define entry(x)\
pascal@23696 1142 case x: bprintf( #x"\n"); break;
pascal@23696 1143 @@ -130,7 +146,9 @@
pascal@23696 1144 entry(Z_DATA_ERROR)
pascal@23696 1145 #undef entry
pascal@23696 1146 default: bprintf("Z_UNKNOWN_ERROR\n");
pascal@23696 1147 + }
pascal@23696 1148 }
pascal@23696 1149 + else bprintf("Unsuppoted compression type\n");
pascal@23696 1150 if(destlen!=c->blocksize)bprintf("Size mismatch\n");
pascal@23696 1151 return 0;
pascal@23696 1152 }
pascal@23696 1153 --- /dev/null
pascal@23696 1154 +++ cloopunpack.c
pascal@23696 1155 @@ -0,0 +1,74 @@
pascal@23696 1156 +
pascal@23696 1157 +static int none_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
pascal@23696 1158 +{
pascal@23696 1159 + memcpy(dest,source,*destLen = sourceLen);
pascal@23696 1160 + return Z_OK;
pascal@23696 1161 +}
pascal@23696 1162 +
pascal@23696 1163 +#include <lzma.h>
pascal@23696 1164 +static int xz_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
pascal@23696 1165 +{
pascal@23696 1166 + size_t src_pos = 0;
pascal@23696 1167 + size_t dest_pos = 0;
pascal@23696 1168 + uint64_t memlimit = 32*1024*1024;
pascal@23696 1169 +
pascal@23696 1170 + lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL,
pascal@23696 1171 + source, &src_pos, sourceLen, dest, &dest_pos, *destLen);
pascal@23696 1172 +
pascal@23696 1173 + if(res == LZMA_OK && sourceLen == (int) src_pos) {
pascal@23696 1174 + *destLen = dest_pos;
pascal@23696 1175 + return Z_OK;
pascal@23696 1176 + }
pascal@23696 1177 + else return Z_ERRNO;
pascal@23696 1178 +}
pascal@23696 1179 +
pascal@23696 1180 +#include <lz4.h>
pascal@23696 1181 +static int lz4_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
pascal@23696 1182 +{
pascal@23696 1183 + long size = LZ4_decompress_safe((const char *) source, (char *) dest, sourceLen, *destLen);
pascal@23696 1184 +
pascal@23696 1185 + if (size < 0) return Z_ERRNO;
pascal@23696 1186 + *destLen = size;
pascal@23696 1187 + return Z_OK;
pascal@23696 1188 +}
pascal@23696 1189 +
pascal@23696 1190 +#include <lzo/lzo1x.h>
pascal@23696 1191 +static int lzo_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
pascal@23696 1192 +{
pascal@23696 1193 + lzo_uint outlen = *destLen;
pascal@23696 1194 +
pascal@23696 1195 + if (lzo1x_decompress_safe(source, sourceLen, dest, &outlen, NULL) == LZO_E_OK) {
pascal@23696 1196 + *destLen = outlen;
pascal@23696 1197 + return Z_OK;
pascal@23696 1198 + }
pascal@23696 1199 + else return Z_ERRNO;
pascal@23696 1200 +}
pascal@23696 1201 +
pascal@23696 1202 +#include <zstd.h>
pascal@23696 1203 +static int zstd_uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen)
pascal@23696 1204 +{
pascal@23696 1205 + const size_t res = ZSTD_decompress(dest, *destLen, source, sourceLen);
pascal@23696 1206 +
pascal@23696 1207 + if (ZSTD_isError(res)) {
pascal@23696 1208 + return Z_ERRNO;
pascal@23696 1209 + }
pascal@23696 1210 +
pascal@23696 1211 + *destLen = res;
pascal@23696 1212 + return Z_OK;
pascal@23696 1213 +}
pascal@23696 1214 +
pascal@23696 1215 +#define CLOOP_COMPRESSOR_ZSTD 0x5
pascal@23696 1216 +
pascal@23696 1217 +#define CLOOP_COMPRESSOR_MAX CLOOP_COMPRESSOR_ZSTD
pascal@23696 1218 +
pascal@23696 1219 +#define CLOOP_COMPRESSOR_NAMES "gzip","copy","xz","lz4","lzo","zstd"
pascal@23696 1220 +
pascal@23696 1221 +static int (*unpack[CLOOP_COMPRESSOR_MAX+1])(Bytef *dest, uLongf *destLen, const Bytef *source, uLongf sourceLen) = {
pascal@23696 1222 + uncompress,
pascal@23696 1223 + none_uncompress,
pascal@23696 1224 + xz_uncompress,
pascal@23696 1225 + lz4_uncompress,
pascal@23696 1226 + lzo_uncompress,
pascal@23696 1227 + zstd_uncompress
pascal@23696 1228 +};
pascal@23696 1229 +
pascal@23696 1230 --- extract_compressed_fs.c
pascal@23696 1231 +++ extract_compressed_fs.c
pascal@23757 1232 @@ -3,14 +3,78 @@
pascal@23696 1233 #include "common_header.h"
pascal@23696 1234 #define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@23696 1235
pascal@23696 1236 +#include "cloopunpack.c"
pascal@23696 1237 +static char *packnames[CLOOP_COMPRESSOR_MAX+1] = { CLOOP_COMPRESSOR_NAMES };
pascal@23696 1238 +
pascal@23757 1239 +static unsigned char *out;
pascal@23757 1240 +static unsigned long zblock_maxsize, block_size, num_blocks;
pascal@23757 1241 +static int handle, err;
pascal@23757 1242 +static unsigned char *buffer, *clear_buffer;
pascal@23757 1243 +static struct block_info *offsets;
pascal@23757 1244 +
pascal@23757 1245 +static uLongf get_block(unsigned int i, int argc)
pascal@23757 1246 +{
pascal@23757 1247 + unsigned long len;
pascal@23757 1248 + uLongf ulen;
pascal@23757 1249 + int flags = offsets[i].flags;
pascal@23757 1250 +
pascal@23757 1251 + if (flags > CLOOP_COMPRESSOR_MAX) {
pascal@23757 1252 + fprintf(stderr, "Block %u: unsupported compression %d \n",
pascal@23757 1253 + i, flags);
pascal@23757 1254 + exit(1);
pascal@23757 1255 + }
pascal@23757 1256 +
pascal@23757 1257 + len = offsets[i].size;
pascal@23757 1258 + if (len > zblock_maxsize) {
pascal@23757 1259 + fprintf(stderr,
pascal@23757 1260 + "Size %lu for block %u (offset %Lu) too big\n",
pascal@23757 1261 + len, i, offsets[i].offset);
pascal@23757 1262 + exit(1);
pascal@23757 1263 + }
pascal@23757 1264 +
pascal@23757 1265 + if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
pascal@23757 1266 + fprintf(stderr, "lseek to %Lu: %s\n",
pascal@23757 1267 + offsets[i].offset, strerror(errno));
pascal@23757 1268 + exit(1);
pascal@23757 1269 + }
pascal@23757 1270 +
pascal@23757 1271 + read(handle, out = buffer, ulen = len);
pascal@23757 1272 +
pascal@23757 1273 + if (flags != CLOOP_COMPRESSOR_ZLIB || argc <= 2) {
pascal@23757 1274 + ulen = block_size;
pascal@23757 1275 + err = unpack[flags](out = clear_buffer, &ulen, buffer, len);
pascal@23757 1276 + if (err != Z_OK) {
pascal@23757 1277 + fprintf(stderr, "Unpack %s block %u error %d \n",
pascal@23757 1278 + packnames[flags], i, err);
pascal@23757 1279 + exit(1);
pascal@23757 1280 + }
pascal@23757 1281 + if (ulen != block_size && i != num_blocks - 1) {
pascal@23757 1282 + fprintf(stderr, "Uncomp %s: bad len %u (%lu not %lu)\n",
pascal@23757 1283 + packnames[flags], i, ulen, block_size);
pascal@23757 1284 + exit(1);
pascal@23757 1285 + }
pascal@23757 1286 + if (argc > 2) {
pascal@23757 1287 + len = ulen;
pascal@23757 1288 + ulen = zblock_maxsize;
pascal@23757 1289 + err = compress2(out = buffer, &ulen, clear_buffer, len, Z_BEST_SPEED);
pascal@23757 1290 + if (err != Z_OK) {
pascal@23757 1291 + fprintf(stderr, "Compress %s block %u error %d \n",
pascal@23757 1292 + packnames[flags], i, err);
pascal@23757 1293 + exit(1);
pascal@23757 1294 + }
pascal@23757 1295 + }
pascal@23757 1296 + }
pascal@23757 1297 + return ulen;
pascal@23757 1298 +}
pascal@23757 1299 +
pascal@23696 1300 int main(int argc, char *argv[])
pascal@23696 1301 {
pascal@23696 1302 - int handle;
pascal@23757 1303 + int flags;
pascal@23696 1304 struct cloop_head head;
pascal@23696 1305 - unsigned int i;
pascal@23696 1306 - unsigned long num_blocks, block_size, zblock_maxsize, lastlen = 0;
pascal@23757 1307 - unsigned char *buffer, *clear_buffer;
pascal@23757 1308 - struct block_info *offsets;
pascal@23696 1309 + unsigned int i, v4_header_last, global_flags;
pascal@23757 1310 + unsigned long n, len;
pascal@23696 1311 + uLongf ulen;
pascal@23696 1312 + loff_t end;
pascal@23696 1313
pascal@23696 1314 if (argc < 2 || argv[1][0] == '-') {
pascal@23696 1315 fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
pascal@23757 1316 @@ -23,23 +87,35 @@
pascal@23696 1317 exit(1);
pascal@23696 1318 }
pascal@23696 1319
pascal@23696 1320 - if (read(handle, &head, sizeof(head)) != sizeof(head)) {
pascal@23696 1321 - perror("Reading compressed file header\n");
pascal@23696 1322 + for (v4_header_last=0;; v4_header_last++) {
pascal@23696 1323 + if (read(handle, &head, sizeof(head)) != sizeof(head)) {
pascal@23696 1324 + perror("Reading compressed file header\n");
pascal@23696 1325 + exit(1);
pascal@23696 1326 + }
pascal@23696 1327 + if (!memcmp(&head,"#!/bin/sh",9)) break;
pascal@23696 1328 + end = lseek64(handle, 0, SEEK_END);
pascal@23696 1329 + lseek64(handle, end - sizeof(head), SEEK_SET);
pascal@23696 1330 + if (v4_header_last == 0) continue;
pascal@23696 1331 + perror("Not a cloop file\n");
pascal@23696 1332 exit(1);
pascal@23696 1333 }
pascal@23696 1334
pascal@23696 1335 num_blocks = ntohl(head.num_blocks);
pascal@23696 1336 block_size = ntohl(head.block_size);
pascal@23696 1337 - zblock_maxsize = block_size + block_size/1000 + 12 + 4;
pascal@23696 1338 + zblock_maxsize = GZIP_MAX_BUFFER(block_size);
pascal@23696 1339 buffer = malloc(zblock_maxsize);
pascal@23696 1340 clear_buffer = malloc(block_size);
pascal@23696 1341
pascal@23696 1342 + if (v4_header_last) {
pascal@23696 1343 + lseek64(handle, end - ((num_blocks+1) * sizeof(loff_t)) - sizeof(head), SEEK_SET);
pascal@23696 1344 + }
pascal@23696 1345 +
pascal@23696 1346 if (num_blocks == 0xFFFFFFFF) {
pascal@23696 1347 void *table;
pascal@23696 1348 struct cloop_tail tail;
pascal@23696 1349 - unsigned long len, table_size;
pascal@23696 1350 - loff_t end = lseek64(handle, 0, SEEK_END);
pascal@23696 1351 + unsigned long table_size;
pascal@23696 1352
pascal@23696 1353 + end = lseek64(handle, 0, SEEK_END);
pascal@23696 1354 if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
pascal@23696 1355 read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
pascal@23696 1356 lseek64(handle, end - sizeof(tail) -
pascal@23757 1357 @@ -51,105 +127,108 @@
pascal@23696 1358 num_blocks = ntohl(head.num_blocks);
pascal@23696 1359 table_size = ntohl(tail.table_size);
pascal@23696 1360 table = malloc(table_size);
pascal@23696 1361 - len = i = num_blocks * (ntohl(tail.index_size) & 255);
pascal@23696 1362 - lastlen = ntohl(tail.index_size) / 256;
pascal@23696 1363 - offsets = malloc(num_blocks * sizeof(*offsets));
pascal@23696 1364 - if (!table || !offsets ||
pascal@23696 1365 - read(handle, table, table_size) != table_size ||
pascal@23696 1366 - uncompress((void *)offsets, &len, table, table_size) != Z_OK ||
pascal@23696 1367 - len != i) {
pascal@23696 1368 + len = num_blocks * CLOOP3_INDEX_SIZE(ntohl(tail.index_size));
pascal@23696 1369 + global_flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
pascal@23696 1370 + if (global_flags > CLOOP_COMPRESSOR_MAX) {
pascal@23696 1371 + fprintf(stderr, "Unsupported compression %d\n",
pascal@23696 1372 + global_flags);
pascal@23696 1373 + exit(1);
pascal@23696 1374 + }
pascal@23696 1375 + ulen = num_blocks * sizeof(*offsets);
pascal@23696 1376 + offsets = malloc(ulen);
pascal@23696 1377 + if (!table || !offsets || !buffer || !clear_buffer) {
pascal@23696 1378 + fprintf(stderr,"Out of memory\n");
pascal@23696 1379 + exit(1);
pascal@23696 1380 + }
pascal@23696 1381 + if (read(handle, table, table_size) != table_size) {
pascal@23696 1382 perror("Reading index\n");
pascal@23696 1383 exit(1);
pascal@23696 1384 }
pascal@23696 1385 + err = unpack[global_flags]((void *) offsets, &ulen, table, table_size);
pascal@23696 1386 + if (err != Z_OK) {
pascal@23696 1387 + fprintf(stderr, "Unpack %s index error %d\n",
pascal@23696 1388 + packnames[global_flags],err);
pascal@23696 1389 + exit(1);
pascal@23696 1390 + }
pascal@23696 1391 free(table);
pascal@23696 1392 }
pascal@23696 1393 else {
pascal@23696 1394 - offsets = malloc(i = num_blocks * sizeof(*offsets));
pascal@23696 1395 - if (!offsets || read(handle, offsets, i) != i) {
pascal@23696 1396 + global_flags = 0;
pascal@23696 1397 + len = num_blocks * sizeof(*offsets);
pascal@23696 1398 + offsets = malloc(len);
pascal@23696 1399 + if (v4_header_last) {
pascal@23696 1400 + len = (num_blocks+1) * sizeof(loff_t);
pascal@23696 1401 + }
pascal@23696 1402 + if (!offsets || !buffer || !clear_buffer) {
pascal@23696 1403 + fprintf(stderr,"Out of memory\n");
pascal@23696 1404 + exit(1);
pascal@23696 1405 + }
pascal@23696 1406 + if (read(handle, offsets, len) != len) {
pascal@23696 1407 perror("Reading index\n");
pascal@23696 1408 exit(1);
pascal@23696 1409 }
pascal@23696 1410 }
pascal@23696 1411
pascal@23696 1412 + if (v4_header_last) {
pascal@23696 1413 + lseek64(handle, 0, SEEK_SET);
pascal@23696 1414 + }
pascal@23696 1415 +
pascal@23696 1416 fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
pascal@23696 1417 num_blocks, block_size, head.preamble);
pascal@23696 1418 - fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
pascal@23696 1419 +#if 1
pascal@23696 1420 + if (getenv("CLOOP_INDEX") != NULL) {
pascal@23696 1421 + fprintf(stderr, "CLOOP_INDEX: binary\n");
pascal@23696 1422 + write(STDOUT_FILENO, offsets, len);
pascal@23696 1423 + exit(0);
pascal@23696 1424 + }
pascal@23696 1425 +#endif
pascal@23696 1426 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size, global_flags));
pascal@23696 1427
pascal@23696 1428 +#if 1
pascal@23696 1429 + if (getenv("CLOOP_TABLE") != NULL) {
pascal@23696 1430 + fprintf(stderr, "CLOOP_TABLE ascii: offset, size, flags\n");
pascal@23696 1431 + for (i = 0; i < num_blocks; i++) {
pascal@23696 1432 + printf("%llu %u %u\n",
pascal@23696 1433 + offsets[i].offset,
pascal@23696 1434 + offsets[i].size,
pascal@23696 1435 + offsets[i].flags);
pascal@23696 1436 + }
pascal@23696 1437 + exit(0);
pascal@23696 1438 + }
pascal@23696 1439 +#endif
pascal@23696 1440 +
pascal@23696 1441 if (argc > 2) {
pascal@23696 1442 - unsigned n;
pascal@23696 1443 - loff_t data, offset = ((num_blocks + 1) * sizeof(offset)) + sizeof(head);
pascal@23696 1444 + loff_t data, ofs = ((num_blocks + 1) * sizeof(ofs)) + sizeof(head);
pascal@23696 1445
pascal@23696 1446 strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@23696 1447 write(STDOUT_FILENO, &head, n = sizeof(head));
pascal@23757 1448 for (i = 0; i < num_blocks; i++) {
pascal@23696 1449 - data = __be64_to_cpu(offset);
pascal@23696 1450 + data = __be64_to_cpu(ofs);
pascal@23696 1451 write(STDOUT_FILENO, &data, sizeof(data));
pascal@23696 1452 n += sizeof(data);
pascal@23696 1453 - offset += offsets[i].size;
pascal@23757 1454 + if (offsets[i].flags == CLOOP_COMPRESSOR_ZLIB)
pascal@23757 1455 + ofs += offsets[i].size;
pascal@23757 1456 + else
pascal@23757 1457 + ofs += get_block(i,argc);
pascal@23696 1458 }
pascal@23696 1459 - data = __be64_to_cpu(offset);
pascal@23757 1460 + data = __be64_to_cpu(ofs);
pascal@23757 1461 write(STDOUT_FILENO, &data, sizeof(data));
pascal@23696 1462 - for (i = 0; i < num_blocks && lseek64(handle, offsets[i].offset, SEEK_SET) >= 0; i++) {
pascal@23696 1463 - read(handle, buffer, offsets[i].size);
pascal@23696 1464 - write(STDOUT_FILENO, buffer, offsets[i].size);
pascal@23696 1465 - n += offsets[i].size;
pascal@23696 1466 - }
pascal@23757 1467 + }
pascal@23757 1468 + for (i = 0; i < num_blocks; i++) {
pascal@23757 1469 +
pascal@23757 1470 + fprintf(stderr, "Block %u %s at %llu length %lu ",
pascal@23765 1471 + i, packnames[offsets[i].flags], offsets[i].offset, offsets[i].size);
pascal@23757 1472 + ulen = get_block(i, argc);
pascal@23757 1473 + fprintf(stderr, " => %lu\n", ulen);
pascal@23757 1474 + write(STDOUT_FILENO, out, ulen);
pascal@23757 1475 + n += ulen;
pascal@23757 1476 + }
pascal@23757 1477 + if (argc > 2) {
pascal@23757 1478 n &= 0x1FF;
pascal@23757 1479 if (n) {
pascal@23757 1480 memset(buffer, 0, 512);
pascal@23757 1481 write(STDOUT_FILENO, buffer, 512 - n);
pascal@23757 1482 }
pascal@23696 1483 - return 0;
pascal@23757 1484 - }
pascal@23696 1485 -
pascal@23757 1486 - for (i = 0; i < num_blocks; i++) {
pascal@23696 1487 - unsigned long destlen = block_size;
pascal@23696 1488 - unsigned int size = offsets[i].size;
pascal@23757 1489 -
pascal@23757 1490 - if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
pascal@23757 1491 - fprintf(stderr, "lseek to %Lu: %s\n",
pascal@23757 1492 - offsets[i].offset, strerror(errno));
pascal@23757 1493 - exit(1);
pascal@23757 1494 - }
pascal@23757 1495 -
pascal@23696 1496 - if (size > zblock_maxsize) {
pascal@23757 1497 - fprintf(stderr,
pascal@23702 1498 - "Size %u for block %u (offset %Lu) too big\n",
pascal@23696 1499 - size, i, offsets[i].offset);
pascal@23757 1500 - exit(1);
pascal@23757 1501 - }
pascal@23696 1502 - read(handle, buffer, size);
pascal@23696 1503 -
pascal@23696 1504 - fprintf(stderr, "Block %u at %llu length %u",
pascal@23696 1505 - i, offsets[i].offset, size);
pascal@23696 1506 - switch (uncompress(clear_buffer, &destlen,
pascal@23696 1507 - buffer, size)) {
pascal@23696 1508 - case Z_OK:
pascal@23696 1509 - break;
pascal@23757 1510 -
pascal@23696 1511 - case Z_MEM_ERROR:
pascal@23696 1512 - fprintf(stderr, "Uncomp: oom block %u\n", i);
pascal@23696 1513 - exit(1);
pascal@23696 1514 -
pascal@23696 1515 - case Z_BUF_ERROR:
pascal@23696 1516 - fprintf(stderr, "Uncomp: not enough out room %u\n", i);
pascal@23696 1517 - exit(1);
pascal@23696 1518 -
pascal@23696 1519 - case Z_DATA_ERROR:
pascal@23696 1520 - fprintf(stderr, "Uncomp: input corrupt %u\n", i);
pascal@23757 1521 - exit(1);
pascal@23757 1522 -
pascal@23696 1523 - default:
pascal@23696 1524 - fprintf(stderr, "Uncomp: unknown error %u\n", i);
pascal@23696 1525 - exit(1);
pascal@23757 1526 - }
pascal@23696 1527 - fprintf(stderr, " => %lu\n", destlen);
pascal@23696 1528 - if (destlen != block_size && i != num_blocks - 1) {
pascal@23696 1529 - fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
pascal@23696 1530 - destlen, block_size);
pascal@23696 1531 - exit(1);
pascal@23757 1532 - }
pascal@23696 1533 - write(STDOUT_FILENO, clear_buffer, (lastlen != 0 && (i+1) == num_blocks) ? lastlen : block_size);
pascal@23696 1534 }
pascal@23696 1535 return 0;
pascal@23696 1536 }
pascal@23696 1537 --- create_compressed_fs.c
pascal@23696 1538 +++ create_compressed_fs.c
pascal@23696 1539 @@ -7,10 +7,11 @@
pascal@23696 1540 #define ZMAX 9
pascal@23696 1541 static shrink_t level;
pascal@23696 1542 static int pass, iter;
pascal@23696 1543 -static int best_compress(unsigned char *compressed,
pascal@23696 1544 - unsigned long *compressed_len,
pascal@23696 1545 - unsigned char *uncompressed,
pascal@23696 1546 - unsigned long uncompressed_len)
pascal@23696 1547 +static int best_compress(Bytef *compressed,
pascal@23696 1548 + uLongf *compressed_len,
pascal@23696 1549 + const Bytef *uncompressed,
pascal@23696 1550 + uLong uncompressed_len,
pascal@23696 1551 + int dummy)
pascal@23696 1552 {
pascal@23696 1553 int i, j, err;
pascal@23696 1554 unsigned char *buf[2];
pascal@23696 1555 @@ -19,6 +20,7 @@
pascal@23696 1556 static unsigned char *buffer;
pascal@23696 1557 static unsigned long buffersz;
pascal@23696 1558
pascal@23696 1559 + (void) dummy;
pascal@23696 1560 if (buffersz < *compressed_len) {
pascal@23696 1561 if (buffer) free(buffer);
pascal@23696 1562 buffer = (unsigned char *) malloc(buffersz = *compressed_len);
pascal@23696 1563 @@ -50,9 +52,95 @@
pascal@23696 1564 memcpy(compressed, buffer, best);
pascal@23696 1565 return err;
pascal@23696 1566 }
pascal@23696 1567 -#define compress2(a,b,c,d,e) best_compress(a,b,c,d)
pascal@23696 1568 +#else
pascal@23696 1569 +#include <stdlib.h>
pascal@23696 1570 +#include <string.h>
pascal@23696 1571 +#include <zlib.h>
pascal@23696 1572 #endif
pascal@23696 1573 +
pascal@23696 1574 +#include <lzma.h>
pascal@23696 1575 +static int xz_compress(Bytef *compressed,
pascal@23696 1576 + uLongf *compressed_len,
pascal@23696 1577 + const Bytef *uncompressed,
pascal@23696 1578 + uLong uncompressed_len,
pascal@23696 1579 + int level)
pascal@23696 1580 +{
pascal@23696 1581 + int res = Z_ERRNO;
pascal@23696 1582 + lzma_stream strm = LZMA_STREAM_INIT;
pascal@23696 1583 +
pascal@23696 1584 + if (lzma_easy_encoder(&strm, LZMA_PRESET_DEFAULT, LZMA_CHECK_CRC32) == LZMA_OK) {
pascal@23696 1585 +
pascal@23696 1586 + strm.next_in = uncompressed;
pascal@23696 1587 + strm.avail_in = uncompressed_len;
pascal@23696 1588 + strm.next_out = compressed;
pascal@23696 1589 + strm.avail_out = *compressed_len;
pascal@23696 1590 +
pascal@23696 1591 + if (lzma_code(&strm, LZMA_FINISH) == LZMA_STREAM_END) {
pascal@23696 1592 +
pascal@23696 1593 + *compressed_len -= strm.avail_out;
pascal@23696 1594 + res = Z_OK;
pascal@23696 1595 + }
pascal@23696 1596 + }
pascal@23696 1597 + lzma_end(&strm);
pascal@23696 1598 + return res;
pascal@23696 1599 +}
pascal@23696 1600 +
pascal@23696 1601 +#include <lz4.h>
pascal@23696 1602 +#include <lz4hc.h>
pascal@23696 1603 +#ifndef LZ4HC_CLEVEL_MAX
pascal@23696 1604 +#define LZ4HC_CLEVEL_MAX 12
pascal@23696 1605 +#endif
pascal@23696 1606 +static int lz4_compress(Bytef *compressed,
pascal@23696 1607 + uLongf *compressed_len,
pascal@23696 1608 + const Bytef *uncompressed,
pascal@23696 1609 + uLong uncompressed_len,
pascal@23696 1610 + int level)
pascal@23696 1611 +{
pascal@23696 1612 + int res = LZ4_compress_HC((const char *) uncompressed, (char *) compressed,
pascal@23696 1613 + uncompressed_len, *compressed_len, LZ4HC_CLEVEL_MAX);
pascal@23696 1614 + (void) level;
pascal@23696 1615 + if (res <= 0) return Z_ERRNO;
pascal@23696 1616 + *compressed_len = res;
pascal@23696 1617 + return Z_OK;
pascal@23696 1618 +}
pascal@23696 1619 +
pascal@23696 1620 +static int setup_compress(Bytef *compressed,
pascal@23696 1621 + uLongf *compressed_len,
pascal@23696 1622 + const Bytef *uncompressed,
pascal@23696 1623 + uLong uncompressed_len,
pascal@23696 1624 + int level);
pascal@23696 1625 +
pascal@23696 1626 +static int (*compress3)(Bytef *compressed,
pascal@23696 1627 + uLongf *compressed_len,
pascal@23696 1628 + const Bytef *uncompressed,
pascal@23696 1629 + uLong uncompressed_len,
pascal@23696 1630 + int level) = setup_compress;
pascal@23696 1631
pascal@23696 1632 +static int setup_compress(Bytef *compressed,
pascal@23696 1633 + uLongf *compressed_len,
pascal@23696 1634 + const Bytef *uncompressed,
pascal@23696 1635 + uLong uncompressed_len,
pascal@23696 1636 + int level)
pascal@23696 1637 +{
pascal@23696 1638 + char *s = getenv("CLOOP_COMP");
pascal@23696 1639 + if (s && !strcmp(s,"XZ")) {
pascal@23696 1640 + compress3 = xz_compress;
pascal@23696 1641 + }
pascal@23696 1642 + else if (s && !strcmp(s,"LZ4")) {
pascal@23696 1643 + compress3 = lz4_compress;
pascal@23696 1644 + }
pascal@23696 1645 + else
pascal@23696 1646 +#ifdef FIND_BEST_COMPRESSION
pascal@23696 1647 + if (s && !strcmp(s,"GZIP")) {
pascal@23696 1648 + compress3 = compress2;
pascal@23696 1649 + }
pascal@23765 1650 + else compress3 = best_compress;
pascal@23696 1651 +#else
pascal@23696 1652 + compress3 = compress2;
pascal@23696 1653 +#endif
pascal@23696 1654 + return compress3(compressed,compressed_len,uncompressed,uncompressed_len,level);
pascal@23696 1655 +}
pascal@23696 1656 +
pascal@23696 1657 #include <signal.h>
pascal@23696 1658
pascal@23696 1659 /* Creates a compressed file */
pascal@23696 1660 @@ -82,9 +170,7 @@
pascal@23696 1661 return i;
pascal@23696 1662 }
pascal@23696 1663
pascal@23696 1664 -#ifdef FIND_BEST_COMPRESSION
pascal@23696 1665 #include "md5sum.c"
pascal@23696 1666 -#endif
pascal@23696 1667
pascal@23696 1668 static unsigned n;
pascal@23696 1669 static unsigned long lastlen, pos;
pascal@23696 1670 @@ -96,15 +182,23 @@
pascal@23696 1671 static char padding[512];
pascal@23696 1672 struct cloop_tail tail;
pascal@23696 1673 unsigned long len;
pascal@23696 1674 + int flags = 0;
pascal@23696 1675
pascal@23696 1676 - fprintf(stderr, "Write index for %lu blocks\n", n);
pascal@23696 1677 + fprintf(stderr, "Write index for %u blocks\n", n);
pascal@23696 1678 if (block_size >= 0x1000000) lastlen = 0;
pascal@23696 1679 - tail.index_size = ntohl(sizeof(*block_index) + 256*(lastlen % 0xFFffFF));
pascal@23696 1680 + if (sig) flags = 0x80;
pascal@23696 1681 + if (compress3 == xz_compress) {
pascal@23696 1682 + flags |= (CLOOP_COMPRESSOR_XZ << 4);
pascal@23696 1683 + }
pascal@23696 1684 + if (compress3 == lz4_compress) {
pascal@23696 1685 + flags |= (CLOOP_COMPRESSOR_LZ4 << 4);
pascal@23696 1686 + }
pascal@23696 1687 + tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
pascal@23696 1688 tail.num_blocks = ntohl(n);
pascal@23696 1689 n *= sizeof(*block_index);
pascal@23696 1690 - len = n + n/1000 + 12;
pascal@23696 1691 + len = GZIP_MAX_BUFFER(n);
pascal@23696 1692 compressed = (unsigned char *) realloc(compressed, len);
pascal@23696 1693 - if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
pascal@23696 1694 + if (!compressed || compress3(compressed, &len, (unsigned char *) block_index,
pascal@23696 1695 n, Z_BEST_SPEED) != Z_OK)
pascal@23696 1696 quit("Index compression failed");
pascal@23696 1697 tail.table_size = ntohl(len);
pascal@23696 1698 @@ -122,11 +216,10 @@
pascal@23696 1699 unsigned char *uncompressed;
pascal@23696 1700 unsigned long len;
pascal@23696 1701 unsigned indexmax, zlenmax;
pascal@23696 1702 -#ifdef FIND_BEST_COMPRESSION
pascal@23696 1703 - unsigned i, j, hashmax;
pascal@23696 1704 + unsigned i, j, hashmax, domd5;
pascal@23696 1705 md5hash *hash;
pascal@23696 1706 -#endif
pascal@23696 1707
pascal@23696 1708 + domd5 = getenv("CLOOP_NOMD5") == NULL;
pascal@23696 1709 #ifdef FIND_BEST_COMPRESSION
pascal@23696 1710 while (argc > 1) {
pascal@23696 1711 if (argv[1][0] == '-') {
pascal@23696 1712 @@ -141,11 +234,11 @@
pascal@23696 1713 }
pascal@23696 1714 argc--;
pascal@23696 1715 if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@23696 1716 - quit("Usage : create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
pascal@23696 1717 + quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5] create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
pascal@23696 1718 #else
pascal@23696 1719 if (argc > 1) {
pascal@23696 1720 if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@23696 1721 - quit("Usage : create_compressed_fs [block size] < input > output");
pascal@23696 1722 + quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] create_compressed_fs [block size] < input > output");
pascal@23696 1723 #endif
pascal@23696 1724 block_size = atoi(argv[1]);
pascal@23696 1725 }
pascal@23696 1726 @@ -164,12 +257,8 @@
pascal@23696 1727 compressed = (unsigned char *) malloc(zlenmax);
pascal@23696 1728 uncompressed = (unsigned char *) malloc(block_size);
pascal@23696 1729 block_index = (u_int32_t *) malloc(indexmax = CHUNK);
pascal@23696 1730 -#ifdef FIND_BEST_COMPRESSION
pascal@23696 1731 hash = (md5hash *) malloc(hashmax = CHUNK);
pascal@23696 1732 if (!compressed || !uncompressed || !block_index || !hash)
pascal@23696 1733 -#else
pascal@23696 1734 - if (!compressed || !uncompressed || !block_index)
pascal@23696 1735 -#endif
pascal@23696 1736 quit("Malloc failed");
pascal@23696 1737
pascal@23696 1738 signal(SIGINT,flush_index);
pascal@23696 1739 @@ -184,39 +273,42 @@
pascal@23696 1740 if (!block_index)
pascal@23696 1741 quit("Realloc");
pascal@23696 1742 }
pascal@23696 1743 -#ifdef FIND_BEST_COMPRESSION
pascal@23696 1744 - if (n * sizeof(*hash) >= hashmax) {
pascal@23696 1745 - hash = (md5hash *) realloc(hash, hashmax += CHUNK);
pascal@23696 1746 - if (!hash)
pascal@23696 1747 - quit("Realloc hash");
pascal@23696 1748 - }
pascal@23696 1749 - hash[n] = md5sum(uncompressed, len);
pascal@23696 1750 - j = 0x7FFFFFFF;
pascal@23696 1751 - if (n < j)
pascal@23696 1752 - j = n;
pascal@23696 1753 - for (i = 0; i < j; i++) {
pascal@23696 1754 - if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
pascal@23696 1755 - && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
pascal@23696 1756 - break;
pascal@23696 1757 - }
pascal@23696 1758 - if (i != j) {
pascal@23696 1759 - block_index[n] = ntohl(0x80000000 | i);
pascal@23696 1760 - fprintf(stderr, "Block %u length %lu => duplicate %lu\n",
pascal@23696 1761 - n, block_size, i);
pascal@23696 1762 + if (domd5) {
pascal@23696 1763 + if (n * sizeof(*hash) >= hashmax) {
pascal@23696 1764 + hash = (md5hash *) realloc(hash, hashmax += CHUNK);
pascal@23696 1765 + if (!hash)
pascal@23696 1766 + quit("Realloc hash");
pascal@23696 1767 + }
pascal@23696 1768 + hash[n] = md5sum(uncompressed, len);
pascal@23696 1769 + j = 0x7FFFFFFF;
pascal@23696 1770 + if (n < j)
pascal@23696 1771 + j = n;
pascal@23696 1772 + for (i = 0; i < j; i++) {
pascal@23696 1773 + if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
pascal@23696 1774 + && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
pascal@23696 1775 + break;
pascal@23696 1776 + }
pascal@23696 1777 + if (i != j) {
pascal@23696 1778 + block_index[n] = ntohl(0x80000000 | i);
pascal@23696 1779 + fprintf(stderr, "Block %u length %lu => duplicate %u\n",
pascal@23696 1780 + n, block_size, i);
pascal@23696 1781 + continue;
pascal@23696 1782 + }
pascal@23696 1783 }
pascal@23696 1784 - else
pascal@23696 1785 -#endif
pascal@23696 1786 - {
pascal@23696 1787 - len = zlenmax;
pascal@23696 1788 - if (compress2(compressed, &len, uncompressed, lastlen,
pascal@23696 1789 - Z_BEST_SPEED) != Z_OK)
pascal@23696 1790 - quit("Compression failed");
pascal@23696 1791 - fprintf(stderr, "Block %u length %lu => %lu\n",
pascal@23696 1792 - n, block_size, len);
pascal@23696 1793 - write(STDOUT_FILENO, compressed, len);
pascal@23696 1794 - pos += len;
pascal@23696 1795 + len = zlenmax;
pascal@23696 1796 + if (compress3(compressed, &len, uncompressed, lastlen,
pascal@23696 1797 + Z_BEST_SPEED) != Z_OK || len >= lastlen) {
pascal@23696 1798 + len = lastlen;
pascal@23696 1799 + block_index[n] = ntohl(0xFFFFFFFF);
pascal@23696 1800 + write(STDOUT_FILENO, uncompressed, len);
pascal@23696 1801 + }
pascal@23696 1802 + else {
pascal@23696 1803 block_index[n] = ntohl(len);
pascal@23696 1804 + write(STDOUT_FILENO, compressed, len);
pascal@23696 1805 }
pascal@23696 1806 + fprintf(stderr, "Block %u length %lu => %lu\n",
pascal@23696 1807 + n, block_size, len);
pascal@23696 1808 + pos += len;
pascal@23696 1809 }
pascal@23696 1810 flush_index(0);
pascal@23696 1811 return 0;
pascal@23762 1812 --- compressed_loop.h
pascal@23762 1813 +++ compressed_loop.h
pascal@23762 1814 @@ -92,9 +92,8 @@
pascal@23762 1815 struct cloop_tail
pascal@23762 1816 {
pascal@23762 1817 u_int32_t table_size;
pascal@23762 1818 - u_int32_t index_size; /* size:4 comp:3 ctrl-c:1 lastlen:24 */
pascal@23762 1819 + u_int32_t index_size; /* size:4 unused:3 ctrl-c:1 lastlen:24 */
pascal@23762 1820 #define CLOOP3_INDEX_SIZE(x) ((unsigned int)((x) & 0xF))
pascal@23762 1821 -#define CLOOP3_BLOCKS_FLAGS(x) ((unsigned int)((x) & 0x70) >> 4)
pascal@23762 1822 #define CLOOP3_TRUNCATED(x) ((unsigned int)((x) & 0x80) >> 7)
pascal@23762 1823 #define CLOOP3_LASTLEN(x) (unsigned int)((x) >> 8)
pascal@23762 1824 u_int32_t num_blocks;
pascal@23762 1825 @@ -107,12 +106,12 @@
pascal@23762 1826 loff_t offset; /* 64-bit offsets of compressed block */
pascal@23762 1827 u_int32_t size; /* 32-bit compressed block size */
pascal@23762 1828 u_int32_t flags; /* 32-bit compression flags */
pascal@23762 1829 -
pascal@23762 1830 };
pascal@23762 1831
pascal@23762 1832 -static inline char *build_index(struct block_info *offsets, unsigned long n,
pascal@23762 1833 - unsigned long block_size, unsigned global_flags)
pascal@23762 1834 +static inline char *build_index(struct block_info *offsets, unsigned long n, unsigned long block_size)
pascal@23762 1835 {
pascal@23762 1836 + static char v[sizeof("64BE v4.0a")];
pascal@23762 1837 + u_int32_t flags;
pascal@23762 1838 u_int32_t *ofs32 = (u_int32_t *) offsets;
pascal@23762 1839 loff_t *ofs64 = (loff_t *) offsets;
pascal@23762 1840
pascal@23762 1841 @@ -137,8 +136,6 @@
pascal@23762 1842 }
pascal@23762 1843 else { /* V2.0/V4.0 */
pascal@23762 1844 loff_t last = CLOOP_BLOCK_OFFSET(__be64_to_cpu(ofs64[n]));
pascal@23762 1845 - u_int32_t flags;
pascal@23762 1846 - static char v4[11];
pascal@23762 1847 unsigned long i = n;
pascal@23762 1848
pascal@23762 1849 for (flags = 0; n-- ;) {
pascal@23762 1850 @@ -156,12 +153,7 @@
pascal@23762 1851 offsets[i] = offsets[offsets[i].offset];
pascal@23762 1852 }
pascal@23762 1853 }
pascal@23762 1854 - strcpy(v4, (char *) "64BE v4.0a");
pascal@23762 1855 - v4[10] = 'a' + ((flags-1) & 0xF); // compressors used
pascal@23762 1856 - if (flags > 0x10) { // with links ?
pascal@23762 1857 - v4[10] += 'A' - 'a';
pascal@23762 1858 - }
pascal@23762 1859 - return v4;
pascal@23762 1860 + strcpy(v, (char *) "64BE v4.0a");
pascal@23762 1861 }
pascal@23762 1862 }
pascal@23762 1863 else if (ofs32[1] == 0 && v3_64 == 0) { /* V1.0 */
pascal@23762 1864 @@ -174,44 +166,50 @@
pascal@23762 1865 }
pascal@23762 1866 return (char *) "64LE v1.0";
pascal@23762 1867 }
pascal@23762 1868 - else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
pascal@23762 1869 - loff_t last = ntohl(ofs32[n]);
pascal@23762 1870 - while (n--) {
pascal@23762 1871 - offsets[n].size = last -
pascal@23762 1872 - (offsets[n].offset = ntohl(ofs32[n]));
pascal@23762 1873 - last = offsets[n].offset;
pascal@23762 1874 - offsets[n].flags = 0;
pascal@23762 1875 - }
pascal@23762 1876 - return (char *) "32BE v0.68";
pascal@23762 1877 - }
pascal@23762 1878 - else { /* V3.0 */
pascal@23762 1879 + else { /* V3.0 or V0.68 */
pascal@23762 1880 unsigned long i;
pascal@23762 1881 loff_t j;
pascal@23762 1882 - static char v3[11];
pascal@23762 1883
pascal@23762 1884 - v3_64 = (ofs32[1] == 0) ? 2 : 1;
pascal@23762 1885 - for (i = n; i-- != 0; )
pascal@23762 1886 - offsets[i].size = ntohl(ofs32[i*v3_64]);
pascal@23762 1887 + for (i = 0; i < n && ntohl(ofs32[i]) < ntohl(ofs32[i+1]); i++);
pascal@23762 1888 + if (i == n && ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
pascal@23762 1889 + loff_t last = ntohl(ofs32[n]);
pascal@23762 1890 + while (n--) {
pascal@23762 1891 + offsets[n].size = last -
pascal@23762 1892 + (offsets[n].offset = ntohl(ofs32[n]));
pascal@23762 1893 + last = offsets[n].offset;
pascal@23762 1894 + offsets[n].flags = 0;
pascal@23762 1895 + }
pascal@23762 1896 + return (char *) "32BE v0.68";
pascal@23762 1897 + }
pascal@23762 1898 +
pascal@23762 1899 + v3_64 = (ofs32[1] == 0);
pascal@23762 1900 + for (i = n; i-- != 0; ) {
pascal@23762 1901 + offsets[i].size = ntohl(ofs32[i << v3_64]);
pascal@23762 1902 + if (offsets[i].size == 0xFFFFFFFF) {
pascal@23762 1903 + offsets[i].size = 0x10000000 | block_size;
pascal@23762 1904 + }
pascal@23762 1905 + offsets[i].flags = (offsets[i].size >> 28);
pascal@23762 1906 + offsets[i].size &= 0x0FFFFFFF;
pascal@23762 1907 + }
pascal@23762 1908 for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
pascal@23762 1909 offsets[i].offset = j;
pascal@23762 1910 - offsets[i].flags = global_flags;
pascal@23762 1911 - if ((offsets[i].size & 0x80000000) == 0) {
pascal@23762 1912 + if (offsets[i].flags < 8) {
pascal@23762 1913 j += offsets[i].size;
pascal@23762 1914 }
pascal@23762 1915 - else if (offsets[i].size == 0xFFFFFFFF) {
pascal@23762 1916 - offsets[i].flags = CLOOP_COMPRESSOR_NONE;
pascal@23762 1917 - j += offsets[i].size = block_size;
pascal@23762 1918 - }
pascal@23762 1919 }
pascal@23762 1920 for (i = 0; i < n; i++) {
pascal@23762 1921 - if (offsets[i].size & 0x80000000) {
pascal@23762 1922 - offsets[i] = offsets[offsets[i].size & 0x7FFFFFFF];
pascal@23762 1923 + flags |= 1 << offsets[i].flags;
pascal@23762 1924 + if (offsets[i].flags >= 8) {
pascal@23762 1925 + offsets[i] = offsets[offsets[i].size];
pascal@23762 1926 }
pascal@23762 1927 }
pascal@23762 1928 - strcpy(v3, (char *) (--v3_64) ? "64BE v3.0a" : "32BE v3.0a");
pascal@23762 1929 - v3[10] += global_flags;
pascal@23762 1930 - return v3;
pascal@23762 1931 + strcpy(v, (char *) (v3_64) ? "64BE v3.0a" : "32BE v3.0a");
pascal@23762 1932 + }
pascal@23765 1933 + v[9] = 'a' + ((flags-1) & 0xF); // compressors used
pascal@23762 1934 + if (flags > 0x10) { // with links ?
pascal@23765 1935 + v[9] += 'A' - 'a';
pascal@23762 1936 }
pascal@23762 1937 + return v;
pascal@23762 1938 }
pascal@23762 1939
pascal@23762 1940 /* Cloop suspend IOCTL */
pascal@23762 1941 --- cloopreader.c
pascal@23762 1942 +++ cloopreader.c
pascal@23762 1943 @@ -52,7 +52,7 @@
pascal@23762 1944 bfuncinfo("fh=%d",fh);
pascal@23762 1945 c->fh=fh;
pascal@23762 1946 struct cloop_head head;
pascal@23762 1947 - int v4_header_last, flags;
pascal@23762 1948 + int v4_header_last;
pascal@23762 1949 loff_t end;
pascal@23762 1950
pascal@23762 1951 for (v4_header_last=0;;v4_header_last++) {
pascal@23762 1952 @@ -83,23 +83,21 @@
pascal@23762 1953 c->numblocks = ntohl(tail.num_blocks);
pascal@23762 1954 c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@23762 1955 len = ntohl(tail.table_size);
pascal@23762 1956 - flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
pascal@23762 1957 toclen = CLOOP3_INDEX_SIZE(ntohl(tail.index_size)) * c->numblocks;
pascal@23762 1958 OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
pascal@23762 1959 ALLOC(c->toc, sizeof(*c->toc) * c->numblocks);
pascal@23762 1960 ALLOC(p,len);
pascal@23762 1961 OP(read_all(c->fh,p,len)); /* read Data Index */
pascal@23762 1962 - if (unpack[flags]((void *)c->toc,&toclen,p,len) != Z_OK)
pascal@23762 1963 + if (unpack[CLOOP_COMPRESSOR_ZLIB]((void *)c->toc,&toclen,p,len) != Z_OK)
pascal@23762 1964 exit(1);
pascal@23762 1965 free(p);
pascal@23762 1966 }
pascal@23762 1967 else {
pascal@23762 1968 - flags = 0;
pascal@23762 1969 c->tocsize = sizeof(*c->toc) * c->numblocks;
pascal@23762 1970 ALLOC(c->toc,c->tocsize);
pascal@23762 1971 OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
pascal@23762 1972 }
pascal@23762 1973 - build_index(c->toc, c->numblocks, c->blocksize, flags);
pascal@23762 1974 + build_index(c->toc, c->numblocks, c->blocksize);
pascal@23762 1975 c->cblocksizecur=0;
pascal@23762 1976 c->curblock=-1;
pascal@23762 1977 return 0;
pascal@23762 1978 --- create_compressed_fs.c
pascal@23762 1979 +++ create_compressed_fs.c
pascal@23764 1980 @@ -58,6 +58,9 @@
pascal@23764 1981 #include <zlib.h>
pascal@23764 1982 #endif
pascal@23764 1983
pascal@23764 1984 +/* Creates a compressed file */
pascal@23764 1985 +#include "common_header.h"
pascal@23764 1986 +
pascal@23764 1987 #include <lzma.h>
pascal@23764 1988 static int xz_compress(Bytef *compressed,
pascal@23764 1989 uLongf *compressed_len,
pascal@23764 1990 @@ -78,6 +81,7 @@
pascal@23763 1991 if (lzma_code(&strm, LZMA_FINISH) == LZMA_STREAM_END) {
pascal@23763 1992
pascal@23763 1993 *compressed_len -= strm.avail_out;
pascal@23763 1994 + *compressed_len |= (CLOOP_COMPRESSOR_XZ << 28);
pascal@23763 1995 res = Z_OK;
pascal@23763 1996 }
pascal@23763 1997 }
pascal@23764 1998 @@ -101,6 +105,7 @@
pascal@23763 1999 (void) level;
pascal@23763 2000 if (res <= 0) return Z_ERRNO;
pascal@23763 2001 *compressed_len = res;
pascal@23763 2002 + *compressed_len |= (CLOOP_COMPRESSOR_LZ4 << 28);
pascal@23763 2003 return Z_OK;
pascal@23763 2004 }
pascal@23763 2005
pascal@23764 2006 @@ -143,9 +148,6 @@
pascal@23764 2007
pascal@23764 2008 #include <signal.h>
pascal@23764 2009
pascal@23764 2010 -/* Creates a compressed file */
pascal@23764 2011 -#include "common_header.h"
pascal@23764 2012 -
pascal@23764 2013 #define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@23764 2014 #define CHUNK 65536
pascal@23764 2015 #define DEFAULT_BLOCKSIZE 65536
pascal@23763 2016 @@ -179,7 +181,6 @@
pascal@23762 2017 static unsigned long block_size = 0;
pascal@23762 2018 static void flush_index(int sig)
pascal@23762 2019 {
pascal@23762 2020 - static char padding[512];
pascal@23762 2021 struct cloop_tail tail;
pascal@23762 2022 unsigned long len;
pascal@23762 2023 int flags = 0;
pascal@23763 2024 @@ -187,24 +188,21 @@
pascal@23762 2025 fprintf(stderr, "Write index for %u blocks\n", n);
pascal@23762 2026 if (block_size >= 0x1000000) lastlen = 0;
pascal@23762 2027 if (sig) flags = 0x80;
pascal@23762 2028 - if (compress3 == xz_compress) {
pascal@23762 2029 - flags |= (CLOOP_COMPRESSOR_XZ << 4);
pascal@23762 2030 - }
pascal@23762 2031 - if (compress3 == lz4_compress) {
pascal@23762 2032 - flags |= (CLOOP_COMPRESSOR_LZ4 << 4);
pascal@23762 2033 - }
pascal@23762 2034 tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
pascal@23762 2035 tail.num_blocks = ntohl(n);
pascal@23762 2036 n *= sizeof(*block_index);
pascal@23762 2037 len = GZIP_MAX_BUFFER(n);
pascal@23762 2038 compressed = (unsigned char *) realloc(compressed, len);
pascal@23762 2039 - if (!compressed || compress3(compressed, &len, (unsigned char *) block_index,
pascal@23762 2040 +#ifdef FIND_BEST_COMPRESSION
pascal@23762 2041 + if (!compressed || best_compress(compressed, &len, (unsigned char *) block_index,
pascal@23762 2042 +#else
pascal@23762 2043 + if (!compressed || compress2(compressed, &len, (unsigned char *) block_index,
pascal@23762 2044 +#endif
pascal@23762 2045 n, Z_BEST_SPEED) != Z_OK)
pascal@23762 2046 quit("Index compression failed");
pascal@23762 2047 tail.table_size = ntohl(len);
pascal@23762 2048 pos += len + sizeof(tail);
pascal@23762 2049 n = pos & 511;
pascal@23762 2050 - // if (n) write(STDOUT_FILENO, padding, 512 - n);
pascal@23762 2051 write(STDOUT_FILENO, compressed, len);
pascal@23762 2052 write(STDOUT_FILENO, &tail, sizeof(tail));
pascal@23762 2053 exit(sig != 0);
pascal@23763 2054 @@ -242,7 +240,7 @@
pascal@23762 2055 #endif
pascal@23762 2056 block_size = atoi(argv[1]);
pascal@23762 2057 }
pascal@23762 2058 - if (block_size < 4096)
pascal@23762 2059 + if (block_size < 4096 || block_size > 0x0FFFFFFF)
pascal@23762 2060 block_size = DEFAULT_BLOCKSIZE;
pascal@23762 2061 fprintf(stderr, "Block size is %lu\n", block_size);
pascal@23762 2062 zlenmax = block_size + block_size/1000 + 12;
pascal@23763 2063 @@ -280,7 +278,7 @@
pascal@23762 2064 quit("Realloc hash");
pascal@23762 2065 }
pascal@23762 2066 hash[n] = md5sum(uncompressed, len);
pascal@23762 2067 - j = 0x7FFFFFFF;
pascal@23762 2068 + j = 0x0FFFFFFF;
pascal@23762 2069 if (n < j)
pascal@23762 2070 j = n;
pascal@23762 2071 for (i = 0; i < j; i++) {
pascal@23763 2072 @@ -289,7 +287,7 @@
pascal@23762 2073 break;
pascal@23762 2074 }
pascal@23762 2075 if (i != j) {
pascal@23762 2076 - block_index[n] = ntohl(0x80000000 | i);
pascal@23763 2077 + block_index[n] = ntohl((CLOOP_COMPRESSOR_LINK << 28) | i);
pascal@23762 2078 fprintf(stderr, "Block %u length %lu => duplicate %u\n",
pascal@23762 2079 n, block_size, i);
pascal@23762 2080 continue;
pascal@23763 2081 @@ -297,14 +295,14 @@
pascal@23763 2082 }
pascal@23763 2083 len = zlenmax;
pascal@23762 2084 if (compress3(compressed, &len, uncompressed, lastlen,
pascal@23763 2085 - Z_BEST_SPEED) != Z_OK || len >= lastlen) {
pascal@23763 2086 + Z_BEST_SPEED) != Z_OK || (len & 0x0FFFFFFF) >= lastlen) {
pascal@23762 2087 len = lastlen;
pascal@23762 2088 - block_index[n] = ntohl(0xFFFFFFFF);
pascal@23763 2089 + block_index[n] = ntohl((CLOOP_COMPRESSOR_NONE << 28) | lastlen);
pascal@23762 2090 write(STDOUT_FILENO, uncompressed, len);
pascal@23762 2091 }
pascal@23762 2092 else {
pascal@23763 2093 block_index[n] = ntohl(len);
pascal@23763 2094 - write(STDOUT_FILENO, compressed, len);
pascal@23763 2095 + write(STDOUT_FILENO, compressed, len &= 0x0FFFFFFF);
pascal@23763 2096 }
pascal@23763 2097 fprintf(stderr, "Block %u length %lu => %lu\n",
pascal@23763 2098 n, block_size, len);
pascal@23762 2099 --- extract_compressed_fs.c
pascal@23762 2100 +++ extract_compressed_fs.c
pascal@23762 2101 @@ -69,9 +69,8 @@
pascal@23762 2102
pascal@23762 2103 int main(int argc, char *argv[])
pascal@23762 2104 {
pascal@23762 2105 - int flags;
pascal@23762 2106 struct cloop_head head;
pascal@23762 2107 - unsigned int i, v4_header_last, global_flags;
pascal@23762 2108 + unsigned int i, v4_header_last;
pascal@23762 2109 unsigned long n, len;
pascal@23762 2110 uLongf ulen;
pascal@23762 2111 loff_t end;
pascal@23762 2112 @@ -128,12 +127,6 @@
pascal@23762 2113 table_size = ntohl(tail.table_size);
pascal@23762 2114 table = malloc(table_size);
pascal@23762 2115 len = num_blocks * CLOOP3_INDEX_SIZE(ntohl(tail.index_size));
pascal@23762 2116 - global_flags = CLOOP3_BLOCKS_FLAGS(ntohl(tail.index_size));
pascal@23762 2117 - if (global_flags > CLOOP_COMPRESSOR_MAX) {
pascal@23762 2118 - fprintf(stderr, "Unsupported compression %d\n",
pascal@23762 2119 - global_flags);
pascal@23762 2120 - exit(1);
pascal@23762 2121 - }
pascal@23762 2122 ulen = num_blocks * sizeof(*offsets);
pascal@23762 2123 offsets = malloc(ulen);
pascal@23762 2124 if (!table || !offsets || !buffer || !clear_buffer) {
pascal@23762 2125 @@ -144,16 +137,15 @@
pascal@23762 2126 perror("Reading index\n");
pascal@23762 2127 exit(1);
pascal@23762 2128 }
pascal@23762 2129 - err = unpack[global_flags]((void *) offsets, &ulen, table, table_size);
pascal@23762 2130 + err = unpack[CLOOP_COMPRESSOR_ZLIB]((void *) offsets, &ulen, table, table_size);
pascal@23762 2131 if (err != Z_OK) {
pascal@23762 2132 fprintf(stderr, "Unpack %s index error %d\n",
pascal@23762 2133 - packnames[global_flags],err);
pascal@23762 2134 + packnames[CLOOP_COMPRESSOR_ZLIB],err);
pascal@23762 2135 exit(1);
pascal@23762 2136 }
pascal@23762 2137 free(table);
pascal@23762 2138 }
pascal@23762 2139 else {
pascal@23762 2140 - global_flags = 0;
pascal@23762 2141 len = num_blocks * sizeof(*offsets);
pascal@23762 2142 offsets = malloc(len);
pascal@23762 2143 if (v4_header_last) {
pascal@23762 2144 @@ -182,7 +174,7 @@
pascal@23762 2145 exit(0);
pascal@23762 2146 }
pascal@23762 2147 #endif
pascal@23762 2148 - fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size, global_flags));
pascal@23762 2149 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size));
pascal@23762 2150
pascal@23762 2151 #if 1
pascal@23762 2152 if (getenv("CLOOP_TABLE") != NULL) {
pascal@24163 2153 === Add v4 support
pascal@24163 2154 --- compressed_loop.h
pascal@24163 2155 +++ compressed_loop.h
pascal@24163 2156 @@ -64,6 +64,7 @@
pascal@24163 2157 * 2 xz compression (currently best space saver) *
pascal@24163 2158 * 3 lz4 compression *
pascal@24163 2159 * 4 lzo compression (fastest) *
pascal@24163 2160 +* 5 zstd compression *
pascal@24163 2161 * 15 block link *
pascal@24163 2162 */
pascal@24163 2163 /* Get value of first 4 bits */
pascal@24163 2164 @@ -76,6 +77,7 @@
pascal@24163 2165 #define CLOOP_COMPRESSOR_XZ 0x2
pascal@24163 2166 #define CLOOP_COMPRESSOR_LZ4 0x3
pascal@24163 2167 #define CLOOP_COMPRESSOR_LZO 0x4
pascal@24163 2168 +#define CLOOP_COMPRESSOR_ZSTD 0x5
pascal@24163 2169 #define CLOOP_COMPRESSOR_LINK 0xF
pascal@24163 2170
pascal@24163 2171
pascal@24163 2172 --- create_compressed_fs.c
pascal@24163 2173 +++ create_compressed_fs.c
pascal@24163 2174 @@ -6,7 +6,9 @@
pascal@24163 2175
pascal@24163 2176 #define ZMAX 9
pascal@24163 2177 static shrink_t level;
pascal@24163 2178 -static int pass, iter;
pascal@24163 2179 +static struct {
pascal@24163 2180 + int pass, iter, v4;
pascal@24163 2181 +} conf;
pascal@24163 2182 static int best_compress(Bytef *compressed,
pascal@24163 2183 uLongf *compressed_len,
pascal@24163 2184 const Bytef *uncompressed,
pascal@24163 2185 @@ -28,7 +30,7 @@
pascal@24163 2186 }
pascal@24163 2187 buf[0] = compressed;
pascal@24163 2188 buf[1] = buffer;
pascal@24163 2189 - for (i = j = 0; i <= ZMAX+3 && (pass == 0 || i < pass); i++) {
pascal@24163 2190 + for (i = j = 0; i <= ZMAX+3 && (conf.pass == 0 || i < conf.pass); i++) {
pascal@24163 2191 llen = len = *compressed_len;
pascal@24163 2192 if (i >= ZMAX+1) {
pascal@24163 2193 level.level = (i == ZMAX+1) ? shrink_normal :
pascal@24163 2194 @@ -56,6 +58,9 @@
pascal@24163 2195 #include <stdlib.h>
pascal@24163 2196 #include <string.h>
pascal@24163 2197 #include <zlib.h>
pascal@24163 2198 +static struct {
pascal@24163 2199 + int v4;
pascal@24163 2200 +} conf;
pascal@24163 2201 #endif
pascal@24163 2202
pascal@24163 2203 /* Creates a compressed file */
pascal@24163 2204 @@ -148,7 +153,8 @@
pascal@24163 2205
pascal@24163 2206 #include <signal.h>
pascal@24163 2207
pascal@24163 2208 -#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@24163 2209 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@24163 2210 +#define CLOOP_PREAMBLE_VERSION 12
pascal@24163 2211 #define CHUNK 65536
pascal@24163 2212 #define DEFAULT_BLOCKSIZE 65536
pascal@24163 2213
pascal@24163 2214 @@ -174,11 +180,22 @@
pascal@24163 2215
pascal@24163 2216 #include "md5sum.c"
pascal@24163 2217
pascal@24163 2218 +//#define CLOOP_DEBUG
pascal@24163 2219 +#ifdef CLOOP_DEBUG
pascal@24163 2220 +static void write_file(const char *s, void *data, unsigned n)
pascal@24163 2221 +{
pascal@24163 2222 + int fd = open(s,O_WRONLY|O_CREAT|O_TRUNC,0600);
pascal@24163 2223 + if (fd == 0) return;
pascal@24163 2224 + write(fd,data,n);
pascal@24163 2225 + close(fd);
pascal@24163 2226 +}
pascal@24163 2227 +#endif
pascal@24163 2228 static unsigned n;
pascal@24163 2229 -static unsigned long lastlen, pos;
pascal@24163 2230 +static unsigned long lastlen, pos = 0;
pascal@24163 2231 static u_int32_t *block_index;
pascal@24163 2232 static unsigned char *compressed;
pascal@24163 2233 static unsigned long block_size = 0;
pascal@24163 2234 +static struct cloop_head head;
pascal@24163 2235 static void flush_index(int sig)
pascal@24163 2236 {
pascal@24163 2237 struct cloop_tail tail;
pascal@24163 2238 @@ -187,11 +204,26 @@
pascal@24163 2239
pascal@24163 2240 fprintf(stderr, "Write index for %u blocks\n", n);
pascal@24163 2241 if (block_size >= 0x1000000) lastlen = 0;
pascal@24163 2242 + if (conf.v4 == 1) {
pascal@24163 2243 + unsigned i;
pascal@24163 2244 + loff_t ofs64;
pascal@24163 2245 + for (i = 0, ofs64 = 0; i < n; ofs64 += (loff_t) ntohl(block_index[i++] & 0xFFffFFf0)) {
pascal@24163 2246 + loff_t index64 = ((block_index[i] & 0xF) == CLOOP_COMPRESSOR_NONE) ? ((loff_t) CLOOP_COMPRESSOR_NONE << 60) : 0;
pascal@24163 2247 + index64 = __le64_to_cpu(index64 + ofs64);
pascal@24163 2248 + write(STDOUT_FILENO, &index64, sizeof(index64));
pascal@24163 2249 + }
pascal@24163 2250 + head.num_blocks = htonl(n);
pascal@24163 2251 + write(STDOUT_FILENO, &head, sizeof(head));
pascal@24163 2252 + exit(sig != 0);
pascal@24163 2253 + }
pascal@24163 2254 if (sig) flags = 0x80;
pascal@24163 2255 tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF));
pascal@24163 2256 tail.num_blocks = ntohl(n);
pascal@24163 2257 n *= sizeof(*block_index);
pascal@24163 2258 len = GZIP_MAX_BUFFER(n);
pascal@24163 2259 +#ifdef CLOOP_DEBUG
pascal@24163 2260 + write_file("cloop.index",block_index,n);
pascal@24163 2261 +#endif
pascal@24163 2262 compressed = (unsigned char *) realloc(compressed, len);
pascal@24163 2263 #ifdef FIND_BEST_COMPRESSION
pascal@24163 2264 if (!compressed || best_compress(compressed, &len, (unsigned char *) block_index,
pascal@24163 2265 @@ -201,8 +233,10 @@
pascal@24163 2266 n, Z_BEST_SPEED) != Z_OK)
pascal@24163 2267 quit("Index compression failed");
pascal@24163 2268 tail.table_size = ntohl(len);
pascal@24163 2269 - pos += len + sizeof(tail);
pascal@24163 2270 - n = pos & 511;
pascal@24163 2271 +#ifdef CLOOP_DEBUG
pascal@24163 2272 + write_file("cloop.zindex",compressed,len);
pascal@24163 2273 + write_file("cloop.tail",&tail,sizeof(tail));
pascal@24163 2274 +#endif
pascal@24163 2275 write(STDOUT_FILENO, compressed, len);
pascal@24163 2276 write(STDOUT_FILENO, &tail, sizeof(tail));
pascal@24163 2277 exit(sig != 0);
pascal@24163 2278 @@ -210,7 +244,6 @@
pascal@24163 2279
pascal@24163 2280 int main(int argc, char *argv[])
pascal@24163 2281 {
pascal@24163 2282 - struct cloop_head head;
pascal@24163 2283 unsigned char *uncompressed;
pascal@24163 2284 unsigned long len;
pascal@24163 2285 unsigned indexmax, zlenmax;
pascal@24163 2286 @@ -221,9 +254,12 @@
pascal@24163 2287 #ifdef FIND_BEST_COMPRESSION
pascal@24163 2288 while (argc > 1) {
pascal@24163 2289 if (argv[1][0] == '-') {
pascal@24163 2290 - int *p = &pass;
pascal@24163 2291 + int *p = &conf.pass;
pascal@24163 2292 switch (argv[1][1]) {
pascal@24163 2293 - case 'i' : p = &iter;
pascal@24163 2294 + case 'v' : conf.v4++;
pascal@24163 2295 + argv++; argc--;
pascal@24163 2296 + continue;
pascal@24163 2297 + case 'i' : p = &conf.iter;
pascal@24163 2298 case 'n' : *p = atoi(argv[2]);
pascal@24163 2299 argc -= 2;
pascal@24163 2300 argv += 2;
pascal@24163 2301 @@ -232,11 +268,23 @@
pascal@24163 2302 }
pascal@24163 2303 argc--;
pascal@24163 2304 if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@24163 2305 - quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5] create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output");
pascal@24163 2306 + quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5=1] "
pascal@24163 2307 +# ifdef CLOOP_DEBUG
pascal@24163 2308 + "[CLOOP_DEBUG=1] "
pascal@24163 2309 +# endif
pascal@24163 2310 + "create_compressed_fs [-v4] [-n <pass>][ -i <iter>] [block size] < input > output");
pascal@24163 2311 #else
pascal@24163 2312 if (argc > 1) {
pascal@24163 2313 + if (strcmp(argv[1],"-v4") == 0) {
pascal@24163 2314 + conf.v4++;
pascal@24163 2315 + argv++; argc--;
pascal@24163 2316 + }
pascal@24163 2317 if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@24163 2318 - quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] create_compressed_fs [block size] < input > output");
pascal@24163 2319 + quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] "
pascal@24163 2320 +# ifdef CLOOP_DEBUG
pascal@24163 2321 + "[CLOOP_DEBUG=1] "
pascal@24163 2322 +# endif
pascal@24163 2323 + "create_compressed_fs [-v4] [block size] < input > output");
pascal@24163 2324 #endif
pascal@24163 2325 block_size = atoi(argv[1]);
pascal@24163 2326 }
pascal@24163 2327 @@ -247,10 +295,13 @@
pascal@24163 2328
pascal@24163 2329 memset(&head, 0, sizeof(head));
pascal@24163 2330 strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@24163 2331 - head.num_blocks = -1;
pascal@24163 2332 head.block_size = htonl(block_size);
pascal@24163 2333 - write(STDOUT_FILENO, &head, sizeof(head));
pascal@24163 2334 - pos = sizeof(head);
pascal@24163 2335 + if (conf.v4 == 0) {
pascal@24163 2336 + head.num_blocks = -1;
pascal@24163 2337 + write(STDOUT_FILENO, &head, sizeof(head));
pascal@24163 2338 + pos = sizeof(head);
pascal@24163 2339 + }
pascal@24163 2340 + else head.preamble[CLOOP_PREAMBLE_VERSION] = '4';
pascal@24163 2341
pascal@24163 2342 compressed = (unsigned char *) malloc(zlenmax);
pascal@24163 2343 uncompressed = (unsigned char *) malloc(block_size);
pascal@24163 2344 @@ -277,14 +328,20 @@
pascal@24163 2345 if (!hash)
pascal@24163 2346 quit("Realloc hash");
pascal@24163 2347 }
pascal@24163 2348 - hash[n] = md5sum(uncompressed, len);
pascal@24163 2349 + hash[n] = md5sum(uncompressed, lastlen);
pascal@24163 2350 j = 0x0FFFFFFF;
pascal@24163 2351 if (n < j)
pascal@24163 2352 j = n;
pascal@24163 2353 for (i = 0; i < j; i++) {
pascal@24163 2354 - if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n]
pascal@24163 2355 - && !memcmp(&hash[i],&hash[n],sizeof(*hash)))
pascal@24163 2356 - break;
pascal@24163 2357 + if (* (uint32_t *) &hash[i] != * (uint32_t *) &hash[n]
pascal@24163 2358 + || memcmp(&hash[i],&hash[n],sizeof(*hash)))
pascal@24163 2359 + continue;
pascal@24163 2360 + len = zlenmax;
pascal@24163 2361 + if (compress3(compressed, &len, uncompressed, lastlen,
pascal@24163 2362 + Z_BEST_SPEED) != Z_OK || (len & 0x0FFFFFFF) >= lastlen) {
pascal@24163 2363 + len = (CLOOP_COMPRESSOR_NONE << 28) | lastlen;
pascal@24163 2364 + }
pascal@24163 2365 + if (block_index[i] == ntohl(len)) break;
pascal@24163 2366 }
pascal@24163 2367 if (i != j) {
pascal@24163 2368 block_index[n] = ntohl((CLOOP_COMPRESSOR_LINK << 28) | i);
pascal@24163 2369 --- extract_compressed_fs.c
pascal@24163 2370 +++ extract_compressed_fs.c
pascal@24163 2371 @@ -67,16 +67,21 @@
pascal@24163 2372 return ulen;
pascal@24163 2373 }
pascal@24163 2374
pascal@24163 2375 +//#define CLOOP_DEBUG
pascal@24163 2376 int main(int argc, char *argv[])
pascal@24163 2377 {
pascal@24163 2378 struct cloop_head head;
pascal@24163 2379 unsigned int i, v4_header_last;
pascal@24163 2380 unsigned long n, len;
pascal@24163 2381 uLongf ulen;
pascal@24163 2382 - loff_t end;
pascal@24163 2383 + off64_t end;
pascal@24163 2384
pascal@24163 2385 if (argc < 2 || argv[1][0] == '-') {
pascal@24163 2386 - fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
pascal@24163 2387 + fprintf(stderr, "Usage: "
pascal@24163 2388 +#ifdef CLOOP_DEBUG
pascal@24163 2389 + "[CLOOP_TAIL=1|CLOOP_ZINDEX=1|CLOOP_INDEX=1|CLOOP_TABLE=1] "
pascal@24163 2390 +#endif
pascal@24163 2391 + "extract_compressed_fs file [--convert-to-v2] > output\n");
pascal@24163 2392 exit(1);
pascal@24163 2393 }
pascal@24163 2394
pascal@24163 2395 @@ -122,6 +127,17 @@
pascal@24163 2396 perror("Reading tail\n");
pascal@24163 2397 exit(1);
pascal@24163 2398 }
pascal@24163 2399 +#ifdef CLOOP_DEBUG
pascal@24163 2400 + if (getenv("CLOOP_TAIL") != NULL) {
pascal@24163 2401 + fprintf(stderr, "Table size:32 %u, index_size:4 %u, unused:3\n",
pascal@24163 2402 + tail.table_size,CLOOP3_INDEX_SIZE(tail.index_size));
pascal@24163 2403 + fprintf(stderr, "Truncated:1 %u, lastlen:24 %u, num_blocks:32 %u end=%llu\n",
pascal@24163 2404 + CLOOP3_TRUNCATED(tail.index_size),CLOOP3_LASTLEN(tail.index_size),tail.num_blocks,end);
pascal@24163 2405 + fprintf(stderr, "CLOOP_TAIL: binary\n");
pascal@24163 2406 + write(STDOUT_FILENO, &tail, sizeof(tail));
pascal@24163 2407 + exit(0);
pascal@24163 2408 + }
pascal@24163 2409 +#endif
pascal@24163 2410 head.num_blocks = tail.num_blocks;
pascal@24163 2411 num_blocks = ntohl(head.num_blocks);
pascal@24163 2412 table_size = ntohl(tail.table_size);
pascal@24163 2413 @@ -137,6 +153,13 @@
pascal@24163 2414 perror("Reading index\n");
pascal@24163 2415 exit(1);
pascal@24163 2416 }
pascal@24163 2417 +#ifdef CLOOP_DEBUG
pascal@24163 2418 + if (getenv("CLOOP_ZINDEX") != NULL) {
pascal@24163 2419 + fprintf(stderr, "CLOOP_ZINDEX: binary\n");
pascal@24163 2420 + write(STDOUT_FILENO, table, table_size);
pascal@24163 2421 + exit(0);
pascal@24163 2422 + }
pascal@24163 2423 +#endif
pascal@24163 2424 err = unpack[CLOOP_COMPRESSOR_ZLIB]((void *) offsets, &ulen, table, table_size);
pascal@24163 2425 if (err != Z_OK) {
pascal@24163 2426 fprintf(stderr, "Unpack %s index error %d\n",
pascal@24163 2427 @@ -167,7 +190,7 @@
pascal@24163 2428
pascal@24163 2429 fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
pascal@24163 2430 num_blocks, block_size, head.preamble);
pascal@24163 2431 -#if 1
pascal@24163 2432 +#ifdef CLOOP_DEBUG
pascal@24163 2433 if (getenv("CLOOP_INDEX") != NULL) {
pascal@24163 2434 fprintf(stderr, "CLOOP_INDEX: binary\n");
pascal@24163 2435 write(STDOUT_FILENO, offsets, len);
pascal@24163 2436 @@ -175,8 +198,8 @@
pascal@24163 2437 }
pascal@24163 2438 #endif
pascal@24163 2439 fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size));
pascal@24163 2440 -
pascal@24163 2441 -#if 1
pascal@24163 2442 +
pascal@24163 2443 +#ifdef CLOOP_DEBUG
pascal@24163 2444 if (getenv("CLOOP_TABLE") != NULL) {
pascal@24163 2445 fprintf(stderr, "CLOOP_TABLE ascii: offset, size, flags\n");
pascal@24163 2446 for (i = 0; i < num_blocks; i++) {