wok-next annotate fusecloop/stuff/fusecloop.u @ rev 10937

fusecloop: large file support
author Pascal Bellard <pascal.bellard@slitaz.org>
date Sun Aug 28 10:58:22 2011 +0200 (2011-08-28)
parents e5f7f1b3d407
children 1e7452b326f8
rev   line source
pascal@10932 1 --- compressed_loop.h
pascal@10932 2 +++ compressed_loop.h
pascal@10935 3 @@ -41,6 +41,73 @@
pascal@10932 4 /* data_index (num_blocks 64bit pointers, network order)... */
pascal@10932 5 /* compressed data (gzip block compressed format)... */
pascal@10932 6
pascal@10933 7 +struct cloop_tail
pascal@10933 8 +{
pascal@10933 9 + u_int32_t index_size;
pascal@10933 10 + u_int32_t num_blocks;
pascal@10933 11 +};
pascal@10933 12 +
pascal@10932 13 +struct block_info
pascal@10932 14 +{
pascal@10932 15 + loff_t offset; /* 64-bit offsets of compressed block */
pascal@10932 16 + u_int32_t size; /* 32-bit compressed block size */
pascal@10932 17 + u_int32_t optidx; /* 32-bit index number */
pascal@10932 18 +};
pascal@10932 19 +
pascal@10935 20 +static inline char *build_index(struct block_info *offsets, unsigned long n)
pascal@10932 21 +{
pascal@10932 22 + u_int32_t *ofs32 = (u_int32_t *) offsets;
pascal@10932 23 + loff_t *ofs64 = (loff_t *) offsets;
pascal@10933 24 +
pascal@10932 25 + if (ofs32[0] == 0) {
pascal@10932 26 + if (ofs32[2]) { /* ACCELERATED KNOPPIX V1.0 */
pascal@10933 27 + while (n--) {
pascal@10932 28 + offsets[n].offset = __be64_to_cpu(offsets[n].offset);
pascal@10932 29 + offsets[n].size = ntohl(offsets[n].size);
pascal@10933 30 + }
pascal@10932 31 + return "128BE accelerated knoppix 1.0";
pascal@10932 32 + }
pascal@10932 33 + else { /* V2.0 */
pascal@10933 34 + loff_t last = __be64_to_cpu(ofs64[n]);
pascal@10933 35 + while (n--) {
pascal@10932 36 + offsets[n].size = last -
pascal@10932 37 + (offsets[n].offset = __be64_to_cpu(ofs64[n]));
pascal@10932 38 + last = offsets[n].offset;
pascal@10933 39 + }
pascal@10932 40 + return "64BE v2.0";
pascal@10932 41 + }
pascal@10932 42 + }
pascal@10932 43 + else if (ofs32[1] == 0) { /* V1.0 */
pascal@10933 44 + loff_t last = __be64_to_cpu(ofs64[n]);
pascal@10933 45 + while (n--) {
pascal@10932 46 + offsets[n].size = last -
pascal@10932 47 + (offsets[n].offset = __le64_to_cpu(ofs64[n]));
pascal@10932 48 + last = offsets[n].offset;
pascal@10933 49 + }
pascal@10932 50 + return "64LE v1.0";
pascal@10932 51 + }
pascal@10933 52 + else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
pascal@10933 53 + loff_t last = ntohl(ofs32[n]);
pascal@10933 54 + while (n--) {
pascal@10932 55 + offsets[n].size = last -
pascal@10932 56 + (offsets[n].offset = ntohl(ofs32[n]));
pascal@10932 57 + last = offsets[n].offset;
pascal@10933 58 + }
pascal@10932 59 + return "32BE v0.68";
pascal@10932 60 + }
pascal@10933 61 + else { /* V3.0 */
pascal@10935 62 + int i, j;
pascal@10933 63 +
pascal@10935 64 + for (i = n; i-- > 0; )
pascal@10935 65 + offsets[i].size = ntohl(ofs32[i]);
pascal@10935 66 + for (i = 0, j = sizeof(struct cloop_head); i < n; i++) {
pascal@10933 67 + offsets[i].offset = j;
pascal@10933 68 + j += offsets[i].size;
pascal@10933 69 + }
pascal@10935 70 + return "32BE v3.0";
pascal@10933 71 + }
pascal@10932 72 +}
pascal@10932 73 +
pascal@10932 74 /* Cloop suspend IOCTL */
pascal@10932 75 #define CLOOP_SUSPEND 0x4C07
pascal@10932 76
pascal@10932 77
pascal@10932 78 --- cloopreader.h
pascal@10932 79 +++ cloopreader.h
pascal@10932 80 @@ -33,7 +33,7 @@
pascal@10932 81 int numblocks;
pascal@10932 82 ulong blocksize;
pascal@10932 83
pascal@10932 84 - loff_t* toc; /* Data index */
pascal@10932 85 + struct block_info *toc; /* Data index */
pascal@10932 86 size_t tocsize;
pascal@10932 87
pascal@10932 88 unsigned char* cblock; /* Compressed block */
pascal@10932 89
pascal@10932 90 --- cloopreader.c
pascal@10932 91 +++ cloopreader.c
pascal@10935 92 @@ -59,10 +59,21 @@
pascal@10932 93
pascal@10932 94 ALLOC(c->pblock,c->blocksize);
pascal@10932 95
pascal@10932 96 - c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */
pascal@10932 97 + c->tocsize=sizeof(*c->toc) * c->numblocks;
pascal@10933 98 + if (c->numblocks == -1) {
pascal@10933 99 + struct cloop_tail tail;
pascal@10935 100 + loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
pascal@10933 101 +
pascal@10935 102 + OP(lseek(c->fh, end - sizeof(tail), SEEK_SET));
pascal@10933 103 + OP(read_all(c->fh, &tail, sizeof(tail)));
pascal@10933 104 + c->numblocks = ntohl(tail.num_blocks);
pascal@10933 105 + c->tocsize = ntohl(tail.index_size) * c->numblocks;
pascal@10935 106 + OP(lseek(c->fh, end - sizeof(tail) - c->tocsize, SEEK_SET));
pascal@10933 107 + }
pascal@10932 108 ALLOC(c->toc,c->tocsize);
pascal@10932 109
pascal@10932 110 OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */
pascal@10935 111 + build_index(c->toc, c->numblocks);
pascal@10932 112 c->cblocksizecur=0;
pascal@10932 113 c->curblock=-1;
pascal@10932 114 return 0;
pascal@10935 115 @@ -79,10 +90,10 @@
pascal@10932 116 if(page>=c->numblocks){errno=EFAULT;return -1;}
pascal@10932 117 c->curblock=page;
pascal@10932 118
pascal@10932 119 - bprintf("Seeking to 0x%Lx\n",btc(c->toc[page]));
pascal@10932 120 - OP(lseek(c->fh,btc(c->toc[page]), SEEK_SET));
pascal@10932 121 + bprintf("Seeking to 0x%Lx\n",c->toc[page].offset);
pascal@10932 122 + OP(lseek(c->fh,c->toc[page].offset, SEEK_SET));
pascal@10932 123
pascal@10932 124 - c->cblocksize=btc(c->toc[page+1]) - btc(c->toc[page]);
pascal@10932 125 + c->cblocksize=c->toc[page].size;
pascal@10932 126 bprintf("Compressed size=%lu\n",c->cblocksize);
pascal@10932 127 if(c->cblocksize > c->cblocksizecur){
pascal@10932 128 if(c->cblocksizecur)free(c->cblock);
pascal@10933 129
pascal@10932 130 --- extract_compressed_fs.c
pascal@10932 131 +++ extract_compressed_fs.c
pascal@10937 132 @@ -1,15 +1,19 @@
pascal@10935 133 /* Extracts a filesystem back from a compressed fs file */
pascal@10937 134 +#define _LARGEFILE64_SOURCE
pascal@10935 135 #include "common_header.h"
pascal@10935 136 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@10935 137
pascal@10935 138 int main(int argc, char *argv[])
pascal@10935 139 {
pascal@10935 140 int handle;
pascal@10932 141 struct cloop_head head;
pascal@10932 142 unsigned int i;
pascal@10935 143 + unsigned long num_blocks, block_size, zblock_maxsize;
pascal@10932 144 unsigned char *buffer, *clear_buffer;
pascal@10932 145 + struct block_info *offsets;
pascal@10932 146
pascal@10935 147 - if (argc != 2) {
pascal@10935 148 - fprintf(stderr, "Need filename\n");
pascal@10935 149 + if (argc < 2 || argv[1][0] == '-') {
pascal@10935 150 + fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n");
pascal@10935 151 exit(1);
pascal@10935 152 }
pascal@10932 153
pascal@10937 154 @@ -24,44 +28,77 @@
pascal@10935 155 exit(1);
pascal@10935 156 }
pascal@10935 157
pascal@10935 158 - buffer = malloc(ntohl(head.block_size) + ntohl(head.block_size)/1000
pascal@10935 159 - + 12 + 4);
pascal@10935 160 - clear_buffer = malloc(ntohl(head.block_size));
pascal@10935 161 - fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n",
pascal@10935 162 - ntohl(head.num_blocks), ntohl(head.block_size), head.preamble);
pascal@10935 163 -
pascal@10935 164 - for (i = 0; i < ntohl(head.num_blocks); i++) {
pascal@10935 165 - int currpos;
pascal@10935 166 - unsigned long destlen = ntohl(head.block_size);
pascal@10935 167 - loff_t offset[2];
pascal@10935 168 - unsigned int size;
pascal@10935 169 + num_blocks = ntohl(head.num_blocks);
pascal@10935 170 + block_size = ntohl(head.block_size);
pascal@10935 171 + zblock_maxsize = block_size + block_size/1000 + 12 + 4;
pascal@10935 172 + buffer = malloc(zblock_maxsize);
pascal@10935 173 + clear_buffer = malloc(block_size);
pascal@10935 174 + fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n",
pascal@10935 175 + num_blocks, block_size, head.preamble);
pascal@10935 176 +
pascal@10935 177 + if (num_blocks == -1) {
pascal@10933 178 + struct cloop_tail tail;
pascal@10937 179 + loff_t end = lseek64(handle, 0, SEEK_END);
pascal@10937 180 + if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
pascal@10933 181 + read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
pascal@10937 182 + lseek64(handle, end - sizeof(tail) -
pascal@10933 183 + (ntohl(tail.num_blocks) * ntohl(tail.index_size)),
pascal@10937 184 + SEEK_SET) < 0) {
pascal@10933 185 + perror("Reading tail\n");
pascal@10933 186 + exit(1);
pascal@10933 187 + }
pascal@10933 188 + head.num_blocks = tail.num_blocks;
pascal@10935 189 + num_blocks = ntohl(head.num_blocks);
pascal@10935 190 + i = num_blocks * ntohl(tail.index_size);
pascal@10933 191 + }
pascal@10935 192 + else i = num_blocks * sizeof(*offsets);
pascal@10932 193 + offsets = malloc(i);
pascal@10932 194 + if (!offsets || read(handle, offsets, i) != i) {
pascal@10932 195 + perror("Reading index\n");
pascal@10932 196 + exit(1);
pascal@10932 197 + }
pascal@10932 198 +
pascal@10935 199 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
pascal@10932 200 +
pascal@10935 201 + if (argc > 2) {
pascal@10936 202 + loff_t data, offset = ((num_blocks + 1) * sizeof(offset)) + sizeof(head);
pascal@10935 203 +
pascal@10935 204 + strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@10935 205 + write(STDOUT_FILENO, &head, sizeof(head));
pascal@10935 206 + for (i = 0; i < num_blocks; i++) {
pascal@10936 207 + data = __be64_to_cpu(offset);
pascal@10936 208 + write(STDOUT_FILENO, &data, sizeof(data));
pascal@10936 209 + offset += offsets[i].size;
pascal@10935 210 + }
pascal@10936 211 + data = __be64_to_cpu(offset);
pascal@10936 212 + write(STDOUT_FILENO, &data, sizeof(data));
pascal@10937 213 + for (i = 0; i < num_blocks && lseek64(handle, offsets[i].offset, SEEK_SET) >= 0; i++) {
pascal@10935 214 + read(handle, buffer, offsets[i].size);
pascal@10935 215 + write(STDOUT_FILENO, buffer, offsets[i].size);
pascal@10935 216 + }
pascal@10935 217 + return 0;
pascal@10935 218 + }
pascal@10935 219 +
pascal@10935 220 + for (i = 0; i < num_blocks; i++) {
pascal@10935 221 + unsigned long destlen = block_size;
pascal@10932 222 + unsigned int size = offsets[i].size;
pascal@10932 223
pascal@10932 224 - read(handle, &offset, 2*sizeof(loff_t));
pascal@10932 225 - lseek(handle, -sizeof(loff_t), SEEK_CUR);
pascal@10932 226 -
pascal@10932 227 - currpos = lseek(handle, 0, SEEK_CUR);
pascal@10932 228 - if (lseek(handle, __be64_to_cpu(offset[0]), SEEK_SET) < 0) {
pascal@10937 229 + if (lseek64(handle, offsets[i].offset, SEEK_SET) < 0) {
pascal@10932 230 fprintf(stderr, "lseek to %Lu: %s\n",
pascal@10932 231 - __be64_to_cpu(offset[0]), strerror(errno));
pascal@10932 232 + offsets[i].offset, strerror(errno));
pascal@10932 233 exit(1);
pascal@10932 234 }
pascal@10932 235
pascal@10932 236 - size=__be64_to_cpu(offset[1])-__be64_to_cpu(offset[0]);
pascal@10935 237 - if (size > ntohl(head.block_size) + ntohl(head.block_size)/1000
pascal@10935 238 - + 12 + 4) {
pascal@10935 239 + if (size > zblock_maxsize) {
pascal@10932 240 fprintf(stderr,
pascal@10932 241 "Size %u for block %u (offset %Lu) too big\n",
pascal@10932 242 - size, i, __be64_to_cpu(offset[0]));
pascal@10932 243 + size, i, offsets[i].offset);
pascal@10932 244 exit(1);
pascal@10932 245 }
pascal@10932 246 read(handle, buffer, size);
pascal@10932 247 - if (lseek(handle, currpos, SEEK_SET) < 0) {
pascal@10932 248 - perror("seeking");
pascal@10932 249 - exit(1);
pascal@10932 250 - }
pascal@10932 251
pascal@10935 252 - fprintf(stderr, "Block %u length %u => %lu\n",
pascal@10935 253 - i, size, destlen);
pascal@10935 254 + fprintf(stderr, "Block %u at %llu length %u => %lu\n",
pascal@10935 255 + i, offsets[i].offset, size, destlen);
pascal@10935 256 if (i == 3) {
pascal@10935 257 fprintf(stderr,
pascal@10935 258 "Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n",
pascal@10937 259 @@ -105,12 +142,12 @@
pascal@10937 260 fprintf(stderr, "Uncomp: unknown error %u\n", i);
pascal@10937 261 exit(1);
pascal@10937 262 }
pascal@10937 263 - if (destlen != ntohl(head.block_size)) {
pascal@10937 264 - fprintf(stderr, "Uncomp: bad len %u (%lu not %u)\n", i,
pascal@10937 265 - destlen, ntohl(head.block_size));
pascal@10937 266 + if (destlen != block_size) {
pascal@10937 267 + fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
pascal@10937 268 + destlen, block_size);
pascal@10937 269 exit(1);
pascal@10937 270 }
pascal@10937 271 - write(STDOUT_FILENO, clear_buffer, ntohl(head.block_size));
pascal@10937 272 + write(STDOUT_FILENO, clear_buffer, block_size);
pascal@10937 273 }
pascal@10937 274 return 0;
pascal@10937 275 }
pascal@10936 276 @@ -105,12 +140,12 @@
pascal@10935 277 fprintf(stderr, "Uncomp: unknown error %u\n", i);
pascal@10935 278 exit(1);
pascal@10935 279 }
pascal@10935 280 - if (destlen != ntohl(head.block_size)) {
pascal@10935 281 - fprintf(stderr, "Uncomp: bad len %u (%lu not %u)\n", i,
pascal@10935 282 - destlen, ntohl(head.block_size));
pascal@10935 283 + if (destlen != block_size) {
pascal@10935 284 + fprintf(stderr, "Uncomp: bad len %u (%lu not %lu)\n", i,
pascal@10935 285 + destlen, block_size);
pascal@10935 286 exit(1);
pascal@10935 287 }
pascal@10935 288 - write(STDOUT_FILENO, clear_buffer, ntohl(head.block_size));
pascal@10935 289 + write(STDOUT_FILENO, clear_buffer, block_size);
pascal@10935 290 }
pascal@10935 291 return 0;
pascal@10935 292 }
pascal@10935 293
pascal@10935 294 --- Makefile
pascal@10935 295 +++ Makefile
pascal@10935 296 @@ -1,16 +1,19 @@
pascal@10935 297 PROGNAME=fusecloop
pascal@10935 298 ARCFILES=*.c *.h *.pl Makefile configure README VERSION HELP INSTALL typescript *.cloop COPYING
pascal@10935 299 -PROGS=fusecloop cloopreaderdemo extract_compressed_fs
pascal@10935 300 +PROGS=fusecloop cloopreaderdemo extract_compressed_fs create_compressed_fs
pascal@10935 301 FUSECFLAGS=`pkg-config fuse --cflags`
pascal@10935 302 FUSELDFLAGS=`pkg-config fuse --libs`
pascal@10935 303
pascal@10935 304 CFLAGS= -Wall
pascal@10935 305
pascal@10935 306 -all: fusecloop extract_compressed_fs
pascal@10935 307 +all: fusecloop extract_compressed_fs create_compressed_fs
pascal@10935 308
pascal@10935 309 extract_compressed_fs: extract_compressed_fs.c
pascal@10935 310 ${CC} ${CFLAGS} ${LDFLAGS} -lz extract_compressed_fs.c -o extract_compressed_fs
pascal@10935 311
pascal@10935 312 +create_compressed_fs: create_compressed_fs.c
pascal@10935 313 + ${CC} ${CFLAGS} ${LDFLAGS} -lz create_compressed_fs.c -o create_compressed_fs
pascal@10935 314 +
pascal@10935 315 fusecloop: fusecloop.c cloopreader.o strver debug.o
pascal@10935 316 ${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop
pascal@10935 317
pascal@10935 318
pascal@10935 319 --- create_compressed_fs.c
pascal@10935 320 +++ create_compressed_fs.c
pascal@10935 321 @@ -0,0 +1,80 @@
pascal@10935 322 +/* Creates a compressed file */
pascal@10935 323 +#include "common_header.h"
pascal@10935 324 +
pascal@10935 325 +#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
pascal@10935 326 +#define CHUNK 65536
pascal@10935 327 +#define DEFAULT_BLOCKSIZE 65536
pascal@10935 328 +
pascal@10935 329 +static void quit(char *s)
pascal@10935 330 +{
pascal@10935 331 + fprintf(stderr, "%s\n", s);
pascal@10935 332 + exit(1);
pascal@10935 333 +}
pascal@10935 334 +
pascal@10935 335 +static int readblock(unsigned char *buffer, int n)
pascal@10935 336 +{
pascal@10935 337 + int i;
pascal@10935 338 +
pascal@10935 339 + memset(buffer, 0, n);
pascal@10935 340 + for (i = 0 ; i < n;) {
pascal@10935 341 + int j = read(STDIN_FILENO, buffer + i, n - i);
pascal@10935 342 + if (j < 0 && errno == EINTR) continue;
pascal@10935 343 + if (j <= 0) break;
pascal@10935 344 + i += j;
pascal@10935 345 + }
pascal@10935 346 + return i;
pascal@10935 347 +}
pascal@10935 348 +
pascal@10935 349 +int main(int argc, char *argv[])
pascal@10935 350 +{
pascal@10935 351 + struct cloop_head head;
pascal@10935 352 + struct cloop_tail tail;
pascal@10935 353 + unsigned long block_size = 0;
pascal@10935 354 + unsigned char *compressed, *uncompressed;
pascal@10935 355 + unsigned long *index;
pascal@10935 356 + int n, indexmax, zlenmax;
pascal@10935 357 +
pascal@10935 358 + if (argc > 1) {
pascal@10935 359 + if (argv[1][0] < '0' || argv[1][0] > '9')
pascal@10935 360 + quit("Usage : create_compressed_fs [block size] < input > output");
pascal@10935 361 + block_size = atoi(argv[1]);
pascal@10935 362 + }
pascal@10935 363 + if (block_size < 4096)
pascal@10935 364 + block_size = DEFAULT_BLOCKSIZE;
pascal@10935 365 + fprintf(stderr, "Block size is %lu\n", block_size);
pascal@10935 366 + zlenmax = block_size + block_size/1000 + 12;
pascal@10935 367 +
pascal@10935 368 + memset(&head, 0, sizeof(head));
pascal@10935 369 + strcpy(head.preamble, CLOOP_PREAMBLE);
pascal@10935 370 + head.num_blocks = -1;
pascal@10935 371 + head.block_size = htonl(block_size);
pascal@10935 372 + write(STDOUT_FILENO, &head, sizeof(head));
pascal@10935 373 +
pascal@10935 374 + compressed = malloc(zlenmax);
pascal@10935 375 + uncompressed = malloc(block_size);
pascal@10935 376 + index = malloc(indexmax = CHUNK);
pascal@10935 377 + if (!compressed || !uncompressed || !index)
pascal@10935 378 + quit("Malloc failed");
pascal@10935 379 +
pascal@10935 380 + for (n = 0; readblock(uncompressed, block_size); n++) {
pascal@10935 381 + unsigned long len = zlenmax;
pascal@10935 382 +
pascal@10935 383 + if (compress2(compressed, &len, uncompressed, block_size,
pascal@10935 384 + Z_BEST_COMPRESSION) != Z_OK)
pascal@10935 385 + quit("Compression failed");
pascal@10935 386 + fprintf(stderr, "Block %u length %lu => %lu\n",
pascal@10935 387 + n, block_size, len);
pascal@10935 388 + write(STDOUT_FILENO, compressed, len);
pascal@10935 389 + if (n * sizeof(*index) >= indexmax) {
pascal@10935 390 + index = realloc(index, indexmax += CHUNK);
pascal@10935 391 + if (!index)
pascal@10935 392 + quit("Realloc");
pascal@10935 393 + }
pascal@10935 394 + index[n] = ntohl(len);
pascal@10935 395 + }
pascal@10935 396 + write(STDOUT_FILENO, index, n * sizeof(*index));
pascal@10935 397 + tail.index_size = ntohl(sizeof(*index));
pascal@10935 398 + tail.num_blocks = ntohl(n);
pascal@10935 399 + write(STDOUT_FILENO, &tail, sizeof(tail));
pascal@10935 400 + return 0;
pascal@10935 401 +}