wok-current diff fusecloop/stuff/fusecloop.u @ rev 12214
fusecloop: compress index too
author | Pascal Bellard <pascal.bellard@slitaz.org> |
---|---|
date | Mon Apr 02 22:48:21 2012 +0200 (2012-04-02) |
parents | 071c9a026a01 |
children | 1e434243dc31 |
line diff
1.1 --- a/fusecloop/stuff/fusecloop.u Tue Oct 25 08:48:31 2011 +0200 1.2 +++ b/fusecloop/stuff/fusecloop.u Mon Apr 02 22:48:21 2012 +0200 1.3 @@ -1,11 +1,12 @@ 1.4 --- compressed_loop.h 1.5 +++ compressed_loop.h 1.6 -@@ -41,6 +41,73 @@ 1.7 +@@ -41,6 +41,74 @@ 1.8 /* data_index (num_blocks 64bit pointers, network order)... */ 1.9 /* compressed data (gzip block compressed format)... */ 1.10 1.11 +struct cloop_tail 1.12 +{ 1.13 ++ u_int32_t table_size; 1.14 + u_int32_t index_size; 1.15 + u_int32_t num_blocks; 1.16 +}; 1.17 @@ -28,7 +29,7 @@ 1.18 + offsets[n].offset = __be64_to_cpu(offsets[n].offset); 1.19 + offsets[n].size = ntohl(offsets[n].size); 1.20 + } 1.21 -+ return "128BE accelerated knoppix 1.0"; 1.22 ++ return (char *) "128BE accelerated knoppix 1.0"; 1.23 + } 1.24 + else { /* V2.0 */ 1.25 + loff_t last = __be64_to_cpu(ofs64[n]); 1.26 @@ -37,7 +38,7 @@ 1.27 + (offsets[n].offset = __be64_to_cpu(ofs64[n])); 1.28 + last = offsets[n].offset; 1.29 + } 1.30 -+ return "64BE v2.0"; 1.31 ++ return (char *) "64BE v2.0"; 1.32 + } 1.33 + } 1.34 + else if (ofs32[1] == 0) { /* V1.0 */ 1.35 @@ -47,7 +48,7 @@ 1.36 + (offsets[n].offset = __le64_to_cpu(ofs64[n])); 1.37 + last = offsets[n].offset; 1.38 + } 1.39 -+ return "64LE v1.0"; 1.40 ++ return (char *) "64LE v1.0"; 1.41 + } 1.42 + else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */ 1.43 + loff_t last = ntohl(ofs32[n]); 1.44 @@ -56,7 +57,7 @@ 1.45 + (offsets[n].offset = ntohl(ofs32[n])); 1.46 + last = offsets[n].offset; 1.47 + } 1.48 -+ return "32BE v0.68"; 1.49 ++ return (char *) "32BE v0.68"; 1.50 + } 1.51 + else { /* V3.0 */ 1.52 + int i, j; 1.53 @@ -67,7 +68,7 @@ 1.54 + offsets[i].offset = j; 1.55 + j += offsets[i].size; 1.56 + } 1.57 -+ return "32BE v3.0"; 1.58 ++ return (char *) "32BE v3.0"; 1.59 + } 1.60 +} 1.61 + 1.62 @@ -89,30 +90,43 @@ 1.63 1.64 --- cloopreader.c 1.65 +++ cloopreader.c 1.66 -@@ -59,10 +59,21 @@ 1.67 +@@ -59,10 +59,32 @@ 1.68 1.69 ALLOC(c->pblock,c->blocksize); 1.70 1.71 - c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */ 1.72 -+ c->tocsize=sizeof(*c->toc) * c->numblocks; 1.73 -+ if (c->numblocks == -1) { 1.74 +- ALLOC(c->toc,c->tocsize); 1.75 ++ if (c->numblocks + 1 == 0) { 1.76 + struct cloop_tail tail; 1.77 + loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */ 1.78 -+ 1.79 ++ void *p; 1.80 ++ ulong toclen, len; 1.81 + 1.82 +- OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 1.83 + OP(lseek(c->fh, end - sizeof(tail), SEEK_SET)); 1.84 + OP(read_all(c->fh, &tail, sizeof(tail))); 1.85 + c->numblocks = ntohl(tail.num_blocks); 1.86 -+ c->tocsize = ntohl(tail.index_size) * c->numblocks; 1.87 -+ OP(lseek(c->fh, end - sizeof(tail) - c->tocsize, SEEK_SET)); 1.88 ++ c->tocsize = sizeof(*c->toc) * c->numblocks; 1.89 ++ len = ntohl(tail.table_size); 1.90 ++ toclen = (ntohl(tail.index_size) & 255) * c->numblocks; 1.91 ++ OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET)); 1.92 ++ ALLOC(c->toc,c->tocsize); 1.93 ++ ALLOC(p,len); 1.94 ++ OP(read_all(c->fh,p,len)); /* read Data Index */ 1.95 ++ if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK) 1.96 ++ exit(1); 1.97 ++ free(p); 1.98 + } 1.99 - ALLOC(c->toc,c->tocsize); 1.100 - 1.101 - OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 1.102 ++ else { 1.103 ++ c->tocsize = sizeof(*c->toc) * c->numblocks; 1.104 ++ ALLOC(c->toc,c->tocsize); 1.105 ++ OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 1.106 ++ } 1.107 + build_index(c->toc, c->numblocks); 1.108 c->cblocksizecur=0; 1.109 c->curblock=-1; 1.110 return 0; 1.111 -@@ -79,10 +90,10 @@ 1.112 +@@ -79,10 +101,10 @@ 1.113 if(page>=c->numblocks){errno=EFAULT;return -1;} 1.114 c->curblock=page; 1.115 1.116 @@ -133,7 +147,7 @@ 1.117 /* Extracts a filesystem back from a compressed fs file */ 1.118 +#define _LARGEFILE64_SOURCE 1.119 #include "common_header.h" 1.120 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.121 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.122 1.123 int main(int argc, char *argv[]) 1.124 { 1.125 @@ -151,7 +165,7 @@ 1.126 exit(1); 1.127 } 1.128 1.129 -@@ -24,44 +28,77 @@ 1.130 +@@ -24,44 +28,91 @@ 1.131 exit(1); 1.132 } 1.133 1.134 @@ -160,42 +174,55 @@ 1.135 - clear_buffer = malloc(ntohl(head.block_size)); 1.136 - fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n", 1.137 - ntohl(head.num_blocks), ntohl(head.block_size), head.preamble); 1.138 -- 1.139 ++ num_blocks = ntohl(head.num_blocks); 1.140 ++ block_size = ntohl(head.block_size); 1.141 ++ zblock_maxsize = block_size + block_size/1000 + 12 + 4; 1.142 ++ buffer = malloc(zblock_maxsize); 1.143 ++ clear_buffer = malloc(block_size); 1.144 + 1.145 - for (i = 0; i < ntohl(head.num_blocks); i++) { 1.146 - int currpos; 1.147 - unsigned long destlen = ntohl(head.block_size); 1.148 - loff_t offset[2]; 1.149 - unsigned int size; 1.150 -+ num_blocks = ntohl(head.num_blocks); 1.151 -+ block_size = ntohl(head.block_size); 1.152 -+ zblock_maxsize = block_size + block_size/1000 + 12 + 4; 1.153 -+ buffer = malloc(zblock_maxsize); 1.154 -+ clear_buffer = malloc(block_size); 1.155 -+ fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 1.156 -+ num_blocks, block_size, head.preamble); 1.157 -+ 1.158 -+ if (num_blocks == -1) { 1.159 ++ if (num_blocks == (unsigned long) -1) { 1.160 ++ void *table; 1.161 + struct cloop_tail tail; 1.162 ++ unsigned long len, table_size; 1.163 + loff_t end = lseek64(handle, 0, SEEK_END); 1.164 ++ 1.165 + if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 || 1.166 + read(handle, &tail, sizeof(tail)) != sizeof(tail) || 1.167 + lseek64(handle, end - sizeof(tail) - 1.168 -+ (ntohl(tail.num_blocks) * ntohl(tail.index_size)), 1.169 -+ SEEK_SET) < 0) { 1.170 ++ ntohl(tail.table_size), SEEK_SET) < 0) { 1.171 + perror("Reading tail\n"); 1.172 + exit(1); 1.173 + } 1.174 + head.num_blocks = tail.num_blocks; 1.175 + num_blocks = ntohl(head.num_blocks); 1.176 -+ i = num_blocks * ntohl(tail.index_size); 1.177 ++ table_size = ntohl(tail.table_size); 1.178 ++ table = malloc(table_size); 1.179 ++ len = i = num_blocks * (ntohl(tail.index_size) & 255); 1.180 ++ offsets = malloc(num_blocks * sizeof(*offsets)); 1.181 ++ if (!table || !offsets || 1.182 ++ read(handle, table, table_size) != table_size || 1.183 ++ uncompress((void *)offsets, &len, table, table_size) != Z_OK || 1.184 ++ len != i) { 1.185 ++ perror("Reading index\n"); 1.186 ++ exit(1); 1.187 ++ } 1.188 ++ free(table); 1.189 + } 1.190 -+ else i = num_blocks * sizeof(*offsets); 1.191 -+ offsets = malloc(i); 1.192 -+ if (!offsets || read(handle, offsets, i) != i) { 1.193 -+ perror("Reading index\n"); 1.194 -+ exit(1); 1.195 ++ else { 1.196 ++ offsets = malloc(i = num_blocks * sizeof(*offsets)); 1.197 ++ if (!offsets || read(handle, offsets, i) != i) { 1.198 ++ perror("Reading index\n"); 1.199 ++ exit(1); 1.200 ++ } 1.201 + } 1.202 + 1.203 ++ fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 1.204 ++ num_blocks, block_size, head.preamble); 1.205 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks)); 1.206 + 1.207 + if (argc > 2) { 1.208 @@ -256,7 +283,7 @@ 1.209 if (i == 3) { 1.210 fprintf(stderr, 1.211 "Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n", 1.212 -@@ -105,12 +142,12 @@ 1.213 +@@ -105,12 +156,12 @@ 1.214 fprintf(stderr, "Uncomp: unknown error %u\n", i); 1.215 exit(1); 1.216 } 1.217 @@ -299,17 +326,68 @@ 1.218 ${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop 1.219 1.220 1.221 + 1.222 --- create_compressed_fs.c 1.223 +++ create_compressed_fs.c 1.224 -@@ -0,0 +1,80 @@ 1.225 +@@ -0,0 +1,147 @@ 1.226 ++#ifdef FIND_BEST_COMPRESSION 1.227 ++#include <compress.h> 1.228 ++extern "C" { 1.229 ++#include <stdlib.h> 1.230 ++#include <string.h> 1.231 ++ 1.232 ++static int best_compress(unsigned char *compressed, 1.233 ++ unsigned long *compressed_len, 1.234 ++ unsigned char *uncompressed, 1.235 ++ unsigned long uncompressed_len) 1.236 ++{ 1.237 ++ int i, j, err; 1.238 ++ unsigned char *buf[2]; 1.239 ++ unsigned len; 1.240 ++ unsigned long llen, best = *compressed_len * 2; 1.241 ++ static unsigned char *buffer; 1.242 ++ static unsigned long buffersz; 1.243 ++ 1.244 ++ if (buffersz < *compressed_len) { 1.245 ++ if (buffer) free(buffer); 1.246 ++ buffer = (unsigned char *) malloc(buffersz = *compressed_len); 1.247 ++ if (!buffer) return Z_MEM_ERROR; 1.248 ++ } 1.249 ++ buf[0] = compressed; 1.250 ++ buf[1] = buffer; 1.251 ++ for (i = j = 0; i <= 10; i++) { 1.252 ++ llen = len = *compressed_len; 1.253 ++ if (i == 10) 1.254 ++ err = (compress_zlib(shrink_extreme, buf[j], 1.255 ++ len, uncompressed, 1.256 ++ uncompressed_len)) ? Z_OK : Z_DATA_ERROR; 1.257 ++ else { 1.258 ++ err = compress2(buf[j], &llen, uncompressed, 1.259 ++ uncompressed_len, i); 1.260 ++ len = llen; 1.261 ++ } 1.262 ++ if (err != Z_OK) return err; 1.263 ++ if (len < best) { 1.264 ++ best = len; 1.265 ++ j = 1 - j; 1.266 ++ } 1.267 ++ } 1.268 ++ *compressed_len = best; 1.269 ++ if (j == 0) 1.270 ++ memcpy(compressed, buffer, best); 1.271 ++ return err; 1.272 ++} 1.273 ++#define compress2(a,b,c,d,e) best_compress(a,b,c,d) 1.274 ++#endif 1.275 ++ 1.276 +/* Creates a compressed file */ 1.277 +#include "common_header.h" 1.278 + 1.279 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.280 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.281 +#define CHUNK 65536 1.282 +#define DEFAULT_BLOCKSIZE 65536 1.283 + 1.284 -+static void quit(char *s) 1.285 ++static void quit(const char *s) 1.286 +{ 1.287 + fprintf(stderr, "%s\n", s); 1.288 + exit(1); 1.289 @@ -337,6 +415,8 @@ 1.290 + unsigned char *compressed, *uncompressed; 1.291 + unsigned long *index; 1.292 + int n, indexmax, zlenmax; 1.293 ++ unsigned long len, pos; 1.294 ++ static char padding[512]; 1.295 + 1.296 + if (argc > 1) { 1.297 + if (argv[1][0] < '0' || argv[1][0] > '9') 1.298 @@ -353,35 +433,51 @@ 1.299 + head.num_blocks = -1; 1.300 + head.block_size = htonl(block_size); 1.301 + write(STDOUT_FILENO, &head, sizeof(head)); 1.302 ++ pos = sizeof(head); 1.303 + 1.304 -+ compressed = malloc(zlenmax); 1.305 -+ uncompressed = malloc(block_size); 1.306 -+ index = malloc(indexmax = CHUNK); 1.307 ++ compressed = (unsigned char *) malloc(zlenmax); 1.308 ++ uncompressed = (unsigned char *) malloc(block_size); 1.309 ++ index = (unsigned long *) malloc(indexmax = CHUNK); 1.310 + if (!compressed || !uncompressed || !index) 1.311 + quit("Malloc failed"); 1.312 + 1.313 + for (n = 0; readblock(uncompressed, block_size); n++) { 1.314 -+ unsigned long len = zlenmax; 1.315 -+ 1.316 ++ len = zlenmax; 1.317 + if (compress2(compressed, &len, uncompressed, block_size, 1.318 + Z_BEST_COMPRESSION) != Z_OK) 1.319 + quit("Compression failed"); 1.320 + fprintf(stderr, "Block %u length %lu => %lu\n", 1.321 + n, block_size, len); 1.322 + write(STDOUT_FILENO, compressed, len); 1.323 ++ pos += len; 1.324 + if (n * sizeof(*index) >= indexmax) { 1.325 -+ index = realloc(index, indexmax += CHUNK); 1.326 ++ index = (unsigned long *) realloc(index, 1.327 ++ indexmax += CHUNK); 1.328 + if (!index) 1.329 + quit("Realloc"); 1.330 + } 1.331 + index[n] = ntohl(len); 1.332 + } 1.333 -+ write(STDOUT_FILENO, index, n * sizeof(*index)); 1.334 + tail.index_size = ntohl(sizeof(*index)); 1.335 + tail.num_blocks = ntohl(n); 1.336 ++ n *= sizeof(*index); 1.337 ++ len = n + n/1000 + 12; 1.338 ++ compressed = (unsigned char *) realloc(compressed, n); 1.339 ++ if (!compressed || compress2(compressed, &len, (unsigned char *) index, 1.340 ++ n, Z_BEST_COMPRESSION) != Z_OK) 1.341 ++ quit("Index compression failed"); 1.342 ++ tail.table_size = ntohl(len); 1.343 ++ pos += len + sizeof(tail); 1.344 ++ n = pos & 511; 1.345 ++ if (n) write(STDOUT_FILENO, padding, 512 - n); 1.346 ++ write(STDOUT_FILENO, compressed, len); 1.347 + write(STDOUT_FILENO, &tail, sizeof(tail)); 1.348 + return 0; 1.349 +} 1.350 ++#ifdef FIND_BEST_COMPRESSION 1.351 ++} 1.352 ++#endif 1.353 + 1.354 --- fusecloop.c 1.355 +++ fusecloop.c 1.356 @@ -65,7 +65,7 @@