wok-next rev 12214
fusecloop: compress index too
| author | Pascal Bellard <pascal.bellard@slitaz.org> |
|---|---|
| date | Mon Apr 02 22:48:21 2012 +0200 (2012-04-02) |
| parents | 2dde678153c7 |
| children | 188ed036ce84 |
| files | fusecloop/receipt fusecloop/stuff/fusecloop.u |
line diff
1.1 --- a/fusecloop/receipt Sun Apr 01 12:41:48 2012 +0200 1.2 +++ b/fusecloop/receipt Mon Apr 02 22:48:21 2012 +0200 1.3 @@ -8,7 +8,10 @@ 1.4 TARBALL="$PACKAGE-$VERSION.tar.gz" 1.5 WEB_SITE="http://fusecloop.sourceforge.net/" 1.6 WGET_URL="$SF_MIRROR/$PACKAGE/$TARBALL" 1.7 -DEPENDS="fuse zlib" 1.8 +ADVANCECOMP_VERSION="1.15" 1.9 +ADVANCECOMP_TARBALL="advancecomp-$ADVANCECOMP_VERSION.tar.gz" 1.10 +ADVANCECOMP_URL="$SF_MIRROR/advancemame/$ADVANCECOMP_TARBALL" 1.11 +DEPENDS="fuse zlib gcc-lib-base" 1.12 BUILD_DEPENDS="fuse-dev zlib-dev" 1.13 SUGGESTED="fuseiso" 1.14 1.15 @@ -16,11 +19,22 @@ 1.16 compile_rules() 1.17 { 1.18 cd $src 1.19 + [ -s $SOURCES_REPOSITORY/$ADVANCECOMP_TARBALL ] || 1.20 + wget -P $SOURCES_REPOSITORY $ADVANCECOMP_URL 1.21 + tar xzf $SOURCES_REPOSITORY/$ADVANCECOMP_TARBALL 1.22 sed -i 's/dprintf/d_printf/g' *.h *.c 1.23 patch -p0 < $stuff/fusecloop.u 1.24 + ADVANCECOMP=advancecomp-$ADVANCECOMP_VERSION 1.25 + cp *.h *.c $ADVANCECOMP 1.26 + cp create_compressed_fs.c $ADVANCECOMP/redef.cc 1.27 + sed -i 's/def FIND_BEST_COMPRESSION/ 1/' $ADVANCECOMP/redef.cc 1.28 ./configure --prefix=/usr --infodir=/usr/share/info \ 1.29 --mandir=/usr/share/man $CONFIGURE_ARGS && 1.30 - make 1.31 + make && 1.32 + cd $ADVANCECOMP && 1.33 + ./configure --prefix=/usr --infodir=/usr/share/info \ 1.34 + --mandir=/usr/share/man $CONFIGURE_ARGS && 1.35 + make advdef 1.36 } 1.37 1.38 # Rules to gen a SliTaz package suitable for Tazpkg. 1.39 @@ -29,6 +43,6 @@ 1.40 mkdir -p $fs/usr/bin 1.41 cp $src/fusecloop $fs/usr/bin 1.42 cp $src/extract_compressed_fs $fs/usr/bin 1.43 - cp $src/create_compressed_fs $fs/usr/bin 1.44 + cp $src/advancecomp-*/advdef $fs/usr/bin/create_compressed_fs 1.45 } 1.46
2.1 --- a/fusecloop/stuff/fusecloop.u Sun Apr 01 12:41:48 2012 +0200 2.2 +++ b/fusecloop/stuff/fusecloop.u Mon Apr 02 22:48:21 2012 +0200 2.3 @@ -1,11 +1,12 @@ 2.4 --- compressed_loop.h 2.5 +++ compressed_loop.h 2.6 -@@ -41,6 +41,73 @@ 2.7 +@@ -41,6 +41,74 @@ 2.8 /* data_index (num_blocks 64bit pointers, network order)... */ 2.9 /* compressed data (gzip block compressed format)... */ 2.10 2.11 +struct cloop_tail 2.12 +{ 2.13 ++ u_int32_t table_size; 2.14 + u_int32_t index_size; 2.15 + u_int32_t num_blocks; 2.16 +}; 2.17 @@ -28,7 +29,7 @@ 2.18 + offsets[n].offset = __be64_to_cpu(offsets[n].offset); 2.19 + offsets[n].size = ntohl(offsets[n].size); 2.20 + } 2.21 -+ return "128BE accelerated knoppix 1.0"; 2.22 ++ return (char *) "128BE accelerated knoppix 1.0"; 2.23 + } 2.24 + else { /* V2.0 */ 2.25 + loff_t last = __be64_to_cpu(ofs64[n]); 2.26 @@ -37,7 +38,7 @@ 2.27 + (offsets[n].offset = __be64_to_cpu(ofs64[n])); 2.28 + last = offsets[n].offset; 2.29 + } 2.30 -+ return "64BE v2.0"; 2.31 ++ return (char *) "64BE v2.0"; 2.32 + } 2.33 + } 2.34 + else if (ofs32[1] == 0) { /* V1.0 */ 2.35 @@ -47,7 +48,7 @@ 2.36 + (offsets[n].offset = __le64_to_cpu(ofs64[n])); 2.37 + last = offsets[n].offset; 2.38 + } 2.39 -+ return "64LE v1.0"; 2.40 ++ return (char *) "64LE v1.0"; 2.41 + } 2.42 + else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */ 2.43 + loff_t last = ntohl(ofs32[n]); 2.44 @@ -56,7 +57,7 @@ 2.45 + (offsets[n].offset = ntohl(ofs32[n])); 2.46 + last = offsets[n].offset; 2.47 + } 2.48 -+ return "32BE v0.68"; 2.49 ++ return (char *) "32BE v0.68"; 2.50 + } 2.51 + else { /* V3.0 */ 2.52 + int i, j; 2.53 @@ -67,7 +68,7 @@ 2.54 + offsets[i].offset = j; 2.55 + j += offsets[i].size; 2.56 + } 2.57 -+ return "32BE v3.0"; 2.58 ++ return (char *) "32BE v3.0"; 2.59 + } 2.60 +} 2.61 + 2.62 @@ -89,30 +90,43 @@ 2.63 2.64 --- cloopreader.c 2.65 +++ cloopreader.c 2.66 -@@ -59,10 +59,21 @@ 2.67 +@@ -59,10 +59,32 @@ 2.68 2.69 ALLOC(c->pblock,c->blocksize); 2.70 2.71 - c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */ 2.72 -+ c->tocsize=sizeof(*c->toc) * c->numblocks; 2.73 -+ if (c->numblocks == -1) { 2.74 +- ALLOC(c->toc,c->tocsize); 2.75 ++ if (c->numblocks + 1 == 0) { 2.76 + struct cloop_tail tail; 2.77 + loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */ 2.78 -+ 2.79 ++ void *p; 2.80 ++ ulong toclen, len; 2.81 + 2.82 +- OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 2.83 + OP(lseek(c->fh, end - sizeof(tail), SEEK_SET)); 2.84 + OP(read_all(c->fh, &tail, sizeof(tail))); 2.85 + c->numblocks = ntohl(tail.num_blocks); 2.86 -+ c->tocsize = ntohl(tail.index_size) * c->numblocks; 2.87 -+ OP(lseek(c->fh, end - sizeof(tail) - c->tocsize, SEEK_SET)); 2.88 ++ c->tocsize = sizeof(*c->toc) * c->numblocks; 2.89 ++ len = ntohl(tail.table_size); 2.90 ++ toclen = (ntohl(tail.index_size) & 255) * c->numblocks; 2.91 ++ OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET)); 2.92 ++ ALLOC(c->toc,c->tocsize); 2.93 ++ ALLOC(p,len); 2.94 ++ OP(read_all(c->fh,p,len)); /* read Data Index */ 2.95 ++ if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK) 2.96 ++ exit(1); 2.97 ++ free(p); 2.98 + } 2.99 - ALLOC(c->toc,c->tocsize); 2.100 - 2.101 - OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 2.102 ++ else { 2.103 ++ c->tocsize = sizeof(*c->toc) * c->numblocks; 2.104 ++ ALLOC(c->toc,c->tocsize); 2.105 ++ OP(read_all(c->fh,c->toc,c->tocsize)); /* read Data Index */ 2.106 ++ } 2.107 + build_index(c->toc, c->numblocks); 2.108 c->cblocksizecur=0; 2.109 c->curblock=-1; 2.110 return 0; 2.111 -@@ -79,10 +90,10 @@ 2.112 +@@ -79,10 +101,10 @@ 2.113 if(page>=c->numblocks){errno=EFAULT;return -1;} 2.114 c->curblock=page; 2.115 2.116 @@ -133,7 +147,7 @@ 2.117 /* Extracts a filesystem back from a compressed fs file */ 2.118 +#define _LARGEFILE64_SOURCE 2.119 #include "common_header.h" 2.120 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 2.121 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 2.122 2.123 int main(int argc, char *argv[]) 2.124 { 2.125 @@ -151,7 +165,7 @@ 2.126 exit(1); 2.127 } 2.128 2.129 -@@ -24,44 +28,77 @@ 2.130 +@@ -24,44 +28,91 @@ 2.131 exit(1); 2.132 } 2.133 2.134 @@ -160,42 +174,55 @@ 2.135 - clear_buffer = malloc(ntohl(head.block_size)); 2.136 - fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n", 2.137 - ntohl(head.num_blocks), ntohl(head.block_size), head.preamble); 2.138 -- 2.139 ++ num_blocks = ntohl(head.num_blocks); 2.140 ++ block_size = ntohl(head.block_size); 2.141 ++ zblock_maxsize = block_size + block_size/1000 + 12 + 4; 2.142 ++ buffer = malloc(zblock_maxsize); 2.143 ++ clear_buffer = malloc(block_size); 2.144 + 2.145 - for (i = 0; i < ntohl(head.num_blocks); i++) { 2.146 - int currpos; 2.147 - unsigned long destlen = ntohl(head.block_size); 2.148 - loff_t offset[2]; 2.149 - unsigned int size; 2.150 -+ num_blocks = ntohl(head.num_blocks); 2.151 -+ block_size = ntohl(head.block_size); 2.152 -+ zblock_maxsize = block_size + block_size/1000 + 12 + 4; 2.153 -+ buffer = malloc(zblock_maxsize); 2.154 -+ clear_buffer = malloc(block_size); 2.155 -+ fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 2.156 -+ num_blocks, block_size, head.preamble); 2.157 -+ 2.158 -+ if (num_blocks == -1) { 2.159 ++ if (num_blocks == (unsigned long) -1) { 2.160 ++ void *table; 2.161 + struct cloop_tail tail; 2.162 ++ unsigned long len, table_size; 2.163 + loff_t end = lseek64(handle, 0, SEEK_END); 2.164 ++ 2.165 + if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 || 2.166 + read(handle, &tail, sizeof(tail)) != sizeof(tail) || 2.167 + lseek64(handle, end - sizeof(tail) - 2.168 -+ (ntohl(tail.num_blocks) * ntohl(tail.index_size)), 2.169 -+ SEEK_SET) < 0) { 2.170 ++ ntohl(tail.table_size), SEEK_SET) < 0) { 2.171 + perror("Reading tail\n"); 2.172 + exit(1); 2.173 + } 2.174 + head.num_blocks = tail.num_blocks; 2.175 + num_blocks = ntohl(head.num_blocks); 2.176 -+ i = num_blocks * ntohl(tail.index_size); 2.177 ++ table_size = ntohl(tail.table_size); 2.178 ++ table = malloc(table_size); 2.179 ++ len = i = num_blocks * (ntohl(tail.index_size) & 255); 2.180 ++ offsets = malloc(num_blocks * sizeof(*offsets)); 2.181 ++ if (!table || !offsets || 2.182 ++ read(handle, table, table_size) != table_size || 2.183 ++ uncompress((void *)offsets, &len, table, table_size) != Z_OK || 2.184 ++ len != i) { 2.185 ++ perror("Reading index\n"); 2.186 ++ exit(1); 2.187 ++ } 2.188 ++ free(table); 2.189 + } 2.190 -+ else i = num_blocks * sizeof(*offsets); 2.191 -+ offsets = malloc(i); 2.192 -+ if (!offsets || read(handle, offsets, i) != i) { 2.193 -+ perror("Reading index\n"); 2.194 -+ exit(1); 2.195 ++ else { 2.196 ++ offsets = malloc(i = num_blocks * sizeof(*offsets)); 2.197 ++ if (!offsets || read(handle, offsets, i) != i) { 2.198 ++ perror("Reading index\n"); 2.199 ++ exit(1); 2.200 ++ } 2.201 + } 2.202 + 2.203 ++ fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 2.204 ++ num_blocks, block_size, head.preamble); 2.205 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks)); 2.206 + 2.207 + if (argc > 2) { 2.208 @@ -256,7 +283,7 @@ 2.209 if (i == 3) { 2.210 fprintf(stderr, 2.211 "Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n", 2.212 -@@ -105,12 +142,12 @@ 2.213 +@@ -105,12 +156,12 @@ 2.214 fprintf(stderr, "Uncomp: unknown error %u\n", i); 2.215 exit(1); 2.216 } 2.217 @@ -299,17 +326,68 @@ 2.218 ${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop 2.219 2.220 2.221 + 2.222 --- create_compressed_fs.c 2.223 +++ create_compressed_fs.c 2.224 -@@ -0,0 +1,80 @@ 2.225 +@@ -0,0 +1,147 @@ 2.226 ++#ifdef FIND_BEST_COMPRESSION 2.227 ++#include <compress.h> 2.228 ++extern "C" { 2.229 ++#include <stdlib.h> 2.230 ++#include <string.h> 2.231 ++ 2.232 ++static int best_compress(unsigned char *compressed, 2.233 ++ unsigned long *compressed_len, 2.234 ++ unsigned char *uncompressed, 2.235 ++ unsigned long uncompressed_len) 2.236 ++{ 2.237 ++ int i, j, err; 2.238 ++ unsigned char *buf[2]; 2.239 ++ unsigned len; 2.240 ++ unsigned long llen, best = *compressed_len * 2; 2.241 ++ static unsigned char *buffer; 2.242 ++ static unsigned long buffersz; 2.243 ++ 2.244 ++ if (buffersz < *compressed_len) { 2.245 ++ if (buffer) free(buffer); 2.246 ++ buffer = (unsigned char *) malloc(buffersz = *compressed_len); 2.247 ++ if (!buffer) return Z_MEM_ERROR; 2.248 ++ } 2.249 ++ buf[0] = compressed; 2.250 ++ buf[1] = buffer; 2.251 ++ for (i = j = 0; i <= 10; i++) { 2.252 ++ llen = len = *compressed_len; 2.253 ++ if (i == 10) 2.254 ++ err = (compress_zlib(shrink_extreme, buf[j], 2.255 ++ len, uncompressed, 2.256 ++ uncompressed_len)) ? Z_OK : Z_DATA_ERROR; 2.257 ++ else { 2.258 ++ err = compress2(buf[j], &llen, uncompressed, 2.259 ++ uncompressed_len, i); 2.260 ++ len = llen; 2.261 ++ } 2.262 ++ if (err != Z_OK) return err; 2.263 ++ if (len < best) { 2.264 ++ best = len; 2.265 ++ j = 1 - j; 2.266 ++ } 2.267 ++ } 2.268 ++ *compressed_len = best; 2.269 ++ if (j == 0) 2.270 ++ memcpy(compressed, buffer, best); 2.271 ++ return err; 2.272 ++} 2.273 ++#define compress2(a,b,c,d,e) best_compress(a,b,c,d) 2.274 ++#endif 2.275 ++ 2.276 +/* Creates a compressed file */ 2.277 +#include "common_header.h" 2.278 + 2.279 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 2.280 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 2.281 +#define CHUNK 65536 2.282 +#define DEFAULT_BLOCKSIZE 65536 2.283 + 2.284 -+static void quit(char *s) 2.285 ++static void quit(const char *s) 2.286 +{ 2.287 + fprintf(stderr, "%s\n", s); 2.288 + exit(1); 2.289 @@ -337,6 +415,8 @@ 2.290 + unsigned char *compressed, *uncompressed; 2.291 + unsigned long *index; 2.292 + int n, indexmax, zlenmax; 2.293 ++ unsigned long len, pos; 2.294 ++ static char padding[512]; 2.295 + 2.296 + if (argc > 1) { 2.297 + if (argv[1][0] < '0' || argv[1][0] > '9') 2.298 @@ -353,35 +433,51 @@ 2.299 + head.num_blocks = -1; 2.300 + head.block_size = htonl(block_size); 2.301 + write(STDOUT_FILENO, &head, sizeof(head)); 2.302 ++ pos = sizeof(head); 2.303 + 2.304 -+ compressed = malloc(zlenmax); 2.305 -+ uncompressed = malloc(block_size); 2.306 -+ index = malloc(indexmax = CHUNK); 2.307 ++ compressed = (unsigned char *) malloc(zlenmax); 2.308 ++ uncompressed = (unsigned char *) malloc(block_size); 2.309 ++ index = (unsigned long *) malloc(indexmax = CHUNK); 2.310 + if (!compressed || !uncompressed || !index) 2.311 + quit("Malloc failed"); 2.312 + 2.313 + for (n = 0; readblock(uncompressed, block_size); n++) { 2.314 -+ unsigned long len = zlenmax; 2.315 -+ 2.316 ++ len = zlenmax; 2.317 + if (compress2(compressed, &len, uncompressed, block_size, 2.318 + Z_BEST_COMPRESSION) != Z_OK) 2.319 + quit("Compression failed"); 2.320 + fprintf(stderr, "Block %u length %lu => %lu\n", 2.321 + n, block_size, len); 2.322 + write(STDOUT_FILENO, compressed, len); 2.323 ++ pos += len; 2.324 + if (n * sizeof(*index) >= indexmax) { 2.325 -+ index = realloc(index, indexmax += CHUNK); 2.326 ++ index = (unsigned long *) realloc(index, 2.327 ++ indexmax += CHUNK); 2.328 + if (!index) 2.329 + quit("Realloc"); 2.330 + } 2.331 + index[n] = ntohl(len); 2.332 + } 2.333 -+ write(STDOUT_FILENO, index, n * sizeof(*index)); 2.334 + tail.index_size = ntohl(sizeof(*index)); 2.335 + tail.num_blocks = ntohl(n); 2.336 ++ n *= sizeof(*index); 2.337 ++ len = n + n/1000 + 12; 2.338 ++ compressed = (unsigned char *) realloc(compressed, n); 2.339 ++ if (!compressed || compress2(compressed, &len, (unsigned char *) index, 2.340 ++ n, Z_BEST_COMPRESSION) != Z_OK) 2.341 ++ quit("Index compression failed"); 2.342 ++ tail.table_size = ntohl(len); 2.343 ++ pos += len + sizeof(tail); 2.344 ++ n = pos & 511; 2.345 ++ if (n) write(STDOUT_FILENO, padding, 512 - n); 2.346 ++ write(STDOUT_FILENO, compressed, len); 2.347 + write(STDOUT_FILENO, &tail, sizeof(tail)); 2.348 + return 0; 2.349 +} 2.350 ++#ifdef FIND_BEST_COMPRESSION 2.351 ++} 2.352 ++#endif 2.353 + 2.354 --- fusecloop.c 2.355 +++ fusecloop.c 2.356 @@ -65,7 +65,7 @@