wok-current rev 12214

fusecloop: compress index too
author Pascal Bellard <pascal.bellard@slitaz.org>
date Mon Apr 02 22:48:21 2012 +0200 (2012-04-02)
parents 2dde678153c7
children 188ed036ce84
files fusecloop/receipt fusecloop/stuff/fusecloop.u
line diff
     1.1 --- a/fusecloop/receipt	Sun Apr 01 12:41:48 2012 +0200
     1.2 +++ b/fusecloop/receipt	Mon Apr 02 22:48:21 2012 +0200
     1.3 @@ -8,7 +8,10 @@
     1.4  TARBALL="$PACKAGE-$VERSION.tar.gz"
     1.5  WEB_SITE="http://fusecloop.sourceforge.net/"
     1.6  WGET_URL="$SF_MIRROR/$PACKAGE/$TARBALL"
     1.7 -DEPENDS="fuse zlib"
     1.8 +ADVANCECOMP_VERSION="1.15"
     1.9 +ADVANCECOMP_TARBALL="advancecomp-$ADVANCECOMP_VERSION.tar.gz"
    1.10 +ADVANCECOMP_URL="$SF_MIRROR/advancemame/$ADVANCECOMP_TARBALL"
    1.11 +DEPENDS="fuse zlib gcc-lib-base"
    1.12  BUILD_DEPENDS="fuse-dev zlib-dev"
    1.13  SUGGESTED="fuseiso"
    1.14  
    1.15 @@ -16,11 +19,22 @@
    1.16  compile_rules()
    1.17  {
    1.18  	cd $src
    1.19 +	[ -s $SOURCES_REPOSITORY/$ADVANCECOMP_TARBALL ] ||
    1.20 +		wget -P $SOURCES_REPOSITORY $ADVANCECOMP_URL
    1.21 +	tar xzf $SOURCES_REPOSITORY/$ADVANCECOMP_TARBALL
    1.22  	sed -i 's/dprintf/d_printf/g' *.h *.c
    1.23  	patch -p0 < $stuff/fusecloop.u
    1.24 +	ADVANCECOMP=advancecomp-$ADVANCECOMP_VERSION
    1.25 +	cp *.h *.c $ADVANCECOMP
    1.26 +	cp create_compressed_fs.c $ADVANCECOMP/redef.cc
    1.27 +	sed -i 's/def FIND_BEST_COMPRESSION/ 1/' $ADVANCECOMP/redef.cc
    1.28  	./configure --prefix=/usr --infodir=/usr/share/info \
    1.29  	--mandir=/usr/share/man $CONFIGURE_ARGS &&
    1.30 -	make
    1.31 +	make &&
    1.32 +	cd $ADVANCECOMP &&
    1.33 +	./configure --prefix=/usr --infodir=/usr/share/info \
    1.34 +	--mandir=/usr/share/man $CONFIGURE_ARGS &&
    1.35 +	make advdef
    1.36  }
    1.37  
    1.38  # Rules to gen a SliTaz package suitable for Tazpkg.
    1.39 @@ -29,6 +43,6 @@
    1.40  	mkdir -p $fs/usr/bin
    1.41  	cp $src/fusecloop $fs/usr/bin
    1.42  	cp $src/extract_compressed_fs $fs/usr/bin
    1.43 -	cp $src/create_compressed_fs $fs/usr/bin
    1.44 +	cp $src/advancecomp-*/advdef $fs/usr/bin/create_compressed_fs
    1.45  }
    1.46  
     2.1 --- a/fusecloop/stuff/fusecloop.u	Sun Apr 01 12:41:48 2012 +0200
     2.2 +++ b/fusecloop/stuff/fusecloop.u	Mon Apr 02 22:48:21 2012 +0200
     2.3 @@ -1,11 +1,12 @@
     2.4  --- compressed_loop.h
     2.5  +++ compressed_loop.h
     2.6 -@@ -41,6 +41,73 @@
     2.7 +@@ -41,6 +41,74 @@
     2.8   /* data_index (num_blocks 64bit pointers, network order)...      */
     2.9   /* compressed data (gzip block compressed format)...             */
    2.10   
    2.11  +struct cloop_tail
    2.12  +{
    2.13 ++	u_int32_t table_size;
    2.14  +	u_int32_t index_size;
    2.15  +	u_int32_t num_blocks;
    2.16  +};
    2.17 @@ -28,7 +29,7 @@
    2.18  +				offsets[n].offset = __be64_to_cpu(offsets[n].offset);
    2.19  +				offsets[n].size = ntohl(offsets[n].size);
    2.20  +			}
    2.21 -+			return "128BE accelerated knoppix 1.0";
    2.22 ++			return (char *) "128BE accelerated knoppix 1.0";
    2.23  +		}
    2.24  +		else { /* V2.0 */
    2.25  +			loff_t last = __be64_to_cpu(ofs64[n]);
    2.26 @@ -37,7 +38,7 @@
    2.27  +					(offsets[n].offset = __be64_to_cpu(ofs64[n])); 
    2.28  +				last = offsets[n].offset;
    2.29  +			}
    2.30 -+			return "64BE v2.0";
    2.31 ++			return (char *) "64BE v2.0";
    2.32  +		}
    2.33  +	}
    2.34  +	else if (ofs32[1] == 0) { /* V1.0 */
    2.35 @@ -47,7 +48,7 @@
    2.36  +				(offsets[n].offset = __le64_to_cpu(ofs64[n])); 
    2.37  +			last = offsets[n].offset;
    2.38  +		}
    2.39 -+		return "64LE v1.0";
    2.40 ++		return (char *) "64LE v1.0";
    2.41  +	}
    2.42  +	else if (ntohl(ofs32[0]) == (4*n) + 0x8C) { /* V0.68 */
    2.43  +		loff_t last = ntohl(ofs32[n]);
    2.44 @@ -56,7 +57,7 @@
    2.45  +				(offsets[n].offset = ntohl(ofs32[n])); 
    2.46  +			last = offsets[n].offset;
    2.47  +		}
    2.48 -+		return "32BE v0.68";
    2.49 ++		return (char *) "32BE v0.68";
    2.50  +	}
    2.51  +	else { /* V3.0 */
    2.52  +		int i, j;
    2.53 @@ -67,7 +68,7 @@
    2.54  +			offsets[i].offset = j;
    2.55  +			j += offsets[i].size;
    2.56  +		}
    2.57 -+		return "32BE v3.0";
    2.58 ++		return (char *) "32BE v3.0";
    2.59  +	}
    2.60  +}
    2.61  +
    2.62 @@ -89,30 +90,43 @@
    2.63  
    2.64  --- cloopreader.c
    2.65  +++ cloopreader.c
    2.66 -@@ -59,10 +59,21 @@
    2.67 +@@ -59,10 +59,32 @@
    2.68   
    2.69       ALLOC(c->pblock,c->blocksize);
    2.70   
    2.71  -    c->tocsize=sizeof *c->toc * (c->numblocks+1); /* One extra address is position of EOF */
    2.72 -+    c->tocsize=sizeof(*c->toc) * c->numblocks;
    2.73 -+    if (c->numblocks == -1) {
    2.74 +-    ALLOC(c->toc,c->tocsize);
    2.75 ++    if (c->numblocks + 1 == 0) {
    2.76  +	struct cloop_tail tail;
    2.77  +	loff_t end = lseek(c->fh,0,SEEK_END); /* lseek(,-n,SEEK_END) buggy ? */
    2.78 -+
    2.79 ++	void *p;
    2.80 ++	ulong toclen, len; 
    2.81 + 
    2.82 +-    OP(read_all(c->fh,c->toc,c->tocsize));  /* read Data Index */
    2.83  +	OP(lseek(c->fh, end - sizeof(tail), SEEK_SET)); 
    2.84  +	OP(read_all(c->fh, &tail, sizeof(tail)));
    2.85  +	c->numblocks = ntohl(tail.num_blocks);
    2.86 -+	c->tocsize = ntohl(tail.index_size) * c->numblocks;
    2.87 -+	OP(lseek(c->fh, end - sizeof(tail) - c->tocsize, SEEK_SET));
    2.88 ++	c->tocsize = sizeof(*c->toc) * c->numblocks;
    2.89 ++	len = ntohl(tail.table_size);
    2.90 ++	toclen = (ntohl(tail.index_size) & 255) * c->numblocks;
    2.91 ++	OP(lseek(c->fh, end - sizeof(tail) - len, SEEK_SET));
    2.92 ++        ALLOC(c->toc,c->tocsize);
    2.93 ++        ALLOC(p,len);
    2.94 ++        OP(read_all(c->fh,p,len));  /* read Data Index */
    2.95 ++	if (uncompress((void *)c->toc,&toclen,p,len) != Z_OK)
    2.96 ++		exit(1);
    2.97 ++	free(p);
    2.98  +    }
    2.99 -     ALLOC(c->toc,c->tocsize);
   2.100 - 
   2.101 -     OP(read_all(c->fh,c->toc,c->tocsize));  /* read Data Index */
   2.102 ++    else {
   2.103 ++	c->tocsize = sizeof(*c->toc) * c->numblocks;
   2.104 ++        ALLOC(c->toc,c->tocsize);
   2.105 ++        OP(read_all(c->fh,c->toc,c->tocsize));  /* read Data Index */
   2.106 ++    }
   2.107  +    build_index(c->toc, c->numblocks);
   2.108       c->cblocksizecur=0;
   2.109       c->curblock=-1;
   2.110       return 0;
   2.111 -@@ -79,10 +90,10 @@
   2.112 +@@ -79,10 +101,10 @@
   2.113       if(page>=c->numblocks){errno=EFAULT;return -1;}
   2.114       c->curblock=page;
   2.115   
   2.116 @@ -133,7 +147,7 @@
   2.117   /* Extracts a filesystem back from a compressed fs file */
   2.118  +#define _LARGEFILE64_SOURCE
   2.119   #include "common_header.h"
   2.120 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
   2.121 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V2.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
   2.122   
   2.123   int main(int argc, char *argv[])
   2.124   {
   2.125 @@ -151,7 +165,7 @@
   2.126   		exit(1);
   2.127   	}
   2.128   
   2.129 -@@ -24,44 +28,77 @@
   2.130 +@@ -24,44 +28,91 @@
   2.131   		exit(1);
   2.132   	}
   2.133   
   2.134 @@ -160,42 +174,55 @@
   2.135  -	clear_buffer = malloc(ntohl(head.block_size));
   2.136  -	fprintf(stderr, "%u blocks of size %u. Preamble:\n%s\n", 
   2.137  -		ntohl(head.num_blocks), ntohl(head.block_size), head.preamble);
   2.138 --
   2.139 ++	num_blocks = ntohl(head.num_blocks);
   2.140 ++	block_size = ntohl(head.block_size);
   2.141 ++	zblock_maxsize =  block_size + block_size/1000 + 12 + 4;
   2.142 ++	buffer = malloc(zblock_maxsize);
   2.143 ++	clear_buffer = malloc(block_size);
   2.144 + 
   2.145  -	for (i = 0; i < ntohl(head.num_blocks); i++) {
   2.146  -		int currpos;
   2.147  -		unsigned long destlen = ntohl(head.block_size);
   2.148  -		loff_t offset[2];
   2.149  -		unsigned int size;
   2.150 -+	num_blocks = ntohl(head.num_blocks);
   2.151 -+	block_size = ntohl(head.block_size);
   2.152 -+	zblock_maxsize =  block_size + block_size/1000 + 12 + 4;
   2.153 -+	buffer = malloc(zblock_maxsize);
   2.154 -+	clear_buffer = malloc(block_size);
   2.155 -+	fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 
   2.156 -+		num_blocks, block_size, head.preamble);
   2.157 -+
   2.158 -+	if (num_blocks == -1) {
   2.159 ++	if (num_blocks == (unsigned long) -1) {
   2.160 ++		void *table;
   2.161  +		struct cloop_tail tail;
   2.162 ++		unsigned long len, table_size;
   2.163  +		loff_t end = lseek64(handle, 0, SEEK_END);
   2.164 ++		
   2.165  +		if (lseek64(handle, end - sizeof(tail), SEEK_SET) < 0 ||
   2.166  +		    read(handle, &tail, sizeof(tail)) != sizeof(tail) ||
   2.167  +		    lseek64(handle, end - sizeof(tail) - 
   2.168 -+		    	  (ntohl(tail.num_blocks) * ntohl(tail.index_size)), 
   2.169 -+		    	  SEEK_SET) < 0) {
   2.170 ++		    	  ntohl(tail.table_size), SEEK_SET) < 0) {
   2.171  +			perror("Reading tail\n");
   2.172  +			exit(1);
   2.173  +		}
   2.174  +		head.num_blocks = tail.num_blocks;
   2.175  +		num_blocks = ntohl(head.num_blocks);
   2.176 -+		i = num_blocks * ntohl(tail.index_size);
   2.177 ++		table_size = ntohl(tail.table_size);
   2.178 ++		table = malloc(table_size);
   2.179 ++		len = i = num_blocks * (ntohl(tail.index_size) & 255);
   2.180 ++		offsets = malloc(num_blocks * sizeof(*offsets));
   2.181 ++		if (!table || !offsets || 
   2.182 ++		    read(handle, table, table_size) != table_size ||
   2.183 ++		    uncompress((void *)offsets, &len, table, table_size) != Z_OK ||
   2.184 ++		    len != i) {
   2.185 ++			perror("Reading index\n");
   2.186 ++			exit(1);
   2.187 ++		}
   2.188 ++		free(table);
   2.189  +	}
   2.190 -+	else i = num_blocks * sizeof(*offsets);
   2.191 -+	offsets = malloc(i);
   2.192 -+	if (!offsets || read(handle, offsets, i) != i) {
   2.193 -+		perror("Reading index\n");
   2.194 -+		exit(1);
   2.195 ++	else {
   2.196 ++		offsets = malloc(i = num_blocks * sizeof(*offsets));
   2.197 ++		if (!offsets || read(handle, offsets, i) != i) {
   2.198 ++			perror("Reading index\n");
   2.199 ++			exit(1);
   2.200 ++		}
   2.201  +	}
   2.202  +	
   2.203 ++	fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 
   2.204 ++		num_blocks, block_size, head.preamble);
   2.205  +	fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks));
   2.206  +	
   2.207  +	if (argc > 2) {
   2.208 @@ -256,7 +283,7 @@
   2.209   		if (i == 3) {
   2.210   			fprintf(stderr,
   2.211   				"Block head:%02X%02X%02X%02X%02X%02X%02X%02X\n",
   2.212 -@@ -105,12 +142,12 @@
   2.213 +@@ -105,12 +156,12 @@
   2.214   			fprintf(stderr, "Uncomp: unknown error %u\n", i);
   2.215   			exit(1);
   2.216   		}
   2.217 @@ -299,17 +326,68 @@
   2.218   	${CC} ${CFLAGS} ${LDFLAGS} -lz cloopreader.o ${FUSECFLAGS} ${FUSELDFLAGS} fusecloop.c debug.o -o fusecloop
   2.219   
   2.220  
   2.221 +
   2.222  --- create_compressed_fs.c
   2.223  +++ create_compressed_fs.c
   2.224 -@@ -0,0 +1,80 @@
   2.225 +@@ -0,0 +1,147 @@
   2.226 ++#ifdef FIND_BEST_COMPRESSION
   2.227 ++#include <compress.h>
   2.228 ++extern "C" {
   2.229 ++#include <stdlib.h>
   2.230 ++#include <string.h>
   2.231 ++
   2.232 ++static int best_compress(unsigned char *compressed,
   2.233 ++			 unsigned long *compressed_len,
   2.234 ++			 unsigned char *uncompressed,
   2.235 ++			 unsigned long uncompressed_len)
   2.236 ++{
   2.237 ++	int i, j, err;
   2.238 ++	unsigned char *buf[2];
   2.239 ++	unsigned len;
   2.240 ++	unsigned long llen,  best = *compressed_len * 2;
   2.241 ++	static unsigned char *buffer;
   2.242 ++	static unsigned long buffersz;
   2.243 ++
   2.244 ++	if (buffersz < *compressed_len) {
   2.245 ++		if (buffer) free(buffer);
   2.246 ++		buffer = (unsigned char *)  malloc(buffersz = *compressed_len);
   2.247 ++		if (!buffer) return Z_MEM_ERROR;
   2.248 ++	}
   2.249 ++	buf[0] = compressed;
   2.250 ++	buf[1] = buffer;
   2.251 ++	for (i = j = 0; i <= 10; i++) {
   2.252 ++		llen = len = *compressed_len;
   2.253 ++		if (i == 10) 
   2.254 ++			err = (compress_zlib(shrink_extreme, buf[j],
   2.255 ++		     		len, uncompressed,
   2.256 ++		     		uncompressed_len)) ? Z_OK : Z_DATA_ERROR;
   2.257 ++		else {
   2.258 ++			err = compress2(buf[j], &llen, uncompressed,
   2.259 ++				     uncompressed_len, i);
   2.260 ++			len = llen;
   2.261 ++		}
   2.262 ++		if (err != Z_OK) return err;
   2.263 ++		if (len < best) {
   2.264 ++			best = len;
   2.265 ++			j = 1 - j;
   2.266 ++		}
   2.267 ++	}
   2.268 ++	*compressed_len = best;
   2.269 ++	if (j == 0)
   2.270 ++		memcpy(compressed, buffer, best);
   2.271 ++	return err;
   2.272 ++}
   2.273 ++#define compress2(a,b,c,d,e) best_compress(a,b,c,d)
   2.274 ++#endif
   2.275 ++                        
   2.276  +/* Creates a compressed file */
   2.277  +#include "common_header.h"
   2.278  +
   2.279 -+#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "insmod cloop.o file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
   2.280 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n"
   2.281  +#define CHUNK 65536
   2.282  +#define DEFAULT_BLOCKSIZE 65536
   2.283  +
   2.284 -+static void quit(char *s)
   2.285 ++static void quit(const char *s)
   2.286  +{
   2.287  +	fprintf(stderr, "%s\n", s);
   2.288  +	exit(1);
   2.289 @@ -337,6 +415,8 @@
   2.290  +	unsigned char *compressed, *uncompressed;
   2.291  +	unsigned long *index;
   2.292  +	int n, indexmax, zlenmax;
   2.293 ++	unsigned long len, pos;
   2.294 ++	static char padding[512];
   2.295  +	
   2.296  +	if (argc > 1) {
   2.297  +		if (argv[1][0] < '0' || argv[1][0] > '9')
   2.298 @@ -353,35 +433,51 @@
   2.299  +	head.num_blocks = -1;
   2.300  +	head.block_size = htonl(block_size);
   2.301  +	write(STDOUT_FILENO, &head, sizeof(head));
   2.302 ++	pos = sizeof(head);
   2.303  +	
   2.304 -+	compressed = malloc(zlenmax);
   2.305 -+	uncompressed = malloc(block_size);
   2.306 -+	index = malloc(indexmax = CHUNK);
   2.307 ++	compressed = (unsigned char *) malloc(zlenmax);
   2.308 ++	uncompressed = (unsigned char *) malloc(block_size);
   2.309 ++	index = (unsigned long *) malloc(indexmax = CHUNK);
   2.310  +	if (!compressed || !uncompressed || !index)
   2.311  +		quit("Malloc failed");
   2.312  +	
   2.313  +	for (n = 0; readblock(uncompressed, block_size); n++) {
   2.314 -+		unsigned long len = zlenmax;
   2.315 -+		
   2.316 ++		len = zlenmax;
   2.317  +		if (compress2(compressed, &len, uncompressed, block_size, 
   2.318  +				Z_BEST_COMPRESSION) != Z_OK)
   2.319  +			quit("Compression failed");
   2.320  +		fprintf(stderr, "Block %u length %lu => %lu\n",
   2.321  +			n, block_size, len);
   2.322  +		write(STDOUT_FILENO, compressed, len);
   2.323 ++		pos += len;
   2.324  +		if (n * sizeof(*index) >= indexmax) {
   2.325 -+			index = realloc(index, indexmax += CHUNK);
   2.326 ++			index = (unsigned long *) realloc(index,
   2.327 ++							  indexmax += CHUNK);
   2.328  +			if (!index)
   2.329  +				quit("Realloc");
   2.330  +		}
   2.331  +		index[n] = ntohl(len);
   2.332  +	}
   2.333 -+	write(STDOUT_FILENO, index, n * sizeof(*index));
   2.334  +	tail.index_size = ntohl(sizeof(*index));
   2.335  +	tail.num_blocks = ntohl(n);
   2.336 ++	n *= sizeof(*index);
   2.337 ++	len = n + n/1000 + 12;
   2.338 ++	compressed = (unsigned char *) realloc(compressed, n);
   2.339 ++	if (!compressed || compress2(compressed, &len, (unsigned char *) index,
   2.340 ++				     n, Z_BEST_COMPRESSION) != Z_OK)
   2.341 ++		quit("Index compression failed");
   2.342 ++	tail.table_size = ntohl(len);
   2.343 ++	pos += len + sizeof(tail);
   2.344 ++	n = pos & 511;
   2.345 ++	if (n) write(STDOUT_FILENO, padding, 512 - n);
   2.346 ++	write(STDOUT_FILENO, compressed, len);
   2.347  +	write(STDOUT_FILENO, &tail, sizeof(tail));
   2.348  +	return 0;
   2.349  +}
   2.350 ++#ifdef FIND_BEST_COMPRESSION
   2.351 ++}
   2.352 ++#endif
   2.353 +
   2.354  --- fusecloop.c
   2.355  +++ fusecloop.c
   2.356  @@ -65,7 +65,7 @@