wok-6.x diff fusecloop/stuff/fusecloop.u @ rev 25017
updated openjpeg, openjpeg-dev and openjpeg-tools (1.5.0 -> 1.5.2)
author | Hans-G?nter Theisgen |
---|---|
date | Wed May 18 06:55:41 2022 +0100 (2022-05-18) |
parents | c6796c6304f6 |
children |
line diff
1.1 --- a/fusecloop/stuff/fusecloop.u Wed May 06 09:10:52 2020 +0000 1.2 +++ b/fusecloop/stuff/fusecloop.u Wed May 18 06:55:41 2022 +0100 1.3 @@ -2150,3 +2150,297 @@ 1.4 1.5 #if 1 1.6 if (getenv("CLOOP_TABLE") != NULL) { 1.7 +=== Add v4 support 1.8 +--- compressed_loop.h 1.9 ++++ compressed_loop.h 1.10 +@@ -64,6 +64,7 @@ 1.11 + * 2 xz compression (currently best space saver) * 1.12 + * 3 lz4 compression * 1.13 + * 4 lzo compression (fastest) * 1.14 ++* 5 zstd compression * 1.15 + * 15 block link * 1.16 + */ 1.17 + /* Get value of first 4 bits */ 1.18 +@@ -76,6 +77,7 @@ 1.19 + #define CLOOP_COMPRESSOR_XZ 0x2 1.20 + #define CLOOP_COMPRESSOR_LZ4 0x3 1.21 + #define CLOOP_COMPRESSOR_LZO 0x4 1.22 ++#define CLOOP_COMPRESSOR_ZSTD 0x5 1.23 + #define CLOOP_COMPRESSOR_LINK 0xF 1.24 + 1.25 + 1.26 +--- create_compressed_fs.c 1.27 ++++ create_compressed_fs.c 1.28 +@@ -6,7 +6,9 @@ 1.29 + 1.30 + #define ZMAX 9 1.31 + static shrink_t level; 1.32 +-static int pass, iter; 1.33 ++static struct { 1.34 ++ int pass, iter, v4; 1.35 ++} conf; 1.36 + static int best_compress(Bytef *compressed, 1.37 + uLongf *compressed_len, 1.38 + const Bytef *uncompressed, 1.39 +@@ -28,7 +30,7 @@ 1.40 + } 1.41 + buf[0] = compressed; 1.42 + buf[1] = buffer; 1.43 +- for (i = j = 0; i <= ZMAX+3 && (pass == 0 || i < pass); i++) { 1.44 ++ for (i = j = 0; i <= ZMAX+3 && (conf.pass == 0 || i < conf.pass); i++) { 1.45 + llen = len = *compressed_len; 1.46 + if (i >= ZMAX+1) { 1.47 + level.level = (i == ZMAX+1) ? shrink_normal : 1.48 +@@ -56,6 +58,9 @@ 1.49 + #include <stdlib.h> 1.50 + #include <string.h> 1.51 + #include <zlib.h> 1.52 ++static struct { 1.53 ++ int v4; 1.54 ++} conf; 1.55 + #endif 1.56 + 1.57 + /* Creates a compressed file */ 1.58 +@@ -148,7 +153,8 @@ 1.59 + 1.60 + #include <signal.h> 1.61 + 1.62 +-#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.63 ++#define CLOOP_PREAMBLE "#!/bin/sh\n" "#V3.0 Format\n" "modprobe cloop file=$0 && mount -r -t iso9660 /dev/cloop $1\n" "exit $?\n" 1.64 ++#define CLOOP_PREAMBLE_VERSION 12 1.65 + #define CHUNK 65536 1.66 + #define DEFAULT_BLOCKSIZE 65536 1.67 + 1.68 +@@ -174,11 +180,22 @@ 1.69 + 1.70 + #include "md5sum.c" 1.71 + 1.72 ++//#define CLOOP_DEBUG 1.73 ++#ifdef CLOOP_DEBUG 1.74 ++static void write_file(const char *s, void *data, unsigned n) 1.75 ++{ 1.76 ++ int fd = open(s,O_WRONLY|O_CREAT|O_TRUNC,0600); 1.77 ++ if (fd == 0) return; 1.78 ++ write(fd,data,n); 1.79 ++ close(fd); 1.80 ++} 1.81 ++#endif 1.82 + static unsigned n; 1.83 +-static unsigned long lastlen, pos; 1.84 ++static unsigned long lastlen, pos = 0; 1.85 + static u_int32_t *block_index; 1.86 + static unsigned char *compressed; 1.87 + static unsigned long block_size = 0; 1.88 ++static struct cloop_head head; 1.89 + static void flush_index(int sig) 1.90 + { 1.91 + struct cloop_tail tail; 1.92 +@@ -187,11 +204,26 @@ 1.93 + 1.94 + fprintf(stderr, "Write index for %u blocks\n", n); 1.95 + if (block_size >= 0x1000000) lastlen = 0; 1.96 ++ if (conf.v4 == 1) { 1.97 ++ unsigned i; 1.98 ++ loff_t ofs64; 1.99 ++ for (i = 0, ofs64 = 0; i < n; ofs64 += (loff_t) ntohl(block_index[i++] & 0xFFffFFf0)) { 1.100 ++ loff_t index64 = ((block_index[i] & 0xF) == CLOOP_COMPRESSOR_NONE) ? ((loff_t) CLOOP_COMPRESSOR_NONE << 60) : 0; 1.101 ++ index64 = __le64_to_cpu(index64 + ofs64); 1.102 ++ write(STDOUT_FILENO, &index64, sizeof(index64)); 1.103 ++ } 1.104 ++ head.num_blocks = htonl(n); 1.105 ++ write(STDOUT_FILENO, &head, sizeof(head)); 1.106 ++ exit(sig != 0); 1.107 ++ } 1.108 + if (sig) flags = 0x80; 1.109 + tail.index_size = ntohl(sizeof(*block_index) + flags + 256*(lastlen % 0xFFffFF)); 1.110 + tail.num_blocks = ntohl(n); 1.111 + n *= sizeof(*block_index); 1.112 + len = GZIP_MAX_BUFFER(n); 1.113 ++#ifdef CLOOP_DEBUG 1.114 ++ write_file("cloop.index",block_index,n); 1.115 ++#endif 1.116 + compressed = (unsigned char *) realloc(compressed, len); 1.117 + #ifdef FIND_BEST_COMPRESSION 1.118 + if (!compressed || best_compress(compressed, &len, (unsigned char *) block_index, 1.119 +@@ -201,8 +233,10 @@ 1.120 + n, Z_BEST_SPEED) != Z_OK) 1.121 + quit("Index compression failed"); 1.122 + tail.table_size = ntohl(len); 1.123 +- pos += len + sizeof(tail); 1.124 +- n = pos & 511; 1.125 ++#ifdef CLOOP_DEBUG 1.126 ++ write_file("cloop.zindex",compressed,len); 1.127 ++ write_file("cloop.tail",&tail,sizeof(tail)); 1.128 ++#endif 1.129 + write(STDOUT_FILENO, compressed, len); 1.130 + write(STDOUT_FILENO, &tail, sizeof(tail)); 1.131 + exit(sig != 0); 1.132 +@@ -210,7 +244,6 @@ 1.133 + 1.134 + int main(int argc, char *argv[]) 1.135 + { 1.136 +- struct cloop_head head; 1.137 + unsigned char *uncompressed; 1.138 + unsigned long len; 1.139 + unsigned indexmax, zlenmax; 1.140 +@@ -221,9 +254,12 @@ 1.141 + #ifdef FIND_BEST_COMPRESSION 1.142 + while (argc > 1) { 1.143 + if (argv[1][0] == '-') { 1.144 +- int *p = &pass; 1.145 ++ int *p = &conf.pass; 1.146 + switch (argv[1][1]) { 1.147 +- case 'i' : p = &iter; 1.148 ++ case 'v' : conf.v4++; 1.149 ++ argv++; argc--; 1.150 ++ continue; 1.151 ++ case 'i' : p = &conf.iter; 1.152 + case 'n' : *p = atoi(argv[2]); 1.153 + argc -= 2; 1.154 + argv += 2; 1.155 +@@ -232,11 +268,23 @@ 1.156 + } 1.157 + argc--; 1.158 + if (argv[1][0] < '0' || argv[1][0] > '9') 1.159 +- quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5] create_compressed_fs [-n <pass>][ -i <iter>] [block size] < input > output"); 1.160 ++ quit("Usage : [CLOOP_COMP=XZ|GZIP|LZ4] [CLOOP_NOMD5=1] " 1.161 ++# ifdef CLOOP_DEBUG 1.162 ++ "[CLOOP_DEBUG=1] " 1.163 ++# endif 1.164 ++ "create_compressed_fs [-v4] [-n <pass>][ -i <iter>] [block size] < input > output"); 1.165 + #else 1.166 + if (argc > 1) { 1.167 ++ if (strcmp(argv[1],"-v4") == 0) { 1.168 ++ conf.v4++; 1.169 ++ argv++; argc--; 1.170 ++ } 1.171 + if (argv[1][0] < '0' || argv[1][0] > '9') 1.172 +- quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] create_compressed_fs [block size] < input > output"); 1.173 ++ quit("Usage : [CLOOP_COMP=XZ|LZ4] [CLOOP_NOMD5=1] " 1.174 ++# ifdef CLOOP_DEBUG 1.175 ++ "[CLOOP_DEBUG=1] " 1.176 ++# endif 1.177 ++ "create_compressed_fs [-v4] [block size] < input > output"); 1.178 + #endif 1.179 + block_size = atoi(argv[1]); 1.180 + } 1.181 +@@ -247,10 +295,13 @@ 1.182 + 1.183 + memset(&head, 0, sizeof(head)); 1.184 + strcpy(head.preamble, CLOOP_PREAMBLE); 1.185 +- head.num_blocks = -1; 1.186 + head.block_size = htonl(block_size); 1.187 +- write(STDOUT_FILENO, &head, sizeof(head)); 1.188 +- pos = sizeof(head); 1.189 ++ if (conf.v4 == 0) { 1.190 ++ head.num_blocks = -1; 1.191 ++ write(STDOUT_FILENO, &head, sizeof(head)); 1.192 ++ pos = sizeof(head); 1.193 ++ } 1.194 ++ else head.preamble[CLOOP_PREAMBLE_VERSION] = '4'; 1.195 + 1.196 + compressed = (unsigned char *) malloc(zlenmax); 1.197 + uncompressed = (unsigned char *) malloc(block_size); 1.198 +@@ -277,14 +328,20 @@ 1.199 + if (!hash) 1.200 + quit("Realloc hash"); 1.201 + } 1.202 +- hash[n] = md5sum(uncompressed, len); 1.203 ++ hash[n] = md5sum(uncompressed, lastlen); 1.204 + j = 0x0FFFFFFF; 1.205 + if (n < j) 1.206 + j = n; 1.207 + for (i = 0; i < j; i++) { 1.208 +- if (* (uint32_t *) &hash[i] == * (uint32_t *) &hash[n] 1.209 +- && !memcmp(&hash[i],&hash[n],sizeof(*hash))) 1.210 +- break; 1.211 ++ if (* (uint32_t *) &hash[i] != * (uint32_t *) &hash[n] 1.212 ++ || memcmp(&hash[i],&hash[n],sizeof(*hash))) 1.213 ++ continue; 1.214 ++ len = zlenmax; 1.215 ++ if (compress3(compressed, &len, uncompressed, lastlen, 1.216 ++ Z_BEST_SPEED) != Z_OK || (len & 0x0FFFFFFF) >= lastlen) { 1.217 ++ len = (CLOOP_COMPRESSOR_NONE << 28) | lastlen; 1.218 ++ } 1.219 ++ if (block_index[i] == ntohl(len)) break; 1.220 + } 1.221 + if (i != j) { 1.222 + block_index[n] = ntohl((CLOOP_COMPRESSOR_LINK << 28) | i); 1.223 +--- extract_compressed_fs.c 1.224 ++++ extract_compressed_fs.c 1.225 +@@ -67,16 +67,21 @@ 1.226 + return ulen; 1.227 + } 1.228 + 1.229 ++//#define CLOOP_DEBUG 1.230 + int main(int argc, char *argv[]) 1.231 + { 1.232 + struct cloop_head head; 1.233 + unsigned int i, v4_header_last; 1.234 + unsigned long n, len; 1.235 + uLongf ulen; 1.236 +- loff_t end; 1.237 ++ off64_t end; 1.238 + 1.239 + if (argc < 2 || argv[1][0] == '-') { 1.240 +- fprintf(stderr, "Usage: extract_compressed_fs file [--convert-to-v2] > output\n"); 1.241 ++ fprintf(stderr, "Usage: " 1.242 ++#ifdef CLOOP_DEBUG 1.243 ++ "[CLOOP_TAIL=1|CLOOP_ZINDEX=1|CLOOP_INDEX=1|CLOOP_TABLE=1] " 1.244 ++#endif 1.245 ++ "extract_compressed_fs file [--convert-to-v2] > output\n"); 1.246 + exit(1); 1.247 + } 1.248 + 1.249 +@@ -122,6 +127,17 @@ 1.250 + perror("Reading tail\n"); 1.251 + exit(1); 1.252 + } 1.253 ++#ifdef CLOOP_DEBUG 1.254 ++ if (getenv("CLOOP_TAIL") != NULL) { 1.255 ++ fprintf(stderr, "Table size:32 %u, index_size:4 %u, unused:3\n", 1.256 ++ tail.table_size,CLOOP3_INDEX_SIZE(tail.index_size)); 1.257 ++ fprintf(stderr, "Truncated:1 %u, lastlen:24 %u, num_blocks:32 %u end=%llu\n", 1.258 ++ CLOOP3_TRUNCATED(tail.index_size),CLOOP3_LASTLEN(tail.index_size),tail.num_blocks,end); 1.259 ++ fprintf(stderr, "CLOOP_TAIL: binary\n"); 1.260 ++ write(STDOUT_FILENO, &tail, sizeof(tail)); 1.261 ++ exit(0); 1.262 ++ } 1.263 ++#endif 1.264 + head.num_blocks = tail.num_blocks; 1.265 + num_blocks = ntohl(head.num_blocks); 1.266 + table_size = ntohl(tail.table_size); 1.267 +@@ -137,6 +153,13 @@ 1.268 + perror("Reading index\n"); 1.269 + exit(1); 1.270 + } 1.271 ++#ifdef CLOOP_DEBUG 1.272 ++ if (getenv("CLOOP_ZINDEX") != NULL) { 1.273 ++ fprintf(stderr, "CLOOP_ZINDEX: binary\n"); 1.274 ++ write(STDOUT_FILENO, table, table_size); 1.275 ++ exit(0); 1.276 ++ } 1.277 ++#endif 1.278 + err = unpack[CLOOP_COMPRESSOR_ZLIB]((void *) offsets, &ulen, table, table_size); 1.279 + if (err != Z_OK) { 1.280 + fprintf(stderr, "Unpack %s index error %d\n", 1.281 +@@ -167,7 +190,7 @@ 1.282 + 1.283 + fprintf(stderr, "%lu blocks of size %lu. Preamble:\n%s\n", 1.284 + num_blocks, block_size, head.preamble); 1.285 +-#if 1 1.286 ++#ifdef CLOOP_DEBUG 1.287 + if (getenv("CLOOP_INDEX") != NULL) { 1.288 + fprintf(stderr, "CLOOP_INDEX: binary\n"); 1.289 + write(STDOUT_FILENO, offsets, len); 1.290 +@@ -175,8 +198,8 @@ 1.291 + } 1.292 + #endif 1.293 + fprintf(stderr, "Index %s.\n", build_index(offsets, num_blocks, block_size)); 1.294 +- 1.295 +-#if 1 1.296 ++ 1.297 ++#ifdef CLOOP_DEBUG 1.298 + if (getenv("CLOOP_TABLE") != NULL) { 1.299 + fprintf(stderr, "CLOOP_TABLE ascii: offset, size, flags\n"); 1.300 + for (i = 0; i < num_blocks; i++) {