cookutils view modules/compressor @ rev 893

web/cooker.cgi: work with HTTP headers "Last-Modified", "If-Modified-Since", "HTTP 304 Not Modified", "HTTP 404 Not Found". Other small improvements.
SliTaz Next Cooker is working right now with this version of cooker.cgi.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Mon Mar 20 05:12:29 2017 +0200 (2017-03-20)
parents 3f0ca4492ec7
children ea11d3c96873
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9
10 # Compressor cache stuff
11
12 comp_cache_root='/home/slitaz/cache/cook'
13 mkdir -p "$comp_cache_root"
14 cache_stat=$(mktemp)
15
16 # Cache notes.
17 # Do not do the same job twice. Getting the file from the cache is much faster
18 # than compressing the file one more time. In addition, this cache is trying not
19 # to take extra space, using the hardlinks. Although the files from the cache
20 # without reference to itself should be removed periodically.
21
22
23
24
25 #
26 # Functions
27 #
28
29
30 # Working with time (with hundredths precision)
31
32 get_time() {
33 cut -d" " -f2 /proc/uptime
34 }
35
36 calc_time() {
37 # L10n: 's' is for seconds, 'm' is for minutes
38 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
39 BEGIN{
40 time = b - a;
41 if (time < 30)
42 printf("%.2f%s\n", time, s);
43 else
44 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
45 }'
46 }
47
48
49 # Compressor mini summary
50
51 comp_summary() {
52 # "$time0" "$size0" "$size1"
53 status
54 [ "$2" -eq 0 ] && return
55 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
56 cache_msg=''
57 if [ -s "$cache_stat" ]; then
58 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
59 echo -n > $cache_stat
60 fi
61 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
62 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
63 }
64
65
66 # Calculating different sizes
67
68 sizes() {
69 case $1 in
70 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
71 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
72 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
73 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
74 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
75 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
76 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
77 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
78 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
79 esac | awk '{s+=$5}END{print s}'
80 }
81
82
83 # Query cache for already existing compressed file; substitute original file with the cached
84 # compressed one if so.
85 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
86
87 query_cache() {
88 md5=$(md5sum "$2")
89 cachefile="$comp_cache_root/$1/${md5%% *}"
90 echo "$cachefile"
91 if [ -f "$cachefile" ]; then
92 ln -f "$cachefile" "$2"
93 echo '+' >> "$cache_stat"
94 else
95 echo '-' >> "$cache_stat"
96 false
97 fi
98 }
99
100
101 # Store compressed file to the cache
102 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
103
104 store_cache() {
105 mkdir -p "${1%/*}"
106 mv "$2" "$1"
107 ln "$1" "$2"
108 }
109
110
111 # Function to compress all man pages
112 # Compressing can be disabled with COOKOPTS="!manz"
113
114 compress_manpages() {
115 time0=$(get_time)
116 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
117 manpath="$install/usr/share/man"
118 [ -d "$manpath" ] || return
119 size0=$(sizes man); [ -z "$size0" ] && return
120
121 tazpkg -gi advancecomp --quiet --cookmode
122
123 action 'Compressing man pages...'
124
125 # We'll use only Gzip compression, so decompress other formats first
126 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
127 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
128
129 # Fast compress with gzip
130 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
131
132 # Fix symlinks
133 for i in $(find $manpath -type l); do
134 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
135 link=$(echo $i | sed 's|\.[gbx]z2*$||')
136 rm $i; ln -s $dest.gz $link.gz
137 done
138
139 # Recompress with advdef (it can't compress, only recompress)
140 IFS=$'\n'
141 for i in $(find $manpath -type f); do
142 if ! cached_path=$(query_cache mangz "$i"); then
143 advdef -z4q "$i"
144 store_cache "$cached_path" "$i"
145 fi
146 done
147
148 comp_summary "$time0" "$size0" "$(sizes man)"
149 }
150
151
152 # Function to recompress all gzip archives
153 # Recompressing can be disabled with COOKOPTS="!gz"
154
155 recompress_gz() {
156 time0=$(get_time)
157 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
158 size0=$(sizes gz); [ -z "$size0" ] && return
159
160 tazpkg -gi advancecomp --quiet --cookmode
161
162 action 'Recompressing gzip files...'
163
164 # Recompress with advdef
165 IFS=$'\n'
166 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
167 if ! cached_path=$(query_cache gz "$i"); then
168 advdef -z4q "$i"
169 store_cache "$cached_path" "$i"
170 fi
171 done
172
173 comp_summary "$time0" "$size0" "$(sizes gz)"
174 }
175
176
177 # Function used after compile_rules() to compress all png images
178 # Compressing can be disabled with COOKOPTS="!pngz"
179
180 compress_png() {
181 time0=$(get_time)
182 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
183 size0=$(sizes png); [ -z "$size0" ] && return
184
185 use_pq=true
186 use_op=true
187 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
188 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
189 $use_pq && tazpkg -gi pngquant --quiet --cookmode
190 $use_op && tazpkg -gi optipng --quiet --cookmode
191
192 action 'Compressing png images...'
193
194 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
195 [ -z "$oplevel" ] && oplevel='2'
196
197 cache_section="png$oplevel"
198 $use_pq && cache_section="${cache_section}p"
199 $use_op && cache_section="${cache_section}o"
200
201 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
202
203 IFS=$'\n'
204 for i in $(find $install -type f -name '*.png'); do
205 if ! cached_path=$(query_cache $cache_section "$i"); then
206 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
207 $use_op && optipng -quiet -strip all -o$oplevel "$i"
208 store_cache "$cached_path" "$i"
209 fi
210 done
211
212 comp_summary "$time0" "$size0" "$(sizes png)"
213 }
214
215
216 # Function used after compile_rules() to compress all svg images
217 # Compressing can be disabled with COOKOPTS="!svgz"
218
219 compress_svg() {
220 time0=$(get_time)
221 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
222 size0=$(sizes svg); [ -z "$size0" ] && return
223
224 tazpkg -gi svgcleaner --quiet --cookmode
225
226 action 'Compressing svg images...'
227
228 cleaner_log="$(mktemp)"
229 IFS=$'\n'
230 for i in $(find $install -type f -name '*.svg'); do
231 echo -n "$i: " >> "$cleaner_log"
232 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
233 done
234
235 comp_summary "$time0" "$size0" "$(sizes svg)"
236
237 sed -i '/: $/d' "$cleaner_log"
238 if [ -s "$cleaner_log" ]; then
239 _ 'Cleaner warnings and errors:'
240 awk '{printf " %s\n", $0;}' "$cleaner_log"
241 echo
242 fi
243 rm "$cleaner_log"
244 }
245
246
247 # Function used after compile_rules() to shrink all *.ui and *.glade files:
248 # remove insignificant spaces and comments
249 # Compressing can be disabled with COOKOPTS="!uiz"
250
251 compress_ui() {
252 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
253 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
254
255 tazpkg -gi xmlstarlet --quiet --cookmode
256
257 action 'Compressing ui files...'
258
259 size0=$(sizes xml)
260 time0=$(get_time)
261 temp_ui="$(mktemp)"
262 IFS=$'\n'
263 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
264 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
265 cat "$temp_ui" > "$ui"
266 done
267
268 comp_summary "$time0" "$size0" "$(sizes xml)"
269 rm "$temp_ui"
270 }
271
272
273 # Get list of supported locales...
274
275 get_supported_locales() {
276 lpc='/slitaz-i18n/stuff/locale-pack.conf'
277 if [ -e "$WOK$lpc" ]; then
278 # ... from package in the local wok
279 . "$WOK$lpc"
280 else
281 # ... from Hg
282 temp_conf=$(mktemp)
283 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
284 if [ -s $temp_conf ]; then
285 . $temp_conf
286 else
287 # Give up and use hardcoded list
288 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
289 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
290 fi
291 rm $temp_conf
292 fi
293 echo $LOCALE_PACK
294 }
295
296
297 # Fix common errors and warnings in the .desktop files
298 # Fixing can be disabled with COOKOPTS="!fixdesktops"
299
300 fix_desktop_files() {
301 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
302 deskpath="$install/usr/share/applications"
303 [ -d "$deskpath" ] || return
304 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
305
306 size0=$(sizes des)
307 time0=$(get_time)
308
309 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
310 tazpkg -gi desktop-file-utils-extra --quiet --cookmode
311 fi
312
313 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
314 # Default value is "" (empty). That means for us that we'll use the full
315 # list of supported locales here.
316 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
317
318 IFS=$'\n'
319 for desktop in $(find $deskpath -type f -name '*.desktop'); do
320 cp "$desktop" "$desktop.orig"
321
322 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
323 sdft "$desktop" -i
324
325 # Fix common errors in .desktop file
326 fix-desktop-file "$desktop"
327
328 # Strip unsupported locales from .desktop file
329 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
330 sdft "$desktop" -i -k "$LOCALE"
331
332 # Extra-strip
333 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
334 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
335
336 if [ -n "$QA" ]; then
337 # Check the rest of errors, warnings and tips
338 _ 'QA: Checking %s...' "$(basename $desktop)"
339 diff "$desktop.orig" "$desktop"
340 desktop-file-validate "$desktop" | busybox fold -s
341 echo
342 fi
343
344 rm "$desktop.orig"
345 done
346
347 comp_summary "$time0" "$size0" "$(sizes des)"
348 }
349
350
351 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
352 # to the translation (msgid = msgstr)
353 # Normalization can be disabled with COOKOPTS="!monorm"
354
355 normalize_mo() {
356 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
357 [ -z "$(find $install -type f -name '*.mo')" ] && return
358
359 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
360 tazpkg -gi gettext --quiet --cookmode
361 # Gconv modules (convert to UTF-8)
362 tazpkg -gi glibc-locale --quiet --cookmode
363
364 action 'Normalizing mo files...'
365
366 size0=$(sizes mo1)
367 time0=$(get_time)
368
369 # Process all existing *.mo files
370 IFS=$'\n'
371 for mo in $(find "$install" -type f -name '*.mo'); do
372 tmpfile="$(mktemp)"
373
374 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
375 # add newline
376 echo >> "$tmpfile"
377
378 # get Plural-Forms
379 awk '
380 BEGIN { skip = ""; }
381 {
382 if (! skip) {
383 s = $0;
384 gsub(/^[^\"]*\"/, "", s);
385 gsub(/\"$/, "", s);
386 printf("%s", s);
387 }
388 if (! $0) skip = "yes";
389 }
390 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
391
392 if ! grep -q 'msgid_plural' "$tmpfile"; then
393 echo > "$tmpfile.pf"
394 fi
395
396 # main
397 awk -v pf="$(cat "$tmpfile.pf")" '
398 function clean() {
399 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
400 }
401
402 function getstring() {
403 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
404 s = $0;
405 gsub(/^[^\"]*\"/, "", s);
406 gsub(/\"$/, "", s);
407 return s;
408 }
409
410 BEGIN {
411 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
412 if (pf)
413 printf("\"%s\\n\"\n", pf);
414 printf("\n");
415 skip = 1;
416 clean();
417 }
418
419 {
420 # Skip the entire header
421 if (!skip) {
422 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
423 mode = $1;
424 if ($1 == "msgstr[0]") mode = "msgstr0";
425 if ($1 == "msgstr[1]") mode = "msgstr1";
426 if ($1 == "msgstr[2]") mode = "msgstr2";
427 if ($1 == "msgstr[3]") mode = "msgstr3";
428 if ($1 == "msgstr[4]") mode = "msgstr4";
429 if ($1 == "msgstr[5]") mode = "msgstr5";
430
431 if (mode == "msgctxt") msgctxt = msgctxt getstring();
432 if (mode == "msgid") msgid = msgid getstring();
433 if (mode == "msgstr") msgstr = msgstr getstring();
434 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
435 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
436 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
437 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
438 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
439 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
440 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
441
442 if (! $0) {
443 if (msgid != msgstr) {
444 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
445 printf("msgid \"%s\"\n", msgid);
446 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
447 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
448 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
449 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
450 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
451 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
452 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
453 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
454 printf("\n");
455 }
456 clean();
457 }
458 }
459 if ($0 == "") skip = "";
460 }
461 ' "$tmpfile" > "$tmpfile.awk"
462
463 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
464
465 if [ -s "$tmpfile.mo" ]; then
466 rm "$mo"; mv "$tmpfile.mo" "$mo"
467 else
468 _ 'Error processing %s' "$mo"
469 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
470 fi
471
472 # Clean
473 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
474 done
475
476 comp_summary "$time0" "$size0" "$(sizes mo1)"
477 }
478
479
480 # Strip locale definitions: normalize whitespace and remove comments
481 # Stripping can be disabled with COOKOPTS="!locdef"
482
483 strip_locale_def() {
484 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
485 [ ! -d "$install/usr/share/i18n/locales" ] && return
486 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
487
488 action 'Stripping locale definitions...'
489 size0=$(sizes loc)
490 time0=$(get_time)
491
492 for i in $(find $install/usr/share/i18n/locales -type f); do
493 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
494 done
495
496 comp_summary "$time0" "$size0" "$(sizes loc)"
497 }
498
499
500 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
501 # as removing unneeded files like in Python packages. Cross compiled binaries
502 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
503 # Stripping can be disabled with COOKOPTS="!strip"
504
505 strip_package() {
506 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
507
508 case "$ARCH" in
509 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
510 *) export STRIP='strip' ;;
511 esac
512 action 'Executing strip on all files...'
513 size0=0
514 size1=0
515 time0=$(get_time)
516
517 # Strip executable files
518 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
519 if [ -d "$dir" ]; then
520 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
521 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
522 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
523 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
524 fi
525 done
526
527 # Strip shared and static libraries
528 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist
529 oldsize=$(find $fs -type f \( \
530 -name '*.so*' -o -name '*.a' -o \
531 -name '*.pyc' -o -name '*.pyo' -o \
532 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
533
534 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
535 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
536 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
537 find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null
538
539 newsize=$(find $fs -type f \( \
540 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
541
542 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
543 }
544
545
546 # Strip unsupported locales (.mo files)
547
548 strip_mo_i18n() {
549 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
550
551 [ ! -d "$fs/usr/share/locale" ] && return
552 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
553
554 action 'Thin out translation files...'
555 size0=$(sizes mo2)
556 time0=$(get_time)
557
558 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
559 # Default value is "" (empty). That means for us that we'll use the full
560 # list of supported locales here.
561 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
562
563 # Existing locales
564 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
565
566 # Thin out the list of existing locales. At the end there will be only locales that need
567 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
568 for keep_locale in $LOCALE; do
569 elocales=${elocales//$keep_locale}
570 done
571
572 # Remove the unsupported locales
573 for rem_locale in $elocales; do
574 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
575 rm -r "$fs/usr/share/locale/$rem_locale"
576 done
577
578 comp_summary "$time0" "$size0" "$(sizes mo2)"
579 }
580
581
582
583
584 case $1 in
585 install)
586 # Compressors working in the $install
587 case "$ARCH" in
588 arm*) ;;
589 *)
590 recompress_gz
591 compress_manpages
592 compress_png
593 compress_svg
594 compress_ui
595 ;;
596 esac
597
598 fix_desktop_files
599 normalize_mo
600 strip_locale_def
601 ;;
602 fs)
603 # Compressors working in the $fs
604 strip_package
605 strip_mo_i18n
606 ;;
607 esac
608
609 # Clean
610 rm "$cache_stat"
611 find $comp_cache_root -type f -links -2 -delete