cookutils view modules/compressor @ rev 893
web/cooker.cgi: work with HTTP headers "Last-Modified", "If-Modified-Since", "HTTP 304 Not Modified", "HTTP 404 Not Found". Other small improvements.
SliTaz Next Cooker is working right now with this version of cooker.cgi.
SliTaz Next Cooker is working right now with this version of cooker.cgi.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Mon Mar 20 05:12:29 2017 +0200 (2017-03-20) |
parents | 3f0ca4492ec7 |
children | ea11d3c96873 |
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 # Compressor cache stuff
12 comp_cache_root='/home/slitaz/cache/cook'
13 mkdir -p "$comp_cache_root"
14 cache_stat=$(mktemp)
16 # Cache notes.
17 # Do not do the same job twice. Getting the file from the cache is much faster
18 # than compressing the file one more time. In addition, this cache is trying not
19 # to take extra space, using the hardlinks. Although the files from the cache
20 # without reference to itself should be removed periodically.
25 #
26 # Functions
27 #
30 # Working with time (with hundredths precision)
32 get_time() {
33 cut -d" " -f2 /proc/uptime
34 }
36 calc_time() {
37 # L10n: 's' is for seconds, 'm' is for minutes
38 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
39 BEGIN{
40 time = b - a;
41 if (time < 30)
42 printf("%.2f%s\n", time, s);
43 else
44 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
45 }'
46 }
49 # Compressor mini summary
51 comp_summary() {
52 # "$time0" "$size0" "$size1"
53 status
54 [ "$2" -eq 0 ] && return
55 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
56 cache_msg=''
57 if [ -s "$cache_stat" ]; then
58 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
59 echo -n > $cache_stat
60 fi
61 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
62 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
63 }
66 # Calculating different sizes
68 sizes() {
69 case $1 in
70 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
71 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
72 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
73 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
74 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
75 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
76 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
77 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
78 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
79 esac | awk '{s+=$5}END{print s}'
80 }
83 # Query cache for already existing compressed file; substitute original file with the cached
84 # compressed one if so.
85 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
87 query_cache() {
88 md5=$(md5sum "$2")
89 cachefile="$comp_cache_root/$1/${md5%% *}"
90 echo "$cachefile"
91 if [ -f "$cachefile" ]; then
92 ln -f "$cachefile" "$2"
93 echo '+' >> "$cache_stat"
94 else
95 echo '-' >> "$cache_stat"
96 false
97 fi
98 }
101 # Store compressed file to the cache
102 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
104 store_cache() {
105 mkdir -p "${1%/*}"
106 mv "$2" "$1"
107 ln "$1" "$2"
108 }
111 # Function to compress all man pages
112 # Compressing can be disabled with COOKOPTS="!manz"
114 compress_manpages() {
115 time0=$(get_time)
116 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
117 manpath="$install/usr/share/man"
118 [ -d "$manpath" ] || return
119 size0=$(sizes man); [ -z "$size0" ] && return
121 tazpkg -gi advancecomp --quiet --cookmode
123 action 'Compressing man pages...'
125 # We'll use only Gzip compression, so decompress other formats first
126 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
127 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
129 # Fast compress with gzip
130 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
132 # Fix symlinks
133 for i in $(find $manpath -type l); do
134 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
135 link=$(echo $i | sed 's|\.[gbx]z2*$||')
136 rm $i; ln -s $dest.gz $link.gz
137 done
139 # Recompress with advdef (it can't compress, only recompress)
140 IFS=$'\n'
141 for i in $(find $manpath -type f); do
142 if ! cached_path=$(query_cache mangz "$i"); then
143 advdef -z4q "$i"
144 store_cache "$cached_path" "$i"
145 fi
146 done
148 comp_summary "$time0" "$size0" "$(sizes man)"
149 }
152 # Function to recompress all gzip archives
153 # Recompressing can be disabled with COOKOPTS="!gz"
155 recompress_gz() {
156 time0=$(get_time)
157 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
158 size0=$(sizes gz); [ -z "$size0" ] && return
160 tazpkg -gi advancecomp --quiet --cookmode
162 action 'Recompressing gzip files...'
164 # Recompress with advdef
165 IFS=$'\n'
166 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
167 if ! cached_path=$(query_cache gz "$i"); then
168 advdef -z4q "$i"
169 store_cache "$cached_path" "$i"
170 fi
171 done
173 comp_summary "$time0" "$size0" "$(sizes gz)"
174 }
177 # Function used after compile_rules() to compress all png images
178 # Compressing can be disabled with COOKOPTS="!pngz"
180 compress_png() {
181 time0=$(get_time)
182 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
183 size0=$(sizes png); [ -z "$size0" ] && return
185 use_pq=true
186 use_op=true
187 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
188 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
189 $use_pq && tazpkg -gi pngquant --quiet --cookmode
190 $use_op && tazpkg -gi optipng --quiet --cookmode
192 action 'Compressing png images...'
194 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
195 [ -z "$oplevel" ] && oplevel='2'
197 cache_section="png$oplevel"
198 $use_pq && cache_section="${cache_section}p"
199 $use_op && cache_section="${cache_section}o"
201 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
203 IFS=$'\n'
204 for i in $(find $install -type f -name '*.png'); do
205 if ! cached_path=$(query_cache $cache_section "$i"); then
206 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
207 $use_op && optipng -quiet -strip all -o$oplevel "$i"
208 store_cache "$cached_path" "$i"
209 fi
210 done
212 comp_summary "$time0" "$size0" "$(sizes png)"
213 }
216 # Function used after compile_rules() to compress all svg images
217 # Compressing can be disabled with COOKOPTS="!svgz"
219 compress_svg() {
220 time0=$(get_time)
221 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
222 size0=$(sizes svg); [ -z "$size0" ] && return
224 tazpkg -gi svgcleaner --quiet --cookmode
226 action 'Compressing svg images...'
228 cleaner_log="$(mktemp)"
229 IFS=$'\n'
230 for i in $(find $install -type f -name '*.svg'); do
231 echo -n "$i: " >> "$cleaner_log"
232 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
233 done
235 comp_summary "$time0" "$size0" "$(sizes svg)"
237 sed -i '/: $/d' "$cleaner_log"
238 if [ -s "$cleaner_log" ]; then
239 _ 'Cleaner warnings and errors:'
240 awk '{printf " %s\n", $0;}' "$cleaner_log"
241 echo
242 fi
243 rm "$cleaner_log"
244 }
247 # Function used after compile_rules() to shrink all *.ui and *.glade files:
248 # remove insignificant spaces and comments
249 # Compressing can be disabled with COOKOPTS="!uiz"
251 compress_ui() {
252 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
253 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
255 tazpkg -gi xmlstarlet --quiet --cookmode
257 action 'Compressing ui files...'
259 size0=$(sizes xml)
260 time0=$(get_time)
261 temp_ui="$(mktemp)"
262 IFS=$'\n'
263 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
264 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
265 cat "$temp_ui" > "$ui"
266 done
268 comp_summary "$time0" "$size0" "$(sizes xml)"
269 rm "$temp_ui"
270 }
273 # Get list of supported locales...
275 get_supported_locales() {
276 lpc='/slitaz-i18n/stuff/locale-pack.conf'
277 if [ -e "$WOK$lpc" ]; then
278 # ... from package in the local wok
279 . "$WOK$lpc"
280 else
281 # ... from Hg
282 temp_conf=$(mktemp)
283 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
284 if [ -s $temp_conf ]; then
285 . $temp_conf
286 else
287 # Give up and use hardcoded list
288 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
289 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
290 fi
291 rm $temp_conf
292 fi
293 echo $LOCALE_PACK
294 }
297 # Fix common errors and warnings in the .desktop files
298 # Fixing can be disabled with COOKOPTS="!fixdesktops"
300 fix_desktop_files() {
301 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
302 deskpath="$install/usr/share/applications"
303 [ -d "$deskpath" ] || return
304 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
306 size0=$(sizes des)
307 time0=$(get_time)
309 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
310 tazpkg -gi desktop-file-utils-extra --quiet --cookmode
311 fi
313 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
314 # Default value is "" (empty). That means for us that we'll use the full
315 # list of supported locales here.
316 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
318 IFS=$'\n'
319 for desktop in $(find $deskpath -type f -name '*.desktop'); do
320 cp "$desktop" "$desktop.orig"
322 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
323 sdft "$desktop" -i
325 # Fix common errors in .desktop file
326 fix-desktop-file "$desktop"
328 # Strip unsupported locales from .desktop file
329 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
330 sdft "$desktop" -i -k "$LOCALE"
332 # Extra-strip
333 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
334 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
336 if [ -n "$QA" ]; then
337 # Check the rest of errors, warnings and tips
338 _ 'QA: Checking %s...' "$(basename $desktop)"
339 diff "$desktop.orig" "$desktop"
340 desktop-file-validate "$desktop" | busybox fold -s
341 echo
342 fi
344 rm "$desktop.orig"
345 done
347 comp_summary "$time0" "$size0" "$(sizes des)"
348 }
351 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
352 # to the translation (msgid = msgstr)
353 # Normalization can be disabled with COOKOPTS="!monorm"
355 normalize_mo() {
356 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
357 [ -z "$(find $install -type f -name '*.mo')" ] && return
359 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
360 tazpkg -gi gettext --quiet --cookmode
361 # Gconv modules (convert to UTF-8)
362 tazpkg -gi glibc-locale --quiet --cookmode
364 action 'Normalizing mo files...'
366 size0=$(sizes mo1)
367 time0=$(get_time)
369 # Process all existing *.mo files
370 IFS=$'\n'
371 for mo in $(find "$install" -type f -name '*.mo'); do
372 tmpfile="$(mktemp)"
374 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
375 # add newline
376 echo >> "$tmpfile"
378 # get Plural-Forms
379 awk '
380 BEGIN { skip = ""; }
381 {
382 if (! skip) {
383 s = $0;
384 gsub(/^[^\"]*\"/, "", s);
385 gsub(/\"$/, "", s);
386 printf("%s", s);
387 }
388 if (! $0) skip = "yes";
389 }
390 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
392 if ! grep -q 'msgid_plural' "$tmpfile"; then
393 echo > "$tmpfile.pf"
394 fi
396 # main
397 awk -v pf="$(cat "$tmpfile.pf")" '
398 function clean() {
399 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
400 }
402 function getstring() {
403 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
404 s = $0;
405 gsub(/^[^\"]*\"/, "", s);
406 gsub(/\"$/, "", s);
407 return s;
408 }
410 BEGIN {
411 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
412 if (pf)
413 printf("\"%s\\n\"\n", pf);
414 printf("\n");
415 skip = 1;
416 clean();
417 }
419 {
420 # Skip the entire header
421 if (!skip) {
422 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
423 mode = $1;
424 if ($1 == "msgstr[0]") mode = "msgstr0";
425 if ($1 == "msgstr[1]") mode = "msgstr1";
426 if ($1 == "msgstr[2]") mode = "msgstr2";
427 if ($1 == "msgstr[3]") mode = "msgstr3";
428 if ($1 == "msgstr[4]") mode = "msgstr4";
429 if ($1 == "msgstr[5]") mode = "msgstr5";
431 if (mode == "msgctxt") msgctxt = msgctxt getstring();
432 if (mode == "msgid") msgid = msgid getstring();
433 if (mode == "msgstr") msgstr = msgstr getstring();
434 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
435 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
436 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
437 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
438 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
439 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
440 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
442 if (! $0) {
443 if (msgid != msgstr) {
444 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
445 printf("msgid \"%s\"\n", msgid);
446 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
447 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
448 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
449 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
450 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
451 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
452 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
453 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
454 printf("\n");
455 }
456 clean();
457 }
458 }
459 if ($0 == "") skip = "";
460 }
461 ' "$tmpfile" > "$tmpfile.awk"
463 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
465 if [ -s "$tmpfile.mo" ]; then
466 rm "$mo"; mv "$tmpfile.mo" "$mo"
467 else
468 _ 'Error processing %s' "$mo"
469 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
470 fi
472 # Clean
473 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
474 done
476 comp_summary "$time0" "$size0" "$(sizes mo1)"
477 }
480 # Strip locale definitions: normalize whitespace and remove comments
481 # Stripping can be disabled with COOKOPTS="!locdef"
483 strip_locale_def() {
484 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
485 [ ! -d "$install/usr/share/i18n/locales" ] && return
486 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
488 action 'Stripping locale definitions...'
489 size0=$(sizes loc)
490 time0=$(get_time)
492 for i in $(find $install/usr/share/i18n/locales -type f); do
493 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
494 done
496 comp_summary "$time0" "$size0" "$(sizes loc)"
497 }
500 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
501 # as removing unneeded files like in Python packages. Cross compiled binaries
502 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
503 # Stripping can be disabled with COOKOPTS="!strip"
505 strip_package() {
506 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
508 case "$ARCH" in
509 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
510 *) export STRIP='strip' ;;
511 esac
512 action 'Executing strip on all files...'
513 size0=0
514 size1=0
515 time0=$(get_time)
517 # Strip executable files
518 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
519 if [ -d "$dir" ]; then
520 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
521 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
522 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
523 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
524 fi
525 done
527 # Strip shared and static libraries
528 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist
529 oldsize=$(find $fs -type f \( \
530 -name '*.so*' -o -name '*.a' -o \
531 -name '*.pyc' -o -name '*.pyo' -o \
532 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
534 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
535 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
536 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
537 find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null
539 newsize=$(find $fs -type f \( \
540 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
542 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
543 }
546 # Strip unsupported locales (.mo files)
548 strip_mo_i18n() {
549 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
551 [ ! -d "$fs/usr/share/locale" ] && return
552 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
554 action 'Thin out translation files...'
555 size0=$(sizes mo2)
556 time0=$(get_time)
558 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
559 # Default value is "" (empty). That means for us that we'll use the full
560 # list of supported locales here.
561 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
563 # Existing locales
564 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
566 # Thin out the list of existing locales. At the end there will be only locales that need
567 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
568 for keep_locale in $LOCALE; do
569 elocales=${elocales//$keep_locale}
570 done
572 # Remove the unsupported locales
573 for rem_locale in $elocales; do
574 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
575 rm -r "$fs/usr/share/locale/$rem_locale"
576 done
578 comp_summary "$time0" "$size0" "$(sizes mo2)"
579 }
584 case $1 in
585 install)
586 # Compressors working in the $install
587 case "$ARCH" in
588 arm*) ;;
589 *)
590 recompress_gz
591 compress_manpages
592 compress_png
593 compress_svg
594 compress_ui
595 ;;
596 esac
598 fix_desktop_files
599 normalize_mo
600 strip_locale_def
601 ;;
602 fs)
603 # Compressors working in the $fs
604 strip_package
605 strip_mo_i18n
606 ;;
607 esac
609 # Clean
610 rm "$cache_stat"
611 find $comp_cache_root -type f -links -2 -delete