cookutils view modules/compressor @ rev 912
cook: remake copy_generic_files()/copy_generic_stuff() logic; change build dep installation (AUTO_COOK isn't work now) with tazpkg cookmode; improve mk_pkg_receipt(), copy(); update 'cook package --pack'; add docs about copy(); lighttpd/index.cgi: summary on unpackaged files; tiny edits on lighttpd/cooker.css and modules/compressor.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon Jun 05 15:59:11 2017 +0300 (2017-06-05) |
| parents | 39feb4e7243d |
| children | 5f884df71b44 |
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
9 # Compressor cache stuff
11 comp_cache_root='/home/slitaz/cache/cook'
12 mkdir -p "$comp_cache_root"
13 cache_stat=$(mktemp)
15 # Cache notes.
16 # Do not do the same job twice. Getting the file from the cache is much faster
17 # than compressing the file one more time. In addition, this cache is trying not
18 # to take extra space, using the hardlinks. Although the files from the cache
19 # without reference to itself should be removed periodically.
24 #
25 # Functions
26 #
29 # tazpkg install command
31 tpi() {
32 tazpkg -gi --quiet --local --cookmode $1
33 }
36 # Working with time (with hundredths precision)
38 get_time() {
39 cut -d" " -f2 /proc/uptime
40 }
42 calc_time() {
43 # L10n: 's' is for seconds, 'm' is for minutes
44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
45 BEGIN{
46 time = b - a;
47 if (time < 30)
48 printf("%.2f%s\n", time, s);
49 else
50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
51 }'
52 }
55 # Compressor mini summary
57 comp_summary() {
58 # "$time0" "$size0" "$size1"
59 status
60 [ "$2" -eq 0 ] && return
61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
62 cache_msg=''
63 if [ -s "$cache_stat" ]; then
64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
65 echo -n > $cache_stat
66 fi
67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
69 }
72 # Calculating different sizes
74 sizes() {
75 case $1 in
76 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
77 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
78 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
79 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
80 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
81 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
82 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
83 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
84 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
85 str) find $fs -type f \( -name '*.so*' -o -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
86 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \) -exec ls -l \{\} \; ;;
87 esac | awk '{s+=$5}END{print s}'
88 }
91 # Query cache for already existing compressed file; substitute original file with the cached
92 # compressed one if so.
93 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
95 query_cache() {
96 md5=$(md5sum "$2")
97 cachefile="$comp_cache_root/$1/${md5%% *}"
98 echo "$cachefile"
99 if [ -f "$cachefile" ]; then
100 ln -f "$cachefile" "$2"
101 echo '+' >> "$cache_stat"
102 else
103 echo '-' >> "$cache_stat"
104 false
105 fi
106 }
109 # Store compressed file to the cache
110 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
112 store_cache() {
113 mkdir -p "${1%/*}"
114 mv "$2" "$1"
115 ln "$1" "$2"
116 }
119 # Function to compress all man pages
120 # Compressing can be disabled with COOKOPTS="!manz"
122 compress_manpages() {
123 time0=$(get_time)
124 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
125 manpath="$install/usr/share/man"
126 [ -d "$manpath" ] || return
127 size0=$(sizes man); [ -z "$size0" ] && return
129 tpi advancecomp
131 action 'Compressing man pages...'
133 # We'll use only Gzip compression, so decompress other formats first
134 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
135 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
137 # Fast compress with gzip
138 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
140 # Fix symlinks
141 for i in $(find $manpath -type l); do
142 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
143 link=$(echo $i | sed 's|\.[gbx]z2*$||')
144 rm $i; ln -s $dest.gz $link.gz
145 done
147 # Recompress with advdef (it can't compress, only recompress)
148 IFS=$'\n'
149 for i in $(find $manpath -type f); do
150 if ! cached_path=$(query_cache mangz "$i"); then
151 advdef -z4q "$i"
152 store_cache "$cached_path" "$i"
153 fi
154 done
156 comp_summary "$time0" "$size0" "$(sizes man)"
157 }
160 # Function to recompress all gzip archives
161 # Recompressing can be disabled with COOKOPTS="!gz"
163 recompress_gz() {
164 time0=$(get_time)
165 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
166 size0=$(sizes gz); [ -z "$size0" ] && return
168 tpi advancecomp
170 action 'Recompressing gzip files...'
172 # Recompress with advdef
173 IFS=$'\n'
174 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
175 if ! cached_path=$(query_cache gz "$i"); then
176 advdef -z4q "$i"
177 store_cache "$cached_path" "$i"
178 fi
179 done
181 comp_summary "$time0" "$size0" "$(sizes gz)"
182 }
185 # Function used after compile_rules() to compress all png images
186 # Compressing can be disabled with COOKOPTS="!pngz"
188 compress_png() {
189 time0=$(get_time)
190 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
191 size0=$(sizes png); [ -z "$size0" ] && return
193 use_pq=true
194 use_op=true
195 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
196 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
197 $use_pq && tpi pngquant
198 $use_op && tpi optipng
200 action 'Compressing png images...'
202 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
203 [ -z "$oplevel" ] && oplevel='2'
205 cache_section="png$oplevel"
206 $use_pq && cache_section="${cache_section}p"
207 $use_op && cache_section="${cache_section}o"
209 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
211 IFS=$'\n'
212 for i in $(find $install -type f -name '*.png'); do
213 if ! cached_path=$(query_cache $cache_section "$i"); then
214 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
215 $use_op && optipng -quiet -strip all -o$oplevel "$i"
216 store_cache "$cached_path" "$i"
217 fi
218 done
220 comp_summary "$time0" "$size0" "$(sizes png)"
221 }
224 # Function used after compile_rules() to compress all svg images
225 # Compressing can be disabled with COOKOPTS="!svgz"
227 compress_svg() {
228 time0=$(get_time)
229 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
230 size0=$(sizes svg); [ -z "$size0" ] && return
232 tpi svgcleaner
234 action 'Compressing svg images...'
236 cleaner_log="$(mktemp)"
237 IFS=$'\n'
238 for i in $(find $install -type f -name '*.svg'); do
239 echo -n "$i: " >> "$cleaner_log"
240 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
241 done
243 comp_summary "$time0" "$size0" "$(sizes svg)"
245 sed -i '/: $/d' "$cleaner_log"
246 if [ -s "$cleaner_log" ]; then
247 _ 'Cleaner warnings and errors:'
248 awk '{printf " %s\n", $0;}' "$cleaner_log"
249 echo
250 fi
251 rm "$cleaner_log"
252 }
255 # Function used after compile_rules() to shrink all *.ui and *.glade files:
256 # remove insignificant spaces and comments
257 # Compressing can be disabled with COOKOPTS="!uiz"
259 compress_ui() {
260 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
261 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
263 tpi xmlstarlet
265 action 'Compressing ui files...'
267 size0=$(sizes xml)
268 time0=$(get_time)
269 temp_ui="$(mktemp)"
270 IFS=$'\n'
271 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
272 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
273 cat "$temp_ui" > "$ui"
274 done
276 comp_summary "$time0" "$size0" "$(sizes xml)"
277 rm "$temp_ui"
278 }
281 # Get list of supported locales...
283 get_supported_locales() {
284 lpc='/slitaz-i18n/stuff/locale-pack.conf'
285 if [ -e "$WOK$lpc" ]; then
286 # ... from package in the local wok
287 . "$WOK$lpc"
288 else
289 # ... from Hg
290 temp_conf=$(mktemp)
291 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
292 if [ -s $temp_conf ]; then
293 . $temp_conf
294 else
295 # Give up and use hardcoded list
296 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
297 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
298 fi
299 rm $temp_conf
300 fi
301 echo $LOCALE_PACK
302 }
305 # Fix common errors and warnings in the .desktop files
306 # Fixing can be disabled with COOKOPTS="!fixdesktops"
308 fix_desktop_files() {
309 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
310 deskpath="$install/usr/share/applications"
311 [ -d "$deskpath" ] || return
312 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
314 size0=$(sizes des)
315 time0=$(get_time)
317 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
318 tpi desktop-file-utils-extra
319 fi
321 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
322 # Default value is "" (empty). That means for us that we'll use the full
323 # list of supported locales here.
324 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
326 IFS=$'\n'
327 for desktop in $(find $deskpath -type f -name '*.desktop'); do
328 cp "$desktop" "$desktop.orig"
330 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
331 sdft "$desktop" -i
333 # Fix common errors in .desktop file
334 fix-desktop-file "$desktop"
336 # Strip unsupported locales from .desktop file
337 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
338 sdft "$desktop" -i -k "$LOCALE"
340 # Extra-strip
341 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
342 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
344 if [ -n "$QA" ]; then
345 # Check the rest of errors, warnings and tips
346 _ 'QA: Checking %s...' "$(basename $desktop)"
347 diff "$desktop.orig" "$desktop"
348 desktop-file-validate "$desktop" | busybox fold -s
349 echo
350 fi
352 rm "$desktop.orig"
353 done
355 comp_summary "$time0" "$size0" "$(sizes des)"
356 }
359 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
360 # to the translation (msgid = msgstr)
361 # Normalization can be disabled with COOKOPTS="!monorm"
363 normalize_mo() {
364 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
365 [ -z "$(find $install -type f -name '*.mo')" ] && return
367 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
368 tpi gettext
369 # Gconv modules (convert to UTF-8)
370 tpi glibc-locale
372 action 'Normalizing mo files...'
374 size0=$(sizes mo1)
375 time0=$(get_time)
377 # Process all existing *.mo files
378 IFS=$'\n'
379 for mo in $(find "$install" -type f -name '*.mo'); do
380 tmpfile="$(mktemp)"
382 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
383 # add newline
384 echo >> "$tmpfile"
386 # get Plural-Forms
387 awk '
388 BEGIN { skip = ""; }
389 {
390 if (! skip) {
391 s = $0;
392 gsub(/^[^\"]*\"/, "", s);
393 gsub(/\"$/, "", s);
394 printf("%s", s);
395 }
396 if (! $0) skip = "yes";
397 }
398 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
400 if ! grep -q 'msgid_plural' "$tmpfile"; then
401 echo > "$tmpfile.pf"
402 fi
404 # main
405 awk -v pf="$(cat "$tmpfile.pf")" '
406 function clean() {
407 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
408 }
410 function getstring() {
411 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
412 s = $0;
413 gsub(/^[^\"]*\"/, "", s);
414 gsub(/\"$/, "", s);
415 return s;
416 }
418 BEGIN {
419 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
420 if (pf)
421 printf("\"%s\\n\"\n", pf);
422 printf("\n");
423 skip = 1;
424 clean();
425 }
427 {
428 # Skip the entire header
429 if (!skip) {
430 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
431 mode = $1;
432 if ($1 == "msgstr[0]") mode = "msgstr0";
433 if ($1 == "msgstr[1]") mode = "msgstr1";
434 if ($1 == "msgstr[2]") mode = "msgstr2";
435 if ($1 == "msgstr[3]") mode = "msgstr3";
436 if ($1 == "msgstr[4]") mode = "msgstr4";
437 if ($1 == "msgstr[5]") mode = "msgstr5";
439 if (mode == "msgctxt") msgctxt = msgctxt getstring();
440 if (mode == "msgid") msgid = msgid getstring();
441 if (mode == "msgstr") msgstr = msgstr getstring();
442 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
443 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
444 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
445 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
446 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
447 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
448 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
450 if (! $0) {
451 if (msgid != msgstr) {
452 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
453 printf("msgid \"%s\"\n", msgid);
454 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
455 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
456 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
457 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
458 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
459 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
460 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
461 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
462 printf("\n");
463 }
464 clean();
465 }
466 }
467 if ($0 == "") skip = "";
468 }
469 ' "$tmpfile" > "$tmpfile.awk"
471 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
473 if [ -s "$tmpfile.mo" ]; then
474 rm "$mo"; mv "$tmpfile.mo" "$mo"
475 else
476 _ 'Error processing %s' "$mo"
477 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
478 fi
480 # Clean
481 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
482 done
484 comp_summary "$time0" "$size0" "$(sizes mo1)"
485 }
488 # Strip locale definitions: normalize whitespace and remove comments
489 # Stripping can be disabled with COOKOPTS="!locdef"
491 strip_locale_def() {
492 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
493 [ ! -d "$install/usr/share/i18n/locales" ] && return
494 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
496 action 'Stripping locale definitions...'
497 size0=$(sizes loc)
498 time0=$(get_time)
500 for i in $(find $install/usr/share/i18n/locales -type f); do
501 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
502 done
504 comp_summary "$time0" "$size0" "$(sizes loc)"
505 }
508 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
509 # as removing unneeded files like in Python packages. Cross compiled binaries
510 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
511 # Stripping can be disabled with COOKOPTS="!strip"
513 strip_package() {
514 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
516 case "$ARCH" in
517 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
518 *) export STRIP='strip' ;;
519 esac
520 action 'Executing strip on all files...'
521 size0=0
522 size1=0
523 time0=$(get_time)
525 # Strip executable files
526 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
527 if [ -d "$dir" ]; then
528 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
529 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
530 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
531 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
532 fi
533 done
535 oldsize=$(sizes str)
537 # Strip shared and static libraries
538 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
539 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
541 # Remove Python *.pyc and *.pyo
542 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
544 # Remove both with the empty subfolders:
545 # 1. Perl perllocal.pod and .packlist (unconditionally)
546 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
547 # 2. Perl *.pod (if not disabled)
548 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
549 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')"
550 echo "$perlfiles" | xargs rm -f 2>/dev/null
551 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null
553 # Strip Perl files (*.pm and *.pl) from documentation inside (if not disabled)
554 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
555 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
557 newsize=$(sizes str)
559 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
560 }
563 # Strip unsupported locales (.mo files)
565 strip_mo_i18n() {
566 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
568 [ ! -d "$fs/usr/share/locale" ] && return
569 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
571 action 'Thin out translation files...'
572 size0=$(sizes mo2)
573 time0=$(get_time)
575 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
576 # Default value is "" (empty). That means for us that we'll use the full
577 # list of supported locales here.
578 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
580 # Existing locales
581 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
583 # Thin out the list of existing locales. At the end there will be only locales that need
584 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
585 for keep_locale in $LOCALE; do
586 elocales=${elocales//$keep_locale}
587 done
589 # Remove the unsupported locales
590 for rem_locale in $elocales; do
591 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
592 rm -r "$fs/usr/share/locale/$rem_locale"
593 done
595 comp_summary "$time0" "$size0" "$(sizes mo2)"
596 }
601 case $1 in
602 install)
603 # Compressors working in the $install
604 case "$ARCH" in
605 arm*) ;;
606 *)
607 recompress_gz
608 compress_manpages
609 compress_png
610 compress_svg
611 compress_ui
612 ;;
613 esac
615 fix_desktop_files
616 normalize_mo
617 strip_locale_def
618 ;;
619 fs)
620 # Compressors working in the $fs
621 strip_package
622 strip_mo_i18n
623 ;;
624 esac
626 # Clean
627 rm "$cache_stat"
628 find $comp_cache_root -type f -links -2 -delete