cookutils view modules/compressor @ rev 908
cook: support per-package CONFIG_FILES, fix pre_install etc. extracting; lighttpd/index.cgi: display all descriptions.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon May 29 04:48:50 2017 +0300 (2017-05-29) |
| parents | 897914bd4c94 |
| children | 39feb4e7243d |
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 # Compressor cache stuff
12 comp_cache_root='/home/slitaz/cache/cook'
13 mkdir -p "$comp_cache_root"
14 cache_stat=$(mktemp)
16 # Cache notes.
17 # Do not do the same job twice. Getting the file from the cache is much faster
18 # than compressing the file one more time. In addition, this cache is trying not
19 # to take extra space, using the hardlinks. Although the files from the cache
20 # without reference to itself should be removed periodically.
25 #
26 # Functions
27 #
30 # Working with time (with hundredths precision)
32 get_time() {
33 cut -d" " -f2 /proc/uptime
34 }
36 calc_time() {
37 # L10n: 's' is for seconds, 'm' is for minutes
38 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
39 BEGIN{
40 time = b - a;
41 if (time < 30)
42 printf("%.2f%s\n", time, s);
43 else
44 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
45 }'
46 }
49 # Compressor mini summary
51 comp_summary() {
52 # "$time0" "$size0" "$size1"
53 status
54 [ "$2" -eq 0 ] && return
55 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
56 cache_msg=''
57 if [ -s "$cache_stat" ]; then
58 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
59 echo -n > $cache_stat
60 fi
61 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
62 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
63 }
66 # Calculating different sizes
68 sizes() {
69 case $1 in
70 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
71 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
72 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
73 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
74 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
75 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
76 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
77 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
78 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
79 str) find $fs -type f \( -name '*.so*' -o -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
80 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \) -exec ls -l \{\} \; ;;
81 esac | awk '{s+=$5}END{print s}'
82 }
85 # Query cache for already existing compressed file; substitute original file with the cached
86 # compressed one if so.
87 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
89 query_cache() {
90 md5=$(md5sum "$2")
91 cachefile="$comp_cache_root/$1/${md5%% *}"
92 echo "$cachefile"
93 if [ -f "$cachefile" ]; then
94 ln -f "$cachefile" "$2"
95 echo '+' >> "$cache_stat"
96 else
97 echo '-' >> "$cache_stat"
98 false
99 fi
100 }
103 # Store compressed file to the cache
104 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
106 store_cache() {
107 mkdir -p "${1%/*}"
108 mv "$2" "$1"
109 ln "$1" "$2"
110 }
113 # Function to compress all man pages
114 # Compressing can be disabled with COOKOPTS="!manz"
116 compress_manpages() {
117 time0=$(get_time)
118 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
119 manpath="$install/usr/share/man"
120 [ -d "$manpath" ] || return
121 size0=$(sizes man); [ -z "$size0" ] && return
123 tazpkg -gi advancecomp --quiet --cookmode
125 action 'Compressing man pages...'
127 # We'll use only Gzip compression, so decompress other formats first
128 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
129 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
131 # Fast compress with gzip
132 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
134 # Fix symlinks
135 for i in $(find $manpath -type l); do
136 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
137 link=$(echo $i | sed 's|\.[gbx]z2*$||')
138 rm $i; ln -s $dest.gz $link.gz
139 done
141 # Recompress with advdef (it can't compress, only recompress)
142 IFS=$'\n'
143 for i in $(find $manpath -type f); do
144 if ! cached_path=$(query_cache mangz "$i"); then
145 advdef -z4q "$i"
146 store_cache "$cached_path" "$i"
147 fi
148 done
150 comp_summary "$time0" "$size0" "$(sizes man)"
151 }
154 # Function to recompress all gzip archives
155 # Recompressing can be disabled with COOKOPTS="!gz"
157 recompress_gz() {
158 time0=$(get_time)
159 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
160 size0=$(sizes gz); [ -z "$size0" ] && return
162 tazpkg -gi advancecomp --quiet --cookmode
164 action 'Recompressing gzip files...'
166 # Recompress with advdef
167 IFS=$'\n'
168 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
169 if ! cached_path=$(query_cache gz "$i"); then
170 advdef -z4q "$i"
171 store_cache "$cached_path" "$i"
172 fi
173 done
175 comp_summary "$time0" "$size0" "$(sizes gz)"
176 }
179 # Function used after compile_rules() to compress all png images
180 # Compressing can be disabled with COOKOPTS="!pngz"
182 compress_png() {
183 time0=$(get_time)
184 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
185 size0=$(sizes png); [ -z "$size0" ] && return
187 use_pq=true
188 use_op=true
189 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
190 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
191 $use_pq && tazpkg -gi pngquant --quiet --cookmode
192 $use_op && tazpkg -gi optipng --quiet --cookmode
194 action 'Compressing png images...'
196 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
197 [ -z "$oplevel" ] && oplevel='2'
199 cache_section="png$oplevel"
200 $use_pq && cache_section="${cache_section}p"
201 $use_op && cache_section="${cache_section}o"
203 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
205 IFS=$'\n'
206 for i in $(find $install -type f -name '*.png'); do
207 if ! cached_path=$(query_cache $cache_section "$i"); then
208 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
209 $use_op && optipng -quiet -strip all -o$oplevel "$i"
210 store_cache "$cached_path" "$i"
211 fi
212 done
214 comp_summary "$time0" "$size0" "$(sizes png)"
215 }
218 # Function used after compile_rules() to compress all svg images
219 # Compressing can be disabled with COOKOPTS="!svgz"
221 compress_svg() {
222 time0=$(get_time)
223 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
224 size0=$(sizes svg); [ -z "$size0" ] && return
226 tazpkg -gi svgcleaner --quiet --cookmode
228 action 'Compressing svg images...'
230 cleaner_log="$(mktemp)"
231 IFS=$'\n'
232 for i in $(find $install -type f -name '*.svg'); do
233 echo -n "$i: " >> "$cleaner_log"
234 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
235 done
237 comp_summary "$time0" "$size0" "$(sizes svg)"
239 sed -i '/: $/d' "$cleaner_log"
240 if [ -s "$cleaner_log" ]; then
241 _ 'Cleaner warnings and errors:'
242 awk '{printf " %s\n", $0;}' "$cleaner_log"
243 echo
244 fi
245 rm "$cleaner_log"
246 }
249 # Function used after compile_rules() to shrink all *.ui and *.glade files:
250 # remove insignificant spaces and comments
251 # Compressing can be disabled with COOKOPTS="!uiz"
253 compress_ui() {
254 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
255 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
257 tazpkg -gi xmlstarlet --quiet --cookmode
259 action 'Compressing ui files...'
261 size0=$(sizes xml)
262 time0=$(get_time)
263 temp_ui="$(mktemp)"
264 IFS=$'\n'
265 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
266 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
267 cat "$temp_ui" > "$ui"
268 done
270 comp_summary "$time0" "$size0" "$(sizes xml)"
271 rm "$temp_ui"
272 }
275 # Get list of supported locales...
277 get_supported_locales() {
278 lpc='/slitaz-i18n/stuff/locale-pack.conf'
279 if [ -e "$WOK$lpc" ]; then
280 # ... from package in the local wok
281 . "$WOK$lpc"
282 else
283 # ... from Hg
284 temp_conf=$(mktemp)
285 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
286 if [ -s $temp_conf ]; then
287 . $temp_conf
288 else
289 # Give up and use hardcoded list
290 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
291 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
292 fi
293 rm $temp_conf
294 fi
295 echo $LOCALE_PACK
296 }
299 # Fix common errors and warnings in the .desktop files
300 # Fixing can be disabled with COOKOPTS="!fixdesktops"
302 fix_desktop_files() {
303 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
304 deskpath="$install/usr/share/applications"
305 [ -d "$deskpath" ] || return
306 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
308 size0=$(sizes des)
309 time0=$(get_time)
311 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
312 tazpkg -gi desktop-file-utils-extra --quiet --cookmode
313 fi
315 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
316 # Default value is "" (empty). That means for us that we'll use the full
317 # list of supported locales here.
318 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
320 IFS=$'\n'
321 for desktop in $(find $deskpath -type f -name '*.desktop'); do
322 cp "$desktop" "$desktop.orig"
324 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
325 sdft "$desktop" -i
327 # Fix common errors in .desktop file
328 fix-desktop-file "$desktop"
330 # Strip unsupported locales from .desktop file
331 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
332 sdft "$desktop" -i -k "$LOCALE"
334 # Extra-strip
335 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
336 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
338 if [ -n "$QA" ]; then
339 # Check the rest of errors, warnings and tips
340 _ 'QA: Checking %s...' "$(basename $desktop)"
341 diff "$desktop.orig" "$desktop"
342 desktop-file-validate "$desktop" | busybox fold -s
343 echo
344 fi
346 rm "$desktop.orig"
347 done
349 comp_summary "$time0" "$size0" "$(sizes des)"
350 }
353 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
354 # to the translation (msgid = msgstr)
355 # Normalization can be disabled with COOKOPTS="!monorm"
357 normalize_mo() {
358 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
359 [ -z "$(find $install -type f -name '*.mo')" ] && return
361 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
362 tazpkg -gi gettext --quiet --cookmode
363 # Gconv modules (convert to UTF-8)
364 tazpkg -gi glibc-locale --quiet --cookmode
366 action 'Normalizing mo files...'
368 size0=$(sizes mo1)
369 time0=$(get_time)
371 # Process all existing *.mo files
372 IFS=$'\n'
373 for mo in $(find "$install" -type f -name '*.mo'); do
374 tmpfile="$(mktemp)"
376 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
377 # add newline
378 echo >> "$tmpfile"
380 # get Plural-Forms
381 awk '
382 BEGIN { skip = ""; }
383 {
384 if (! skip) {
385 s = $0;
386 gsub(/^[^\"]*\"/, "", s);
387 gsub(/\"$/, "", s);
388 printf("%s", s);
389 }
390 if (! $0) skip = "yes";
391 }
392 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
394 if ! grep -q 'msgid_plural' "$tmpfile"; then
395 echo > "$tmpfile.pf"
396 fi
398 # main
399 awk -v pf="$(cat "$tmpfile.pf")" '
400 function clean() {
401 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
402 }
404 function getstring() {
405 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
406 s = $0;
407 gsub(/^[^\"]*\"/, "", s);
408 gsub(/\"$/, "", s);
409 return s;
410 }
412 BEGIN {
413 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
414 if (pf)
415 printf("\"%s\\n\"\n", pf);
416 printf("\n");
417 skip = 1;
418 clean();
419 }
421 {
422 # Skip the entire header
423 if (!skip) {
424 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
425 mode = $1;
426 if ($1 == "msgstr[0]") mode = "msgstr0";
427 if ($1 == "msgstr[1]") mode = "msgstr1";
428 if ($1 == "msgstr[2]") mode = "msgstr2";
429 if ($1 == "msgstr[3]") mode = "msgstr3";
430 if ($1 == "msgstr[4]") mode = "msgstr4";
431 if ($1 == "msgstr[5]") mode = "msgstr5";
433 if (mode == "msgctxt") msgctxt = msgctxt getstring();
434 if (mode == "msgid") msgid = msgid getstring();
435 if (mode == "msgstr") msgstr = msgstr getstring();
436 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
437 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
438 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
439 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
440 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
441 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
442 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
444 if (! $0) {
445 if (msgid != msgstr) {
446 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
447 printf("msgid \"%s\"\n", msgid);
448 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
449 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
450 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
451 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
452 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
453 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
454 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
455 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
456 printf("\n");
457 }
458 clean();
459 }
460 }
461 if ($0 == "") skip = "";
462 }
463 ' "$tmpfile" > "$tmpfile.awk"
465 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
467 if [ -s "$tmpfile.mo" ]; then
468 rm "$mo"; mv "$tmpfile.mo" "$mo"
469 else
470 _ 'Error processing %s' "$mo"
471 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
472 fi
474 # Clean
475 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
476 done
478 comp_summary "$time0" "$size0" "$(sizes mo1)"
479 }
482 # Strip locale definitions: normalize whitespace and remove comments
483 # Stripping can be disabled with COOKOPTS="!locdef"
485 strip_locale_def() {
486 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
487 [ ! -d "$install/usr/share/i18n/locales" ] && return
488 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
490 action 'Stripping locale definitions...'
491 size0=$(sizes loc)
492 time0=$(get_time)
494 for i in $(find $install/usr/share/i18n/locales -type f); do
495 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
496 done
498 comp_summary "$time0" "$size0" "$(sizes loc)"
499 }
502 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
503 # as removing unneeded files like in Python packages. Cross compiled binaries
504 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
505 # Stripping can be disabled with COOKOPTS="!strip"
507 strip_package() {
508 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
510 case "$ARCH" in
511 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
512 *) export STRIP='strip' ;;
513 esac
514 action 'Executing strip on all files...'
515 size0=0
516 size1=0
517 time0=$(get_time)
519 # Strip executable files
520 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
521 if [ -d "$dir" ]; then
522 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
523 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
524 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
525 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
526 fi
527 done
529 oldsize=$(sizes str)
531 # Strip shared and static libraries
532 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
533 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
535 # Remove Python *.pyc and *.pyo
536 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
538 # Remove both with the empty subfolders:
539 # 1. Perl perllocal.pod and .packlist (unconditionally)
540 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
541 # 2. Perl *.pod (if not disabled)
542 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
543 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')"
544 echo "$perlfiles" | xargs rm -f 2>/dev/null
545 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null
547 # Strip Perl files (*.pm and *.pl) from documentation inside (if not disabled)
548 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
549 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
551 newsize=$(sizes str)
553 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
554 }
557 # Strip unsupported locales (.mo files)
559 strip_mo_i18n() {
560 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
562 [ ! -d "$fs/usr/share/locale" ] && return
563 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
565 action 'Thin out translation files...'
566 size0=$(sizes mo2)
567 time0=$(get_time)
569 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
570 # Default value is "" (empty). That means for us that we'll use the full
571 # list of supported locales here.
572 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
574 # Existing locales
575 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
577 # Thin out the list of existing locales. At the end there will be only locales that need
578 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
579 for keep_locale in $LOCALE; do
580 elocales=${elocales//$keep_locale}
581 done
583 # Remove the unsupported locales
584 for rem_locale in $elocales; do
585 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
586 rm -r "$fs/usr/share/locale/$rem_locale"
587 done
589 comp_summary "$time0" "$size0" "$(sizes mo2)"
590 }
595 case $1 in
596 install)
597 # Compressors working in the $install
598 case "$ARCH" in
599 arm*) ;;
600 *)
601 recompress_gz
602 compress_manpages
603 compress_png
604 compress_svg
605 compress_ui
606 ;;
607 esac
609 fix_desktop_files
610 normalize_mo
611 strip_locale_def
612 ;;
613 fs)
614 # Compressors working in the $fs
615 strip_package
616 strip_mo_i18n
617 ;;
618 esac
620 # Clean
621 rm "$cache_stat"
622 find $comp_cache_root -type f -links -2 -delete