cookutils view modules/compressor @ rev 868

modules/compressor: make the caching system for *.man.gz and *.png.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Sun Jan 22 16:18:08 2017 +0200 (2017-01-22)
parents 5f6be706ab4f
children 604b1ab4c992
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9
10 # Compressor cache stuff
11
12 comp_cache_root='/var/cache/cook'
13 cache_stat=$(mktemp)
14
15 # Cache notes.
16 # Do not do the same job twice. Getting the file from the cache is much faster
17 # than compressing the file one more time. In addition, this cache is trying not
18 # to take extra space, using the hardlinks. Although the files from the cache
19 # without reference to himself should be removed periodically (to be done).
20
21
22
23
24 #
25 # Functions
26 #
27
28
29 # Display time.
30
31 disp_time() {
32 div=$(( ($1 + 30) / 60))
33 case $div in
34 0) min='';;
35 # L10n: 'm' is for minutes (approximate time)
36 *) min=$(_n ' ~ %dm' "$div");;
37 esac
38
39 # L10n: 's' is for seconds (cooking time)
40 _ '%ds%s' "$1" "$min"
41 }
42
43
44 # Compressor mini summary
45
46 comp_summary() {
47 # "$time0" "$size0" "$size1"
48 time1=$(date +%s)
49 status
50 [ "$2" -eq 0 ] && return
51 time=$(($time1 - $1))
52 saving=$(( ($2 - $3) / 1024 ))
53 cache_msg=''
54 if [ -s "$cache_stat" ]; then
55 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
56 echo -n > $cache_stat
57 fi
58 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
59 "$(disp_time $time)" "$2" "$3" "$saving" "$cache_msg"
60 }
61
62
63 # Calculating different sizes
64
65 sizes() {
66 case $1 in
67 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
68 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
69 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
70 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
71 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
72 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
73 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
74 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
75 esac | awk '{s+=$5}END{print s}'
76 }
77
78
79 # Query cache for already existing compressed file; substitute original file with the cached
80 # compressed one if so.
81 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
82
83 query_cache() {
84 md5=$(md5sum "$2")
85 cachefile="$comp_cache_root/$1/${md5%% *}"
86 echo "$cachefile"
87 if [ -f "$cachefile" ]; then
88 ln -f "$cachefile" "$2"
89 echo '+' >> "$cache_stat"
90 else
91 echo '-' >> "$cache_stat"
92 false
93 fi
94 }
95
96
97 # Store compressed file to the cache
98 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
99
100 store_cache() {
101 mkdir -p "${1%/*}"
102 mv "$2" "$1"
103 ln "$1" "$2"
104 }
105
106
107 # Function to compress all man pages
108 # Compressing can be disabled with COOKOPTS="!manz"
109
110 compress_manpages() {
111 time0=$(date +%s)
112 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
113 manpath="$install/usr/share/man"
114 [ -d "$manpath" ] || return
115 size0=$(sizes man); [ -z "$size0" ] && return
116
117 tazpkg -gi advancecomp --quiet --cookmode
118
119 action 'Compressing man pages...'
120
121 # We'll use only Gzip compression, so decompress other formats first
122 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
123 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
124
125 # Fast compress with gzip
126 find $manpath -type f -name '*.[1-9]*' -exec gzip \{\} \;
127
128 # Fix symlinks
129 for i in $(find $install/usr/share/man -type l); do
130 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
131 link=$(echo $i | sed 's|\.[gbx]z2*$||')
132 rm $i; ln -s $dest.gz $link.gz
133 done
134
135 # Recompress with advdef (it can't compress, only recompress)
136 for i in $(find $install/usr/share/man -type f); do
137 if ! cached_path=$(query_cache mangz "$i"); then
138 advdef -z4q "$i"
139 store_cache "$cached_path" "$i"
140 fi
141 done
142
143 comp_summary "$time0" "$size0" "$(sizes man)"
144 }
145
146
147 # Function used after compile_rules() to compress all png images
148 # Compressing can be disabled with COOKOPTS="!pngz"
149
150 compress_png() {
151 time0=$(date +%s)
152 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
153 size0=$(sizes png); [ -z "$size0" ] && return
154
155 use_pq=true
156 use_op=true
157 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
158 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
159 $use_pq && tazpkg -gi pngquant --quiet --cookmode
160 $use_op && tazpkg -gi optipng --quiet --cookmode
161
162 action 'Compressing png images...'
163
164 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
165 [ -z "$oplevel" ] && oplevel='2'
166
167 cache_section="png$oplevel"
168 $use_pq && cache_section="${cache_section}p"
169 $use_op && cache_section="${cache_section}o"
170
171 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
172
173 for i in $(find $install -type f -name '*.png'); do
174 if ! cached_path=$(query_cache $cache_section "$i"); then
175 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
176 $use_op && optipng -quiet -strip all -o$oplevel "$i"
177 store_cache "$cached_path" "$i"
178 fi
179 done
180
181 comp_summary "$time0" "$size0" "$(sizes png)"
182 }
183
184
185 # Function used after compile_rules() to compress all svg images
186 # Compressing can be disabled with COOKOPTS="!svgz"
187
188 compress_svg() {
189 time0=$(date +%s)
190 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
191 size0=$(sizes svg); [ -z "$size0" ] && return
192
193 tazpkg -gi svgcleaner --quiet --cookmode
194
195 action 'Compressing svg images...'
196
197 cleaner_log="$(mktemp)"
198 for i in $(find $install -type f -name '*.svg'); do
199 echo -n "$i: " >> "$cleaner_log"
200 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
201 done
202
203 comp_summary "$time0" "$size0" "$(sizes svg)"
204
205 sed -i '/: $/d' "$cleaner_log"
206 if [ -s "$cleaner_log" ]; then
207 _ 'Cleaner warnings and errors:'
208 awk '{printf " %s\n", $0;}' "$cleaner_log"
209 echo
210 fi
211 rm "$cleaner_log"
212 }
213
214
215 # Function used after compile_rules() to shrink all *.ui and *.glade files:
216 # remove insignificant spaces and comments
217 # Compressing can be disabled with COOKOPTS="!uiz"
218
219 compress_ui() {
220 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
221 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
222
223 tazpkg -gi xmlstarlet --quiet --cookmode
224
225 action 'Compressing ui files...'
226
227 size0=$(sizes xml)
228 time0=$(date +%s)
229 temp_ui="$(mktemp)"
230 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
231 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
232 cat "$temp_ui" > "$ui"
233 done
234
235 comp_summary "$time0" "$size0" "$(sizes xml)"
236 rm "$temp_ui"
237 }
238
239
240 # Get list of supported locales...
241
242 get_supported_locales() {
243 lpc='/slitaz-i18n/stuff/locale-pack.conf'
244 if [ -e "$WOK$lpc" ]; then
245 # ... from package in the local wok
246 . "$WOK$lpc"
247 else
248 # ... from Hg
249 temp_conf=$(mktemp)
250 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
251 if [ -s $temp_conf ]; then
252 . $temp_conf
253 else
254 # Give up and use hardcoded list
255 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
256 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
257 fi
258 rm $temp_conf
259 fi
260 echo $LOCALE_PACK
261 }
262
263
264 # Fix common errors and warnings in the .desktop files
265 # Fixing can be disabled with COOKOPTS="!fixdesktops"
266
267 fix_desktop_files() {
268 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
269 [ -z "$(find $install -type f -name '*.desktop')" ] && return
270
271 size0=$(sizes des)
272 time0=$(date +%s)
273
274 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
275 tazpkg -gi desktop-file-utils-extra --quiet --cookmode
276 fi
277
278 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
279 # Default value is "" (empty). That means for us that we'll use the full
280 # list of supported locales here.
281 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
282
283 for desktop in $(find $install -type f -name '*.desktop'); do
284 cp "$desktop" "$desktop.orig"
285
286 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
287 sdft "$desktop" -i
288
289 # Fix common errors in .desktop file
290 fix-desktop-file "$desktop"
291
292 # Strip unsupported locales from .desktop file
293 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
294 sdft "$desktop" -i -k "$LOCALE"
295
296 # Extra-strip
297 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
298 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
299
300 if [ -n "$QA" ]; then
301 # Check the rest of errors, warnings and tips
302 _ 'QA: Checking %s...' "$(basename $desktop)"
303 diff "$desktop.orig" "$desktop"
304 desktop-file-validate "$desktop" | busybox fold -s
305 echo
306 fi
307
308 rm "$desktop.orig"
309 done
310
311 comp_summary "$time0" "$size0" "$(sizes des)"
312 }
313
314
315 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really added
316 # to the translation (msgid = msgstr)
317 # Normalization can be disabled with COOKOPTS="!monorm"
318
319 normalize_mo() {
320 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
321 [ -z "$(find $install -type f -name '*.mo')" ] && return
322
323 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
324 tazpkg -gi gettext --quiet --cookmode
325 # Gconv modules (convert to UTF-8)
326 tazpkg -gi glibc-locale --quiet --cookmode
327
328 action 'Normalizing mo files...'
329
330 size0=$(sizes mo1)
331 time0=$(date +%s)
332
333 # Process all existing *.mo files
334 for mo in $(find "$install" -type f -name '*.mo'); do
335 tmpfile="$(mktemp)"
336
337 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
338 # add newline
339 echo >> "$tmpfile"
340
341 # get Plural-Forms
342 awk '
343 BEGIN { skip = ""; }
344 {
345 if (! skip) {
346 s = $0;
347 gsub(/^[^\"]*\"/, "", s);
348 gsub(/\"$/, "", s);
349 printf("%s", s);
350 }
351 if (! $0) skip = "yes";
352 }
353 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
354
355 if ! grep -q 'msgid_plural' "$tmpfile"; then
356 echo > "$tmpfile.pf"
357 fi
358
359 # main
360 awk -v pf="$(cat "$tmpfile.pf")" '
361 function clean() {
362 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
363 }
364
365 function getstring() {
366 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
367 s = $0;
368 gsub(/^[^\"]*\"/, "", s);
369 gsub(/\"$/, "", s);
370 return s;
371 }
372
373 BEGIN {
374 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
375 if (pf)
376 printf("\"%s\\n\"\n", pf);
377 printf("\n");
378 skip = 1;
379 clean();
380 }
381
382 {
383 # Skip the entire header
384 if (!skip) {
385 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
386 mode = $1;
387 if ($1 == "msgstr[0]") mode = "msgstr0";
388 if ($1 == "msgstr[1]") mode = "msgstr1";
389 if ($1 == "msgstr[2]") mode = "msgstr2";
390 if ($1 == "msgstr[3]") mode = "msgstr3";
391 if ($1 == "msgstr[4]") mode = "msgstr4";
392 if ($1 == "msgstr[5]") mode = "msgstr5";
393
394 if (mode == "msgctxt") msgctxt = msgctxt getstring();
395 if (mode == "msgid") msgid = msgid getstring();
396 if (mode == "msgstr") msgstr = msgstr getstring();
397 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
398 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
399 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
400 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
401 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
402 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
403 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
404
405 if (! $0) {
406 if (msgid != msgstr) {
407 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
408 printf("msgid \"%s\"\n", msgid);
409 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
410 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
411 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
412 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
413 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
414 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
415 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
416 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
417 printf("\n");
418 }
419 clean();
420 }
421 }
422 if ($0 == "") skip = "";
423 }
424 ' "$tmpfile" > "$tmpfile.awk"
425
426 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
427
428 if [ -s "$tmpfile.mo" ]; then
429 rm "$mo"; mv "$tmpfile.mo" "$mo"
430 else
431 _ 'Error processing %s' "$mo"
432 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
433 fi
434
435 # Clean
436 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
437 done
438
439 comp_summary "$time0" "$size0" "$(sizes mo1)"
440 }
441
442
443 # Strip locale definitions: normalize whitespace and remove comments
444 # Stripping can be disabled with COOKOPTS="!locdef"
445
446 strip_locale_def() {
447 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
448 [ ! -d "$install/usr/share/i18n/locales" ] && return
449 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
450
451 action 'Stripping locale definitions...'
452 size0=$(sizes loc)
453 time0=$(date +%s)
454
455 for i in $(find $install/usr/share/i18n/locales -type f); do
456 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
457 done
458
459 comp_summary "$time0" "$size0" "$(sizes loc)"
460 }
461
462
463 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
464 # as removing unneeded files like in Python packages. Cross compiled binaries
465 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
466 # Stripping can be disabled with COOKOPTS="!strip"
467
468 strip_package() {
469 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
470
471 case "$ARCH" in
472 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
473 *) export STRIP='strip' ;;
474 esac
475 action 'Executing strip on all files...'
476 size0=0
477 size1=0
478 time0=$(date +%s)
479
480 # Strip executable files
481 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
482 if [ -d "$dir" ]; then
483 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
484 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
485 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
486 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
487 fi
488 done
489
490 # Strip shared and static libraries
491 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist
492 oldsize=$(find $fs -type f \( \
493 -name '*.so*' -o -name '*.a' -o \
494 -name '*.pyc' -o -name '*.pyo' -o \
495 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
496
497 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
498 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
499 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
500 find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null
501
502 newsize=$(find $fs -type f \( \
503 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
504
505 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
506 }
507
508
509 # Strip unsupported locales (.mo files)
510
511 strip_mo_i18n() {
512 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
513
514 [ ! -d "$fs/usr/share/locale" ] && return
515 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
516
517 action 'Thin out translation files...'
518 size0=$(sizes mo2)
519 time0=$(date +%s)
520
521 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
522 # Default value is "" (empty). That means for us that we'll use the full
523 # list of supported locales here.
524 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
525
526 # Existing locales
527 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
528
529 # Thin out the list of existing locales. At the end there will be only locales that need
530 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
531 for keep_locale in $LOCALE; do
532 elocales=${elocales//$keep_locale}
533 done
534
535 # Remove the unsupported locales
536 for rem_locale in $elocales; do
537 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
538 rm -r "$fs/usr/share/locale/$rem_locale"
539 done
540
541 comp_summary "$time0" "$size0" "$(sizes mo2)"
542 }
543
544
545
546
547 case $1 in
548 install)
549 # Compressors working in the $install
550 case "$ARCH" in
551 arm*) ;;
552 *)
553 compress_manpages
554 compress_png
555 compress_svg
556 compress_ui
557 ;;
558 esac
559
560 fix_desktop_files
561 normalize_mo
562 strip_locale_def
563 ;;
564 fs)
565 # Compressors working in the $fs
566 strip_package
567 strip_mo_i18n
568 ;;
569 esac
570
571 # Clean
572 rm "$cache_stat"