cookutils view modules/compressor @ rev 1066

modules/compressor: compress_gif(): fix optimized files permissions
(don't use mktemp, because it set the rw------- permissions in /tmp)
author Aleksej Bobylev <al.bobylev@gmail.com>
date Wed Jun 06 00:53:16 2018 +0300 (2018-06-06)
parents 0af059297232
children 9c90eaa721d2
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9 # Compressor cache stuff
10
11 comp_cache_root='/home/slitaz/cache/cook'
12 mkdir -p "$comp_cache_root"
13 cache_stat=$(mktemp)
14
15 # Cache notes.
16 # Do not do the same job twice. Getting the file from the cache is much faster
17 # than compressing the file one more time. In addition, this cache is trying not
18 # to take extra space using the hardlinks. Although the files from the cache
19 # without reference to itself should be removed periodically.
20
21
22
23
24 #
25 # Functions
26 #
27
28
29 # tazpkg install command
30
31 tpi() {
32 tazpkg -gi --quiet --local --cookmode $1
33 }
34
35
36 # Working with time (with hundredths precision)
37
38 get_time() {
39 cut -d" " -f2 /proc/uptime
40 }
41
42 calc_time() {
43 # L10n: 's' is for seconds, 'm' is for minutes
44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
45 BEGIN{
46 time = b - a;
47 if (time < 30)
48 printf("%.2f%s\n", time, s);
49 else
50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
51 }'
52 }
53
54
55 # Compressor mini summary
56
57 comp_summary() {
58 # "$time0" "$size0" "$size1" "$log_file"
59 status
60 [ "$2" -eq 0 ] && return
61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
62 cache_msg=''
63 if [ -s "$cache_stat" ]; then
64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
65 echo -n > $cache_stat
66 fi
67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
69
70 if [ -s "$4" ]; then
71 _ 'Compressor warnings and errors:'
72 awk '{printf " %s\n", $0;}' "$4"
73 echo
74 fi
75 # Clean log
76 [ ! -f "$4" ] || rm "$4"
77 }
78
79
80 # Find ELF files
81
82 find_elf() {
83 local i ifs="$IFS"
84 IFS=$'\n'
85 find $fs -type f \
86 | while read i; do
87 # output of `readelf -h <file> is human-readable information,
88 # we are interested in the next line:
89 # Type: EXEC (Executable file)
90 # or
91 # Type: DYN (Shared object file)
92 if [ "$(readelf -h "$i" 2>/dev/null \
93 | sed -n '/Type:/ s|.*: *\([^ ]*\) .*|\1|p')" == "$1" ]; then
94 echo "$i" # $1 = { EXEC | DYN }
95 fi
96 done
97 IFS="$ifs"
98 }
99
100
101 # Calculating different sizes
102
103 sizes() {
104 local ifs="$IFS"; IFS=$'\n'
105 case $1 in
106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
109 gif) find $install -type f -name '*.gif' -exec ls -l \{\} \; ;;
110 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
111 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
112 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
113 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
114 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
115 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
116 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
117 strip)
118 {
119 find_elf EXEC
120 find_elf DYN
121 find $fs -type f \( -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
122 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \)
123 } \
124 | tr '\n' '\0' \
125 | xargs -0 ls -l
126 ;;
127 esac | awk '{s+=$5}END{print s}'
128 IFS="$ifs"
129 }
130
131
132 # Query cache for already existing compressed file; substitute original file with the cached
133 # compressed one if so.
134 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
135
136 query_cache() {
137 md5=$(md5sum "$2")
138 cachefile="$comp_cache_root/$1/${md5%% *}"
139 echo "$cachefile"
140 if [ -f "$cachefile" ]; then
141 ln -f "$cachefile" "$2"
142 echo '+' >> "$cache_stat"
143 else
144 echo '-' >> "$cache_stat"
145 false
146 fi
147 }
148
149
150 # Store compressed file to the cache
151 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
152
153 store_cache() {
154 mkdir -p "${1%/*}"
155 mv "$2" "$1"
156 ln "$1" "$2"
157 }
158
159
160 # Function to compress all man pages
161 # Compressing can be disabled with COOKOPTS="!manz"
162
163 compress_manpages() {
164 time0=$(get_time)
165 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
166 manpath="$install/usr/share/man"
167 [ -d "$manpath" ] || return
168 size0=$(sizes man); [ -z "$size0" ] && return
169
170 tpi advancecomp-static
171
172 action 'Compressing man pages...'
173
174 # We'll use only Gzip compression, so decompress other formats first
175 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
176 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
177
178 # Fast compress with gzip
179 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
180
181 # Fix symlinks
182 for i in $(find $manpath -type l); do
183 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
184 link=$(echo $i | sed 's|\.[gbx]z2*$||')
185 rm $i; ln -s $dest.gz $link.gz
186 done
187
188 # Recompress with advdef (it can't compress, only recompress)
189 the_log="$(mktemp)"
190 if which advdef >/dev/null; then
191 IFS=$'\n'
192 for i in $(find $manpath -type f); do
193 if ! cached_path=$(query_cache mangz "$i"); then
194 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
195 out="$(advdef -z4q "$i")"
196 if [ -n "$out" ]; then
197 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
198 mv -f "$i.orig$$" "$i" # restore the original
199 else
200 store_cache "$cached_path" "$i"
201 rm -f "$i.orig$$" # clean
202 fi
203 fi
204 done
205 else
206 echo 'Warning: advdef not found.' > "$the_log"
207 fi
208
209 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log"
210 }
211
212
213 # Function to recompress all gzip archives
214 # Recompressing can be disabled with COOKOPTS="!gz"
215
216 recompress_gz() {
217 time0=$(get_time)
218 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
219 size0=$(sizes gz); [ -z "$size0" ] && return
220
221 tpi advancecomp-static
222
223 action 'Recompressing gzip files...'
224
225 # Recompress with advdef
226 the_log="$(mktemp)"
227 if which advdef >/dev/null; then
228 IFS=$'\n'
229 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
230 if ! cached_path=$(query_cache gz "$i"); then
231 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
232 out="$(advdef -z4q "$i")"
233 if [ -n "$out" ]; then
234 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
235 mv -f "$i.orig$$" "$i" # restore the original
236 else
237 store_cache "$cached_path" "$i"
238 rm -f "$i.orig$$" # clean
239 fi
240 fi
241 done
242 else
243 echo 'Warning: advdef not found.' > "$the_log"
244 fi
245
246 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log"
247 }
248
249
250 # Function to recompress all zip archives
251 # Recompressing can be disabled with COOKOPTS="!zip"
252
253 recompress_zip() {
254 time0=$(get_time)
255 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
256 size0=$(sizes zip); [ -z "$size0" ] && return
257
258 tpi advancecomp-static
259
260 action 'Recompressing zip files...'
261
262 # Recompress with advzip
263 the_log="$(mktemp)"
264 if which advzip >/dev/null; then
265 IFS=$'\n'
266 for i in $(find $install -type f -name '*.zip'); do
267 if ! cached_path=$(query_cache zip "$i"); then
268 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
269 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
270 if [ -n "$out" ]; then
271 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
272 mv -f "$i.orig$$" "$i" # restore the original
273 else
274 store_cache "$cached_path" "$i"
275 rm -f "$i.orig$$" # clean
276 fi
277 fi
278 done
279 else
280 echo 'Warning: advzip not found.' > "$the_log"
281 fi
282
283 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log"
284 }
285
286
287 # Function used after compile_rules() to compress all png images
288 # Compressing can be disabled with COOKOPTS="!pngz"
289
290 compress_png() {
291 time0=$(get_time)
292 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
293 size0=$(sizes png); [ -z "$size0" ] && return
294
295 use_pq=true
296 use_op=true
297 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
298 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
299 $use_pq && tpi pngquant-static
300 $use_op && tpi optipng-static
301
302 action 'Compressing png images...'
303
304 the_log="$(mktemp)"
305 $use_pq && if ! which pngquant >/dev/null; then
306 echo 'Warning: pngquant not found.' > "$the_log"
307 use_pq=false
308 fi
309 $use_op && if ! which optipng >/dev/null; then
310 echo 'Warning: optipng not found.' >> "$the_log"
311 use_op=false
312 fi
313
314 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
315 [ -z "$oplevel" ] && oplevel='2'
316
317 cache_section="png$oplevel"
318 $use_pq && cache_section="${cache_section}p"
319 $use_op && cache_section="${cache_section}o"
320
321 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
322
323 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
324 IFS=$'\n'
325 for i in $(find $install -type f -name '*.png'); do
326 unset IFS iserror
327 if ! cached_path=$(query_cache $cache_section "$i"); then
328 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
329 if $use_pq; then
330 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
331 if [ -n "$out" ]; then
332 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log"
333 iserror='yes'
334 [ -e "$i.tmp" ] && rm "$i.tmp" # zero-size file remains on pngquant fail
335 fi
336 fi
337 if $use_op && [ -z "$iserror" ]; then
338 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
339 if [ -n "$out" ]; then
340 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log"
341 iserror='yes'
342 fi
343 fi
344 if [ -n "$iserror" ]; then
345 mv -f "$i.orig$$" "$i" # restore the original
346 else
347 store_cache "$cached_path" "$i"
348 rm -f "$i.orig$$" # clean
349 fi
350 fi
351 done
352
353 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log"
354 }
355
356
357 # Function used after compile_rules() to compress all svg images
358 # Compressing can be disabled with COOKOPTS="!svgz"
359
360 compress_svg() {
361 time0=$(get_time)
362 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
363 size0=$(sizes svg); [ -z "$size0" ] && return
364
365 tpi svgcleaner
366
367 action 'Compressing svg images...'
368
369 if which svgcleaner >/dev/null; then
370 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
371 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
372
373 the_log="$(mktemp)"
374 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
375 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
376 --multipass --remove-unresolved-classes no $opts 2>&1)"
377 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
378 done
379 else
380 echo 'Warning: svgcleaner not found.' > "$the_log"
381 fi
382
383 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log"
384 }
385
386
387 # Function used after compile_rules() to compress all gif images
388 # Compressing can be disabled with COOKOPTS="!gifz"
389
390 compress_gif() {
391 time0=$(get_time)
392 [ "${COOKOPTS/!gifz/}" != "$COOKOPTS" ] && return
393 size0=$(sizes gif); [ -z "$size0" ] && return
394
395 tpi gifsicle
396
397 action 'Compressing gif images...'
398
399 if which gifsicle >/dev/null; then
400 the_log="$(mktemp)"
401 for i in $(IFS=$'\n' find $install -type f -name '*.gif'); do
402 unset IFS
403 # use intermediate file, if all well ($?=0), then substitute the original
404 if gifsicle -O3 "$i" -o "$i.$$" >> "$the_log" 2>&1; then
405 mv "$i.$$" "$i"
406 else
407 rm "$i.$$"
408 fi
409 done
410 else
411 echo 'Warning: gifsicle not found.' > "$the_log"
412 fi
413
414 comp_summary "$time0" "$size0" "$(sizes gif)" "$the_log"
415 }
416
417
418 # Function used after compile_rules() to shrink all *.ui and *.glade files:
419 # remove insignificant spaces and comments
420 # Compressing can be disabled with COOKOPTS="!uiz"
421
422 compress_ui() {
423 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
424 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
425
426 tpi xmlstarlet
427
428 action 'Compressing ui files...'
429
430 if which xmlstarlet >/dev/null; then
431 size0=$(sizes xml)
432 time0=$(get_time)
433 temp_ui="$(mktemp)"
434 the_log="$(mktemp)"
435 IFS=$'\n'
436 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
437 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")"
438 if [ -n "$out" ]; then
439 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log"
440 else
441 cat "$temp_ui" > "$ui"
442 fi
443 done
444 else
445 echo 'Warning: xmlstarlet not found.' > "$the_log"
446 fi
447
448 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log"
449 rm "$temp_ui"
450 }
451
452
453 # Get list of supported locales...
454
455 get_supported_locales() {
456 lpc='/slitaz-i18n/stuff/locale-pack.conf'
457 if [ -e "$WOK$lpc" ]; then
458 # ... from package in the local wok
459 . "$WOK$lpc"
460 else
461 # ... from Hg
462 temp_conf=$(mktemp)
463 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
464 if [ -s $temp_conf ]; then
465 . $temp_conf
466 else
467 # Give up and use hardcoded list
468 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
469 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
470 fi
471 rm $temp_conf
472 fi
473 echo $LOCALE_PACK
474 }
475
476
477 # Fix common errors and warnings in the .desktop files
478 # Fixing can be disabled with COOKOPTS="!fixdesktops"
479
480 fix_desktop_files() {
481 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
482 deskpath="$install/usr/share/applications"
483 [ -d "$deskpath" ] || return
484 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
485
486 size0=$(sizes des)
487 time0=$(get_time)
488
489 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
490 tpi desktop-file-validate-static
491 fi
492
493 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
494 # Default value is "" (empty). That means for us that we'll use the full
495 # list of supported locales here.
496 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
497
498 IFS=$'\n'
499 for desktop in $(find $deskpath -type f -name '*.desktop'); do
500 cp "$desktop" "$desktop.orig"
501
502 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
503 sdft "$desktop" -i
504
505 # Fix common errors in .desktop file
506 fix-desktop-file "$desktop"
507
508 # Strip unsupported locales from .desktop file
509 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
510 sdft "$desktop" -i -k "$LOCALE"
511
512 # Extra-strip
513 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
514 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
515
516 if [ -n "$QA" ]; then
517 # Check the rest of errors, warnings and tips
518 _ 'QA: Checking %s...' "$(basename $desktop)"
519 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!'
520 if which desktop-file-validate >/dev/null; then
521 desktop-file-validate "$desktop" | busybox fold -s
522 else
523 echo 'Warning: desktop-file-validate not found.'
524 fi
525 echo
526 fi
527
528 rm "$desktop.orig"
529 done
530
531 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
532 }
533
534
535 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
536 # to the translation (msgid = msgstr)
537 # Normalization can be disabled with COOKOPTS="!monorm"
538
539 normalize_mo() {
540 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
541 [ -z "$(find $install -type f -name '*.mo')" ] && return
542
543 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
544 tpi gettext
545 # Gconv modules (convert to UTF-8)
546 tpi glibc-locale
547
548 action 'Normalizing mo files...'
549
550 the_log="$(mktemp)"
551 to_continue=true
552 for i in msgunfmt msguniq msgconv msgfmt; do
553 if ! which $i >/dev/null; then
554 echo "Warning: $i not found. Normalizing aborted" > "$the_log"
555 to_continue=false
556 fi
557 done
558
559 size0=$(sizes mo1)
560 time0=$(get_time)
561
562 # Process all existing *.mo files
563 IFS=$'\n'
564 $to_continue &&
565 for mo in $(find "$install" -type f -name '*.mo'); do
566 tmpfile="$(mktemp)"
567
568 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
569 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
570 if [ -n "$out" ]; then
571 # using literal $'\n' here instead of using `echo -e "...\n..."` because
572 # $out may contain escapes ('\r', '\v') that we should print as-is
573 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log"
574 continue # proceed to next file
575 fi
576
577 # add newline
578 echo >> "$tmpfile"
579
580 # get Plural-Forms
581 awk '
582 BEGIN { skip = ""; }
583 {
584 if (! skip) {
585 s = $0;
586 gsub(/^[^\"]*\"/, "", s);
587 gsub(/\"$/, "", s);
588 printf("%s", s);
589 }
590 if (! $0) skip = "yes";
591 }
592 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
593
594 if ! grep -q 'msgid_plural' "$tmpfile"; then
595 echo > "$tmpfile.pf"
596 fi
597
598 # main
599 awk -v pf="$(cat "$tmpfile.pf")" '
600 function clean() {
601 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
602 }
603
604 function getstring() {
605 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
606 s = $0;
607 gsub(/^[^\"]*\"/, "", s);
608 gsub(/\"$/, "", s);
609 return s;
610 }
611
612 BEGIN {
613 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
614 if (pf)
615 printf("\"%s\\n\"\n", pf);
616 printf("\n");
617 skip = 1;
618 clean();
619 }
620
621 {
622 # Skip the entire header
623 if (!skip) {
624 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
625 mode = $1;
626 if ($1 == "msgstr[0]") mode = "msgstr0";
627 if ($1 == "msgstr[1]") mode = "msgstr1";
628 if ($1 == "msgstr[2]") mode = "msgstr2";
629 if ($1 == "msgstr[3]") mode = "msgstr3";
630 if ($1 == "msgstr[4]") mode = "msgstr4";
631 if ($1 == "msgstr[5]") mode = "msgstr5";
632
633 if (mode == "msgctxt") msgctxt = msgctxt getstring();
634 if (mode == "msgid") msgid = msgid getstring();
635 if (mode == "msgstr") msgstr = msgstr getstring();
636 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
637 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
638 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
639 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
640 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
641 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
642 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
643
644 if (! $0) {
645 if (msgid != msgstr) {
646 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
647 printf("msgid \"%s\"\n", msgid);
648 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
649 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
650 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
651 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
652 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
653 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
654 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
655 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
656 printf("\n");
657 }
658 clean();
659 }
660 }
661 if ($0 == "") skip = "";
662 }
663 ' "$tmpfile" > "$tmpfile.awk"
664
665 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
666 if [ -n "$out" ]; then
667 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log"
668 continue # proceed to next file
669 fi
670
671 if [ -s "$tmpfile.mo" ]; then
672 rm "$mo"; mv "$tmpfile.mo" "$mo"
673 else
674 _ 'Error processing %s' "$mo" >> "$the_log"
675 echo >> "$the_log"
676 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
677 fi
678
679 # Clean
680 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
681 done
682
683 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log"
684 }
685
686
687 # Strip locale definitions: normalize whitespace and remove comments
688 # Stripping can be disabled with COOKOPTS="!locdef"
689
690 strip_locale_def() {
691 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
692 [ ! -d "$install/usr/share/i18n/locales" ] && return
693 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
694
695 action 'Stripping locale definitions...'
696 size0=$(sizes loc)
697 time0=$(get_time)
698
699 for i in $(find $install/usr/share/i18n/locales -type f); do
700 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
701 done
702
703 comp_summary "$time0" "$size0" "$(sizes loc)"
704 }
705
706
707 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
708 # as removing unneeded files like in Python packages. Cross compiled binaries
709 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
710 # Stripping can be disabled with COOKOPTS="!strip"
711
712 strip_package() {
713 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
714
715 local i ifs="$IFS"
716 IFS=$'\n'
717
718 case "$ARCH" in
719 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
720 *) export STRIP='strip' ;;
721 esac
722 action 'Executing strip on all files...'
723 size0=0
724 size1=0
725 time0=$(get_time)
726 oldsize=$(sizes strip)
727
728
729 # GNU strip (GNU Binutils)
730 # -p --preserve-dates Copy modified/access timestamps to the output
731 # -s --strip-all Remove all symbols and relocation information
732 # --strip-unneeded Remove all symbols not needed by relocations
733 # -D --enable-deterministic-archives Produce deterministic output when stripping archives
734 # -g -S -d --strip-debug Remove all debugging symbols & sections
735
736 # Strip executable files
737 while read i; do
738 $STRIP -ps "$i" 2>/dev/null
739 done <<EOT
740 $(find_elf EXEC)
741 EOT
742
743 # Strip shared libraries
744 while read i; do
745 case $i in
746 *.dbg) ;; # skip library.so.*.dbg debugging symbols
747 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;;
748 esac
749 done <<EOT
750 $(find_elf DYN)
751 EOT
752
753 # Strip static libraries
754 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries
755 find $fs -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \;
756
757
758 # Remove Python *.pyc and *.pyo
759 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
760
761 # Remove both with the empty subfolders:
762 # 1. Perl perllocal.pod and .packlist (unconditionally)
763 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
764 # 2. Perl *.pod (if not disabled)
765 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
766 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')"
767 echo "$perlfiles" | xargs rm -f 2>/dev/null
768 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null
769
770 # Strip documentation inside Perl files (*.pm and *.pl) (if not disabled)
771 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
772 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
773
774 newsize=$(sizes strip)
775
776 comp_summary "$time0" "$oldsize" "$newsize"
777 IFS="$ifs"
778 }
779
780
781 # Strip unsupported locales (.mo files)
782
783 strip_mo_i18n() {
784 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
785
786 [ ! -d "$fs/usr/share/locale" ] && return
787 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
788
789 action 'Thin out translation files...'
790 size0=$(sizes mo2)
791 time0=$(get_time)
792
793 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
794 # Default value is "" (empty). That means for us that we'll use the full
795 # list of supported locales here.
796 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
797
798 # Existing locales
799 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
800
801 # Thin out the list of existing locales. At the end there will be only locales that need
802 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
803 for keep_locale in $LOCALE; do
804 elocales=${elocales//$keep_locale}
805 done
806
807 # Remove the unsupported locales
808 for rem_locale in $elocales; do
809 [ ! -d "$fs/usr/share/locale/$rem_locale" ] ||
810 rm -r "$fs/usr/share/locale/$rem_locale"
811 done
812
813 comp_summary "$time0" "$size0" "$(sizes mo2)"
814 }
815
816
817
818
819 case $1 in
820 install)
821 # Compressors working in the $install
822 case "$ARCH" in
823 arm*) ;;
824 *)
825 recompress_gz
826 recompress_zip
827 compress_manpages
828 compress_png
829 compress_svg
830 compress_gif
831 compress_ui
832 ;;
833 esac
834
835 fix_desktop_files
836 normalize_mo
837 strip_locale_def
838 ;;
839 fs)
840 # Compressors working in the $fs
841 strip_package
842 strip_mo_i18n
843 ;;
844 esac
845
846 # Clean
847 rm "$cache_stat"
848 find $comp_cache_root -type f -links -2 -delete