cookutils view modules/compressor @ rev 1078

cook: allow multi-set & no-src receipts (sources may be downloaded inside compile_rules() individually for each set), allow change SHORT_DESC & WEB_SITE in genpkg_rules() (for differents sets);
lighttpd/cooker.css: add style for log line "Switching to the set...";
lighttpd/index.cgi: remove false-positive coloring with word "fatal", mark line "Switching to the set...", don't hide "Logs" button (fresh log initially empty and it may lasts for many seconds due to buffered "lazy write";
modules/compressor: better remove unwanted Perl files;
modules/mk_pkg_receipt: follow changed SHORT_DESC & WEB_SITE, improve condition speed (from 10 sec to 0.01 sec) when SPLIT is relatively long - looks like ${A/ $B /} is slow.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Thu Jun 21 12:57:52 2018 +0300 (2018-06-21)
parents 23700c523fdb
children 33dbe879cc21
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
9 # Compressor cache stuff
11 comp_cache_root='/home/slitaz/cache/cook'
12 mkdir -p "$comp_cache_root"
13 cache_stat=$(mktemp)
15 # Cache notes.
16 # Do not do the same job twice. Getting the file from the cache is much faster
17 # than compressing the file one more time. In addition, this cache is trying not
18 # to take extra space using the hardlinks. Although the files from the cache
19 # without reference to itself should be removed periodically.
24 #
25 # Functions
26 #
29 # tazpkg install command
31 tpi() {
32 tazpkg -gi --quiet --local --cookmode $1
33 }
36 # Working with time (with hundredths precision)
38 get_time() {
39 cut -d" " -f2 /proc/uptime
40 }
42 calc_time() {
43 # L10n: 's' is for seconds, 'm' is for minutes
44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
45 BEGIN{
46 time = b - a;
47 if (time < 30)
48 printf("%.2f%s\n", time, s);
49 else
50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
51 }'
52 }
55 # Compressor mini summary
57 comp_summary() {
58 # "$time0" "$size0" "$size1" "$log_file"
59 status
60 [ "$2" -eq 0 ] && return
61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
62 cache_msg=''
63 if [ -s "$cache_stat" ]; then
64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
65 echo -n > $cache_stat
66 fi
67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
70 if [ -s "$4" ]; then
71 _ 'Compressor warnings and errors:'
72 awk '{printf " %s\n", $0;}' "$4"
73 echo
74 fi
75 # Clean log
76 [ ! -f "$4" ] || rm "$4"
77 }
80 # Find ELF files
82 find_elf() {
83 local i ifs="$IFS"
84 IFS=$'\n'
85 find $fs -type f \
86 | while read i; do
87 # output of `readelf -h <file> is human-readable information,
88 # we are interested in the next line:
89 # Type: EXEC (Executable file)
90 # or
91 # Type: DYN (Shared object file)
92 if [ "$(readelf -h "$i" 2>/dev/null \
93 | sed -n '/Type:/ s|.*: *\([^ ]*\) .*|\1|p')" == "$1" ]; then
94 echo "$i" # $1 = { EXEC | DYN }
95 fi
96 done
97 IFS="$ifs"
98 }
101 # Calculating different sizes
103 sizes() {
104 local ifs="$IFS"; IFS=$'\n'
105 case $1 in
106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
109 gif) find $install -type f -name '*.gif' -exec ls -l \{\} \; ;;
110 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
111 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
112 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
113 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
114 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
115 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
116 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
117 strip)
118 {
119 find_elf EXEC
120 find_elf DYN
121 find $fs -type f \( -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
122 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \)
123 } \
124 | tr '\n' '\0' \
125 | xargs -0 ls -l
126 ;;
127 esac | awk '{s+=$5}END{print s}'
128 IFS="$ifs"
129 }
132 # Query cache for already existing compressed file; substitute original file with the cached
133 # compressed one if so.
134 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
136 query_cache() {
137 md5=$(md5sum "$2")
138 cachefile="$comp_cache_root/$1/${md5%% *}"
139 echo "$cachefile"
140 if [ -f "$cachefile" ]; then
141 ln -f "$cachefile" "$2"
142 echo '+' >> "$cache_stat"
143 else
144 echo '-' >> "$cache_stat"
145 false
146 fi
147 }
150 # Store compressed file to the cache
151 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
153 store_cache() {
154 mkdir -p "${1%/*}"
155 mv "$2" "$1"
156 ln "$1" "$2"
157 }
160 # Function to compress all man pages
161 # Compressing can be disabled with COOKOPTS="!manz"
163 compress_manpages() {
164 time0=$(get_time)
165 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
166 manpath="$install/usr/share/man"
167 [ -d "$manpath" ] || return
168 size0=$(sizes man); [ -z "$size0" ] && return
170 tpi advancecomp-static
172 action 'Compressing man pages...'
174 # We'll use only Gzip compression, so decompress other formats first
175 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
176 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
178 # Fast compress with gzip
179 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
181 # Fix symlinks
182 for i in $(find $manpath -type l); do
183 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
184 link=$(echo $i | sed 's|\.[gbx]z2*$||')
185 rm $i; ln -s $dest.gz $link.gz
186 done
188 # Recompress with advdef (it can't compress, only recompress)
189 the_log="$(mktemp)"
190 if which advdef >/dev/null; then
191 IFS=$'\n'
192 for i in $(find $manpath -type f); do
193 if ! cached_path=$(query_cache mangz "$i"); then
194 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
195 out="$(advdef -z4q "$i")"
196 if [ -n "$out" ]; then
197 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
198 mv -f "$i.orig$$" "$i" # restore the original
199 else
200 store_cache "$cached_path" "$i"
201 rm -f "$i.orig$$" # clean
202 fi
203 fi
204 done
205 else
206 echo 'Warning: advdef not found.' > "$the_log"
207 fi
209 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log"
210 }
213 # Function to recompress all gzip archives
214 # Recompressing can be disabled with COOKOPTS="!gz"
216 recompress_gz() {
217 time0=$(get_time)
218 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
219 size0=$(sizes gz); [ -z "$size0" ] && return
221 tpi advancecomp-static
223 action 'Recompressing gzip files...'
225 # Recompress with advdef
226 the_log="$(mktemp)"
227 if which advdef >/dev/null; then
228 IFS=$'\n'
229 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
230 if ! cached_path=$(query_cache gz "$i"); then
231 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
232 out="$(advdef -z4q "$i")"
233 if [ -n "$out" ]; then
234 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
235 mv -f "$i.orig$$" "$i" # restore the original
236 else
237 store_cache "$cached_path" "$i"
238 rm -f "$i.orig$$" # clean
239 fi
240 fi
241 done
242 else
243 echo 'Warning: advdef not found.' > "$the_log"
244 fi
246 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log"
247 }
250 # Function to recompress all zip archives
251 # Recompressing can be disabled with COOKOPTS="!zip"
253 recompress_zip() {
254 time0=$(get_time)
255 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
256 size0=$(sizes zip); [ -z "$size0" ] && return
258 tpi advancecomp-static
260 action 'Recompressing zip files...'
262 # Recompress with advzip
263 the_log="$(mktemp)"
264 if which advzip >/dev/null; then
265 IFS=$'\n'
266 for i in $(find $install -type f -name '*.zip'); do
267 if ! cached_path=$(query_cache zip "$i"); then
268 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
269 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
270 if [ -n "$out" ]; then
271 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
272 mv -f "$i.orig$$" "$i" # restore the original
273 else
274 store_cache "$cached_path" "$i"
275 rm -f "$i.orig$$" # clean
276 fi
277 fi
278 done
279 else
280 echo 'Warning: advzip not found.' > "$the_log"
281 fi
283 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log"
284 }
287 # Function used after compile_rules() to compress all png images
288 # Compressing can be disabled with COOKOPTS="!pngz"
290 compress_png() {
291 time0=$(get_time)
292 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
293 size0=$(sizes png); [ -z "$size0" ] && return
295 use_pq=true
296 use_op=true
297 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
298 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
299 $use_pq && tpi pngquant-static
300 $use_op && tpi optipng-static
302 action 'Compressing png images...'
304 the_log="$(mktemp)"
305 $use_pq && if ! which pngquant >/dev/null; then
306 echo 'Warning: pngquant not found.' > "$the_log"
307 use_pq=false
308 fi
309 $use_op && if ! which optipng >/dev/null; then
310 echo 'Warning: optipng not found.' >> "$the_log"
311 use_op=false
312 fi
314 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
315 [ -z "$oplevel" ] && oplevel='2'
317 cache_section="png$oplevel"
318 $use_pq && cache_section="${cache_section}p"
319 $use_op && cache_section="${cache_section}o"
321 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
323 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
324 IFS=$'\n'
325 for i in $(find $install -type f -name '*.png'); do
326 unset IFS iserror
327 if ! cached_path=$(query_cache $cache_section "$i"); then
328 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
329 if $use_pq; then
330 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
331 if [ -n "$out" ]; then
332 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log"
333 iserror='yes'
334 [ -e "$i.tmp" ] && rm "$i.tmp" # zero-size file remains on pngquant fail
335 fi
336 fi
337 if $use_op && [ -z "$iserror" ]; then
338 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
339 if [ -n "$out" ]; then
340 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log"
341 iserror='yes'
342 fi
343 fi
344 if [ -n "$iserror" ]; then
345 mv -f "$i.orig$$" "$i" # restore the original
346 else
347 store_cache "$cached_path" "$i"
348 rm -f "$i.orig$$" # clean
349 fi
350 fi
351 done
353 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log"
354 }
357 # Function used after compile_rules() to compress all svg images
358 # Compressing can be disabled with COOKOPTS="!svgz"
360 compress_svg() {
361 time0=$(get_time)
362 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
363 size0=$(sizes svg); [ -z "$size0" ] && return
365 tpi svgcleaner
367 action 'Compressing svg images...'
369 the_log="$(mktemp)"
370 if which svgcleaner >/dev/null; then
371 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
372 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
374 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
375 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
376 --multipass --remove-unresolved-classes no $opts 2>&1)"
377 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
378 done
379 else
380 echo 'Warning: svgcleaner not found.' > "$the_log"
381 fi
383 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log"
384 }
387 # Function used after compile_rules() to compress all gif images
388 # Compressing can be disabled with COOKOPTS="!gifz"
390 compress_gif() {
391 time0=$(get_time)
392 [ "${COOKOPTS/!gifz/}" != "$COOKOPTS" ] && return
393 size0=$(sizes gif); [ -z "$size0" ] && return
395 tpi gifsicle
397 action 'Compressing gif images...'
399 the_log="$(mktemp)"
400 if which gifsicle >/dev/null; then
401 IFS=$'\n'
402 for i in $(find $install -type f -name '*.gif'); do
403 if ! cached_path=$(query_cache gif "$i"); then
404 unset IFS
405 # use intermediate file, if all well ($?=0), then substitute the original
406 if gifsicle -O3 "$i" -o "$i.$$" >> "$the_log" 2>&1; then
407 if [ -s "$i.$$" ]; then
408 mv "$i.$$" "$i"
409 store_cache "$cached_path" "$i"
410 fi
411 else
412 rm "$i.$$"
413 fi
414 fi
415 done
416 else
417 echo 'Warning: gifsicle not found.' > "$the_log"
418 fi
420 comp_summary "$time0" "$size0" "$(sizes gif)" "$the_log"
421 }
424 # Function used after compile_rules() to shrink all *.ui and *.glade files:
425 # remove insignificant spaces and comments
426 # Compressing can be disabled with COOKOPTS="!uiz"
428 compress_ui() {
429 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
430 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
432 tpi xmlstarlet
434 action 'Compressing ui files...'
436 the_log="$(mktemp)"
437 if which xmlstarlet >/dev/null; then
438 size0=$(sizes xml)
439 time0=$(get_time)
440 temp_ui="$(mktemp)"
441 IFS=$'\n'
442 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
443 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")"
444 if [ -n "$out" ]; then
445 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log"
446 else
447 cat "$temp_ui" > "$ui"
448 fi
449 done
450 else
451 echo 'Warning: xmlstarlet not found.' > "$the_log"
452 fi
454 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log"
455 rm "$temp_ui"
456 }
459 # Get list of supported locales...
461 get_supported_locales() {
462 lpc='/slitaz-i18n/stuff/locale-pack.conf'
463 if [ -e "$WOK$lpc" ]; then
464 # ... from package in the local wok
465 . "$WOK$lpc"
466 else
467 # ... from Hg
468 temp_conf=$(mktemp)
469 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
470 if [ -s $temp_conf ]; then
471 . $temp_conf
472 else
473 # Give up and use hardcoded list
474 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
475 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
476 fi
477 rm $temp_conf
478 fi
479 echo $LOCALE_PACK
480 }
483 # Fix common errors and warnings in the .desktop files
484 # Fixing can be disabled with COOKOPTS="!fixdesktops"
486 fix_desktop_files() {
487 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
488 deskpath="$install/usr/share/applications"
489 [ -d "$deskpath" ] || return
490 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
492 size0=$(sizes des)
493 time0=$(get_time)
495 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
496 tpi desktop-file-validate-static
497 fi
499 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
500 # Default value is "" (empty). That means for us that we'll use the full
501 # list of supported locales here.
502 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
504 IFS=$'\n'
505 for desktop in $(find $deskpath -type f -name '*.desktop'); do
506 cp "$desktop" "$desktop.orig"
508 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
509 sdft "$desktop" -i
511 # Fix common errors in .desktop file
512 fix-desktop-file "$desktop"
514 # Strip unsupported locales from .desktop file
515 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
516 sdft "$desktop" -i -k "$LOCALE"
518 # Extra-strip
519 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
520 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
522 if [ -n "$QA" ]; then
523 # Check the rest of errors, warnings and tips
524 _ 'QA: Checking %s...' "$(basename $desktop)"
525 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!'
526 if which desktop-file-validate >/dev/null; then
527 desktop-file-validate "$desktop" | busybox fold -s
528 else
529 echo 'Warning: desktop-file-validate not found.'
530 fi
531 echo
532 fi
534 rm "$desktop.orig"
535 done
537 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
538 }
541 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
542 # to the translation (msgid = msgstr)
543 # Normalization can be disabled with COOKOPTS="!monorm"
545 normalize_mo() {
546 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
547 [ -z "$(find $install -type f -name '*.mo')" ] && return
549 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
550 tpi gettext
551 # Gconv modules (convert to UTF-8)
552 tpi glibc-locale
554 action 'Normalizing mo files...'
556 the_log="$(mktemp)"
557 to_continue=true
558 for i in msgunfmt msguniq msgconv msgfmt; do
559 if ! which $i >/dev/null; then
560 echo "Warning: $i not found. Normalizing aborted" > "$the_log"
561 to_continue=false
562 fi
563 done
565 size0=$(sizes mo1)
566 time0=$(get_time)
568 # Process all existing *.mo files
569 IFS=$'\n'
570 $to_continue &&
571 for mo in $(find "$install" -type f -name '*.mo'); do
572 tmpfile="$(mktemp)"
574 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
575 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
576 if [ -n "$out" ]; then
577 # using literal $'\n' here instead of using `echo -e "...\n..."` because
578 # $out may contain escapes ('\r', '\v') that we should print as-is
579 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log"
580 continue # proceed to next file
581 fi
583 # add newline
584 echo >> "$tmpfile"
586 # get Plural-Forms
587 awk '
588 BEGIN { skip = ""; }
589 {
590 if (! skip) {
591 s = $0;
592 gsub(/^[^\"]*\"/, "", s);
593 gsub(/\"$/, "", s);
594 printf("%s", s);
595 }
596 if (! $0) skip = "yes";
597 }
598 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
600 if ! grep -q 'msgid_plural' "$tmpfile"; then
601 echo > "$tmpfile.pf"
602 fi
604 # main
605 awk -v pf="$(cat "$tmpfile.pf")" '
606 function clean() {
607 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
608 }
610 function getstring() {
611 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
612 s = $0;
613 gsub(/^[^\"]*\"/, "", s);
614 gsub(/\"$/, "", s);
615 return s;
616 }
618 BEGIN {
619 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
620 if (pf)
621 printf("\"%s\\n\"\n", pf);
622 printf("\n");
623 skip = 1;
624 clean();
625 }
627 {
628 # Skip the entire header
629 if (!skip) {
630 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
631 mode = $1;
632 if ($1 == "msgstr[0]") mode = "msgstr0";
633 if ($1 == "msgstr[1]") mode = "msgstr1";
634 if ($1 == "msgstr[2]") mode = "msgstr2";
635 if ($1 == "msgstr[3]") mode = "msgstr3";
636 if ($1 == "msgstr[4]") mode = "msgstr4";
637 if ($1 == "msgstr[5]") mode = "msgstr5";
639 if (mode == "msgctxt") msgctxt = msgctxt getstring();
640 if (mode == "msgid") msgid = msgid getstring();
641 if (mode == "msgstr") msgstr = msgstr getstring();
642 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
643 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
644 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
645 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
646 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
647 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
648 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
650 if (! $0) {
651 if (msgid != msgstr) {
652 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
653 printf("msgid \"%s\"\n", msgid);
654 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
655 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
656 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
657 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
658 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
659 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
660 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
661 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
662 printf("\n");
663 }
664 clean();
665 }
666 }
667 if ($0 == "") skip = "";
668 }
669 ' "$tmpfile" > "$tmpfile.awk"
671 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
672 if [ -n "$out" ]; then
673 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log"
674 continue # proceed to next file
675 fi
677 if [ -s "$tmpfile.mo" ]; then
678 rm "$mo"; mv "$tmpfile.mo" "$mo"
679 else
680 _ 'Error processing %s' "$mo" >> "$the_log"
681 echo >> "$the_log"
682 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
683 fi
685 # Clean
686 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
687 done
689 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log"
690 }
693 # Strip locale definitions: normalize whitespace and remove comments
694 # Stripping can be disabled with COOKOPTS="!locdef"
696 strip_locale_def() {
697 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
698 [ ! -d "$install/usr/share/i18n/locales" ] && return
699 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
701 action 'Stripping locale definitions...'
702 size0=$(sizes loc)
703 time0=$(get_time)
705 for i in $(find $install/usr/share/i18n/locales -type f); do
706 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
707 done
709 comp_summary "$time0" "$size0" "$(sizes loc)"
710 }
713 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
714 # as removing unneeded files like in Python packages. Cross compiled binaries
715 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
716 # Stripping can be disabled with COOKOPTS="!strip"
718 strip_package() {
719 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
721 local i ifs="$IFS"
722 IFS=$'\n'
724 case "$ARCH" in
725 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
726 *) export STRIP='strip' ;;
727 esac
728 action 'Executing strip on all files...'
729 size0=0
730 size1=0
731 time0=$(get_time)
732 oldsize=$(sizes strip)
735 # GNU strip (GNU Binutils)
736 # -p --preserve-dates Copy modified/access timestamps to the output
737 # -s --strip-all Remove all symbols and relocation information
738 # --strip-unneeded Remove all symbols not needed by relocations
739 # -D --enable-deterministic-archives Produce deterministic output when stripping archives
740 # -g -S -d --strip-debug Remove all debugging symbols & sections
742 # Strip executable files
743 while read i; do
744 $STRIP -ps "$i" 2>/dev/null
745 done <<EOT
746 $(find_elf EXEC)
747 EOT
749 # Strip shared libraries
750 while read i; do
751 case $i in
752 *.dbg) ;; # skip library.so.*.dbg debugging symbols
753 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;;
754 esac
755 done <<EOT
756 $(find_elf DYN)
757 EOT
759 # Strip static libraries
760 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries
761 find $fs -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \;
764 # Remove Python *.pyc and *.pyo
765 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
767 # Remove both with the empty subfolders:
768 # 1. Perl perllocal.pod and .packlist (unconditionally)
769 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
770 # 2. Perl *.pod (if not disabled)
771 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
772 perlfiles="$perlfiles"$'\n'"$(find $fs -type f -name '*.pod')"
773 echo "$perlfiles" | sort -u | xargs rm -f 2>/dev/null
774 echo "$perlfiles" | sort -u | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' \
775 | xargs rmdir -p --ignore-fail-on-non-empty 2>/dev/null
777 # Strip documentation inside Perl files (*.pm and *.pl) (if not disabled)
778 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
779 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
781 newsize=$(sizes strip)
783 comp_summary "$time0" "$oldsize" "$newsize"
784 IFS="$ifs"
785 }
788 # Strip unsupported locales (.mo files)
790 strip_mo_i18n() {
791 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
793 [ ! -d "$fs/usr/share/locale" ] && return
794 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
796 action 'Thin out translation files...'
797 size0=$(sizes mo2)
798 time0=$(get_time)
800 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
801 # Default value is "" (empty). That means for us that we'll use the full
802 # list of supported locales here.
803 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
805 # Existing locales
806 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
808 # Thin out the list of existing locales. At the end there will be only locales that need
809 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
810 for keep_locale in $LOCALE; do
811 elocales=${elocales//$keep_locale}
812 done
814 # Remove the unsupported locales
815 for rem_locale in $elocales; do
816 [ ! -d "$fs/usr/share/locale/$rem_locale" ] ||
817 rm -r "$fs/usr/share/locale/$rem_locale"
818 done
820 comp_summary "$time0" "$size0" "$(sizes mo2)"
821 }
826 case $1 in
827 install)
828 # Compressors working in the $install
829 case "$ARCH" in
830 arm*) ;;
831 *)
832 recompress_gz
833 recompress_zip
834 compress_manpages
835 compress_png
836 compress_svg
837 compress_gif
838 compress_ui
839 ;;
840 esac
842 fix_desktop_files
843 normalize_mo
844 strip_locale_def
845 ;;
846 fs)
847 # Compressors working in the $fs
848 strip_package
849 strip_mo_i18n
850 ;;
851 esac
853 # Clean
854 rm "$cache_stat"
855 find $comp_cache_root -type f -links -2 -delete