cookutils view modules/compressor @ rev 866

Makefile: fix path
author Aleksej Bobylev <al.bobylev@gmail.com>
date Wed Jan 11 11:58:53 2017 +0200 (2017-01-11)
parents
children 5f6be706ab4f
line source
1 #!/bin/sh
2 #
3 # compressor - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9
10 #
11 # Functions
12 #
13
14
15 # Display time.
16
17 disp_time() {
18 div=$(( ($1 + 30) / 60))
19 case $div in
20 0) min='';;
21 # L10n: 'm' is for minutes (approximate time)
22 *) min=$(_n ' ~ %dm' "$div");;
23 esac
24
25 # L10n: 's' is for seconds (cooking time)
26 _ '%ds%s' "$1" "$min"
27 }
28
29
30 # Compressor mini summary
31
32 comp_summary() {
33 # "$time0" "$size0" "$size1"
34 time1=$(date +%s)
35 status
36 [ "$2" -eq 0 ] && return
37 time=$(($time1 - $1))
38 saving=$(( ($2 - $3) / 1024 ))
39 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB' "$(disp_time $time)" "$2" "$3" "$saving"
40 }
41
42
43 # Calculating different sizes
44
45 sizes() {
46 case $1 in
47 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
48 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
49 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
50 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
51 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
52 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
53 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
54 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
55 esac | awk '{s+=$5}END{print s}'
56 }
57
58
59 # Function to compress all man pages
60 # Compressing can be disabled with COOKOPTS="!manz"
61
62 compress_manpages() {
63 time0=$(date +%s)
64 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
65 manpath="$install/usr/share/man"
66 [ -d "$manpath" ] || return
67 size0=$(sizes man); [ -z "$size0" ] && return
68
69 action 'Compressing man pages...'
70
71 # We'll use only Gzip compression, so decompress other formats first
72 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
73 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
74
75 # Fast compress with gzip
76 find $manpath -type f -name '*.[1-9]*' -exec gzip \{\} \;
77
78 # Fix symlinks
79 for i in $(find $install/usr/share/man -type l); do
80 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
81 link=$(echo $i | sed 's|\.[gbx]z2*$||')
82 rm $i; ln -s $dest.gz $link.gz
83 done
84
85 # Recompress with advdef (it can't compress, only recompress)
86 tazpkg -gi advancecomp --quiet
87 for i in $(find $install/usr/share/man -type f); do
88 advdef -z4q $i
89 done
90
91 comp_summary "$time0" "$size0" "$(sizes man)"
92 }
93
94
95 # Function used after compile_rules() to compress all png images
96 # Compressing can be disabled with COOKOPTS="!pngz"
97
98 compress_png() {
99 time0=$(date +%s)
100 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
101 size0=$(sizes png); [ -z "$size0" ] && return
102
103 action 'Compressing png images...'
104
105 use_pq=true
106 use_op=true
107 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
108 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
109
110 $use_pq && tazpkg -gi pngquant --quiet
111 $use_op && tazpkg -gi optipng --quiet
112
113 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
114 [ -z "$oplevel" ] && oplevel='2'
115 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
116
117 for i in $(find $install -type f -name '*.png'); do
118 $use_pq && pngquant -f --skip-if-larger --ext .png --speed 1 "$i"
119 $use_op && optipng -quiet -strip all -o$oplevel "$i"
120 done
121
122 comp_summary "$time0" "$size0" "$(sizes png)"
123 }
124
125
126 # Function used after compile_rules() to compress all svg images
127 # Compressing can be disabled with COOKOPTS="!svgz"
128
129 compress_svg() {
130 time0=$(date +%s)
131 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
132 size0=$(sizes svg); [ -z "$size0" ] && return
133
134 action 'Compressing svg images...'
135
136 tazpkg -gi svgcleaner --quiet
137 cleaner_log="$(mktemp)"
138 for i in $(find $install -type f -name '*.svg'); do
139 echo -n "$i: " >> "$cleaner_log"
140 svgcleaner "$i" "$i" --remove-unresolved-classes false --quiet true >> "$cleaner_log"
141 done
142
143 comp_summary "$time0" "$size0" "$(sizes svg)"
144
145 sed -i '/: $/d' "$cleaner_log"
146 if [ -s "$cleaner_log" ]; then
147 _ 'Cleaner warnings and errors:'
148 awk '{printf " %s\n", $0;}' "$cleaner_log"
149 echo
150 fi
151 rm "$cleaner_log"
152 }
153
154
155 # Function used after compile_rules() to shrink all *.ui and *.glade files:
156 # remove insignificant spaces and comments
157 # Compressing can be disabled with COOKOPTS="!uiz"
158
159 compress_ui() {
160 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
161 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
162
163 action 'Compressing ui files...'
164 size0=$(sizes xml)
165 time0=$(date +%s)
166 tazpkg -gi xmlstarlet --quiet
167 temp_ui="$(mktemp)"
168 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
169 xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui"
170 cat "$temp_ui" > "$ui"
171 done
172
173 comp_summary "$time0" "$size0" "$(sizes xml)"
174 rm "$temp_ui"
175 }
176
177
178 # Get list of supported locales...
179
180 get_supported_locales() {
181 lpc='/slitaz-i18n/stuff/locale-pack.conf'
182 if [ -e "$WOK$lpc" ]; then
183 # ... from package in the local wok
184 . "$WOK$lpc"
185 else
186 # ... from Hg
187 temp_conf=$(mktemp)
188 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
189 if [ -s $temp_conf ]; then
190 . $temp_conf
191 else
192 # Give up and use hardcoded list
193 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
194 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
195 fi
196 rm $temp_conf
197 fi
198 echo $LOCALE_PACK
199 }
200
201
202 # Fix common errors and warnings in the .desktop files
203 # Fixing can be disabled with COOKOPTS="!fixdesktops"
204
205 fix_desktop_files() {
206 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
207 [ -z "$(find $install -type f -name '*.desktop')" ] && return
208
209 size0=$(sizes des)
210 time0=$(date +%s)
211
212 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
213 tazpkg -gi desktop-file-utils-extra --quiet
214 fi
215
216 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
217 # Default value is "" (empty). That means for us that we'll use the full
218 # list of supported locales here.
219 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
220
221 for desktop in $(find $install -type f -name '*.desktop'); do
222 cp "$desktop" "$desktop.orig"
223
224 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
225 sdft "$desktop" -i
226
227 # Fix common errors in .desktop file
228 fix-desktop-file "$desktop"
229
230 # Strip unsupported locales from .desktop file
231 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
232 sdft "$desktop" -i -k "$LOCALE"
233
234 # Extra-strip
235 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
236 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
237
238 if [ -n "$QA" ]; then
239 # Check the rest of errors, warnings and tips
240 _ 'QA: Checking %s...' "$(basename $desktop)"
241 diff "$desktop.orig" "$desktop"
242 desktop-file-validate "$desktop" | busybox fold -s
243 echo
244 fi
245
246 rm "$desktop.orig"
247 done
248
249 comp_summary "$time0" "$size0" "$(sizes des)"
250 }
251
252
253 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really added
254 # to the translation (msgid = msgstr)
255 # Normalization can be disabled with COOKOPTS="!monorm"
256
257 normalize_mo() {
258 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
259 [ -z "$(find $install -type f -name '*.mo')" ] && return
260
261 action 'Normalizing mo files...'
262 size0=$(sizes mo1)
263 time0=$(date +%s)
264
265 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
266 tazpkg -gi gettext --quiet
267 # Gconv modules (convert to UTF-8)
268 tazpkg -gi glibc-locale --quiet
269
270 # Process all existing *.mo files
271 for mo in $(find "$install" -type f -name '*.mo'); do
272 tmpfile="$(mktemp)"
273
274 msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'
275 # add newline
276 echo >> "$tmpfile"
277
278 # get Plural-Forms
279 awk '
280 BEGIN { skip = ""; }
281 {
282 if (! skip) {
283 s = $0;
284 gsub(/^[^\"]*\"/, "", s);
285 gsub(/\"$/, "", s);
286 printf("%s", s);
287 }
288 if (! $0) skip = "yes";
289 }
290 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
291
292 if ! grep -q 'msgid_plural' "$tmpfile"; then
293 echo > "$tmpfile.pf"
294 fi
295
296 # main
297 awk -v pf="$(cat "$tmpfile.pf")" '
298 function clean() {
299 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
300 }
301
302 function getstring() {
303 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
304 s = $0;
305 gsub(/^[^\"]*\"/, "", s);
306 gsub(/\"$/, "", s);
307 return s;
308 }
309
310 BEGIN {
311 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
312 if (pf)
313 printf("\"%s\\n\"\n", pf);
314 printf("\n");
315 skip = 1;
316 clean();
317 }
318
319 {
320 # Skip the entire header
321 if (!skip) {
322 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
323 mode = $1;
324 if ($1 == "msgstr[0]") mode = "msgstr0";
325 if ($1 == "msgstr[1]") mode = "msgstr1";
326 if ($1 == "msgstr[2]") mode = "msgstr2";
327 if ($1 == "msgstr[3]") mode = "msgstr3";
328 if ($1 == "msgstr[4]") mode = "msgstr4";
329 if ($1 == "msgstr[5]") mode = "msgstr5";
330
331 if (mode == "msgctxt") msgctxt = msgctxt getstring();
332 if (mode == "msgid") msgid = msgid getstring();
333 if (mode == "msgstr") msgstr = msgstr getstring();
334 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
335 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
336 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
337 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
338 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
339 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
340 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
341
342 if (! $0) {
343 if (msgid != msgstr) {
344 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
345 printf("msgid \"%s\"\n", msgid);
346 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
347 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
348 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
349 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
350 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
351 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
352 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
353 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
354 printf("\n");
355 }
356 clean();
357 }
358 }
359 if ($0 == "") skip = "";
360 }
361 ' "$tmpfile" > "$tmpfile.awk"
362
363 msgfmt "$tmpfile.awk" -o "$tmpfile.mo"
364
365 if [ -s "$tmpfile.mo" ]; then
366 rm "$mo"; mv "$tmpfile.mo" "$mo"
367 else
368 _ 'Error processing %s' "$mo"
369 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
370 fi
371
372 # Clean
373 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
374 done
375
376 comp_summary "$time0" "$size0" "$(sizes mo1)"
377 }
378
379
380 # Strip locale definitions: normalize whitespace and remove comments
381 # Stripping can be disabled with COOKOPTS="!locdef"
382
383 strip_locale_def() {
384 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
385 [ ! -d "$install/usr/share/i18n/locales" ] && return
386 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
387
388 action 'Stripping locale definitions...'
389 size0=$(sizes loc)
390 time0=$(date +%s)
391
392 for i in $(find $install/usr/share/i18n/locales -type f); do
393 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
394 done
395
396 comp_summary "$time0" "$size0" "$(sizes loc)"
397 }
398
399
400 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
401 # as removing unneeded files like in Python packages. Cross compiled binaries
402 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
403 # Stripping can be disabled with COOKOPTS="!strip"
404
405 strip_package() {
406 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
407
408 case "$ARCH" in
409 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
410 *) export STRIP='strip' ;;
411 esac
412 action 'Executing strip on all files...'
413 size0=0
414 size1=0
415 time0=$(date +%s)
416
417 # Strip executable files
418 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
419 if [ -d "$dir" ]; then
420 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
421 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
422 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
423 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
424 fi
425 done
426
427 # Strip shared and static libraries
428 # Remove Python *.pyc and *.pyo, Perl perllocal.pod and .packlist
429 oldsize=$(find $fs -type f \( \
430 -name '*.so*' -o -name '*.a' -o \
431 -name '*.pyc' -o -name '*.pyo' -o \
432 -name 'perllocal.pod' -o -name '.packlist' \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
433
434 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
435 find $fs -name '*.a' -exec $STRIP --strip-debug '{}' 2>/dev/null \;
436 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
437 find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \) -delete 2>/dev/null
438
439 newsize=$(find $fs -type f \( \
440 -name '*.so*' -o -name '*.a' -o \) -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
441
442 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
443 }
444
445
446 # Strip unsupported locales (.mo files)
447
448 strip_mo_i18n() {
449 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
450
451 [ ! -d "$fs/usr/share/locale" ] && return
452 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
453
454 action 'Thin out translation files...'
455 size0=$(sizes mo2)
456 time0=$(date +%s)
457
458 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
459 # Default value is "" (empty). That means for us that we'll use the full
460 # list of supported locales here.
461 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
462
463 # Existing locales
464 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
465
466 # Thin out the list of existing locales. At the end there will be only locales that need
467 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
468 for keep_locale in $LOCALE; do
469 elocales=${elocales//$keep_locale}
470 done
471
472 # Remove the unsupported locales
473 for rem_locale in $elocales; do
474 [ -d "$fs/usr/share/locale/$rem_locale" ] &&
475 rm -r "$fs/usr/share/locale/$rem_locale"
476 done
477
478 comp_summary "$time0" "$size0" "$(sizes mo2)"
479 }
480
481
482
483
484 case $1 in
485 install)
486 # Compressors working in the $install
487 case "$ARCH" in
488 arm*) ;;
489 *)
490 compress_manpages
491 compress_png
492 compress_svg
493 compress_ui
494 ;;
495 esac
496
497 fix_desktop_files
498 normalize_mo
499 strip_locale_def
500 ;;
501 fs)
502 # Compressors working in the $fs
503 strip_package
504 strip_mo_i18n
505 ;;
506 esac