rev |
line source |
al@865
|
1 #!/bin/sh
|
al@865
|
2 #
|
al@865
|
3 # compressor - module of the SliTaz Cook
|
al@865
|
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
|
al@865
|
5 #
|
al@865
|
6
|
al@865
|
7 . /usr/lib/slitaz/libcook.sh
|
al@865
|
8
|
al@868
|
9 # Compressor cache stuff
|
al@868
|
10
|
al@873
|
11 comp_cache_root='/home/slitaz/cache/cook'
|
al@869
|
12 mkdir -p "$comp_cache_root"
|
al@868
|
13 cache_stat=$(mktemp)
|
al@868
|
14
|
al@868
|
15 # Cache notes.
|
al@868
|
16 # Do not do the same job twice. Getting the file from the cache is much faster
|
al@868
|
17 # than compressing the file one more time. In addition, this cache is trying not
|
paul@1006
|
18 # to take extra space using the hardlinks. Although the files from the cache
|
paul@870
|
19 # without reference to itself should be removed periodically.
|
al@868
|
20
|
al@868
|
21
|
al@868
|
22
|
al@868
|
23
|
al@865
|
24 #
|
al@865
|
25 # Functions
|
al@865
|
26 #
|
al@865
|
27
|
al@865
|
28
|
al@912
|
29 # tazpkg install command
|
al@912
|
30
|
al@912
|
31 tpi() {
|
al@912
|
32 tazpkg -gi --quiet --local --cookmode $1
|
al@912
|
33 }
|
al@912
|
34
|
al@912
|
35
|
al@881
|
36 # Working with time (with hundredths precision)
|
al@865
|
37
|
al@881
|
38 get_time() {
|
al@881
|
39 cut -d" " -f2 /proc/uptime
|
al@881
|
40 }
|
al@865
|
41
|
al@881
|
42 calc_time() {
|
al@881
|
43 # L10n: 's' is for seconds, 'm' is for minutes
|
al@881
|
44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
|
al@881
|
45 BEGIN{
|
al@881
|
46 time = b - a;
|
al@881
|
47 if (time < 30)
|
al@881
|
48 printf("%.2f%s\n", time, s);
|
al@881
|
49 else
|
al@881
|
50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
|
al@881
|
51 }'
|
al@865
|
52 }
|
al@865
|
53
|
al@865
|
54
|
al@865
|
55 # Compressor mini summary
|
al@865
|
56
|
al@865
|
57 comp_summary() {
|
al@998
|
58 # "$time0" "$size0" "$size1" "$log_file"
|
al@865
|
59 status
|
al@865
|
60 [ "$2" -eq 0 ] && return
|
al@881
|
61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
|
al@868
|
62 cache_msg=''
|
al@868
|
63 if [ -s "$cache_stat" ]; then
|
al@868
|
64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
|
al@868
|
65 echo -n > $cache_stat
|
al@868
|
66 fi
|
al@868
|
67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
|
al@881
|
68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
|
al@998
|
69
|
al@998
|
70 if [ -s "$4" ]; then
|
al@1024
|
71 _ 'Compressor warnings and errors:'
|
al@998
|
72 awk '{printf " %s\n", $0;}' "$4"
|
al@998
|
73 echo
|
al@998
|
74 fi
|
al@998
|
75 # Clean log
|
al@998
|
76 [ ! -f "$4" ] || rm "$4"
|
al@865
|
77 }
|
al@865
|
78
|
al@865
|
79
|
al@1020
|
80 # Find ELF files
|
al@1020
|
81
|
al@1020
|
82 find_elf() {
|
al@1020
|
83 local i ifs="$IFS"
|
al@1020
|
84 IFS=$'\n'
|
al@1020
|
85 find $fs -type f \
|
al@1020
|
86 | while read i; do
|
al@1020
|
87 # output of `readelf -h <file> is human-readable information,
|
paul@1021
|
88 # we are interested in the next line:
|
al@1020
|
89 # Type: EXEC (Executable file)
|
al@1020
|
90 # or
|
al@1020
|
91 # Type: DYN (Shared object file)
|
al@1020
|
92 if [ "$(readelf -h "$i" 2>/dev/null \
|
al@1020
|
93 | sed -n '/Type:/ s|.*: *\([^ ]*\) .*|\1|p')" == "$1" ]; then
|
al@1020
|
94 echo "$i" # $1 = { EXEC | DYN }
|
al@1020
|
95 fi
|
al@1020
|
96 done
|
al@1020
|
97 IFS="$ifs"
|
al@1020
|
98 }
|
al@1020
|
99
|
al@1020
|
100
|
al@865
|
101 # Calculating different sizes
|
al@865
|
102
|
al@865
|
103 sizes() {
|
al@1020
|
104 local ifs="$IFS"; IFS=$'\n'
|
al@865
|
105 case $1 in
|
al@865
|
106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
|
al@865
|
107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
|
al@865
|
108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
|
al@1064
|
109 gif) find $install -type f -name '*.gif' -exec ls -l \{\} \; ;;
|
al@865
|
110 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
|
al@865
|
111 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
|
al@865
|
112 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
|
al@865
|
113 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
|
al@865
|
114 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
|
al@872
|
115 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
|
al@954
|
116 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
|
al@1020
|
117 strip)
|
al@1020
|
118 {
|
al@1020
|
119 find_elf EXEC
|
al@1020
|
120 find_elf DYN
|
al@1020
|
121 find $fs -type f \( -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
|
al@1020
|
122 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \)
|
al@1020
|
123 } \
|
al@1034
|
124 | tr '\n' '\0' \
|
al@1034
|
125 | xargs -0 ls -l
|
al@1020
|
126 ;;
|
al@865
|
127 esac | awk '{s+=$5}END{print s}'
|
al@1020
|
128 IFS="$ifs"
|
al@865
|
129 }
|
al@865
|
130
|
al@865
|
131
|
al@868
|
132 # Query cache for already existing compressed file; substitute original file with the cached
|
al@868
|
133 # compressed one if so.
|
al@868
|
134 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
|
al@868
|
135
|
al@868
|
136 query_cache() {
|
al@868
|
137 md5=$(md5sum "$2")
|
al@868
|
138 cachefile="$comp_cache_root/$1/${md5%% *}"
|
al@868
|
139 echo "$cachefile"
|
al@868
|
140 if [ -f "$cachefile" ]; then
|
al@868
|
141 ln -f "$cachefile" "$2"
|
al@868
|
142 echo '+' >> "$cache_stat"
|
al@868
|
143 else
|
al@868
|
144 echo '-' >> "$cache_stat"
|
al@868
|
145 false
|
al@868
|
146 fi
|
al@868
|
147 }
|
al@868
|
148
|
al@868
|
149
|
al@868
|
150 # Store compressed file to the cache
|
al@868
|
151 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
|
al@868
|
152
|
al@868
|
153 store_cache() {
|
al@868
|
154 mkdir -p "${1%/*}"
|
al@868
|
155 mv "$2" "$1"
|
al@868
|
156 ln "$1" "$2"
|
al@868
|
157 }
|
al@868
|
158
|
al@868
|
159
|
al@865
|
160 # Function to compress all man pages
|
al@865
|
161 # Compressing can be disabled with COOKOPTS="!manz"
|
al@865
|
162
|
al@865
|
163 compress_manpages() {
|
al@881
|
164 time0=$(get_time)
|
al@865
|
165 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
|
al@874
|
166 manpath="$install/usr/share/man"
|
al@865
|
167 [ -d "$manpath" ] || return
|
al@865
|
168 size0=$(sizes man); [ -z "$size0" ] && return
|
al@865
|
169
|
al@936
|
170 tpi advancecomp-static
|
al@867
|
171
|
al@865
|
172 action 'Compressing man pages...'
|
al@865
|
173
|
al@865
|
174 # We'll use only Gzip compression, so decompress other formats first
|
al@865
|
175 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
|
al@865
|
176 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
|
al@865
|
177
|
al@865
|
178 # Fast compress with gzip
|
al@874
|
179 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
|
al@865
|
180
|
al@865
|
181 # Fix symlinks
|
al@874
|
182 for i in $(find $manpath -type l); do
|
al@865
|
183 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
|
al@865
|
184 link=$(echo $i | sed 's|\.[gbx]z2*$||')
|
al@865
|
185 rm $i; ln -s $dest.gz $link.gz
|
al@865
|
186 done
|
al@865
|
187
|
al@865
|
188 # Recompress with advdef (it can't compress, only recompress)
|
al@1024
|
189 the_log="$(mktemp)"
|
al@1024
|
190 if which advdef >/dev/null; then
|
al@1024
|
191 IFS=$'\n'
|
al@1024
|
192 for i in $(find $manpath -type f); do
|
al@1024
|
193 if ! cached_path=$(query_cache mangz "$i"); then
|
al@1024
|
194 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
|
al@1024
|
195 out="$(advdef -z4q "$i")"
|
al@1024
|
196 if [ -n "$out" ]; then
|
al@1024
|
197 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
|
al@1024
|
198 mv -f "$i.orig$$" "$i" # restore the original
|
al@1024
|
199 else
|
al@1024
|
200 store_cache "$cached_path" "$i"
|
al@1024
|
201 rm -f "$i.orig$$" # clean
|
al@1024
|
202 fi
|
al@998
|
203 fi
|
al@1024
|
204 done
|
al@1024
|
205 else
|
al@1024
|
206 echo 'Warning: advdef not found.' > "$the_log"
|
al@1024
|
207 fi
|
al@865
|
208
|
al@1024
|
209 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log"
|
al@865
|
210 }
|
al@865
|
211
|
al@865
|
212
|
al@872
|
213 # Function to recompress all gzip archives
|
al@872
|
214 # Recompressing can be disabled with COOKOPTS="!gz"
|
al@872
|
215
|
al@872
|
216 recompress_gz() {
|
al@881
|
217 time0=$(get_time)
|
al@872
|
218 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
|
al@872
|
219 size0=$(sizes gz); [ -z "$size0" ] && return
|
al@872
|
220
|
al@936
|
221 tpi advancecomp-static
|
al@872
|
222
|
al@872
|
223 action 'Recompressing gzip files...'
|
al@872
|
224
|
al@872
|
225 # Recompress with advdef
|
al@1024
|
226 the_log="$(mktemp)"
|
al@1024
|
227 if which advdef >/dev/null; then
|
al@1024
|
228 IFS=$'\n'
|
al@1024
|
229 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
|
al@1024
|
230 if ! cached_path=$(query_cache gz "$i"); then
|
al@1024
|
231 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
|
al@1024
|
232 out="$(advdef -z4q "$i")"
|
al@1024
|
233 if [ -n "$out" ]; then
|
al@1024
|
234 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
|
al@1024
|
235 mv -f "$i.orig$$" "$i" # restore the original
|
al@1024
|
236 else
|
al@1024
|
237 store_cache "$cached_path" "$i"
|
al@1024
|
238 rm -f "$i.orig$$" # clean
|
al@1024
|
239 fi
|
al@998
|
240 fi
|
al@1024
|
241 done
|
al@1024
|
242 else
|
al@1024
|
243 echo 'Warning: advdef not found.' > "$the_log"
|
al@1024
|
244 fi
|
al@872
|
245
|
al@1024
|
246 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log"
|
al@872
|
247 }
|
al@872
|
248
|
al@872
|
249
|
al@954
|
250 # Function to recompress all zip archives
|
al@954
|
251 # Recompressing can be disabled with COOKOPTS="!zip"
|
al@954
|
252
|
al@954
|
253 recompress_zip() {
|
al@954
|
254 time0=$(get_time)
|
al@954
|
255 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
|
al@954
|
256 size0=$(sizes zip); [ -z "$size0" ] && return
|
al@954
|
257
|
al@954
|
258 tpi advancecomp-static
|
al@954
|
259
|
al@954
|
260 action 'Recompressing zip files...'
|
al@954
|
261
|
al@954
|
262 # Recompress with advzip
|
al@1024
|
263 the_log="$(mktemp)"
|
al@1024
|
264 if which advzip >/dev/null; then
|
al@1024
|
265 IFS=$'\n'
|
al@1024
|
266 for i in $(find $install -type f -name '*.zip'); do
|
al@1024
|
267 if ! cached_path=$(query_cache zip "$i"); then
|
al@1024
|
268 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
|
al@1024
|
269 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
|
al@1024
|
270 if [ -n "$out" ]; then
|
al@1024
|
271 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
|
al@1024
|
272 mv -f "$i.orig$$" "$i" # restore the original
|
al@1024
|
273 else
|
al@1024
|
274 store_cache "$cached_path" "$i"
|
al@1024
|
275 rm -f "$i.orig$$" # clean
|
al@1024
|
276 fi
|
al@998
|
277 fi
|
al@1024
|
278 done
|
al@1024
|
279 else
|
al@1024
|
280 echo 'Warning: advzip not found.' > "$the_log"
|
al@1024
|
281 fi
|
al@954
|
282
|
al@1024
|
283 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log"
|
al@954
|
284 }
|
al@954
|
285
|
al@954
|
286
|
al@865
|
287 # Function used after compile_rules() to compress all png images
|
al@865
|
288 # Compressing can be disabled with COOKOPTS="!pngz"
|
al@865
|
289
|
al@865
|
290 compress_png() {
|
al@881
|
291 time0=$(get_time)
|
al@865
|
292 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
|
al@865
|
293 size0=$(sizes png); [ -z "$size0" ] && return
|
al@865
|
294
|
al@865
|
295 use_pq=true
|
al@865
|
296 use_op=true
|
al@865
|
297 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
|
al@865
|
298 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
|
al@932
|
299 $use_pq && tpi pngquant-static
|
al@932
|
300 $use_op && tpi optipng-static
|
al@865
|
301
|
al@867
|
302 action 'Compressing png images...'
|
al@865
|
303
|
al@1024
|
304 the_log="$(mktemp)"
|
al@1024
|
305 $use_pq && if ! which pngquant >/dev/null; then
|
al@1024
|
306 echo 'Warning: pngquant not found.' > "$the_log"
|
al@1024
|
307 use_pq=false
|
al@1024
|
308 fi
|
al@1024
|
309 $use_op && if ! which optipng >/dev/null; then
|
al@1024
|
310 echo 'Warning: optipng not found.' >> "$the_log"
|
al@1024
|
311 use_op=false
|
al@1024
|
312 fi
|
al@1024
|
313
|
al@865
|
314 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
|
al@868
|
315 [ -z "$oplevel" ] && oplevel='2'
|
al@868
|
316
|
al@868
|
317 cache_section="png$oplevel"
|
al@868
|
318 $use_pq && cache_section="${cache_section}p"
|
al@868
|
319 $use_op && cache_section="${cache_section}o"
|
al@868
|
320
|
al@865
|
321 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
|
al@865
|
322
|
al@998
|
323 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
|
al@978
|
324 IFS=$'\n'
|
al@978
|
325 for i in $(find $install -type f -name '*.png'); do
|
al@998
|
326 unset IFS iserror
|
al@868
|
327 if ! cached_path=$(query_cache $cache_section "$i"); then
|
al@998
|
328 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
|
al@998
|
329 if $use_pq; then
|
al@998
|
330 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
|
al@998
|
331 if [ -n "$out" ]; then
|
al@1024
|
332 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log"
|
al@998
|
333 iserror='yes'
|
al@1045
|
334 [ -e "$i.tmp" ] && rm "$i.tmp" # zero-size file remains on pngquant fail
|
al@998
|
335 fi
|
al@998
|
336 fi
|
al@998
|
337 if $use_op && [ -z "$iserror" ]; then
|
al@998
|
338 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
|
al@998
|
339 if [ -n "$out" ]; then
|
al@1024
|
340 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log"
|
al@998
|
341 iserror='yes'
|
al@998
|
342 fi
|
al@998
|
343 fi
|
al@998
|
344 if [ -n "$iserror" ]; then
|
al@998
|
345 mv -f "$i.orig$$" "$i" # restore the original
|
al@998
|
346 else
|
al@998
|
347 store_cache "$cached_path" "$i"
|
al@998
|
348 rm -f "$i.orig$$" # clean
|
al@998
|
349 fi
|
al@868
|
350 fi
|
al@865
|
351 done
|
al@865
|
352
|
al@1024
|
353 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log"
|
al@865
|
354 }
|
al@865
|
355
|
al@865
|
356
|
al@865
|
357 # Function used after compile_rules() to compress all svg images
|
al@865
|
358 # Compressing can be disabled with COOKOPTS="!svgz"
|
al@865
|
359
|
al@865
|
360 compress_svg() {
|
al@881
|
361 time0=$(get_time)
|
al@865
|
362 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
|
al@865
|
363 size0=$(sizes svg); [ -z "$size0" ] && return
|
al@865
|
364
|
al@912
|
365 tpi svgcleaner
|
al@867
|
366
|
al@865
|
367 action 'Compressing svg images...'
|
al@865
|
368
|
al@1024
|
369 if which svgcleaner >/dev/null; then
|
al@1024
|
370 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
|
al@1024
|
371 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
|
al@959
|
372
|
al@1024
|
373 the_log="$(mktemp)"
|
al@1024
|
374 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
|
al@1024
|
375 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
|
al@1024
|
376 --multipass --remove-unresolved-classes no $opts 2>&1)"
|
al@1024
|
377 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
|
al@1024
|
378 done
|
al@1024
|
379 else
|
al@1024
|
380 echo 'Warning: svgcleaner not found.' > "$the_log"
|
al@1024
|
381 fi
|
al@865
|
382
|
al@1024
|
383 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log"
|
al@865
|
384 }
|
al@865
|
385
|
al@865
|
386
|
al@1064
|
387 # Function used after compile_rules() to compress all gif images
|
al@1064
|
388 # Compressing can be disabled with COOKOPTS="!gifz"
|
al@1064
|
389
|
al@1064
|
390 compress_gif() {
|
al@1064
|
391 time0=$(get_time)
|
al@1064
|
392 [ "${COOKOPTS/!gifz/}" != "$COOKOPTS" ] && return
|
al@1064
|
393 size0=$(sizes gif); [ -z "$size0" ] && return
|
al@1064
|
394
|
al@1064
|
395 tpi gifsicle
|
al@1064
|
396
|
al@1064
|
397 action 'Compressing gif images...'
|
al@1064
|
398
|
al@1064
|
399 if which gifsicle >/dev/null; then
|
al@1064
|
400 the_log="$(mktemp)"
|
al@1067
|
401 IFS=$'\n'
|
al@1067
|
402 for i in $(find $install -type f -name '*.gif'); do
|
al@1064
|
403 unset IFS
|
al@1064
|
404 # use intermediate file, if all well ($?=0), then substitute the original
|
al@1066
|
405 if gifsicle -O3 "$i" -o "$i.$$" >> "$the_log" 2>&1; then
|
al@1066
|
406 mv "$i.$$" "$i"
|
al@1064
|
407 else
|
al@1066
|
408 rm "$i.$$"
|
al@1064
|
409 fi
|
al@1064
|
410 done
|
al@1064
|
411 else
|
al@1064
|
412 echo 'Warning: gifsicle not found.' > "$the_log"
|
al@1064
|
413 fi
|
al@1064
|
414
|
al@1064
|
415 comp_summary "$time0" "$size0" "$(sizes gif)" "$the_log"
|
al@1064
|
416 }
|
al@1064
|
417
|
al@1064
|
418
|
al@865
|
419 # Function used after compile_rules() to shrink all *.ui and *.glade files:
|
al@865
|
420 # remove insignificant spaces and comments
|
al@865
|
421 # Compressing can be disabled with COOKOPTS="!uiz"
|
al@865
|
422
|
al@865
|
423 compress_ui() {
|
al@865
|
424 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
|
al@865
|
425 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
|
al@865
|
426
|
al@912
|
427 tpi xmlstarlet
|
al@867
|
428
|
al@865
|
429 action 'Compressing ui files...'
|
al@867
|
430
|
al@1024
|
431 if which xmlstarlet >/dev/null; then
|
al@1024
|
432 size0=$(sizes xml)
|
al@1024
|
433 time0=$(get_time)
|
al@1024
|
434 temp_ui="$(mktemp)"
|
al@1024
|
435 the_log="$(mktemp)"
|
al@1024
|
436 IFS=$'\n'
|
al@1024
|
437 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
|
al@1024
|
438 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")"
|
al@1024
|
439 if [ -n "$out" ]; then
|
al@1024
|
440 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log"
|
al@1024
|
441 else
|
al@1024
|
442 cat "$temp_ui" > "$ui"
|
al@1024
|
443 fi
|
al@1024
|
444 done
|
al@1024
|
445 else
|
al@1024
|
446 echo 'Warning: xmlstarlet not found.' > "$the_log"
|
al@1024
|
447 fi
|
al@865
|
448
|
al@1024
|
449 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log"
|
al@865
|
450 rm "$temp_ui"
|
al@865
|
451 }
|
al@865
|
452
|
al@865
|
453
|
al@865
|
454 # Get list of supported locales...
|
al@865
|
455
|
al@865
|
456 get_supported_locales() {
|
al@865
|
457 lpc='/slitaz-i18n/stuff/locale-pack.conf'
|
al@865
|
458 if [ -e "$WOK$lpc" ]; then
|
al@865
|
459 # ... from package in the local wok
|
al@865
|
460 . "$WOK$lpc"
|
al@865
|
461 else
|
al@865
|
462 # ... from Hg
|
al@865
|
463 temp_conf=$(mktemp)
|
al@865
|
464 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
|
al@865
|
465 if [ -s $temp_conf ]; then
|
al@865
|
466 . $temp_conf
|
al@865
|
467 else
|
al@865
|
468 # Give up and use hardcoded list
|
al@865
|
469 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
|
al@865
|
470 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
|
al@865
|
471 fi
|
al@865
|
472 rm $temp_conf
|
al@865
|
473 fi
|
al@865
|
474 echo $LOCALE_PACK
|
al@865
|
475 }
|
al@865
|
476
|
al@865
|
477
|
al@865
|
478 # Fix common errors and warnings in the .desktop files
|
al@865
|
479 # Fixing can be disabled with COOKOPTS="!fixdesktops"
|
al@865
|
480
|
al@865
|
481 fix_desktop_files() {
|
al@865
|
482 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
|
al@876
|
483 deskpath="$install/usr/share/applications"
|
al@876
|
484 [ -d "$deskpath" ] || return
|
al@876
|
485 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
|
al@865
|
486
|
al@865
|
487 size0=$(sizes des)
|
al@881
|
488 time0=$(get_time)
|
al@865
|
489
|
al@865
|
490 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
|
al@936
|
491 tpi desktop-file-validate-static
|
al@865
|
492 fi
|
al@865
|
493
|
al@865
|
494 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
|
al@865
|
495 # Default value is "" (empty). That means for us that we'll use the full
|
al@865
|
496 # list of supported locales here.
|
al@865
|
497 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
|
al@865
|
498
|
al@876
|
499 IFS=$'\n'
|
al@876
|
500 for desktop in $(find $deskpath -type f -name '*.desktop'); do
|
al@865
|
501 cp "$desktop" "$desktop.orig"
|
al@865
|
502
|
al@865
|
503 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
|
al@865
|
504 sdft "$desktop" -i
|
al@865
|
505
|
al@865
|
506 # Fix common errors in .desktop file
|
al@865
|
507 fix-desktop-file "$desktop"
|
al@865
|
508
|
al@865
|
509 # Strip unsupported locales from .desktop file
|
al@865
|
510 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
|
al@865
|
511 sdft "$desktop" -i -k "$LOCALE"
|
al@865
|
512
|
al@865
|
513 # Extra-strip
|
al@865
|
514 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
|
al@865
|
515 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
|
al@865
|
516
|
al@865
|
517 if [ -n "$QA" ]; then
|
al@865
|
518 # Check the rest of errors, warnings and tips
|
al@865
|
519 _ 'QA: Checking %s...' "$(basename $desktop)"
|
al@1002
|
520 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!'
|
al@1036
|
521 if which desktop-file-validate >/dev/null; then
|
al@1024
|
522 desktop-file-validate "$desktop" | busybox fold -s
|
al@1024
|
523 else
|
al@1024
|
524 echo 'Warning: desktop-file-validate not found.'
|
al@1024
|
525 fi
|
al@865
|
526 echo
|
al@865
|
527 fi
|
al@865
|
528
|
al@865
|
529 rm "$desktop.orig"
|
al@865
|
530 done
|
al@865
|
531
|
al@998
|
532 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
|
al@865
|
533 }
|
al@865
|
534
|
al@865
|
535
|
paul@879
|
536 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
|
al@865
|
537 # to the translation (msgid = msgstr)
|
al@865
|
538 # Normalization can be disabled with COOKOPTS="!monorm"
|
al@865
|
539
|
al@865
|
540 normalize_mo() {
|
al@865
|
541 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
|
al@865
|
542 [ -z "$(find $install -type f -name '*.mo')" ] && return
|
al@865
|
543
|
al@867
|
544 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
|
al@912
|
545 tpi gettext
|
al@867
|
546 # Gconv modules (convert to UTF-8)
|
al@912
|
547 tpi glibc-locale
|
al@867
|
548
|
al@865
|
549 action 'Normalizing mo files...'
|
al@867
|
550
|
al@1024
|
551 the_log="$(mktemp)"
|
al@1024
|
552 to_continue=true
|
al@1024
|
553 for i in msgunfmt msguniq msgconv msgfmt; do
|
al@1024
|
554 if ! which $i >/dev/null; then
|
al@1024
|
555 echo "Warning: $i not found. Normalizing aborted" > "$the_log"
|
al@1024
|
556 to_continue=false
|
al@1024
|
557 fi
|
al@1024
|
558 done
|
al@1024
|
559
|
al@865
|
560 size0=$(sizes mo1)
|
al@881
|
561 time0=$(get_time)
|
al@865
|
562
|
al@865
|
563 # Process all existing *.mo files
|
al@876
|
564 IFS=$'\n'
|
al@1024
|
565 $to_continue &&
|
al@865
|
566 for mo in $(find "$install" -type f -name '*.mo'); do
|
al@865
|
567 tmpfile="$(mktemp)"
|
al@865
|
568
|
al@998
|
569 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
|
al@998
|
570 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
|
al@998
|
571 if [ -n "$out" ]; then
|
al@998
|
572 # using literal $'\n' here instead of using `echo -e "...\n..."` because
|
al@998
|
573 # $out may contain escapes ('\r', '\v') that we should print as-is
|
al@1024
|
574 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log"
|
al@998
|
575 continue # proceed to next file
|
al@998
|
576 fi
|
al@998
|
577
|
al@865
|
578 # add newline
|
al@865
|
579 echo >> "$tmpfile"
|
al@865
|
580
|
al@865
|
581 # get Plural-Forms
|
al@865
|
582 awk '
|
al@865
|
583 BEGIN { skip = ""; }
|
al@865
|
584 {
|
al@865
|
585 if (! skip) {
|
al@865
|
586 s = $0;
|
al@865
|
587 gsub(/^[^\"]*\"/, "", s);
|
al@865
|
588 gsub(/\"$/, "", s);
|
al@865
|
589 printf("%s", s);
|
al@865
|
590 }
|
al@865
|
591 if (! $0) skip = "yes";
|
al@865
|
592 }
|
al@865
|
593 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
|
al@865
|
594
|
al@865
|
595 if ! grep -q 'msgid_plural' "$tmpfile"; then
|
al@865
|
596 echo > "$tmpfile.pf"
|
al@865
|
597 fi
|
al@865
|
598
|
al@865
|
599 # main
|
al@865
|
600 awk -v pf="$(cat "$tmpfile.pf")" '
|
al@865
|
601 function clean() {
|
al@865
|
602 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
|
al@865
|
603 }
|
al@865
|
604
|
al@865
|
605 function getstring() {
|
al@865
|
606 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
|
al@865
|
607 s = $0;
|
al@865
|
608 gsub(/^[^\"]*\"/, "", s);
|
al@865
|
609 gsub(/\"$/, "", s);
|
al@865
|
610 return s;
|
al@865
|
611 }
|
al@865
|
612
|
al@865
|
613 BEGIN {
|
al@865
|
614 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
|
al@865
|
615 if (pf)
|
al@865
|
616 printf("\"%s\\n\"\n", pf);
|
al@865
|
617 printf("\n");
|
al@865
|
618 skip = 1;
|
al@865
|
619 clean();
|
al@865
|
620 }
|
al@865
|
621
|
al@865
|
622 {
|
al@865
|
623 # Skip the entire header
|
al@865
|
624 if (!skip) {
|
al@865
|
625 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
|
al@865
|
626 mode = $1;
|
al@865
|
627 if ($1 == "msgstr[0]") mode = "msgstr0";
|
al@865
|
628 if ($1 == "msgstr[1]") mode = "msgstr1";
|
al@865
|
629 if ($1 == "msgstr[2]") mode = "msgstr2";
|
al@865
|
630 if ($1 == "msgstr[3]") mode = "msgstr3";
|
al@865
|
631 if ($1 == "msgstr[4]") mode = "msgstr4";
|
al@865
|
632 if ($1 == "msgstr[5]") mode = "msgstr5";
|
al@865
|
633
|
al@865
|
634 if (mode == "msgctxt") msgctxt = msgctxt getstring();
|
al@865
|
635 if (mode == "msgid") msgid = msgid getstring();
|
al@865
|
636 if (mode == "msgstr") msgstr = msgstr getstring();
|
al@865
|
637 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
|
al@865
|
638 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
|
al@865
|
639 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
|
al@865
|
640 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
|
al@865
|
641 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
|
al@865
|
642 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
|
al@865
|
643 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
|
al@865
|
644
|
al@865
|
645 if (! $0) {
|
al@865
|
646 if (msgid != msgstr) {
|
al@865
|
647 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
|
al@865
|
648 printf("msgid \"%s\"\n", msgid);
|
al@865
|
649 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
|
al@865
|
650 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
|
al@865
|
651 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
|
al@865
|
652 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
|
al@865
|
653 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
|
al@865
|
654 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
|
al@865
|
655 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
|
al@865
|
656 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
|
al@865
|
657 printf("\n");
|
al@865
|
658 }
|
al@865
|
659 clean();
|
al@865
|
660 }
|
al@865
|
661 }
|
al@865
|
662 if ($0 == "") skip = "";
|
al@865
|
663 }
|
al@865
|
664 ' "$tmpfile" > "$tmpfile.awk"
|
al@865
|
665
|
al@998
|
666 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
|
al@998
|
667 if [ -n "$out" ]; then
|
al@1024
|
668 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log"
|
al@998
|
669 continue # proceed to next file
|
al@998
|
670 fi
|
al@865
|
671
|
al@865
|
672 if [ -s "$tmpfile.mo" ]; then
|
al@865
|
673 rm "$mo"; mv "$tmpfile.mo" "$mo"
|
al@865
|
674 else
|
al@1024
|
675 _ 'Error processing %s' "$mo" >> "$the_log"
|
al@1024
|
676 echo >> "$the_log"
|
al@865
|
677 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
|
al@865
|
678 fi
|
al@865
|
679
|
al@865
|
680 # Clean
|
al@865
|
681 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
|
al@865
|
682 done
|
al@865
|
683
|
al@1024
|
684 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log"
|
al@865
|
685 }
|
al@865
|
686
|
al@865
|
687
|
al@865
|
688 # Strip locale definitions: normalize whitespace and remove comments
|
al@865
|
689 # Stripping can be disabled with COOKOPTS="!locdef"
|
al@865
|
690
|
al@865
|
691 strip_locale_def() {
|
al@865
|
692 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
|
al@865
|
693 [ ! -d "$install/usr/share/i18n/locales" ] && return
|
al@865
|
694 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
|
al@865
|
695
|
al@865
|
696 action 'Stripping locale definitions...'
|
al@865
|
697 size0=$(sizes loc)
|
al@881
|
698 time0=$(get_time)
|
al@865
|
699
|
al@865
|
700 for i in $(find $install/usr/share/i18n/locales -type f); do
|
al@865
|
701 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
|
al@865
|
702 done
|
al@865
|
703
|
al@865
|
704 comp_summary "$time0" "$size0" "$(sizes loc)"
|
al@865
|
705 }
|
al@865
|
706
|
al@865
|
707
|
al@865
|
708 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
|
al@865
|
709 # as removing unneeded files like in Python packages. Cross compiled binaries
|
al@865
|
710 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
|
al@865
|
711 # Stripping can be disabled with COOKOPTS="!strip"
|
al@865
|
712
|
al@865
|
713 strip_package() {
|
al@865
|
714 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
|
al@865
|
715
|
al@1020
|
716 local i ifs="$IFS"
|
al@1020
|
717 IFS=$'\n'
|
al@1020
|
718
|
al@865
|
719 case "$ARCH" in
|
al@865
|
720 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
|
al@865
|
721 *) export STRIP='strip' ;;
|
al@865
|
722 esac
|
al@865
|
723 action 'Executing strip on all files...'
|
al@865
|
724 size0=0
|
al@865
|
725 size1=0
|
al@881
|
726 time0=$(get_time)
|
al@1020
|
727 oldsize=$(sizes strip)
|
al@865
|
728
|
al@1027
|
729
|
al@1027
|
730 # GNU strip (GNU Binutils)
|
al@1027
|
731 # -p --preserve-dates Copy modified/access timestamps to the output
|
paul@1030
|
732 # -s --strip-all Remove all symbols and relocation information
|
al@1027
|
733 # --strip-unneeded Remove all symbols not needed by relocations
|
al@1027
|
734 # -D --enable-deterministic-archives Produce deterministic output when stripping archives
|
al@1027
|
735 # -g -S -d --strip-debug Remove all debugging symbols & sections
|
al@1027
|
736
|
al@1027
|
737 # Strip executable files
|
al@1020
|
738 while read i; do
|
al@1027
|
739 $STRIP -ps "$i" 2>/dev/null
|
al@1020
|
740 done <<EOT
|
al@1027
|
741 $(find_elf EXEC)
|
al@1027
|
742 EOT
|
al@1027
|
743
|
al@1027
|
744 # Strip shared libraries
|
al@1027
|
745 while read i; do
|
al@1027
|
746 case $i in
|
al@1027
|
747 *.dbg) ;; # skip library.so.*.dbg debugging symbols
|
al@1027
|
748 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;;
|
al@1027
|
749 esac
|
al@1027
|
750 done <<EOT
|
al@1027
|
751 $(find_elf DYN)
|
al@1020
|
752 EOT
|
al@865
|
753
|
al@1020
|
754 # Strip static libraries
|
al@1027
|
755 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries
|
al@1027
|
756 find $fs -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \;
|
al@955
|
757
|
al@899
|
758
|
al@899
|
759 # Remove Python *.pyc and *.pyo
|
al@865
|
760 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
|
al@865
|
761
|
al@899
|
762 # Remove both with the empty subfolders:
|
al@899
|
763 # 1. Perl perllocal.pod and .packlist (unconditionally)
|
al@899
|
764 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
|
al@899
|
765 # 2. Perl *.pod (if not disabled)
|
al@899
|
766 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
|
al@899
|
767 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')"
|
al@899
|
768 echo "$perlfiles" | xargs rm -f 2>/dev/null
|
al@899
|
769 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null
|
al@899
|
770
|
al@1020
|
771 # Strip documentation inside Perl files (*.pm and *.pl) (if not disabled)
|
al@899
|
772 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
|
al@899
|
773 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
|
al@899
|
774
|
al@1020
|
775 newsize=$(sizes strip)
|
al@865
|
776
|
al@1020
|
777 comp_summary "$time0" "$oldsize" "$newsize"
|
al@1020
|
778 IFS="$ifs"
|
al@865
|
779 }
|
al@865
|
780
|
al@865
|
781
|
al@865
|
782 # Strip unsupported locales (.mo files)
|
al@865
|
783
|
al@865
|
784 strip_mo_i18n() {
|
al@865
|
785 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
|
al@865
|
786
|
al@865
|
787 [ ! -d "$fs/usr/share/locale" ] && return
|
al@865
|
788 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
|
al@865
|
789
|
al@865
|
790 action 'Thin out translation files...'
|
al@865
|
791 size0=$(sizes mo2)
|
al@881
|
792 time0=$(get_time)
|
al@865
|
793
|
al@865
|
794 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
|
al@865
|
795 # Default value is "" (empty). That means for us that we'll use the full
|
al@865
|
796 # list of supported locales here.
|
al@865
|
797 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
|
al@865
|
798
|
al@865
|
799 # Existing locales
|
al@865
|
800 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
|
al@865
|
801
|
al@865
|
802 # Thin out the list of existing locales. At the end there will be only locales that need
|
al@865
|
803 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
|
al@865
|
804 for keep_locale in $LOCALE; do
|
al@865
|
805 elocales=${elocales//$keep_locale}
|
al@865
|
806 done
|
al@865
|
807
|
al@865
|
808 # Remove the unsupported locales
|
al@865
|
809 for rem_locale in $elocales; do
|
al@998
|
810 [ ! -d "$fs/usr/share/locale/$rem_locale" ] ||
|
al@865
|
811 rm -r "$fs/usr/share/locale/$rem_locale"
|
al@865
|
812 done
|
al@865
|
813
|
al@865
|
814 comp_summary "$time0" "$size0" "$(sizes mo2)"
|
al@865
|
815 }
|
al@865
|
816
|
al@865
|
817
|
al@865
|
818
|
al@865
|
819
|
al@865
|
820 case $1 in
|
al@865
|
821 install)
|
al@865
|
822 # Compressors working in the $install
|
al@865
|
823 case "$ARCH" in
|
al@865
|
824 arm*) ;;
|
al@865
|
825 *)
|
al@872
|
826 recompress_gz
|
al@954
|
827 recompress_zip
|
al@865
|
828 compress_manpages
|
al@865
|
829 compress_png
|
al@865
|
830 compress_svg
|
al@1064
|
831 compress_gif
|
al@865
|
832 compress_ui
|
al@865
|
833 ;;
|
al@865
|
834 esac
|
al@865
|
835
|
al@865
|
836 fix_desktop_files
|
al@865
|
837 normalize_mo
|
al@865
|
838 strip_locale_def
|
al@865
|
839 ;;
|
al@865
|
840 fs)
|
al@865
|
841 # Compressors working in the $fs
|
al@865
|
842 strip_package
|
al@865
|
843 strip_mo_i18n
|
al@865
|
844 ;;
|
al@865
|
845 esac
|
al@868
|
846
|
al@868
|
847 # Clean
|
al@868
|
848 rm "$cache_stat"
|
al@869
|
849 find $comp_cache_root -type f -links -2 -delete
|