cookutils: modules/compressor annotate

cookutils annotate modules/compressor @ rev 1078

cook: allow multi-set & no-src receipts (sources may be downloaded inside compile_rules() individually for each set), allow change SHORT_DESC & WEB_SITE in genpkg_rules() (for differents sets);
lighttpd/cooker.css: add style for log line "Switching to the set...";
lighttpd/index.cgi: remove false-positive coloring with word "fatal", mark line "Switching to the set...", don't hide "Logs" button (fresh log initially empty and it may lasts for many seconds due to buffered "lazy write";
modules/compressor: better remove unwanted Perl files;
modules/mk_pkg_receipt: follow changed SHORT_DESC & WEB_SITE, improve condition speed (from 10 sec to 0.01 sec) when SPLIT is relatively long - looks like ${A/ $B /} is slow.

author	Aleksej Bobylev <al.bobylev@gmail.com>
date	Thu Jun 21 12:57:52 2018 +0300 (2018-06-21)
parents	23700c523fdb
children	33dbe879cc21

rev	line source
al@865	1 #!/bin/sh
al@865	2 #
al@865	3 # compressor - module of the SliTaz Cook
al@865	4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@865	5 #
al@865	6
al@865	7 . /usr/lib/slitaz/libcook.sh
al@865	8
al@868	9 # Compressor cache stuff
al@868	10
al@873	11 comp_cache_root='/home/slitaz/cache/cook'
al@869	12 mkdir -p "$comp_cache_root"
al@868	13 cache_stat=$(mktemp)
al@868	14
al@868	15 # Cache notes.
al@868	16 # Do not do the same job twice. Getting the file from the cache is much faster
al@868	17 # than compressing the file one more time. In addition, this cache is trying not
paul@1006	18 # to take extra space using the hardlinks. Although the files from the cache
paul@870	19 # without reference to itself should be removed periodically.
al@868	20
al@868	21
al@868	22
al@868	23
al@865	24 #
al@865	25 # Functions
al@865	26 #
al@865	27
al@865	28
al@912	29 # tazpkg install command
al@912	30
al@912	31 tpi() {
al@912	32 tazpkg -gi --quiet --local --cookmode $1
al@912	33 }
al@912	34
al@912	35
al@881	36 # Working with time (with hundredths precision)
al@865	37
al@881	38 get_time() {
al@881	39 cut -d" " -f2 /proc/uptime
al@881	40 }
al@865	41
al@881	42 calc_time() {
al@881	43 # L10n: 's' is for seconds, 'm' is for minutes
al@881	44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
al@881	45 BEGIN{
al@881	46 time = b - a;
al@881	47 if (time < 30)
al@881	48 printf("%.2f%s\n", time, s);
al@881	49 else
al@881	50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
al@881	51 }'
al@865	52 }
al@865	53
al@865	54
al@865	55 # Compressor mini summary
al@865	56
al@865	57 comp_summary() {
al@998	58 # "$time0" "$size0" "$size1" "$log_file"
al@865	59 status
al@865	60 [ "$2" -eq 0 ] && return
al@881	61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
al@868	62 cache_msg=''
al@868	63 if [ -s "$cache_stat" ]; then
al@868	64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat \| wc -l)" "$(wc -l < $cache_stat)")
al@868	65 echo -n > $cache_stat
al@868	66 fi
al@868	67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
al@881	68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
al@998	69
al@998	70 if [ -s "$4" ]; then
al@1024	71 _ 'Compressor warnings and errors:'
al@998	72 awk '{printf " %s\n", $0;}' "$4"
al@998	73 echo
al@998	74 fi
al@998	75 # Clean log
al@998	76 [ ! -f "$4" ] \|\| rm "$4"
al@865	77 }
al@865	78
al@865	79
al@1020	80 # Find ELF files
al@1020	81
al@1020	82 find_elf() {
al@1020	83 local i ifs="$IFS"
al@1020	84 IFS=$'\n'
al@1020	85 find $fs -type f \
al@1020	86 \| while read i; do
al@1020	87 # output of `readelf -h <file> is human-readable information,
paul@1021	88 # we are interested in the next line:
al@1020	89 # Type: EXEC (Executable file)
al@1020	90 # or
al@1020	91 # Type: DYN (Shared object file)
al@1020	92 if [ "$(readelf -h "$i" 2>/dev/null \
al@1020	93 \| sed -n '/Type:/ s\|.: $[^ ]$ .\|\1\|p')" == "$1" ]; then
al@1020	94 echo "$i" # $1 = { EXEC \| DYN }
al@1020	95 fi
al@1020	96 done
al@1020	97 IFS="$ifs"
al@1020	98 }
al@1020	99
al@1020	100
al@865	101 # Calculating different sizes
al@865	102
al@865	103 sizes() {
al@1020	104 local ifs="$IFS"; IFS=$'\n'
al@865	105 case $1 in
al@865	106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
al@865	107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
al@865	108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
al@1064	109 gif) find $install -type f -name '*.gif' -exec ls -l \{\} \; ;;
al@865	110 xml) find $install -type f $ -name '.ui' -o -name '.glade' $ -exec ls -l \{\} \; ;;
al@865	111 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
al@865	112 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@865	113 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
al@865	114 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@872	115 gz) find $install -type f -name '.gz' ! -path '/share/man/*' -exec ls -l \{\} \; ;;
al@954	116 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
al@1020	117 strip)
al@1020	118 {
al@1020	119 find_elf EXEC
al@1020	120 find_elf DYN
al@1020	121 find $fs -type f \( -name '.a' -o -name '.pyc' -o -name '*.pyo' \
al@1020	122 -o -name '.packlist' -o -name '.pm' -o -name '.pl' -o -name '*.pod' \)
al@1020	123 } \
al@1034	124 \| tr '\n' '\0' \
al@1034	125 \| xargs -0 ls -l
al@1020	126 ;;
al@865	127 esac \| awk '{s+=$5}END{print s}'
al@1020	128 IFS="$ifs"
al@865	129 }
al@865	130
al@865	131
al@868	132 # Query cache for already existing compressed file; substitute original file with the cached
al@868	133 # compressed one if so.
al@868	134 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
al@868	135
al@868	136 query_cache() {
al@868	137 md5=$(md5sum "$2")
al@868	138 cachefile="$comp_cache_root/$1/${md5%% *}"
al@868	139 echo "$cachefile"
al@868	140 if [ -f "$cachefile" ]; then
al@868	141 ln -f "$cachefile" "$2"
al@868	142 echo '+' >> "$cache_stat"
al@868	143 else
al@868	144 echo '-' >> "$cache_stat"
al@868	145 false
al@868	146 fi
al@868	147 }
al@868	148
al@868	149
al@868	150 # Store compressed file to the cache
al@868	151 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
al@868	152
al@868	153 store_cache() {
al@868	154 mkdir -p "${1%/*}"
al@868	155 mv "$2" "$1"
al@868	156 ln "$1" "$2"
al@868	157 }
al@868	158
al@868	159
al@865	160 # Function to compress all man pages
al@865	161 # Compressing can be disabled with COOKOPTS="!manz"
al@865	162
al@865	163 compress_manpages() {
al@881	164 time0=$(get_time)
al@865	165 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
al@874	166 manpath="$install/usr/share/man"
al@865	167 [ -d "$manpath" ] \|\| return
al@865	168 size0=$(sizes man); [ -z "$size0" ] && return
al@865	169
al@936	170 tpi advancecomp-static
al@867	171
al@865	172 action 'Compressing man pages...'
al@865	173
al@865	174 # We'll use only Gzip compression, so decompress other formats first
al@865	175 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
al@865	176 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
al@865	177
al@865	178 # Fast compress with gzip
al@874	179 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
al@865	180
al@865	181 # Fix symlinks
al@874	182 for i in $(find $manpath -type l); do
al@865	183 dest=$(readlink $i \| sed 's\|\.[gbx]z2*$\|\|')
al@865	184 link=$(echo $i \| sed 's\|\.[gbx]z2*$\|\|')
al@865	185 rm $i; ln -s $dest.gz $link.gz
al@865	186 done
al@865	187
al@865	188 # Recompress with advdef (it can't compress, only recompress)
al@1024	189 the_log="$(mktemp)"
al@1024	190 if which advdef >/dev/null; then
al@1024	191 IFS=$'\n'
al@1024	192 for i in $(find $manpath -type f); do
al@1024	193 if ! cached_path=$(query_cache mangz "$i"); then
al@1024	194 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024	195 out="$(advdef -z4q "$i")"
al@1024	196 if [ -n "$out" ]; then
al@1024	197 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024	198 mv -f "$i.orig$$" "$i" # restore the original
al@1024	199 else
al@1024	200 store_cache "$cached_path" "$i"
al@1024	201 rm -f "$i.orig$$" # clean
al@1024	202 fi
al@998	203 fi
al@1024	204 done
al@1024	205 else
al@1024	206 echo 'Warning: advdef not found.' > "$the_log"
al@1024	207 fi
al@865	208
al@1024	209 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log"
al@865	210 }
al@865	211
al@865	212
al@872	213 # Function to recompress all gzip archives
al@872	214 # Recompressing can be disabled with COOKOPTS="!gz"
al@872	215
al@872	216 recompress_gz() {
al@881	217 time0=$(get_time)
al@872	218 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
al@872	219 size0=$(sizes gz); [ -z "$size0" ] && return
al@872	220
al@936	221 tpi advancecomp-static
al@872	222
al@872	223 action 'Recompressing gzip files...'
al@872	224
al@872	225 # Recompress with advdef
al@1024	226 the_log="$(mktemp)"
al@1024	227 if which advdef >/dev/null; then
al@1024	228 IFS=$'\n'
al@1024	229 for i in $(find $install -type f -name '.gz' ! -path '/share/man/*'); do
al@1024	230 if ! cached_path=$(query_cache gz "$i"); then
al@1024	231 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024	232 out="$(advdef -z4q "$i")"
al@1024	233 if [ -n "$out" ]; then
al@1024	234 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024	235 mv -f "$i.orig$$" "$i" # restore the original
al@1024	236 else
al@1024	237 store_cache "$cached_path" "$i"
al@1024	238 rm -f "$i.orig$$" # clean
al@1024	239 fi
al@998	240 fi
al@1024	241 done
al@1024	242 else
al@1024	243 echo 'Warning: advdef not found.' > "$the_log"
al@1024	244 fi
al@872	245
al@1024	246 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log"
al@872	247 }
al@872	248
al@872	249
al@954	250 # Function to recompress all zip archives
al@954	251 # Recompressing can be disabled with COOKOPTS="!zip"
al@954	252
al@954	253 recompress_zip() {
al@954	254 time0=$(get_time)
al@954	255 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
al@954	256 size0=$(sizes zip); [ -z "$size0" ] && return
al@954	257
al@954	258 tpi advancecomp-static
al@954	259
al@954	260 action 'Recompressing zip files...'
al@954	261
al@954	262 # Recompress with advzip
al@1024	263 the_log="$(mktemp)"
al@1024	264 if which advzip >/dev/null; then
al@1024	265 IFS=$'\n'
al@1024	266 for i in $(find $install -type f -name '*.zip'); do
al@1024	267 if ! cached_path=$(query_cache zip "$i"); then
al@1024	268 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024	269 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
al@1024	270 if [ -n "$out" ]; then
al@1024	271 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024	272 mv -f "$i.orig$$" "$i" # restore the original
al@1024	273 else
al@1024	274 store_cache "$cached_path" "$i"
al@1024	275 rm -f "$i.orig$$" # clean
al@1024	276 fi
al@998	277 fi
al@1024	278 done
al@1024	279 else
al@1024	280 echo 'Warning: advzip not found.' > "$the_log"
al@1024	281 fi
al@954	282
al@1024	283 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log"
al@954	284 }
al@954	285
al@954	286
al@865	287 # Function used after compile_rules() to compress all png images
al@865	288 # Compressing can be disabled with COOKOPTS="!pngz"
al@865	289
al@865	290 compress_png() {
al@881	291 time0=$(get_time)
al@865	292 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
al@865	293 size0=$(sizes png); [ -z "$size0" ] && return
al@865	294
al@865	295 use_pq=true
al@865	296 use_op=true
al@865	297 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
al@865	298 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
al@932	299 $use_pq && tpi pngquant-static
al@932	300 $use_op && tpi optipng-static
al@865	301
al@867	302 action 'Compressing png images...'
al@865	303
al@1024	304 the_log="$(mktemp)"
al@1024	305 $use_pq && if ! which pngquant >/dev/null; then
al@1024	306 echo 'Warning: pngquant not found.' > "$the_log"
al@1024	307 use_pq=false
al@1024	308 fi
al@1024	309 $use_op && if ! which optipng >/dev/null; then
al@1024	310 echo 'Warning: optipng not found.' >> "$the_log"
al@1024	311 use_op=false
al@1024	312 fi
al@1024	313
al@865	314 oplevel=$(echo $COOKOPTS \| grep 'op[0-8]' \| sed 's\|.op$[0-8]$.\|\1\|')
al@868	315 [ -z "$oplevel" ] && oplevel='2'
al@868	316
al@868	317 cache_section="png$oplevel"
al@868	318 $use_pq && cache_section="${cache_section}p"
al@868	319 $use_op && cache_section="${cache_section}o"
al@868	320
al@865	321 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
al@865	322
al@998	323 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
al@978	324 IFS=$'\n'
al@978	325 for i in $(find $install -type f -name '*.png'); do
al@998	326 unset IFS iserror
al@868	327 if ! cached_path=$(query_cache $cache_section "$i"); then
al@998	328 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998	329 if $use_pq; then
al@998	330 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
al@998	331 if [ -n "$out" ]; then
al@1024	332 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log"
al@998	333 iserror='yes'
al@1045	334 [ -e "$i.tmp" ] && rm "$i.tmp" # zero-size file remains on pngquant fail
al@998	335 fi
al@998	336 fi
al@998	337 if $use_op && [ -z "$iserror" ]; then
al@998	338 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
al@998	339 if [ -n "$out" ]; then
al@1024	340 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log"
al@998	341 iserror='yes'
al@998	342 fi
al@998	343 fi
al@998	344 if [ -n "$iserror" ]; then
al@998	345 mv -f "$i.orig$$" "$i" # restore the original
al@998	346 else
al@998	347 store_cache "$cached_path" "$i"
al@998	348 rm -f "$i.orig$$" # clean
al@998	349 fi
al@868	350 fi
al@865	351 done
al@865	352
al@1024	353 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log"
al@865	354 }
al@865	355
al@865	356
al@865	357 # Function used after compile_rules() to compress all svg images
al@865	358 # Compressing can be disabled with COOKOPTS="!svgz"
al@865	359
al@865	360 compress_svg() {
al@881	361 time0=$(get_time)
al@865	362 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
al@865	363 size0=$(sizes svg); [ -z "$size0" ] && return
al@865	364
al@912	365 tpi svgcleaner
al@867	366
al@865	367 action 'Compressing svg images...'
al@865	368
al@1072	369 the_log="$(mktemp)"
al@1024	370 if which svgcleaner >/dev/null; then
al@1024	371 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
al@1024	372 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
al@959	373
al@1024	374 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
al@1024	375 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
al@1024	376 --multipass --remove-unresolved-classes no $opts 2>&1)"
al@1024	377 [ -z "$out" ] \|\| echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024	378 done
al@1024	379 else
al@1024	380 echo 'Warning: svgcleaner not found.' > "$the_log"
al@1024	381 fi
al@865	382
al@1024	383 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log"
al@865	384 }
al@865	385
al@865	386
al@1064	387 # Function used after compile_rules() to compress all gif images
al@1064	388 # Compressing can be disabled with COOKOPTS="!gifz"
al@1064	389
al@1064	390 compress_gif() {
al@1064	391 time0=$(get_time)
al@1064	392 [ "${COOKOPTS/!gifz/}" != "$COOKOPTS" ] && return
al@1064	393 size0=$(sizes gif); [ -z "$size0" ] && return
al@1064	394
al@1064	395 tpi gifsicle
al@1064	396
al@1064	397 action 'Compressing gif images...'
al@1064	398
al@1072	399 the_log="$(mktemp)"
al@1064	400 if which gifsicle >/dev/null; then
al@1067	401 IFS=$'\n'
al@1067	402 for i in $(find $install -type f -name '*.gif'); do
al@1072	403 if ! cached_path=$(query_cache gif "$i"); then
al@1072	404 unset IFS
al@1072	405 # use intermediate file, if all well ($?=0), then substitute the original
al@1072	406 if gifsicle -O3 "$i" -o "$i.$$" >> "$the_log" 2>&1; then
al@1072	407 if [ -s "$i.$$" ]; then
al@1072	408 mv "$i.$$" "$i"
al@1072	409 store_cache "$cached_path" "$i"
al@1072	410 fi
al@1072	411 else
al@1072	412 rm "$i.$$"
al@1072	413 fi
al@1064	414 fi
al@1064	415 done
al@1064	416 else
al@1064	417 echo 'Warning: gifsicle not found.' > "$the_log"
al@1064	418 fi
al@1064	419
al@1064	420 comp_summary "$time0" "$size0" "$(sizes gif)" "$the_log"
al@1064	421 }
al@1064	422
al@1064	423
al@865	424 # Function used after compile_rules() to shrink all .ui and .glade files:
al@865	425 # remove insignificant spaces and comments
al@865	426 # Compressing can be disabled with COOKOPTS="!uiz"
al@865	427
al@865	428 compress_ui() {
al@865	429 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
al@865	430 [ -z "$(find $install -type f $ -name '.ui' -o -name '.glade' $ )" ] && return
al@865	431
al@912	432 tpi xmlstarlet
al@867	433
al@865	434 action 'Compressing ui files...'
al@867	435
al@1072	436 the_log="$(mktemp)"
al@1024	437 if which xmlstarlet >/dev/null; then
al@1024	438 size0=$(sizes xml)
al@1024	439 time0=$(get_time)
al@1024	440 temp_ui="$(mktemp)"
al@1024	441 IFS=$'\n'
al@1024	442 for ui in $(find $install -type f $ -name '.ui' -o -name '.glade' $ ); do
al@1024	443 out="$(xmlstarlet c14n --without-comments "$ui" \| xmlstarlet sel -B -t -c '*' > "$temp_ui")"
al@1024	444 if [ -n "$out" ]; then
al@1024	445 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log"
al@1024	446 else
al@1024	447 cat "$temp_ui" > "$ui"
al@1024	448 fi
al@1024	449 done
al@1024	450 else
al@1024	451 echo 'Warning: xmlstarlet not found.' > "$the_log"
al@1024	452 fi
al@865	453
al@1024	454 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log"
al@865	455 rm "$temp_ui"
al@865	456 }
al@865	457
al@865	458
al@865	459 # Get list of supported locales...
al@865	460
al@865	461 get_supported_locales() {
al@865	462 lpc='/slitaz-i18n/stuff/locale-pack.conf'
al@865	463 if [ -e "$WOK$lpc" ]; then
al@865	464 # ... from package in the local wok
al@865	465 . "$WOK$lpc"
al@865	466 else
al@865	467 # ... from Hg
al@865	468 temp_conf=$(mktemp)
al@865	469 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
al@865	470 if [ -s $temp_conf ]; then
al@865	471 . $temp_conf
al@865	472 else
al@865	473 # Give up and use hardcoded list
al@865	474 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
al@865	475 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
al@865	476 fi
al@865	477 rm $temp_conf
al@865	478 fi
al@865	479 echo $LOCALE_PACK
al@865	480 }
al@865	481
al@865	482
al@865	483 # Fix common errors and warnings in the .desktop files
al@865	484 # Fixing can be disabled with COOKOPTS="!fixdesktops"
al@865	485
al@865	486 fix_desktop_files() {
al@865	487 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
al@876	488 deskpath="$install/usr/share/applications"
al@876	489 [ -d "$deskpath" ] \|\| return
al@876	490 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
al@865	491
al@865	492 size0=$(sizes des)
al@881	493 time0=$(get_time)
al@865	494
al@865	495 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
al@936	496 tpi desktop-file-validate-static
al@865	497 fi
al@865	498
al@865	499 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865	500 # Default value is "" (empty). That means for us that we'll use the full
al@865	501 # list of supported locales here.
al@865	502 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865	503
al@876	504 IFS=$'\n'
al@876	505 for desktop in $(find $deskpath -type f -name '*.desktop'); do
al@865	506 cp "$desktop" "$desktop.orig"
al@865	507
al@865	508 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
al@865	509 sdft "$desktop" -i
al@865	510
al@865	511 # Fix common errors in .desktop file
al@865	512 fix-desktop-file "$desktop"
al@865	513
al@865	514 # Strip unsupported locales from .desktop file
al@865	515 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
al@865	516 sdft "$desktop" -i -k "$LOCALE"
al@865	517
al@865	518 # Extra-strip
al@865	519 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
al@865	520 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
al@865	521
al@865	522 if [ -n "$QA" ]; then
al@865	523 # Check the rest of errors, warnings and tips
al@865	524 _ 'QA: Checking %s...' "$(basename $desktop)"
al@1002	525 busybox diff "$desktop.orig" "$desktop" \| sed 's!^!\|!'
al@1036	526 if which desktop-file-validate >/dev/null; then
al@1024	527 desktop-file-validate "$desktop" \| busybox fold -s
al@1024	528 else
al@1024	529 echo 'Warning: desktop-file-validate not found.'
al@1024	530 fi
al@865	531 echo
al@865	532 fi
al@865	533
al@865	534 rm "$desktop.orig"
al@865	535 done
al@865	536
al@998	537 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
al@865	538 }
al@865	539
al@865	540
paul@879	541 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
al@865	542 # to the translation (msgid = msgstr)
al@865	543 # Normalization can be disabled with COOKOPTS="!monorm"
al@865	544
al@865	545 normalize_mo() {
al@865	546 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
al@865	547 [ -z "$(find $install -type f -name '*.mo')" ] && return
al@865	548
al@867	549 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
al@912	550 tpi gettext
al@867	551 # Gconv modules (convert to UTF-8)
al@912	552 tpi glibc-locale
al@867	553
al@865	554 action 'Normalizing mo files...'
al@867	555
al@1024	556 the_log="$(mktemp)"
al@1024	557 to_continue=true
al@1024	558 for i in msgunfmt msguniq msgconv msgfmt; do
al@1024	559 if ! which $i >/dev/null; then
al@1024	560 echo "Warning: $i not found. Normalizing aborted" > "$the_log"
al@1024	561 to_continue=false
al@1024	562 fi
al@1024	563 done
al@1024	564
al@865	565 size0=$(sizes mo1)
al@881	566 time0=$(get_time)
al@865	567
al@865	568 # Process all existing *.mo files
al@876	569 IFS=$'\n'
al@1024	570 $to_continue &&
al@865	571 for mo in $(find "$install" -type f -name '*.mo'); do
al@865	572 tmpfile="$(mktemp)"
al@865	573
al@998	574 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
al@998	575 out="$({ msgunfmt "$mo" \| msguniq \| msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
al@998	576 if [ -n "$out" ]; then
al@998	577 # using literal $'\n' here instead of using `echo -e "...\n..."` because
al@998	578 # $out may contain escapes ('\r', '\v') that we should print as-is
al@1024	579 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log"
al@998	580 continue # proceed to next file
al@998	581 fi
al@998	582
al@865	583 # add newline
al@865	584 echo >> "$tmpfile"
al@865	585
al@865	586 # get Plural-Forms
al@865	587 awk '
al@865	588 BEGIN { skip = ""; }
al@865	589 {
al@865	590 if (! skip) {
al@865	591 s = $0;
al@865	592 gsub(/^[^\"]*\"/, "", s);
al@865	593 gsub(/\"$/, "", s);
al@865	594 printf("%s", s);
al@865	595 }
al@865	596 if (! $0) skip = "yes";
al@865	597 }
al@865	598 ' "$tmpfile" \| sed 's\|\\n\|\n\|g' \| grep "^Plural-Forms:" > "$tmpfile.pf"
al@865	599
al@865	600 if ! grep -q 'msgid_plural' "$tmpfile"; then
al@865	601 echo > "$tmpfile.pf"
al@865	602 fi
al@865	603
al@865	604 # main
al@865	605 awk -v pf="$(cat "$tmpfile.pf")" '
al@865	606 function clean() {
al@865	607 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
al@865	608 }
al@865	609
al@865	610 function getstring() {
al@865	611 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
al@865	612 s = $0;
al@865	613 gsub(/^[^\"]*\"/, "", s);
al@865	614 gsub(/\"$/, "", s);
al@865	615 return s;
al@865	616 }
al@865	617
al@865	618 BEGIN {
al@865	619 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
al@865	620 if (pf)
al@865	621 printf("\"%s\\n\"\n", pf);
al@865	622 printf("\n");
al@865	623 skip = 1;
al@865	624 clean();
al@865	625 }
al@865	626
al@865	627 {
al@865	628 # Skip the entire header
al@865	629 if (!skip) {
al@865	630 if ($1 == "msgctxt" \|\| $1 == "msgid" \|\| $1 == "msgstr" \|\| $1 == "msgid_plural")
al@865	631 mode = $1;
al@865	632 if ($1 == "msgstr[0]") mode = "msgstr0";
al@865	633 if ($1 == "msgstr[1]") mode = "msgstr1";
al@865	634 if ($1 == "msgstr[2]") mode = "msgstr2";
al@865	635 if ($1 == "msgstr[3]") mode = "msgstr3";
al@865	636 if ($1 == "msgstr[4]") mode = "msgstr4";
al@865	637 if ($1 == "msgstr[5]") mode = "msgstr5";
al@865	638
al@865	639 if (mode == "msgctxt") msgctxt = msgctxt getstring();
al@865	640 if (mode == "msgid") msgid = msgid getstring();
al@865	641 if (mode == "msgstr") msgstr = msgstr getstring();
al@865	642 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
al@865	643 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
al@865	644 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
al@865	645 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
al@865	646 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
al@865	647 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
al@865	648 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
al@865	649
al@865	650 if (! $0) {
al@865	651 if (msgid != msgstr) {
al@865	652 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
al@865	653 printf("msgid \"%s\"\n", msgid);
al@865	654 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
al@865	655 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
al@865	656 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
al@865	657 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
al@865	658 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
al@865	659 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
al@865	660 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
al@865	661 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
al@865	662 printf("\n");
al@865	663 }
al@865	664 clean();
al@865	665 }
al@865	666 }
al@865	667 if ($0 == "") skip = "";
al@865	668 }
al@865	669 ' "$tmpfile" > "$tmpfile.awk"
al@865	670
al@998	671 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
al@998	672 if [ -n "$out" ]; then
al@1024	673 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log"
al@998	674 continue # proceed to next file
al@998	675 fi
al@865	676
al@865	677 if [ -s "$tmpfile.mo" ]; then
al@865	678 rm "$mo"; mv "$tmpfile.mo" "$mo"
al@865	679 else
al@1024	680 _ 'Error processing %s' "$mo" >> "$the_log"
al@1024	681 echo >> "$the_log"
al@865	682 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
al@865	683 fi
al@865	684
al@865	685 # Clean
al@865	686 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
al@865	687 done
al@865	688
al@1024	689 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log"
al@865	690 }
al@865	691
al@865	692
al@865	693 # Strip locale definitions: normalize whitespace and remove comments
al@865	694 # Stripping can be disabled with COOKOPTS="!locdef"
al@865	695
al@865	696 strip_locale_def() {
al@865	697 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
al@865	698 [ ! -d "$install/usr/share/i18n/locales" ] && return
al@865	699 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
al@865	700
al@865	701 action 'Stripping locale definitions...'
al@865	702 size0=$(sizes loc)
al@881	703 time0=$(get_time)
al@865	704
al@865	705 for i in $(find $install/usr/share/i18n/locales -type f); do
al@865	706 sed -i 's\| \| \|g; s\| *\| \|g; s\|^ \|\|; /^%/d' $i
al@865	707 done
al@865	708
al@865	709 comp_summary "$time0" "$size0" "$(sizes loc)"
al@865	710 }
al@865	711
al@865	712
al@865	713 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
al@865	714 # as removing unneeded files like in Python packages. Cross compiled binaries
al@865	715 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
al@865	716 # Stripping can be disabled with COOKOPTS="!strip"
al@865	717
al@865	718 strip_package() {
al@865	719 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
al@865	720
al@1020	721 local i ifs="$IFS"
al@1020	722 IFS=$'\n'
al@1020	723
al@865	724 case "$ARCH" in
al@865	725 arm*\|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
al@865	726 *) export STRIP='strip' ;;
al@865	727 esac
al@865	728 action 'Executing strip on all files...'
al@865	729 size0=0
al@865	730 size1=0
al@881	731 time0=$(get_time)
al@1020	732 oldsize=$(sizes strip)
al@865	733
al@1027	734
al@1027	735 # GNU strip (GNU Binutils)
al@1027	736 # -p --preserve-dates Copy modified/access timestamps to the output
paul@1030	737 # -s --strip-all Remove all symbols and relocation information
al@1027	738 # --strip-unneeded Remove all symbols not needed by relocations
al@1027	739 # -D --enable-deterministic-archives Produce deterministic output when stripping archives
al@1027	740 # -g -S -d --strip-debug Remove all debugging symbols & sections
al@1027	741
al@1027	742 # Strip executable files
al@1020	743 while read i; do
al@1027	744 $STRIP -ps "$i" 2>/dev/null
al@1020	745 done <<EOT
al@1027	746 $(find_elf EXEC)
al@1027	747 EOT
al@1027	748
al@1027	749 # Strip shared libraries
al@1027	750 while read i; do
al@1027	751 case $i in
al@1027	752 .dbg) ;; # skip library.so..dbg debugging symbols
al@1027	753 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;;
al@1027	754 esac
al@1027	755 done <<EOT
al@1027	756 $(find_elf DYN)
al@1020	757 EOT
al@865	758
al@1020	759 # Strip static libraries
al@1027	760 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries
al@1027	761 find $fs -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \;
al@955	762
al@899	763
al@899	764 # Remove Python .pyc and .pyo
al@865	765 find $fs -type f $ -name '.pyc' -o -name '.pyo' $ -delete 2>/dev/null
al@865	766
al@899	767 # Remove both with the empty subfolders:
al@899	768 # 1. Perl perllocal.pod and .packlist (unconditionally)
al@899	769 local perlfiles="$(find $fs -type f $ -name 'perllocal.pod' -o -name '.packlist' $)"
al@899	770 # 2. Perl *.pod (if not disabled)
al@899	771 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
al@1078	772 perlfiles="$perlfiles"$'\n'"$(find $fs -type f -name '*.pod')"
al@1078	773 echo "$perlfiles" \| sort -u \| xargs rm -f 2>/dev/null
al@1078	774 echo "$perlfiles" \| sort -u \| awk 'BEGIN{FS=OFS="/"}{$NF="";print}' \
al@1078	775 \| xargs rmdir -p --ignore-fail-on-non-empty 2>/dev/null
al@899	776
al@1020	777 # Strip documentation inside Perl files (.pm and .pl) (if not disabled)
al@899	778 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
al@899	779 find $fs -type f $ -name '.pm' -o -name '.pl' $ -exec sed -i '/^=/,/^=cut/d' '{}' \;
al@899	780
al@1020	781 newsize=$(sizes strip)
al@865	782
al@1020	783 comp_summary "$time0" "$oldsize" "$newsize"
al@1020	784 IFS="$ifs"
al@865	785 }
al@865	786
al@865	787
al@865	788 # Strip unsupported locales (.mo files)
al@865	789
al@865	790 strip_mo_i18n() {
al@865	791 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
al@865	792
al@865	793 [ ! -d "$fs/usr/share/locale" ] && return
al@865	794 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
al@865	795
al@865	796 action 'Thin out translation files...'
al@865	797 size0=$(sizes mo2)
al@881	798 time0=$(get_time)
al@865	799
al@865	800 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865	801 # Default value is "" (empty). That means for us that we'll use the full
al@865	802 # list of supported locales here.
al@865	803 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865	804
al@865	805 # Existing locales
al@865	806 elocales=" $(ls -1 "$fs/usr/share/locale" \| tr '\n' ' ') "
al@865	807
al@865	808 # Thin out the list of existing locales. At the end there will be only locales that need
al@865	809 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
al@865	810 for keep_locale in $LOCALE; do
al@865	811 elocales=${elocales//$keep_locale}
al@865	812 done
al@865	813
al@865	814 # Remove the unsupported locales
al@865	815 for rem_locale in $elocales; do
al@998	816 [ ! -d "$fs/usr/share/locale/$rem_locale" ] \|\|
al@865	817 rm -r "$fs/usr/share/locale/$rem_locale"
al@865	818 done
al@865	819
al@865	820 comp_summary "$time0" "$size0" "$(sizes mo2)"
al@865	821 }
al@865	822
al@865	823
al@865	824
al@865	825
al@865	826 case $1 in
al@865	827 install)
al@865	828 # Compressors working in the $install
al@865	829 case "$ARCH" in
al@865	830 arm*) ;;
al@865	831 *)
al@872	832 recompress_gz
al@954	833 recompress_zip
al@865	834 compress_manpages
al@865	835 compress_png
al@865	836 compress_svg
al@1064	837 compress_gif
al@865	838 compress_ui
al@865	839 ;;
al@865	840 esac
al@865	841
al@865	842 fix_desktop_files
al@865	843 normalize_mo
al@865	844 strip_locale_def
al@865	845 ;;
al@865	846 fs)
al@865	847 # Compressors working in the $fs
al@865	848 strip_package
al@865	849 strip_mo_i18n
al@865	850 ;;
al@865	851 esac
al@868	852
al@868	853 # Clean
al@868	854 rm "$cache_stat"
al@869	855 find $comp_cache_root -type f -links -2 -delete

SliTaz Repositories

cookutils annotate modules/compressor @ rev 1078