cookutils annotate modules/compressor @ rev 1146

cook: allow built in aufs
author Pascal Bellard <pascal.bellard@slitaz.org>
date Tue Jul 28 07:28:04 2020 +0000 (2020-07-28)
parents 3c6c68093522
children e58a083722aa
rev   line source
al@865 1 #!/bin/sh
al@865 2 #
al@865 3 # compressor - module of the SliTaz Cook
al@865 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@865 5 #
al@865 6
al@865 7 . /usr/lib/slitaz/libcook.sh
al@865 8
al@868 9 # Compressor cache stuff
al@868 10
al@873 11 comp_cache_root='/home/slitaz/cache/cook'
al@869 12 mkdir -p "$comp_cache_root"
al@868 13 cache_stat=$(mktemp)
al@868 14
al@868 15 # Cache notes.
al@868 16 # Do not do the same job twice. Getting the file from the cache is much faster
al@868 17 # than compressing the file one more time. In addition, this cache is trying not
paul@1006 18 # to take extra space using the hardlinks. Although the files from the cache
paul@870 19 # without reference to itself should be removed periodically.
al@868 20
al@868 21
al@868 22
al@868 23
al@865 24 #
al@865 25 # Functions
al@865 26 #
al@865 27
al@865 28
al@912 29 # tazpkg install command
al@912 30
al@912 31 tpi() {
al@912 32 tazpkg -gi --quiet --local --cookmode $1
al@912 33 }
al@912 34
al@912 35
al@881 36 # Working with time (with hundredths precision)
al@865 37
al@881 38 get_time() {
al@881 39 cut -d" " -f2 /proc/uptime
al@881 40 }
al@865 41
al@881 42 calc_time() {
al@881 43 # L10n: 's' is for seconds, 'm' is for minutes
al@881 44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
al@881 45 BEGIN{
al@881 46 time = b - a;
al@881 47 if (time < 30)
al@881 48 printf("%.2f%s\n", time, s);
al@881 49 else
al@881 50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
al@881 51 }'
al@865 52 }
al@865 53
al@865 54
al@865 55 # Compressor mini summary
al@865 56
al@865 57 comp_summary() {
al@998 58 # "$time0" "$size0" "$size1" "$log_file"
al@865 59 status
al@865 60 [ "$2" -eq 0 ] && return
al@881 61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
al@868 62 cache_msg=''
al@868 63 if [ -s "$cache_stat" ]; then
al@868 64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
al@868 65 echo -n > $cache_stat
al@868 66 fi
al@868 67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
al@881 68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
al@998 69
al@998 70 if [ -s "$4" ]; then
al@1024 71 _ 'Compressor warnings and errors:'
al@998 72 awk '{printf " %s\n", $0;}' "$4"
al@998 73 echo
al@998 74 fi
al@998 75 # Clean log
al@998 76 [ ! -f "$4" ] || rm "$4"
al@865 77 }
al@865 78
al@865 79
al@1020 80 # Find ELF files
al@1020 81
al@1020 82 find_elf() {
al@1020 83 local i ifs="$IFS"
al@1020 84 IFS=$'\n'
al@1135 85 find $install -type f \
al@1020 86 | while read i; do
al@1020 87 # output of `readelf -h <file> is human-readable information,
paul@1021 88 # we are interested in the next line:
al@1020 89 # Type: EXEC (Executable file)
al@1020 90 # or
al@1020 91 # Type: DYN (Shared object file)
al@1020 92 if [ "$(readelf -h "$i" 2>/dev/null \
al@1020 93 | sed -n '/Type:/ s|.*: *\([^ ]*\) .*|\1|p')" == "$1" ]; then
al@1020 94 echo "$i" # $1 = { EXEC | DYN }
al@1020 95 fi
al@1020 96 done
al@1020 97 IFS="$ifs"
al@1020 98 }
al@1020 99
al@1020 100
al@865 101 # Calculating different sizes
al@865 102
al@865 103 sizes() {
al@1020 104 local ifs="$IFS"; IFS=$'\n'
al@865 105 case $1 in
al@865 106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
al@865 107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
al@865 108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
al@1064 109 gif) find $install -type f -name '*.gif' -exec ls -l \{\} \; ;;
al@865 110 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
al@865 111 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
al@865 112 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@865 113 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
al@865 114 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@872 115 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
al@954 116 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
al@1102 117 css) find $install -type f -name '*.css' -exec ls -l \{\} \; ;;
al@1020 118 strip)
al@1020 119 {
al@1020 120 find_elf EXEC
al@1020 121 find_elf DYN
al@1135 122 find $install -type f \( -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
al@1020 123 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \)
al@1020 124 } \
al@1034 125 | tr '\n' '\0' \
al@1034 126 | xargs -0 ls -l
al@1020 127 ;;
al@865 128 esac | awk '{s+=$5}END{print s}'
al@1020 129 IFS="$ifs"
al@865 130 }
al@865 131
al@865 132
al@868 133 # Query cache for already existing compressed file; substitute original file with the cached
al@868 134 # compressed one if so.
al@868 135 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
al@868 136
al@868 137 query_cache() {
al@868 138 md5=$(md5sum "$2")
al@868 139 cachefile="$comp_cache_root/$1/${md5%% *}"
al@868 140 echo "$cachefile"
al@1100 141 owner=$(stat -c%u:%g "$2")
al@1100 142 perms=$(stat -c%a "$2")
al@868 143 if [ -f "$cachefile" ]; then
al@868 144 ln -f "$cachefile" "$2"
al@1100 145 chown $owner "$2"
al@1100 146 chmod $perms "$2"
al@868 147 echo '+' >> "$cache_stat"
al@868 148 else
al@868 149 echo '-' >> "$cache_stat"
al@868 150 false
al@868 151 fi
al@868 152 }
al@868 153
al@868 154
al@868 155 # Store compressed file to the cache
al@868 156 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
al@868 157
al@868 158 store_cache() {
al@868 159 mkdir -p "${1%/*}"
al@868 160 mv "$2" "$1"
al@868 161 ln "$1" "$2"
al@868 162 }
al@868 163
al@868 164
al@865 165 # Function to compress all man pages
al@865 166 # Compressing can be disabled with COOKOPTS="!manz"
al@865 167
al@865 168 compress_manpages() {
al@881 169 time0=$(get_time)
al@865 170 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
al@874 171 manpath="$install/usr/share/man"
al@865 172 [ -d "$manpath" ] || return
al@865 173 size0=$(sizes man); [ -z "$size0" ] && return
al@865 174
al@936 175 tpi advancecomp-static
al@867 176
al@865 177 action 'Compressing man pages...'
al@865 178
al@865 179 # We'll use only Gzip compression, so decompress other formats first
al@865 180 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
al@865 181 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
al@865 182
al@865 183 # Fast compress with gzip
al@874 184 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
al@865 185
al@865 186 # Fix symlinks
al@874 187 for i in $(find $manpath -type l); do
al@865 188 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
al@865 189 link=$(echo $i | sed 's|\.[gbx]z2*$||')
al@865 190 rm $i; ln -s $dest.gz $link.gz
al@865 191 done
al@865 192
al@865 193 # Recompress with advdef (it can't compress, only recompress)
al@1024 194 the_log="$(mktemp)"
al@1024 195 if which advdef >/dev/null; then
al@1024 196 IFS=$'\n'
al@1024 197 for i in $(find $manpath -type f); do
al@1024 198 if ! cached_path=$(query_cache mangz "$i"); then
al@1024 199 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024 200 out="$(advdef -z4q "$i")"
al@1024 201 if [ -n "$out" ]; then
al@1024 202 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024 203 mv -f "$i.orig$$" "$i" # restore the original
al@1024 204 else
al@1024 205 store_cache "$cached_path" "$i"
al@1024 206 rm -f "$i.orig$$" # clean
al@1024 207 fi
al@998 208 fi
al@1024 209 done
al@1024 210 else
al@1024 211 echo 'Warning: advdef not found.' > "$the_log"
al@1024 212 fi
al@865 213
al@1024 214 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log"
al@865 215 }
al@865 216
al@865 217
al@872 218 # Function to recompress all gzip archives
al@872 219 # Recompressing can be disabled with COOKOPTS="!gz"
al@872 220
al@872 221 recompress_gz() {
al@881 222 time0=$(get_time)
al@872 223 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
al@872 224 size0=$(sizes gz); [ -z "$size0" ] && return
al@872 225
al@936 226 tpi advancecomp-static
al@872 227
al@872 228 action 'Recompressing gzip files...'
al@872 229
al@872 230 # Recompress with advdef
al@1024 231 the_log="$(mktemp)"
al@1024 232 if which advdef >/dev/null; then
al@1024 233 IFS=$'\n'
al@1024 234 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
al@1024 235 if ! cached_path=$(query_cache gz "$i"); then
al@1024 236 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024 237 out="$(advdef -z4q "$i")"
al@1024 238 if [ -n "$out" ]; then
al@1024 239 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024 240 mv -f "$i.orig$$" "$i" # restore the original
al@1024 241 else
al@1024 242 store_cache "$cached_path" "$i"
al@1024 243 rm -f "$i.orig$$" # clean
al@1024 244 fi
al@998 245 fi
al@1024 246 done
al@1024 247 else
al@1024 248 echo 'Warning: advdef not found.' > "$the_log"
al@1024 249 fi
al@872 250
al@1024 251 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log"
al@872 252 }
al@872 253
al@872 254
al@954 255 # Function to recompress all zip archives
al@954 256 # Recompressing can be disabled with COOKOPTS="!zip"
al@954 257
al@954 258 recompress_zip() {
al@954 259 time0=$(get_time)
al@954 260 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
al@954 261 size0=$(sizes zip); [ -z "$size0" ] && return
al@954 262
al@954 263 tpi advancecomp-static
al@954 264
al@954 265 action 'Recompressing zip files...'
al@954 266
al@954 267 # Recompress with advzip
al@1024 268 the_log="$(mktemp)"
al@1024 269 if which advzip >/dev/null; then
al@1024 270 IFS=$'\n'
al@1024 271 for i in $(find $install -type f -name '*.zip'); do
al@1024 272 if ! cached_path=$(query_cache zip "$i"); then
al@1024 273 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@1024 274 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
al@1024 275 if [ -n "$out" ]; then
al@1024 276 echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024 277 mv -f "$i.orig$$" "$i" # restore the original
al@1024 278 else
al@1024 279 store_cache "$cached_path" "$i"
al@1024 280 rm -f "$i.orig$$" # clean
al@1024 281 fi
al@998 282 fi
al@1024 283 done
al@1024 284 else
al@1024 285 echo 'Warning: advzip not found.' > "$the_log"
al@1024 286 fi
al@954 287
al@1024 288 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log"
al@954 289 }
al@954 290
al@954 291
al@865 292 # Function used after compile_rules() to compress all png images
al@865 293 # Compressing can be disabled with COOKOPTS="!pngz"
al@865 294
al@865 295 compress_png() {
al@881 296 time0=$(get_time)
al@865 297 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
al@865 298 size0=$(sizes png); [ -z "$size0" ] && return
al@865 299
al@865 300 use_pq=true
al@865 301 use_op=true
al@865 302 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
al@865 303 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
al@932 304 $use_pq && tpi pngquant-static
al@932 305 $use_op && tpi optipng-static
al@865 306
al@867 307 action 'Compressing png images...'
al@865 308
al@1024 309 the_log="$(mktemp)"
al@1024 310 $use_pq && if ! which pngquant >/dev/null; then
al@1024 311 echo 'Warning: pngquant not found.' > "$the_log"
al@1024 312 use_pq=false
al@1024 313 fi
al@1024 314 $use_op && if ! which optipng >/dev/null; then
al@1024 315 echo 'Warning: optipng not found.' >> "$the_log"
al@1024 316 use_op=false
al@1024 317 fi
al@1024 318
al@865 319 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
al@868 320 [ -z "$oplevel" ] && oplevel='2'
al@868 321
al@868 322 cache_section="png$oplevel"
al@868 323 $use_pq && cache_section="${cache_section}p"
al@868 324 $use_op && cache_section="${cache_section}o"
al@868 325
al@865 326 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
al@865 327
al@998 328 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
al@978 329 IFS=$'\n'
al@978 330 for i in $(find $install -type f -name '*.png'); do
al@998 331 unset IFS iserror
al@868 332 if ! cached_path=$(query_cache $cache_section "$i"); then
al@998 333 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998 334 if $use_pq; then
al@998 335 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
al@998 336 if [ -n "$out" ]; then
al@1024 337 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log"
al@998 338 iserror='yes'
al@1045 339 [ -e "$i.tmp" ] && rm "$i.tmp" # zero-size file remains on pngquant fail
al@998 340 fi
al@998 341 fi
al@998 342 if $use_op && [ -z "$iserror" ]; then
al@998 343 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
al@998 344 if [ -n "$out" ]; then
al@1024 345 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log"
al@998 346 iserror='yes'
al@998 347 fi
al@998 348 fi
al@998 349 if [ -n "$iserror" ]; then
al@998 350 mv -f "$i.orig$$" "$i" # restore the original
al@998 351 else
al@998 352 store_cache "$cached_path" "$i"
al@998 353 rm -f "$i.orig$$" # clean
al@998 354 fi
al@868 355 fi
al@865 356 done
al@865 357
al@1024 358 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log"
al@865 359 }
al@865 360
al@865 361
al@865 362 # Function used after compile_rules() to compress all svg images
al@865 363 # Compressing can be disabled with COOKOPTS="!svgz"
al@865 364
al@865 365 compress_svg() {
al@881 366 time0=$(get_time)
al@865 367 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
al@865 368 size0=$(sizes svg); [ -z "$size0" ] && return
al@865 369
al@912 370 tpi svgcleaner
al@867 371
al@865 372 action 'Compressing svg images...'
al@865 373
al@1072 374 the_log="$(mktemp)"
al@1024 375 if which svgcleaner >/dev/null; then
al@1024 376 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
al@1024 377 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
al@959 378
al@1024 379 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
al@1024 380 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
al@1024 381 --multipass --remove-unresolved-classes no $opts 2>&1)"
al@1024 382 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$the_log"
al@1024 383 done
al@1024 384 else
al@1024 385 echo 'Warning: svgcleaner not found.' > "$the_log"
al@1024 386 fi
al@865 387
al@1024 388 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log"
al@865 389 }
al@865 390
al@865 391
al@1064 392 # Function used after compile_rules() to compress all gif images
al@1064 393 # Compressing can be disabled with COOKOPTS="!gifz"
al@1064 394
al@1064 395 compress_gif() {
al@1064 396 time0=$(get_time)
al@1064 397 [ "${COOKOPTS/!gifz/}" != "$COOKOPTS" ] && return
al@1064 398 size0=$(sizes gif); [ -z "$size0" ] && return
al@1064 399
al@1064 400 tpi gifsicle
al@1064 401
al@1064 402 action 'Compressing gif images...'
al@1064 403
al@1072 404 the_log="$(mktemp)"
al@1064 405 if which gifsicle >/dev/null; then
al@1067 406 IFS=$'\n'
al@1067 407 for i in $(find $install -type f -name '*.gif'); do
al@1072 408 if ! cached_path=$(query_cache gif "$i"); then
al@1072 409 unset IFS
al@1072 410 # use intermediate file, if all well ($?=0), then substitute the original
al@1072 411 if gifsicle -O3 "$i" -o "$i.$$" >> "$the_log" 2>&1; then
al@1072 412 if [ -s "$i.$$" ]; then
al@1072 413 mv "$i.$$" "$i"
al@1072 414 store_cache "$cached_path" "$i"
al@1072 415 fi
al@1072 416 else
al@1072 417 rm "$i.$$"
al@1072 418 fi
al@1064 419 fi
al@1064 420 done
al@1064 421 else
al@1064 422 echo 'Warning: gifsicle not found.' > "$the_log"
al@1064 423 fi
al@1064 424
al@1064 425 comp_summary "$time0" "$size0" "$(sizes gif)" "$the_log"
al@1064 426 }
al@1064 427
al@1064 428
al@865 429 # Function used after compile_rules() to shrink all *.ui and *.glade files:
al@865 430 # remove insignificant spaces and comments
al@865 431 # Compressing can be disabled with COOKOPTS="!uiz"
al@865 432
al@865 433 compress_ui() {
al@865 434 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
al@865 435 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
al@865 436
al@912 437 tpi xmlstarlet
al@867 438
al@865 439 action 'Compressing ui files...'
al@867 440
al@1072 441 the_log="$(mktemp)"
al@1024 442 if which xmlstarlet >/dev/null; then
al@1024 443 size0=$(sizes xml)
al@1024 444 time0=$(get_time)
al@1024 445 temp_ui="$(mktemp)"
al@1024 446 IFS=$'\n'
al@1024 447 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
al@1024 448 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")"
al@1024 449 if [ -n "$out" ]; then
al@1024 450 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log"
al@1024 451 else
al@1024 452 cat "$temp_ui" > "$ui"
al@1024 453 fi
al@1024 454 done
al@1024 455 else
al@1024 456 echo 'Warning: xmlstarlet not found.' > "$the_log"
al@1024 457 fi
al@865 458
al@1024 459 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log"
al@865 460 rm "$temp_ui"
al@865 461 }
al@865 462
al@865 463
al@1102 464 # Function used after compile_rules() to shrink all *.css files:
al@1102 465 # remove insignificant spaces and comments
al@1102 466 # Compressing can be disabled with COOKOPTS="!cssz"
al@1102 467
al@1102 468 compress_css() {
al@1102 469 [ "${COOKOPTS/!cssz/}" != "$COOKOPTS" ] && return
al@1102 470 size0=$(sizes css); [ -z "$size0" ] && return
al@1102 471
al@1102 472 tpi rcssmin
al@1102 473
al@1102 474 action 'Compressing CSS files...'
al@1102 475
al@1102 476 time0=$(get_time)
al@1102 477 temp_css="$(mktemp)"
al@1102 478 IFS=$'\n'
al@1102 479 for css in $(find $install -type f -name '*.css'); do
al@1102 480 python -mrcssmin < "$css" > "$temp_css"
al@1102 481 cat "$temp_css" > "$css"
al@1102 482 done
al@1102 483 unset IFS
al@1102 484
al@1102 485 comp_summary "$time0" "$size0" "$(sizes css)" '/dev/null'
al@1102 486 rm "$temp_css"
al@1102 487 }
al@1102 488
al@1102 489
al@865 490 # Get list of supported locales...
al@865 491
al@865 492 get_supported_locales() {
al@865 493 lpc='/slitaz-i18n/stuff/locale-pack.conf'
al@865 494 if [ -e "$WOK$lpc" ]; then
al@865 495 # ... from package in the local wok
al@865 496 . "$WOK$lpc"
al@865 497 else
al@865 498 # ... from Hg
al@865 499 temp_conf=$(mktemp)
al@865 500 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
al@865 501 if [ -s $temp_conf ]; then
al@865 502 . $temp_conf
al@865 503 else
al@865 504 # Give up and use hardcoded list
al@865 505 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
al@865 506 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
al@865 507 fi
al@865 508 rm $temp_conf
al@865 509 fi
al@865 510 echo $LOCALE_PACK
al@865 511 }
al@865 512
al@865 513
al@865 514 # Fix common errors and warnings in the .desktop files
al@865 515 # Fixing can be disabled with COOKOPTS="!fixdesktops"
al@865 516
al@865 517 fix_desktop_files() {
al@865 518 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
al@876 519 deskpath="$install/usr/share/applications"
al@876 520 [ -d "$deskpath" ] || return
al@876 521 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
al@865 522
al@865 523 size0=$(sizes des)
al@881 524 time0=$(get_time)
al@865 525
al@865 526 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
al@936 527 tpi desktop-file-validate-static
al@865 528 fi
al@865 529
al@865 530 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865 531 # Default value is "" (empty). That means for us that we'll use the full
al@865 532 # list of supported locales here.
al@865 533 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865 534
al@876 535 IFS=$'\n'
al@876 536 for desktop in $(find $deskpath -type f -name '*.desktop'); do
al@865 537 cp "$desktop" "$desktop.orig"
al@865 538
al@865 539 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
al@865 540 sdft "$desktop" -i
al@865 541
al@865 542 # Fix common errors in .desktop file
al@865 543 fix-desktop-file "$desktop"
al@865 544
al@865 545 # Strip unsupported locales from .desktop file
al@865 546 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
al@865 547 sdft "$desktop" -i -k "$LOCALE"
al@865 548
al@865 549 # Extra-strip
al@865 550 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
al@865 551 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
al@865 552
al@865 553 if [ -n "$QA" ]; then
al@865 554 # Check the rest of errors, warnings and tips
al@865 555 _ 'QA: Checking %s...' "$(basename $desktop)"
al@1002 556 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!'
al@1036 557 if which desktop-file-validate >/dev/null; then
al@1024 558 desktop-file-validate "$desktop" | busybox fold -s
al@1024 559 else
al@1024 560 echo 'Warning: desktop-file-validate not found.'
al@1024 561 fi
al@865 562 echo
al@865 563 fi
al@865 564
al@865 565 rm "$desktop.orig"
al@865 566 done
al@865 567
al@998 568 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
al@865 569 }
al@865 570
al@865 571
paul@879 572 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
al@865 573 # to the translation (msgid = msgstr)
al@865 574 # Normalization can be disabled with COOKOPTS="!monorm"
al@865 575
al@865 576 normalize_mo() {
al@865 577 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
al@865 578 [ -z "$(find $install -type f -name '*.mo')" ] && return
al@865 579
al@867 580 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
al@1103 581 tpi gettext-dev
al@867 582 # Gconv modules (convert to UTF-8)
al@912 583 tpi glibc-locale
al@867 584
al@865 585 action 'Normalizing mo files...'
al@867 586
al@1024 587 the_log="$(mktemp)"
al@1024 588 to_continue=true
al@1024 589 for i in msgunfmt msguniq msgconv msgfmt; do
al@1024 590 if ! which $i >/dev/null; then
al@1024 591 echo "Warning: $i not found. Normalizing aborted" > "$the_log"
al@1024 592 to_continue=false
al@1024 593 fi
al@1024 594 done
al@1024 595
al@865 596 size0=$(sizes mo1)
al@881 597 time0=$(get_time)
al@865 598
al@865 599 # Process all existing *.mo files
al@876 600 IFS=$'\n'
al@1024 601 $to_continue &&
al@865 602 for mo in $(find "$install" -type f -name '*.mo'); do
al@865 603 tmpfile="$(mktemp)"
al@865 604
al@998 605 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
al@998 606 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
al@998 607 if [ -n "$out" ]; then
al@998 608 # using literal $'\n' here instead of using `echo -e "...\n..."` because
al@998 609 # $out may contain escapes ('\r', '\v') that we should print as-is
al@1024 610 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log"
al@998 611 continue # proceed to next file
al@998 612 fi
al@998 613
al@865 614 # add newline
al@865 615 echo >> "$tmpfile"
al@865 616
al@865 617 # get Plural-Forms
al@865 618 awk '
al@865 619 BEGIN { skip = ""; }
al@865 620 {
al@865 621 if (! skip) {
al@865 622 s = $0;
al@865 623 gsub(/^[^\"]*\"/, "", s);
al@865 624 gsub(/\"$/, "", s);
al@865 625 printf("%s", s);
al@865 626 }
al@865 627 if (! $0) skip = "yes";
al@865 628 }
al@865 629 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
al@865 630
al@865 631 if ! grep -q 'msgid_plural' "$tmpfile"; then
al@865 632 echo > "$tmpfile.pf"
al@865 633 fi
al@865 634
al@865 635 # main
al@865 636 awk -v pf="$(cat "$tmpfile.pf")" '
al@865 637 function clean() {
al@865 638 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
al@865 639 }
al@865 640
al@865 641 function getstring() {
al@865 642 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
al@865 643 s = $0;
al@865 644 gsub(/^[^\"]*\"/, "", s);
al@865 645 gsub(/\"$/, "", s);
al@865 646 return s;
al@865 647 }
al@865 648
al@865 649 BEGIN {
al@865 650 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
al@865 651 if (pf)
al@865 652 printf("\"%s\\n\"\n", pf);
al@865 653 printf("\n");
al@865 654 skip = 1;
al@865 655 clean();
al@865 656 }
al@865 657
al@865 658 {
al@865 659 # Skip the entire header
al@865 660 if (!skip) {
al@865 661 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
al@865 662 mode = $1;
al@865 663 if ($1 == "msgstr[0]") mode = "msgstr0";
al@865 664 if ($1 == "msgstr[1]") mode = "msgstr1";
al@865 665 if ($1 == "msgstr[2]") mode = "msgstr2";
al@865 666 if ($1 == "msgstr[3]") mode = "msgstr3";
al@865 667 if ($1 == "msgstr[4]") mode = "msgstr4";
al@865 668 if ($1 == "msgstr[5]") mode = "msgstr5";
al@865 669
al@865 670 if (mode == "msgctxt") msgctxt = msgctxt getstring();
al@865 671 if (mode == "msgid") msgid = msgid getstring();
al@865 672 if (mode == "msgstr") msgstr = msgstr getstring();
al@865 673 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
al@865 674 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
al@865 675 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
al@865 676 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
al@865 677 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
al@865 678 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
al@865 679 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
al@865 680
al@865 681 if (! $0) {
al@865 682 if (msgid != msgstr) {
al@865 683 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
al@865 684 printf("msgid \"%s\"\n", msgid);
al@865 685 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
al@865 686 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
al@865 687 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
al@865 688 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
al@865 689 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
al@865 690 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
al@865 691 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
al@865 692 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
al@865 693 printf("\n");
al@865 694 }
al@865 695 clean();
al@865 696 }
al@865 697 }
al@865 698 if ($0 == "") skip = "";
al@865 699 }
al@865 700 ' "$tmpfile" > "$tmpfile.awk"
al@865 701
al@1135 702 # 64 bytes is the length of prologue added by awk script (here ~ is $'\n'):
al@1135 703 # msgid ""~msgstr ""~"Content-Type: text/plain; charset=UTF-8\n"~~
paul@1138 704 # If size is 64 bit then files have no content, we'll remove it with
al@1135 705 # all empty uplevel directories.
al@1135 706 # For example, look at libbytesize package.
al@1135 707 if [ "$(stat -c%s "$tmpfile.awk")" -eq 64 ]; then
al@1135 708 _ 'Info: %s become empty' "$mo" >> "$the_log"
al@1135 709 rm "$mo" "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
al@1135 710 rmdir --parents --ignore-fail-on-non-empty "$(dirname "$mo")"
al@1135 711 continue # proceed to next file
al@1135 712 fi
al@1135 713
al@998 714 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
al@998 715 if [ -n "$out" ]; then
al@1024 716 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log"
al@1135 717 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
al@998 718 continue # proceed to next file
al@998 719 fi
al@865 720
al@865 721 if [ -s "$tmpfile.mo" ]; then
al@865 722 rm "$mo"; mv "$tmpfile.mo" "$mo"
al@865 723 else
al@1024 724 _ 'Error processing %s' "$mo" >> "$the_log"
al@1024 725 echo >> "$the_log"
al@865 726 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
al@865 727 fi
al@865 728
al@865 729 # Clean
al@865 730 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
al@865 731 done
al@865 732
al@1024 733 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log"
al@865 734 }
al@865 735
al@865 736
al@865 737 # Strip locale definitions: normalize whitespace and remove comments
al@865 738 # Stripping can be disabled with COOKOPTS="!locdef"
al@865 739
al@865 740 strip_locale_def() {
al@865 741 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
al@865 742 [ ! -d "$install/usr/share/i18n/locales" ] && return
al@865 743 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
al@865 744
al@865 745 action 'Stripping locale definitions...'
al@865 746 size0=$(sizes loc)
al@881 747 time0=$(get_time)
al@865 748
al@865 749 for i in $(find $install/usr/share/i18n/locales -type f); do
al@865 750 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
al@865 751 done
al@865 752
al@865 753 comp_summary "$time0" "$size0" "$(sizes loc)"
al@865 754 }
al@865 755
al@865 756
al@865 757 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
al@865 758 # as removing unneeded files like in Python packages. Cross compiled binaries
al@865 759 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
al@865 760 # Stripping can be disabled with COOKOPTS="!strip"
al@865 761
al@865 762 strip_package() {
al@865 763 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
al@865 764
al@1020 765 local i ifs="$IFS"
al@1020 766 IFS=$'\n'
al@1020 767
al@865 768 case "$ARCH" in
al@865 769 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
al@865 770 *) export STRIP='strip' ;;
al@865 771 esac
al@865 772 action 'Executing strip on all files...'
al@865 773 size0=0
al@865 774 size1=0
al@881 775 time0=$(get_time)
al@1020 776 oldsize=$(sizes strip)
al@865 777
al@1027 778
al@1027 779 # GNU strip (GNU Binutils)
al@1027 780 # -p --preserve-dates Copy modified/access timestamps to the output
paul@1030 781 # -s --strip-all Remove all symbols and relocation information
al@1027 782 # --strip-unneeded Remove all symbols not needed by relocations
al@1027 783 # -D --enable-deterministic-archives Produce deterministic output when stripping archives
al@1027 784 # -g -S -d --strip-debug Remove all debugging symbols & sections
al@1027 785
al@1027 786 # Strip executable files
al@1020 787 while read i; do
al@1027 788 $STRIP -ps "$i" 2>/dev/null
al@1020 789 done <<EOT
al@1027 790 $(find_elf EXEC)
al@1027 791 EOT
al@1027 792
al@1027 793 # Strip shared libraries
al@1027 794 while read i; do
al@1027 795 case $i in
al@1027 796 *.dbg) ;; # skip library.so.*.dbg debugging symbols
al@1027 797 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;;
al@1027 798 esac
al@1027 799 done <<EOT
al@1027 800 $(find_elf DYN)
al@1020 801 EOT
al@865 802
al@1020 803 # Strip static libraries
al@1027 804 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries
al@1135 805 find $install -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \;
al@955 806
al@899 807
al@899 808 # Remove Python *.pyc and *.pyo
al@1135 809 find $install -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
al@865 810
al@899 811 # Remove both with the empty subfolders:
al@899 812 # 1. Perl perllocal.pod and .packlist (unconditionally)
al@1135 813 local perlfiles="$(find $install -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
al@899 814 # 2. Perl *.pod (if not disabled)
al@899 815 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
al@1135 816 perlfiles="$perlfiles"$'\n'"$(find $install -type f -name '*.pod')"
al@1078 817 echo "$perlfiles" | sort -u | xargs rm -f 2>/dev/null
al@1078 818 echo "$perlfiles" | sort -u | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' \
al@1078 819 | xargs rmdir -p --ignore-fail-on-non-empty 2>/dev/null
al@899 820
al@1020 821 # Strip documentation inside Perl files (*.pm and *.pl) (if not disabled)
al@899 822 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
al@1135 823 find $install -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
al@899 824
al@1020 825 newsize=$(sizes strip)
al@865 826
al@1020 827 comp_summary "$time0" "$oldsize" "$newsize"
al@1020 828 IFS="$ifs"
al@865 829 }
al@865 830
al@865 831
al@865 832 # Strip unsupported locales (.mo files)
al@865 833
al@865 834 strip_mo_i18n() {
al@865 835 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
al@865 836
al@865 837 [ ! -d "$fs/usr/share/locale" ] && return
al@865 838 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
al@865 839
al@865 840 action 'Thin out translation files...'
al@865 841 size0=$(sizes mo2)
al@881 842 time0=$(get_time)
al@865 843
al@865 844 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865 845 # Default value is "" (empty). That means for us that we'll use the full
al@865 846 # list of supported locales here.
al@865 847 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865 848
al@865 849 # Existing locales
al@865 850 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
al@865 851
al@865 852 # Thin out the list of existing locales. At the end there will be only locales that need
al@865 853 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
al@865 854 for keep_locale in $LOCALE; do
al@865 855 elocales=${elocales//$keep_locale}
al@865 856 done
al@865 857
al@865 858 # Remove the unsupported locales
al@865 859 for rem_locale in $elocales; do
al@998 860 [ ! -d "$fs/usr/share/locale/$rem_locale" ] ||
al@865 861 rm -r "$fs/usr/share/locale/$rem_locale"
al@865 862 done
al@865 863
al@865 864 comp_summary "$time0" "$size0" "$(sizes mo2)"
al@865 865 }
al@865 866
al@865 867
al@865 868
al@865 869
al@865 870 case $1 in
al@865 871 install)
al@865 872 # Compressors working in the $install
al@1135 873 [ -d "$install" ] || return
al@865 874 case "$ARCH" in
al@865 875 arm*) ;;
al@865 876 *)
al@1135 877 strip_package
al@872 878 recompress_gz
al@954 879 recompress_zip
al@865 880 compress_manpages
al@865 881 compress_png
al@865 882 compress_svg
al@1064 883 compress_gif
al@865 884 compress_ui
al@1102 885 compress_css
al@865 886 ;;
al@865 887 esac
al@865 888
al@865 889 fix_desktop_files
al@865 890 normalize_mo
al@865 891 strip_locale_def
al@865 892 ;;
al@865 893 fs)
al@865 894 # Compressors working in the $fs
al@1135 895 [ -d "$fs" ] || return
al@865 896 strip_mo_i18n
al@865 897 ;;
al@865 898 esac
al@868 899
al@868 900 # Clean
al@868 901 rm "$cache_stat"
al@869 902 find $comp_cache_root -type f -links -2 -delete