cookutils annotate modules/compressor @ rev 1006

Tiny edits
author Paul Issott <paul@slitaz.org>
date Sat Nov 11 11:53:55 2017 +0000 (2017-11-11)
parents 8190be7ad294
children dc7238a11470
rev   line source
al@865 1 #!/bin/sh
al@865 2 #
al@865 3 # compressor - module of the SliTaz Cook
al@865 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@865 5 #
al@865 6
al@865 7 . /usr/lib/slitaz/libcook.sh
al@865 8
al@868 9 # Compressor cache stuff
al@868 10
al@873 11 comp_cache_root='/home/slitaz/cache/cook'
al@869 12 mkdir -p "$comp_cache_root"
al@868 13 cache_stat=$(mktemp)
al@868 14
al@868 15 # Cache notes.
al@868 16 # Do not do the same job twice. Getting the file from the cache is much faster
al@868 17 # than compressing the file one more time. In addition, this cache is trying not
paul@1006 18 # to take extra space using the hardlinks. Although the files from the cache
paul@870 19 # without reference to itself should be removed periodically.
al@868 20
al@868 21
al@868 22
al@868 23
al@865 24 #
al@865 25 # Functions
al@865 26 #
al@865 27
al@865 28
al@912 29 # tazpkg install command
al@912 30
al@912 31 tpi() {
al@912 32 tazpkg -gi --quiet --local --cookmode $1
al@912 33 }
al@912 34
al@912 35
al@881 36 # Working with time (with hundredths precision)
al@865 37
al@881 38 get_time() {
al@881 39 cut -d" " -f2 /proc/uptime
al@881 40 }
al@865 41
al@881 42 calc_time() {
al@881 43 # L10n: 's' is for seconds, 'm' is for minutes
al@881 44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" '
al@881 45 BEGIN{
al@881 46 time = b - a;
al@881 47 if (time < 30)
al@881 48 printf("%.2f%s\n", time, s);
al@881 49 else
al@881 50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m);
al@881 51 }'
al@865 52 }
al@865 53
al@865 54
al@865 55 # Compressor mini summary
al@865 56
al@865 57 comp_summary() {
al@998 58 # "$time0" "$size0" "$size1" "$log_file"
al@865 59 status
al@865 60 [ "$2" -eq 0 ] && return
al@881 61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }')
al@868 62 cache_msg=''
al@868 63 if [ -s "$cache_stat" ]; then
al@868 64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)")
al@868 65 echo -n > $cache_stat
al@868 66 fi
al@868 67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \
al@881 68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg"
al@998 69
al@998 70 if [ -s "$4" ]; then
al@998 71 _ 'Cleaner warnings and errors:'
al@998 72 awk '{printf " %s\n", $0;}' "$4"
al@998 73 echo
al@998 74 fi
al@998 75 # Clean log
al@998 76 [ ! -f "$4" ] || rm "$4"
al@865 77 }
al@865 78
al@865 79
al@865 80 # Calculating different sizes
al@865 81
al@865 82 sizes() {
al@865 83 case $1 in
al@865 84 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;;
al@865 85 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;;
al@865 86 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;;
al@865 87 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;;
al@865 88 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;;
al@865 89 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@865 90 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;;
al@865 91 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;;
al@872 92 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;;
al@954 93 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;;
al@899 94 str) find $fs -type f \( -name '*.so*' -o -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \
al@899 95 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \) -exec ls -l \{\} \; ;;
al@865 96 esac | awk '{s+=$5}END{print s}'
al@865 97 }
al@865 98
al@865 99
al@868 100 # Query cache for already existing compressed file; substitute original file with the cached
al@868 101 # compressed one if so.
al@868 102 # $1: cache section (gz, mangz, png, etc.); $2: path to original file
al@868 103
al@868 104 query_cache() {
al@868 105 md5=$(md5sum "$2")
al@868 106 cachefile="$comp_cache_root/$1/${md5%% *}"
al@868 107 echo "$cachefile"
al@868 108 if [ -f "$cachefile" ]; then
al@868 109 ln -f "$cachefile" "$2"
al@868 110 echo '+' >> "$cache_stat"
al@868 111 else
al@868 112 echo '-' >> "$cache_stat"
al@868 113 false
al@868 114 fi
al@868 115 }
al@868 116
al@868 117
al@868 118 # Store compressed file to the cache
al@868 119 # $1: path to cache entry to be stored; $2: path to compressed file to be stored
al@868 120
al@868 121 store_cache() {
al@868 122 mkdir -p "${1%/*}"
al@868 123 mv "$2" "$1"
al@868 124 ln "$1" "$2"
al@868 125 }
al@868 126
al@868 127
al@865 128 # Function to compress all man pages
al@865 129 # Compressing can be disabled with COOKOPTS="!manz"
al@865 130
al@865 131 compress_manpages() {
al@881 132 time0=$(get_time)
al@865 133 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return
al@874 134 manpath="$install/usr/share/man"
al@865 135 [ -d "$manpath" ] || return
al@865 136 size0=$(sizes man); [ -z "$size0" ] && return
al@865 137
al@936 138 tpi advancecomp-static
al@867 139
al@865 140 action 'Compressing man pages...'
al@865 141
al@865 142 # We'll use only Gzip compression, so decompress other formats first
al@865 143 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \;
al@865 144 find $manpath -type f -name '*.xz' -exec unxz \{\} \;
al@865 145
al@865 146 # Fast compress with gzip
al@874 147 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \;
al@865 148
al@865 149 # Fix symlinks
al@874 150 for i in $(find $manpath -type l); do
al@865 151 dest=$(readlink $i | sed 's|\.[gbx]z2*$||')
al@865 152 link=$(echo $i | sed 's|\.[gbx]z2*$||')
al@865 153 rm $i; ln -s $dest.gz $link.gz
al@865 154 done
al@865 155
al@865 156 # Recompress with advdef (it can't compress, only recompress)
al@998 157 cleaner_log="$(mktemp)"
al@876 158 IFS=$'\n'
al@874 159 for i in $(find $manpath -type f); do
al@868 160 if ! cached_path=$(query_cache mangz "$i"); then
al@998 161 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998 162 out="$(advdef -z4q "$i")"
al@998 163 if [ -n "$out" ]; then
al@998 164 echo "$i:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 165 mv -f "$i.orig$$" "$i" # restore the original
al@998 166 else
al@998 167 store_cache "$cached_path" "$i"
al@998 168 rm -f "$i.orig$$" # clean
al@998 169 fi
al@868 170 fi
al@865 171 done
al@865 172
al@998 173 comp_summary "$time0" "$size0" "$(sizes man)" "$cleaner_log"
al@865 174 }
al@865 175
al@865 176
al@872 177 # Function to recompress all gzip archives
al@872 178 # Recompressing can be disabled with COOKOPTS="!gz"
al@872 179
al@872 180 recompress_gz() {
al@881 181 time0=$(get_time)
al@872 182 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return
al@872 183 size0=$(sizes gz); [ -z "$size0" ] && return
al@872 184
al@936 185 tpi advancecomp-static
al@872 186
al@872 187 action 'Recompressing gzip files...'
al@872 188
al@872 189 # Recompress with advdef
al@998 190 cleaner_log="$(mktemp)"
al@876 191 IFS=$'\n'
al@872 192 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do
al@872 193 if ! cached_path=$(query_cache gz "$i"); then
al@998 194 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998 195 out="$(advdef -z4q "$i")"
al@998 196 if [ -n "$out" ]; then
al@998 197 echo "$i:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 198 mv -f "$i.orig$$" "$i" # restore the original
al@998 199 else
al@998 200 store_cache "$cached_path" "$i"
al@998 201 rm -f "$i.orig$$" # clean
al@998 202 fi
al@872 203 fi
al@872 204 done
al@872 205
al@998 206 comp_summary "$time0" "$size0" "$(sizes gz)" "$cleaner_log"
al@872 207 }
al@872 208
al@872 209
al@954 210 # Function to recompress all zip archives
al@954 211 # Recompressing can be disabled with COOKOPTS="!zip"
al@954 212
al@954 213 recompress_zip() {
al@954 214 time0=$(get_time)
al@954 215 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return
al@954 216 size0=$(sizes zip); [ -z "$size0" ] && return
al@954 217
al@954 218 tpi advancecomp-static
al@954 219
al@954 220 action 'Recompressing zip files...'
al@954 221
al@954 222 # Recompress with advzip
al@998 223 cleaner_log="$(mktemp)"
al@954 224 IFS=$'\n'
al@954 225 for i in $(find $install -type f -name '*.zip'); do
al@954 226 if ! cached_path=$(query_cache zip "$i"); then
al@998 227 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998 228 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3'
al@998 229 if [ -n "$out" ]; then
al@998 230 echo "$i:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 231 mv -f "$i.orig$$" "$i" # restore the original
al@998 232 else
al@998 233 store_cache "$cached_path" "$i"
al@998 234 rm -f "$i.orig$$" # clean
al@998 235 fi
al@954 236 fi
al@954 237 done
al@954 238
al@998 239 comp_summary "$time0" "$size0" "$(sizes zip)" "$cleaner_log"
al@954 240 }
al@954 241
al@954 242
al@865 243 # Function used after compile_rules() to compress all png images
al@865 244 # Compressing can be disabled with COOKOPTS="!pngz"
al@865 245
al@865 246 compress_png() {
al@881 247 time0=$(get_time)
al@865 248 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return
al@865 249 size0=$(sizes png); [ -z "$size0" ] && return
al@865 250
al@865 251 use_pq=true
al@865 252 use_op=true
al@865 253 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false
al@865 254 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false
al@932 255 $use_pq && tpi pngquant-static
al@932 256 $use_op && tpi optipng-static
al@865 257
al@867 258 action 'Compressing png images...'
al@865 259
al@865 260 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|')
al@868 261 [ -z "$oplevel" ] && oplevel='2'
al@868 262
al@868 263 cache_section="png$oplevel"
al@868 264 $use_pq && cache_section="${cache_section}p"
al@868 265 $use_op && cache_section="${cache_section}o"
al@868 266
al@865 267 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9'
al@865 268
al@998 269 cleaner_log="$(mktemp)"
al@998 270 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately
al@978 271 IFS=$'\n'
al@978 272 for i in $(find $install -type f -name '*.png'); do
al@998 273 unset IFS iserror
al@868 274 if ! cached_path=$(query_cache $cache_section "$i"); then
al@998 275 cp -a "$i" "$i.orig$$" # save the original if something goes wrong
al@998 276 if $use_pq; then
al@998 277 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)"
al@998 278 if [ -n "$out" ]; then
al@998 279 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 280 iserror='yes'
al@998 281 fi
al@998 282 fi
al@998 283 if $use_op && [ -z "$iserror" ]; then
al@998 284 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)"
al@998 285 if [ -n "$out" ]; then
al@998 286 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 287 iserror='yes'
al@998 288 fi
al@998 289 fi
al@998 290 if [ -n "$iserror" ]; then
al@998 291 mv -f "$i.orig$$" "$i" # restore the original
al@998 292 else
al@998 293 store_cache "$cached_path" "$i"
al@998 294 rm -f "$i.orig$$" # clean
al@998 295 fi
al@868 296 fi
al@865 297 done
al@865 298
al@998 299 comp_summary "$time0" "$size0" "$(sizes png)" "$cleaner_log"
al@865 300 }
al@865 301
al@865 302
al@865 303 # Function used after compile_rules() to compress all svg images
al@865 304 # Compressing can be disabled with COOKOPTS="!svgz"
al@865 305
al@865 306 compress_svg() {
al@881 307 time0=$(get_time)
al@865 308 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return
al@865 309 size0=$(sizes svg); [ -z "$size0" ] && return
al@865 310
al@912 311 tpi svgcleaner
al@867 312
al@865 313 action 'Compressing svg images...'
al@865 314
al@959 315 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] &&
al@959 316 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1"
al@959 317
al@865 318 cleaner_log="$(mktemp)"
al@959 319 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do
al@959 320 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \
al@959 321 --multipass --remove-unresolved-classes no $opts 2>&1)"
al@998 322 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@865 323 done
al@865 324
al@998 325 comp_summary "$time0" "$size0" "$(sizes svg)" "$cleaner_log"
al@865 326 }
al@865 327
al@865 328
al@865 329 # Function used after compile_rules() to shrink all *.ui and *.glade files:
al@865 330 # remove insignificant spaces and comments
al@865 331 # Compressing can be disabled with COOKOPTS="!uiz"
al@865 332
al@865 333 compress_ui() {
al@865 334 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return
al@865 335 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return
al@865 336
al@912 337 tpi xmlstarlet
al@867 338
al@865 339 action 'Compressing ui files...'
al@867 340
al@865 341 size0=$(sizes xml)
al@881 342 time0=$(get_time)
al@865 343 temp_ui="$(mktemp)"
al@998 344 cleaner_log="$(mktemp)"
al@876 345 IFS=$'\n'
al@865 346 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do
al@998 347 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")"
al@998 348 if [ -n "$out" ]; then
al@998 349 echo "$ui:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 350 else
al@998 351 cat "$temp_ui" > "$ui"
al@998 352 fi
al@865 353 done
al@865 354
al@998 355 comp_summary "$time0" "$size0" "$(sizes xml)" "$cleaner_log"
al@865 356 rm "$temp_ui"
al@865 357 }
al@865 358
al@865 359
al@865 360 # Get list of supported locales...
al@865 361
al@865 362 get_supported_locales() {
al@865 363 lpc='/slitaz-i18n/stuff/locale-pack.conf'
al@865 364 if [ -e "$WOK$lpc" ]; then
al@865 365 # ... from package in the local wok
al@865 366 . "$WOK$lpc"
al@865 367 else
al@865 368 # ... from Hg
al@865 369 temp_conf=$(mktemp)
al@865 370 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc"
al@865 371 if [ -s $temp_conf ]; then
al@865 372 . $temp_conf
al@865 373 else
al@865 374 # Give up and use hardcoded list
al@865 375 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \
al@865 376 pt_BR ro ru sl sv tr uk zh_CN zh_TW"
al@865 377 fi
al@865 378 rm $temp_conf
al@865 379 fi
al@865 380 echo $LOCALE_PACK
al@865 381 }
al@865 382
al@865 383
al@865 384 # Fix common errors and warnings in the .desktop files
al@865 385 # Fixing can be disabled with COOKOPTS="!fixdesktops"
al@865 386
al@865 387 fix_desktop_files() {
al@865 388 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return
al@876 389 deskpath="$install/usr/share/applications"
al@876 390 [ -d "$deskpath" ] || return
al@876 391 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return
al@865 392
al@865 393 size0=$(sizes des)
al@881 394 time0=$(get_time)
al@865 395
al@865 396 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then
al@936 397 tpi desktop-file-validate-static
al@865 398 fi
al@865 399
al@865 400 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865 401 # Default value is "" (empty). That means for us that we'll use the full
al@865 402 # list of supported locales here.
al@865 403 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865 404
al@876 405 IFS=$'\n'
al@876 406 for desktop in $(find $deskpath -type f -name '*.desktop'); do
al@865 407 cp "$desktop" "$desktop.orig"
al@865 408
al@865 409 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`)
al@865 410 sdft "$desktop" -i
al@865 411
al@865 412 # Fix common errors in .desktop file
al@865 413 fix-desktop-file "$desktop"
al@865 414
al@865 415 # Strip unsupported locales from .desktop file
al@865 416 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] &&
al@865 417 sdft "$desktop" -i -k "$LOCALE"
al@865 418
al@865 419 # Extra-strip
al@865 420 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] &&
al@865 421 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o
al@865 422
al@865 423 if [ -n "$QA" ]; then
al@865 424 # Check the rest of errors, warnings and tips
al@865 425 _ 'QA: Checking %s...' "$(basename $desktop)"
al@1002 426 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!'
al@865 427 desktop-file-validate "$desktop" | busybox fold -s
al@865 428 echo
al@865 429 fi
al@865 430
al@865 431 rm "$desktop.orig"
al@865 432 done
al@865 433
al@998 434 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null'
al@865 435 }
al@865 436
al@865 437
paul@879 438 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary
al@865 439 # to the translation (msgid = msgstr)
al@865 440 # Normalization can be disabled with COOKOPTS="!monorm"
al@865 441
al@865 442 normalize_mo() {
al@865 443 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return
al@865 444 [ -z "$(find $install -type f -name '*.mo')" ] && return
al@865 445
al@867 446 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt
al@912 447 tpi gettext
al@867 448 # Gconv modules (convert to UTF-8)
al@912 449 tpi glibc-locale
al@867 450
al@865 451 action 'Normalizing mo files...'
al@867 452
al@865 453 size0=$(sizes mo1)
al@881 454 time0=$(get_time)
al@865 455
al@865 456 # Process all existing *.mo files
al@998 457 cleaner_log="$(mktemp)"
al@876 458 IFS=$'\n'
al@865 459 for mo in $(find "$install" -type f -name '*.mo'); do
al@865 460 tmpfile="$(mktemp)"
al@865 461
al@998 462 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME?
al@998 463 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)"
al@998 464 if [ -n "$out" ]; then
al@998 465 # using literal $'\n' here instead of using `echo -e "...\n..."` because
al@998 466 # $out may contain escapes ('\r', '\v') that we should print as-is
al@998 467 echo "$mo:"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 468 continue # proceed to next file
al@998 469 fi
al@998 470
al@865 471 # add newline
al@865 472 echo >> "$tmpfile"
al@865 473
al@865 474 # get Plural-Forms
al@865 475 awk '
al@865 476 BEGIN { skip = ""; }
al@865 477 {
al@865 478 if (! skip) {
al@865 479 s = $0;
al@865 480 gsub(/^[^\"]*\"/, "", s);
al@865 481 gsub(/\"$/, "", s);
al@865 482 printf("%s", s);
al@865 483 }
al@865 484 if (! $0) skip = "yes";
al@865 485 }
al@865 486 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf"
al@865 487
al@865 488 if ! grep -q 'msgid_plural' "$tmpfile"; then
al@865 489 echo > "$tmpfile.pf"
al@865 490 fi
al@865 491
al@865 492 # main
al@865 493 awk -v pf="$(cat "$tmpfile.pf")" '
al@865 494 function clean() {
al@865 495 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = "";
al@865 496 }
al@865 497
al@865 498 function getstring() {
al@865 499 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes
al@865 500 s = $0;
al@865 501 gsub(/^[^\"]*\"/, "", s);
al@865 502 gsub(/\"$/, "", s);
al@865 503 return s;
al@865 504 }
al@865 505
al@865 506 BEGIN {
al@865 507 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n");
al@865 508 if (pf)
al@865 509 printf("\"%s\\n\"\n", pf);
al@865 510 printf("\n");
al@865 511 skip = 1;
al@865 512 clean();
al@865 513 }
al@865 514
al@865 515 {
al@865 516 # Skip the entire header
al@865 517 if (!skip) {
al@865 518 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural")
al@865 519 mode = $1;
al@865 520 if ($1 == "msgstr[0]") mode = "msgstr0";
al@865 521 if ($1 == "msgstr[1]") mode = "msgstr1";
al@865 522 if ($1 == "msgstr[2]") mode = "msgstr2";
al@865 523 if ($1 == "msgstr[3]") mode = "msgstr3";
al@865 524 if ($1 == "msgstr[4]") mode = "msgstr4";
al@865 525 if ($1 == "msgstr[5]") mode = "msgstr5";
al@865 526
al@865 527 if (mode == "msgctxt") msgctxt = msgctxt getstring();
al@865 528 if (mode == "msgid") msgid = msgid getstring();
al@865 529 if (mode == "msgstr") msgstr = msgstr getstring();
al@865 530 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring();
al@865 531 if (mode == "msgstr0") msgstr0 = msgstr0 getstring();
al@865 532 if (mode == "msgstr1") msgstr1 = msgstr1 getstring();
al@865 533 if (mode == "msgstr2") msgstr2 = msgstr2 getstring();
al@865 534 if (mode == "msgstr3") msgstr3 = msgstr3 getstring();
al@865 535 if (mode == "msgstr4") msgstr4 = msgstr4 getstring();
al@865 536 if (mode == "msgstr5") msgstr5 = msgstr5 getstring();
al@865 537
al@865 538 if (! $0) {
al@865 539 if (msgid != msgstr) {
al@865 540 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt);
al@865 541 printf("msgid \"%s\"\n", msgid);
al@865 542 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural);
al@865 543 if (msgstr) printf("msgstr \"%s\"\n", msgstr);
al@865 544 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0);
al@865 545 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1);
al@865 546 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2);
al@865 547 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3);
al@865 548 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4);
al@865 549 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5);
al@865 550 printf("\n");
al@865 551 }
al@865 552 clean();
al@865 553 }
al@865 554 }
al@865 555 if ($0 == "") skip = "";
al@865 556 }
al@865 557 ' "$tmpfile" > "$tmpfile.awk"
al@865 558
al@998 559 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)"
al@998 560 if [ -n "$out" ]; then
al@998 561 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$cleaner_log"
al@998 562 continue # proceed to next file
al@998 563 fi
al@865 564
al@865 565 if [ -s "$tmpfile.mo" ]; then
al@865 566 rm "$mo"; mv "$tmpfile.mo" "$mo"
al@865 567 else
al@998 568 _ 'Error processing %s' "$mo" >> "$cleaner_log"
al@998 569 echo >> "$cleaner_log"
al@865 570 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo"
al@865 571 fi
al@865 572
al@865 573 # Clean
al@865 574 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk"
al@865 575 done
al@865 576
al@998 577 comp_summary "$time0" "$size0" "$(sizes mo1)" "$cleaner_log"
al@865 578 }
al@865 579
al@865 580
al@865 581 # Strip locale definitions: normalize whitespace and remove comments
al@865 582 # Stripping can be disabled with COOKOPTS="!locdef"
al@865 583
al@865 584 strip_locale_def() {
al@865 585 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return
al@865 586 [ ! -d "$install/usr/share/i18n/locales" ] && return
al@865 587 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return
al@865 588
al@865 589 action 'Stripping locale definitions...'
al@865 590 size0=$(sizes loc)
al@881 591 time0=$(get_time)
al@865 592
al@865 593 for i in $(find $install/usr/share/i18n/locales -type f); do
al@865 594 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i
al@865 595 done
al@865 596
al@865 597 comp_summary "$time0" "$size0" "$(sizes loc)"
al@865 598 }
al@865 599
al@865 600
al@865 601 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well
al@865 602 # as removing unneeded files like in Python packages. Cross compiled binaries
al@865 603 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip
al@865 604 # Stripping can be disabled with COOKOPTS="!strip"
al@865 605
al@865 606 strip_package() {
al@865 607 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return
al@865 608
al@865 609 case "$ARCH" in
al@865 610 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;;
al@865 611 *) export STRIP='strip' ;;
al@865 612 esac
al@865 613 action 'Executing strip on all files...'
al@865 614 size0=0
al@865 615 size1=0
al@881 616 time0=$(get_time)
al@865 617
al@865 618 # Strip executable files
al@865 619 for dir in $fs/bin $fs/sbin $fs/usr/bin $fs/usr/sbin $fs/usr/games; do
al@865 620 if [ -d "$dir" ]; then
al@865 621 oldsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
al@865 622 find $dir -type f -exec $STRIP -s '{}' 2>/dev/null \;
al@865 623 newsize=$(find $dir -type f -exec ls -l '{}' \; | awk '{s+=$5}END{print s}')
al@865 624 size0=$((size0 + oldsize)); size1=$((size1 + newsize))
al@865 625 fi
al@865 626 done
al@865 627
al@899 628 oldsize=$(sizes str)
al@899 629
al@865 630 # Strip shared and static libraries
al@865 631 find $fs -name '*.so*' -exec $STRIP -s '{}' 2>/dev/null \;
al@955 632 find $fs -name '*.a' -exec $STRIP -d '{}' 2>/dev/null \;
al@955 633
al@955 634 # Nullify timestamps of files in ar archives
al@957 635 # Skip empty 8-byte archives (hi, musl-libc package)
al@955 636 whereami=$(pwd)
al@957 637 find $fs -name '*.a' -type f -size +8c | \
al@955 638 while read i; do
al@955 639 tempdir=$(mktemp -d); cd $tempdir
al@955 640 ar -x $i; ar -crD $(basename $i) *
al@955 641 mv -f $tempdir/$(basename $i) $i
al@955 642 rm -rf $tempdir
al@955 643 done
al@955 644 cd $whereami; unset whereami
al@899 645
al@899 646 # Remove Python *.pyc and *.pyo
al@865 647 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null
al@865 648
al@899 649 # Remove both with the empty subfolders:
al@899 650 # 1. Perl perllocal.pod and .packlist (unconditionally)
al@899 651 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))"
al@899 652 # 2. Perl *.pod (if not disabled)
al@899 653 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] &&
al@899 654 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')"
al@899 655 echo "$perlfiles" | xargs rm -f 2>/dev/null
al@899 656 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null
al@899 657
al@899 658 # Strip Perl files (*.pm and *.pl) from documentation inside (if not disabled)
al@899 659 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] &&
al@899 660 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \;
al@899 661
al@899 662 newsize=$(sizes str)
al@865 663
al@865 664 comp_summary "$time0" "$((size0 + oldsize))" "$((size1 + newsize))"
al@865 665 }
al@865 666
al@865 667
al@865 668 # Strip unsupported locales (.mo files)
al@865 669
al@865 670 strip_mo_i18n() {
al@865 671 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return
al@865 672
al@865 673 [ ! -d "$fs/usr/share/locale" ] && return
al@865 674 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return
al@865 675
al@865 676 action 'Thin out translation files...'
al@865 677 size0=$(sizes mo2)
al@881 678 time0=$(get_time)
al@865 679
al@865 680 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt.
al@865 681 # Default value is "" (empty). That means for us that we'll use the full
al@865 682 # list of supported locales here.
al@865 683 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales)
al@865 684
al@865 685 # Existing locales
al@865 686 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') "
al@865 687
al@865 688 # Thin out the list of existing locales. At the end there will be only locales that need
al@865 689 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...)
al@865 690 for keep_locale in $LOCALE; do
al@865 691 elocales=${elocales//$keep_locale}
al@865 692 done
al@865 693
al@865 694 # Remove the unsupported locales
al@865 695 for rem_locale in $elocales; do
al@998 696 [ ! -d "$fs/usr/share/locale/$rem_locale" ] ||
al@865 697 rm -r "$fs/usr/share/locale/$rem_locale"
al@865 698 done
al@865 699
al@865 700 comp_summary "$time0" "$size0" "$(sizes mo2)"
al@865 701 }
al@865 702
al@865 703
al@865 704
al@865 705
al@865 706 case $1 in
al@865 707 install)
al@865 708 # Compressors working in the $install
al@865 709 case "$ARCH" in
al@865 710 arm*) ;;
al@865 711 *)
al@872 712 recompress_gz
al@954 713 recompress_zip
al@865 714 compress_manpages
al@865 715 compress_png
al@865 716 compress_svg
al@865 717 compress_ui
al@865 718 ;;
al@865 719 esac
al@865 720
al@865 721 fix_desktop_files
al@865 722 normalize_mo
al@865 723 strip_locale_def
al@865 724 ;;
al@865 725 fs)
al@865 726 # Compressors working in the $fs
al@865 727 strip_package
al@865 728 strip_mo_i18n
al@865 729 ;;
al@865 730 esac
al@868 731
al@868 732 # Clean
al@868 733 rm "$cache_stat"
al@869 734 find $comp_cache_root -type f -links -2 -delete