cookutils annotate modules/compressor @ rev 1027
cook: remove old package in /home/slitaz/packages and upgrade chroot package when $ARCH is used; provide overstriked "Failed" for prepared packages;
cook.site: remove redundant first part, fix ${sysconfdir} and ${docdir};
lighttpd/index.cgi: support for overstriked "Failed";
modules/compressor: strip_package(): consider *.dbg files (from glibc and gcc packages); make deterministic *.a archives in the simpler way.
cook.site: remove redundant first part, fix ${sysconfdir} and ${docdir};
lighttpd/index.cgi: support for overstriked "Failed";
modules/compressor: strip_package(): consider *.dbg files (from glibc and gcc packages); make deterministic *.a archives in the simpler way.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Fri Feb 02 00:51:10 2018 +0200 (2018-02-02) |
parents | be4a05be1a6f |
children | cb2fe4f46a08 |
rev | line source |
---|---|
al@865 | 1 #!/bin/sh |
al@865 | 2 # |
al@865 | 3 # compressor - module of the SliTaz Cook |
al@865 | 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3 |
al@865 | 5 # |
al@865 | 6 |
al@865 | 7 . /usr/lib/slitaz/libcook.sh |
al@865 | 8 |
al@868 | 9 # Compressor cache stuff |
al@868 | 10 |
al@873 | 11 comp_cache_root='/home/slitaz/cache/cook' |
al@869 | 12 mkdir -p "$comp_cache_root" |
al@868 | 13 cache_stat=$(mktemp) |
al@868 | 14 |
al@868 | 15 # Cache notes. |
al@868 | 16 # Do not do the same job twice. Getting the file from the cache is much faster |
al@868 | 17 # than compressing the file one more time. In addition, this cache is trying not |
paul@1006 | 18 # to take extra space using the hardlinks. Although the files from the cache |
paul@870 | 19 # without reference to itself should be removed periodically. |
al@868 | 20 |
al@868 | 21 |
al@868 | 22 |
al@868 | 23 |
al@865 | 24 # |
al@865 | 25 # Functions |
al@865 | 26 # |
al@865 | 27 |
al@865 | 28 |
al@912 | 29 # tazpkg install command |
al@912 | 30 |
al@912 | 31 tpi() { |
al@912 | 32 tazpkg -gi --quiet --local --cookmode $1 |
al@912 | 33 } |
al@912 | 34 |
al@912 | 35 |
al@881 | 36 # Working with time (with hundredths precision) |
al@865 | 37 |
al@881 | 38 get_time() { |
al@881 | 39 cut -d" " -f2 /proc/uptime |
al@881 | 40 } |
al@865 | 41 |
al@881 | 42 calc_time() { |
al@881 | 43 # L10n: 's' is for seconds, 'm' is for minutes |
al@881 | 44 awk -va="$1" -vb="$(get_time)" -vs="$(_ 's')" -vm="$(_ 'm')" ' |
al@881 | 45 BEGIN{ |
al@881 | 46 time = b - a; |
al@881 | 47 if (time < 30) |
al@881 | 48 printf("%.2f%s\n", time, s); |
al@881 | 49 else |
al@881 | 50 printf("%.2f%s ~ %.0f%s\n", time, s, time / 60, m); |
al@881 | 51 }' |
al@865 | 52 } |
al@865 | 53 |
al@865 | 54 |
al@865 | 55 # Compressor mini summary |
al@865 | 56 |
al@865 | 57 comp_summary() { |
al@998 | 58 # "$time0" "$size0" "$size1" "$log_file" |
al@865 | 59 status |
al@865 | 60 [ "$2" -eq 0 ] && return |
al@881 | 61 saving=$(awk -va="$2" -vb="$3" 'BEGIN{ printf("%.0f\n", (a - b) / 1024) }') |
al@868 | 62 cache_msg='' |
al@868 | 63 if [ -s "$cache_stat" ]; then |
al@868 | 64 cache_msg=$(_n ' Cache hit: %d/%d.' "$(fgrep '+' $cache_stat | wc -l)" "$(wc -l < $cache_stat)") |
al@868 | 65 echo -n > $cache_stat |
al@868 | 66 fi |
al@868 | 67 _ ' Time: %s. Size: %s B -> %s B. Save: %s KB.%s' \ |
al@881 | 68 "$(calc_time $1)" "$2" "$3" "$saving" "$cache_msg" |
al@998 | 69 |
al@998 | 70 if [ -s "$4" ]; then |
al@1024 | 71 _ 'Compressor warnings and errors:' |
al@998 | 72 awk '{printf " %s\n", $0;}' "$4" |
al@998 | 73 echo |
al@998 | 74 fi |
al@998 | 75 # Clean log |
al@998 | 76 [ ! -f "$4" ] || rm "$4" |
al@865 | 77 } |
al@865 | 78 |
al@865 | 79 |
al@1020 | 80 # Find ELF files |
al@1020 | 81 |
al@1020 | 82 find_elf() { |
al@1020 | 83 local i ifs="$IFS" |
al@1020 | 84 IFS=$'\n' |
al@1020 | 85 find $fs -type f \ |
al@1020 | 86 | while read i; do |
al@1020 | 87 # output of `readelf -h <file> is human-readable information, |
paul@1021 | 88 # we are interested in the next line: |
al@1020 | 89 # Type: EXEC (Executable file) |
al@1020 | 90 # or |
al@1020 | 91 # Type: DYN (Shared object file) |
al@1020 | 92 if [ "$(readelf -h "$i" 2>/dev/null \ |
al@1020 | 93 | sed -n '/Type:/ s|.*: *\([^ ]*\) .*|\1|p')" == "$1" ]; then |
al@1020 | 94 echo "$i" # $1 = { EXEC | DYN } |
al@1020 | 95 fi |
al@1020 | 96 done |
al@1020 | 97 IFS="$ifs" |
al@1020 | 98 } |
al@1020 | 99 |
al@1020 | 100 |
al@865 | 101 # Calculating different sizes |
al@865 | 102 |
al@865 | 103 sizes() { |
al@1020 | 104 local ifs="$IFS"; IFS=$'\n' |
al@865 | 105 case $1 in |
al@865 | 106 man) find $install/usr/share/man -type f -exec ls -l \{\} \; ;; |
al@865 | 107 png) find $install -type f -name '*.png' -exec ls -l \{\} \; ;; |
al@865 | 108 svg) find $install -type f -name '*.svg' -exec ls -l \{\} \; ;; |
al@865 | 109 xml) find $install -type f \( -name '*.ui' -o -name '*.glade' \) -exec ls -l \{\} \; ;; |
al@865 | 110 des) find $install -type f -name '*.desktop' -exec ls -l \{\} \; ;; |
al@865 | 111 mo1) find $install -type f -name '*.mo' -exec ls -l \{\} \; ;; |
al@865 | 112 loc) find $install/usr/share/i18n/locales -type f -exec ls -l \{\} \; ;; |
al@865 | 113 mo2) find $fs/usr/share/locale -type f -name '*.mo' -exec ls -l \{\} \; ;; |
al@872 | 114 gz) find $install -type f -name '*.gz' ! -path '*/share/man/*' -exec ls -l \{\} \; ;; |
al@954 | 115 zip) find $install -type f -name '*.zip' -exec ls -l \{\} \; ;; |
al@1020 | 116 strip) |
al@1020 | 117 { |
al@1020 | 118 find_elf EXEC |
al@1020 | 119 find_elf DYN |
al@1020 | 120 find $fs -type f \( -name '*.a' -o -name '*.pyc' -o -name '*.pyo' \ |
al@1020 | 121 -o -name '.packlist' -o -name '*.pm' -o -name '*.pl' -o -name '*.pod' \) |
al@1020 | 122 } \ |
al@1020 | 123 | xargs ls -l |
al@1020 | 124 ;; |
al@865 | 125 esac | awk '{s+=$5}END{print s}' |
al@1020 | 126 IFS="$ifs" |
al@865 | 127 } |
al@865 | 128 |
al@865 | 129 |
al@868 | 130 # Query cache for already existing compressed file; substitute original file with the cached |
al@868 | 131 # compressed one if so. |
al@868 | 132 # $1: cache section (gz, mangz, png, etc.); $2: path to original file |
al@868 | 133 |
al@868 | 134 query_cache() { |
al@868 | 135 md5=$(md5sum "$2") |
al@868 | 136 cachefile="$comp_cache_root/$1/${md5%% *}" |
al@868 | 137 echo "$cachefile" |
al@868 | 138 if [ -f "$cachefile" ]; then |
al@868 | 139 ln -f "$cachefile" "$2" |
al@868 | 140 echo '+' >> "$cache_stat" |
al@868 | 141 else |
al@868 | 142 echo '-' >> "$cache_stat" |
al@868 | 143 false |
al@868 | 144 fi |
al@868 | 145 } |
al@868 | 146 |
al@868 | 147 |
al@868 | 148 # Store compressed file to the cache |
al@868 | 149 # $1: path to cache entry to be stored; $2: path to compressed file to be stored |
al@868 | 150 |
al@868 | 151 store_cache() { |
al@868 | 152 mkdir -p "${1%/*}" |
al@868 | 153 mv "$2" "$1" |
al@868 | 154 ln "$1" "$2" |
al@868 | 155 } |
al@868 | 156 |
al@868 | 157 |
al@865 | 158 # Function to compress all man pages |
al@865 | 159 # Compressing can be disabled with COOKOPTS="!manz" |
al@865 | 160 |
al@865 | 161 compress_manpages() { |
al@881 | 162 time0=$(get_time) |
al@865 | 163 [ "${COOKOPTS/!manz/}" != "$COOKOPTS" ] && return |
al@874 | 164 manpath="$install/usr/share/man" |
al@865 | 165 [ -d "$manpath" ] || return |
al@865 | 166 size0=$(sizes man); [ -z "$size0" ] && return |
al@865 | 167 |
al@936 | 168 tpi advancecomp-static |
al@867 | 169 |
al@865 | 170 action 'Compressing man pages...' |
al@865 | 171 |
al@865 | 172 # We'll use only Gzip compression, so decompress other formats first |
al@865 | 173 find $manpath -type f -name '*.bz2' -exec bunzip2 \{\} \; |
al@865 | 174 find $manpath -type f -name '*.xz' -exec unxz \{\} \; |
al@865 | 175 |
al@865 | 176 # Fast compress with gzip |
al@874 | 177 find $manpath -type f ! -name '*.gz' -exec gzip \{\} \; |
al@865 | 178 |
al@865 | 179 # Fix symlinks |
al@874 | 180 for i in $(find $manpath -type l); do |
al@865 | 181 dest=$(readlink $i | sed 's|\.[gbx]z2*$||') |
al@865 | 182 link=$(echo $i | sed 's|\.[gbx]z2*$||') |
al@865 | 183 rm $i; ln -s $dest.gz $link.gz |
al@865 | 184 done |
al@865 | 185 |
al@865 | 186 # Recompress with advdef (it can't compress, only recompress) |
al@1024 | 187 the_log="$(mktemp)" |
al@1024 | 188 if which advdef >/dev/null; then |
al@1024 | 189 IFS=$'\n' |
al@1024 | 190 for i in $(find $manpath -type f); do |
al@1024 | 191 if ! cached_path=$(query_cache mangz "$i"); then |
al@1024 | 192 cp -a "$i" "$i.orig$$" # save the original if something goes wrong |
al@1024 | 193 out="$(advdef -z4q "$i")" |
al@1024 | 194 if [ -n "$out" ]; then |
al@1024 | 195 echo "$i:"$'\n'"$out"$'\n' >> "$the_log" |
al@1024 | 196 mv -f "$i.orig$$" "$i" # restore the original |
al@1024 | 197 else |
al@1024 | 198 store_cache "$cached_path" "$i" |
al@1024 | 199 rm -f "$i.orig$$" # clean |
al@1024 | 200 fi |
al@998 | 201 fi |
al@1024 | 202 done |
al@1024 | 203 else |
al@1024 | 204 echo 'Warning: advdef not found.' > "$the_log" |
al@1024 | 205 fi |
al@865 | 206 |
al@1024 | 207 comp_summary "$time0" "$size0" "$(sizes man)" "$the_log" |
al@865 | 208 } |
al@865 | 209 |
al@865 | 210 |
al@872 | 211 # Function to recompress all gzip archives |
al@872 | 212 # Recompressing can be disabled with COOKOPTS="!gz" |
al@872 | 213 |
al@872 | 214 recompress_gz() { |
al@881 | 215 time0=$(get_time) |
al@872 | 216 [ "${COOKOPTS/!gz/}" != "$COOKOPTS" ] && return |
al@872 | 217 size0=$(sizes gz); [ -z "$size0" ] && return |
al@872 | 218 |
al@936 | 219 tpi advancecomp-static |
al@872 | 220 |
al@872 | 221 action 'Recompressing gzip files...' |
al@872 | 222 |
al@872 | 223 # Recompress with advdef |
al@1024 | 224 the_log="$(mktemp)" |
al@1024 | 225 if which advdef >/dev/null; then |
al@1024 | 226 IFS=$'\n' |
al@1024 | 227 for i in $(find $install -type f -name '*.gz' ! -path '*/share/man/*'); do |
al@1024 | 228 if ! cached_path=$(query_cache gz "$i"); then |
al@1024 | 229 cp -a "$i" "$i.orig$$" # save the original if something goes wrong |
al@1024 | 230 out="$(advdef -z4q "$i")" |
al@1024 | 231 if [ -n "$out" ]; then |
al@1024 | 232 echo "$i:"$'\n'"$out"$'\n' >> "$the_log" |
al@1024 | 233 mv -f "$i.orig$$" "$i" # restore the original |
al@1024 | 234 else |
al@1024 | 235 store_cache "$cached_path" "$i" |
al@1024 | 236 rm -f "$i.orig$$" # clean |
al@1024 | 237 fi |
al@998 | 238 fi |
al@1024 | 239 done |
al@1024 | 240 else |
al@1024 | 241 echo 'Warning: advdef not found.' > "$the_log" |
al@1024 | 242 fi |
al@872 | 243 |
al@1024 | 244 comp_summary "$time0" "$size0" "$(sizes gz)" "$the_log" |
al@872 | 245 } |
al@872 | 246 |
al@872 | 247 |
al@954 | 248 # Function to recompress all zip archives |
al@954 | 249 # Recompressing can be disabled with COOKOPTS="!zip" |
al@954 | 250 |
al@954 | 251 recompress_zip() { |
al@954 | 252 time0=$(get_time) |
al@954 | 253 [ "${COOKOPTS/!zip/}" != "$COOKOPTS" ] && return |
al@954 | 254 size0=$(sizes zip); [ -z "$size0" ] && return |
al@954 | 255 |
al@954 | 256 tpi advancecomp-static |
al@954 | 257 |
al@954 | 258 action 'Recompressing zip files...' |
al@954 | 259 |
al@954 | 260 # Recompress with advzip |
al@1024 | 261 the_log="$(mktemp)" |
al@1024 | 262 if which advzip >/dev/null; then |
al@1024 | 263 IFS=$'\n' |
al@1024 | 264 for i in $(find $install -type f -name '*.zip'); do |
al@1024 | 265 if ! cached_path=$(query_cache zip "$i"); then |
al@1024 | 266 cp -a "$i" "$i.orig$$" # save the original if something goes wrong |
al@1024 | 267 out="$(advzip -z3qk "$i")" # '-4' is more than two orders slower; use '-3' |
al@1024 | 268 if [ -n "$out" ]; then |
al@1024 | 269 echo "$i:"$'\n'"$out"$'\n' >> "$the_log" |
al@1024 | 270 mv -f "$i.orig$$" "$i" # restore the original |
al@1024 | 271 else |
al@1024 | 272 store_cache "$cached_path" "$i" |
al@1024 | 273 rm -f "$i.orig$$" # clean |
al@1024 | 274 fi |
al@998 | 275 fi |
al@1024 | 276 done |
al@1024 | 277 else |
al@1024 | 278 echo 'Warning: advzip not found.' > "$the_log" |
al@1024 | 279 fi |
al@954 | 280 |
al@1024 | 281 comp_summary "$time0" "$size0" "$(sizes zip)" "$the_log" |
al@954 | 282 } |
al@954 | 283 |
al@954 | 284 |
al@865 | 285 # Function used after compile_rules() to compress all png images |
al@865 | 286 # Compressing can be disabled with COOKOPTS="!pngz" |
al@865 | 287 |
al@865 | 288 compress_png() { |
al@881 | 289 time0=$(get_time) |
al@865 | 290 [ "${COOKOPTS/!pngz/}" != "$COOKOPTS" ] && return |
al@865 | 291 size0=$(sizes png); [ -z "$size0" ] && return |
al@865 | 292 |
al@865 | 293 use_pq=true |
al@865 | 294 use_op=true |
al@865 | 295 [ "${COOKOPTS/!pngquant/}" != "$COOKOPTS" ] && use_pq=false |
al@865 | 296 [ "${COOKOPTS/!optipng/}" != "$COOKOPTS" ] && use_op=false |
al@932 | 297 $use_pq && tpi pngquant-static |
al@932 | 298 $use_op && tpi optipng-static |
al@865 | 299 |
al@867 | 300 action 'Compressing png images...' |
al@865 | 301 |
al@1024 | 302 the_log="$(mktemp)" |
al@1024 | 303 $use_pq && if ! which pngquant >/dev/null; then |
al@1024 | 304 echo 'Warning: pngquant not found.' > "$the_log" |
al@1024 | 305 use_pq=false |
al@1024 | 306 fi |
al@1024 | 307 $use_op && if ! which optipng >/dev/null; then |
al@1024 | 308 echo 'Warning: optipng not found.' >> "$the_log" |
al@1024 | 309 use_op=false |
al@1024 | 310 fi |
al@1024 | 311 |
al@865 | 312 oplevel=$(echo $COOKOPTS | grep 'op[0-8]' | sed 's|.*op\([0-8]\).*|\1|') |
al@868 | 313 [ -z "$oplevel" ] && oplevel='2' |
al@868 | 314 |
al@868 | 315 cache_section="png$oplevel" |
al@868 | 316 $use_pq && cache_section="${cache_section}p" |
al@868 | 317 $use_op && cache_section="${cache_section}o" |
al@868 | 318 |
al@865 | 319 [ "$oplevel" == '8' ] && oplevel='7 -zm1-9' |
al@865 | 320 |
al@998 | 321 pq_opt='--skip-if-larger' # Sublime Text is mad about `if` in $(), so put it separately |
al@978 | 322 IFS=$'\n' |
al@978 | 323 for i in $(find $install -type f -name '*.png'); do |
al@998 | 324 unset IFS iserror |
al@868 | 325 if ! cached_path=$(query_cache $cache_section "$i"); then |
al@998 | 326 cp -a "$i" "$i.orig$$" # save the original if something goes wrong |
al@998 | 327 if $use_pq; then |
al@998 | 328 out="$(pngquant -f $pq_opt --ext .png --speed 1 "$i" 2>&1)" |
al@998 | 329 if [ -n "$out" ]; then |
al@1024 | 330 echo "$i (pngquant):"$'\n'"$out"$'\n' >> "$the_log" |
al@998 | 331 iserror='yes' |
al@998 | 332 fi |
al@998 | 333 fi |
al@998 | 334 if $use_op && [ -z "$iserror" ]; then |
al@998 | 335 out="$(optipng -quiet -strip all -o$oplevel "$i" 2>&1)" |
al@998 | 336 if [ -n "$out" ]; then |
al@1024 | 337 echo "$i (optipng):"$'\n'"$out"$'\n' >> "$the_log" |
al@998 | 338 iserror='yes' |
al@998 | 339 fi |
al@998 | 340 fi |
al@998 | 341 if [ -n "$iserror" ]; then |
al@998 | 342 mv -f "$i.orig$$" "$i" # restore the original |
al@998 | 343 else |
al@998 | 344 store_cache "$cached_path" "$i" |
al@998 | 345 rm -f "$i.orig$$" # clean |
al@998 | 346 fi |
al@868 | 347 fi |
al@865 | 348 done |
al@865 | 349 |
al@1024 | 350 comp_summary "$time0" "$size0" "$(sizes png)" "$the_log" |
al@865 | 351 } |
al@865 | 352 |
al@865 | 353 |
al@865 | 354 # Function used after compile_rules() to compress all svg images |
al@865 | 355 # Compressing can be disabled with COOKOPTS="!svgz" |
al@865 | 356 |
al@865 | 357 compress_svg() { |
al@881 | 358 time0=$(get_time) |
al@865 | 359 [ "${COOKOPTS/!svgz/}" != "$COOKOPTS" ] && return |
al@865 | 360 size0=$(sizes svg); [ -z "$size0" ] && return |
al@865 | 361 |
al@912 | 362 tpi svgcleaner |
al@867 | 363 |
al@865 | 364 action 'Compressing svg images...' |
al@865 | 365 |
al@1024 | 366 if which svgcleaner >/dev/null; then |
al@1024 | 367 [ "${COOKOPTS/!svgextra/}" == "$COOKOPTS" ] && |
al@1024 | 368 opts="--apply-transform-to-paths yes --coordinates-precision 1 --paths-coordinates-precision 1" |
al@959 | 369 |
al@1024 | 370 the_log="$(mktemp)" |
al@1024 | 371 for i in $(IFS=$'\n' find $install -type f -name '*.svg'); do |
al@1024 | 372 out="$(unset IFS; svgcleaner "$i" "$i" --copy-on-error --quiet \ |
al@1024 | 373 --multipass --remove-unresolved-classes no $opts 2>&1)" |
al@1024 | 374 [ -z "$out" ] || echo "$i:"$'\n'"$out"$'\n' >> "$the_log" |
al@1024 | 375 done |
al@1024 | 376 else |
al@1024 | 377 echo 'Warning: svgcleaner not found.' > "$the_log" |
al@1024 | 378 fi |
al@865 | 379 |
al@1024 | 380 comp_summary "$time0" "$size0" "$(sizes svg)" "$the_log" |
al@865 | 381 } |
al@865 | 382 |
al@865 | 383 |
al@865 | 384 # Function used after compile_rules() to shrink all *.ui and *.glade files: |
al@865 | 385 # remove insignificant spaces and comments |
al@865 | 386 # Compressing can be disabled with COOKOPTS="!uiz" |
al@865 | 387 |
al@865 | 388 compress_ui() { |
al@865 | 389 [ "${COOKOPTS/!uiz/}" != "$COOKOPTS" ] && return |
al@865 | 390 [ -z "$(find $install -type f \( -name '*.ui' -o -name '*.glade' \) )" ] && return |
al@865 | 391 |
al@912 | 392 tpi xmlstarlet |
al@867 | 393 |
al@865 | 394 action 'Compressing ui files...' |
al@867 | 395 |
al@1024 | 396 if which xmlstarlet >/dev/null; then |
al@1024 | 397 size0=$(sizes xml) |
al@1024 | 398 time0=$(get_time) |
al@1024 | 399 temp_ui="$(mktemp)" |
al@1024 | 400 the_log="$(mktemp)" |
al@1024 | 401 IFS=$'\n' |
al@1024 | 402 for ui in $(find $install -type f \( -name '*.ui' -o -name '*.glade' \) ); do |
al@1024 | 403 out="$(xmlstarlet c14n --without-comments "$ui" | xmlstarlet sel -B -t -c '*' > "$temp_ui")" |
al@1024 | 404 if [ -n "$out" ]; then |
al@1024 | 405 echo "$ui:"$'\n'"$out"$'\n' >> "$the_log" |
al@1024 | 406 else |
al@1024 | 407 cat "$temp_ui" > "$ui" |
al@1024 | 408 fi |
al@1024 | 409 done |
al@1024 | 410 else |
al@1024 | 411 echo 'Warning: xmlstarlet not found.' > "$the_log" |
al@1024 | 412 fi |
al@865 | 413 |
al@1024 | 414 comp_summary "$time0" "$size0" "$(sizes xml)" "$the_log" |
al@865 | 415 rm "$temp_ui" |
al@865 | 416 } |
al@865 | 417 |
al@865 | 418 |
al@865 | 419 # Get list of supported locales... |
al@865 | 420 |
al@865 | 421 get_supported_locales() { |
al@865 | 422 lpc='/slitaz-i18n/stuff/locale-pack.conf' |
al@865 | 423 if [ -e "$WOK$lpc" ]; then |
al@865 | 424 # ... from package in the local wok |
al@865 | 425 . "$WOK$lpc" |
al@865 | 426 else |
al@865 | 427 # ... from Hg |
al@865 | 428 temp_conf=$(mktemp) |
al@865 | 429 wget -q -O $temp_conf -T 10 "http://hg.slitaz.org/wok/raw-file/tip$lpc" |
al@865 | 430 if [ -s $temp_conf ]; then |
al@865 | 431 . $temp_conf |
al@865 | 432 else |
al@865 | 433 # Give up and use hardcoded list |
al@865 | 434 LOCALE_PACK="ar ca cs da de el en es fi fr hr hu id is it ja nb nl nn pl pt \ |
al@865 | 435 pt_BR ro ru sl sv tr uk zh_CN zh_TW" |
al@865 | 436 fi |
al@865 | 437 rm $temp_conf |
al@865 | 438 fi |
al@865 | 439 echo $LOCALE_PACK |
al@865 | 440 } |
al@865 | 441 |
al@865 | 442 |
al@865 | 443 # Fix common errors and warnings in the .desktop files |
al@865 | 444 # Fixing can be disabled with COOKOPTS="!fixdesktops" |
al@865 | 445 |
al@865 | 446 fix_desktop_files() { |
al@865 | 447 [ "${COOKOPTS/!fixdesktops/}" != "$COOKOPTS" ] && return |
al@876 | 448 deskpath="$install/usr/share/applications" |
al@876 | 449 [ -d "$deskpath" ] || return |
al@876 | 450 [ -z "$(find $deskpath -type f -name '*.desktop')" ] && return |
al@865 | 451 |
al@865 | 452 size0=$(sizes des) |
al@881 | 453 time0=$(get_time) |
al@865 | 454 |
al@865 | 455 if [ -n "$QA" -a -z "$(which desktop-file-validate)" ]; then |
al@936 | 456 tpi desktop-file-validate-static |
al@865 | 457 fi |
al@865 | 458 |
al@865 | 459 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt. |
al@865 | 460 # Default value is "" (empty). That means for us that we'll use the full |
al@865 | 461 # list of supported locales here. |
al@865 | 462 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales) |
al@865 | 463 |
al@876 | 464 IFS=$'\n' |
al@876 | 465 for desktop in $(find $deskpath -type f -name '*.desktop'); do |
al@865 | 466 cp "$desktop" "$desktop.orig" |
al@865 | 467 |
al@865 | 468 # Sort out .desktop file (is prerequisite to correct working of `fix-desktop-file`) |
al@865 | 469 sdft "$desktop" -i |
al@865 | 470 |
al@865 | 471 # Fix common errors in .desktop file |
al@865 | 472 fix-desktop-file "$desktop" |
al@865 | 473 |
al@865 | 474 # Strip unsupported locales from .desktop file |
al@865 | 475 [ "${COOKOPTS/!i18nz/}" == "$COOKOPTS" ] && |
al@865 | 476 sdft "$desktop" -i -k "$LOCALE" |
al@865 | 477 |
al@865 | 478 # Extra-strip |
al@865 | 479 [ "${COOKOPTS/!extradesktops/}" == "$COOKOPTS" ] && |
al@865 | 480 sdft "$desktop" -i -g -x -tf -r 'Keywords*' -o |
al@865 | 481 |
al@865 | 482 if [ -n "$QA" ]; then |
al@865 | 483 # Check the rest of errors, warnings and tips |
al@865 | 484 _ 'QA: Checking %s...' "$(basename $desktop)" |
al@1002 | 485 busybox diff "$desktop.orig" "$desktop" | sed 's!^!|!' |
al@1024 | 486 if which xmlstarlet >/dev/null; then |
al@1024 | 487 desktop-file-validate "$desktop" | busybox fold -s |
al@1024 | 488 else |
al@1024 | 489 echo 'Warning: desktop-file-validate not found.' |
al@1024 | 490 fi |
al@865 | 491 echo |
al@865 | 492 fi |
al@865 | 493 |
al@865 | 494 rm "$desktop.orig" |
al@865 | 495 done |
al@865 | 496 |
al@998 | 497 comp_summary "$time0" "$size0" "$(sizes des)" '/dev/null' |
al@865 | 498 } |
al@865 | 499 |
al@865 | 500 |
paul@879 | 501 # Normalize all *.mo files: unconditionally convert to UTF-8; remove strings that are not really necessary |
al@865 | 502 # to the translation (msgid = msgstr) |
al@865 | 503 # Normalization can be disabled with COOKOPTS="!monorm" |
al@865 | 504 |
al@865 | 505 normalize_mo() { |
al@865 | 506 [ "${COOKOPTS/!monorm/}" != "$COOKOPTS" ] && return |
al@865 | 507 [ -z "$(find $install -type f -name '*.mo')" ] && return |
al@865 | 508 |
al@867 | 509 # Gettext functions: msgunfmt, msguniq, msgconv, msgfmt |
al@912 | 510 tpi gettext |
al@867 | 511 # Gconv modules (convert to UTF-8) |
al@912 | 512 tpi glibc-locale |
al@867 | 513 |
al@865 | 514 action 'Normalizing mo files...' |
al@867 | 515 |
al@1024 | 516 the_log="$(mktemp)" |
al@1024 | 517 to_continue=true |
al@1024 | 518 for i in msgunfmt msguniq msgconv msgfmt; do |
al@1024 | 519 if ! which $i >/dev/null; then |
al@1024 | 520 echo "Warning: $i not found. Normalizing aborted" > "$the_log" |
al@1024 | 521 to_continue=false |
al@1024 | 522 fi |
al@1024 | 523 done |
al@1024 | 524 |
al@865 | 525 size0=$(sizes mo1) |
al@881 | 526 time0=$(get_time) |
al@865 | 527 |
al@865 | 528 # Process all existing *.mo files |
al@876 | 529 IFS=$'\n' |
al@1024 | 530 $to_continue && |
al@865 | 531 for mo in $(find "$install" -type f -name '*.mo'); do |
al@865 | 532 tmpfile="$(mktemp)" |
al@865 | 533 |
al@998 | 534 # put ANY errors of {msgunfmt,msguniq,msgconv} to $out. FIXME? |
al@998 | 535 out="$({ msgunfmt "$mo" | msguniq | msgconv -o "$tmpfile" -t 'UTF-8'; } 2>&1)" |
al@998 | 536 if [ -n "$out" ]; then |
al@998 | 537 # using literal $'\n' here instead of using `echo -e "...\n..."` because |
al@998 | 538 # $out may contain escapes ('\r', '\v') that we should print as-is |
al@1024 | 539 echo "$mo:"$'\n'"$out"$'\n' >> "$the_log" |
al@998 | 540 continue # proceed to next file |
al@998 | 541 fi |
al@998 | 542 |
al@865 | 543 # add newline |
al@865 | 544 echo >> "$tmpfile" |
al@865 | 545 |
al@865 | 546 # get Plural-Forms |
al@865 | 547 awk ' |
al@865 | 548 BEGIN { skip = ""; } |
al@865 | 549 { |
al@865 | 550 if (! skip) { |
al@865 | 551 s = $0; |
al@865 | 552 gsub(/^[^\"]*\"/, "", s); |
al@865 | 553 gsub(/\"$/, "", s); |
al@865 | 554 printf("%s", s); |
al@865 | 555 } |
al@865 | 556 if (! $0) skip = "yes"; |
al@865 | 557 } |
al@865 | 558 ' "$tmpfile" | sed 's|\\n|\n|g' | grep "^Plural-Forms:" > "$tmpfile.pf" |
al@865 | 559 |
al@865 | 560 if ! grep -q 'msgid_plural' "$tmpfile"; then |
al@865 | 561 echo > "$tmpfile.pf" |
al@865 | 562 fi |
al@865 | 563 |
al@865 | 564 # main |
al@865 | 565 awk -v pf="$(cat "$tmpfile.pf")" ' |
al@865 | 566 function clean() { |
al@865 | 567 mode = msgctxt = msgid = msgid_plural = msgstr = msgstr0 = msgstr1 = msgstr2 = msgstr3 = msgstr4 = msgstr5 = ""; |
al@865 | 568 } |
al@865 | 569 |
al@865 | 570 function getstring() { |
al@865 | 571 # Skip unquoted words at the beginning (msgid, msgstr...) and get string from inside quotes |
al@865 | 572 s = $0; |
al@865 | 573 gsub(/^[^\"]*\"/, "", s); |
al@865 | 574 gsub(/\"$/, "", s); |
al@865 | 575 return s; |
al@865 | 576 } |
al@865 | 577 |
al@865 | 578 BEGIN { |
al@865 | 579 printf("msgid \"\"\nmsgstr \"\"\n\"Content-Type: text/plain; charset=UTF-8\\n\"\n"); |
al@865 | 580 if (pf) |
al@865 | 581 printf("\"%s\\n\"\n", pf); |
al@865 | 582 printf("\n"); |
al@865 | 583 skip = 1; |
al@865 | 584 clean(); |
al@865 | 585 } |
al@865 | 586 |
al@865 | 587 { |
al@865 | 588 # Skip the entire header |
al@865 | 589 if (!skip) { |
al@865 | 590 if ($1 == "msgctxt" || $1 == "msgid" || $1 == "msgstr" || $1 == "msgid_plural") |
al@865 | 591 mode = $1; |
al@865 | 592 if ($1 == "msgstr[0]") mode = "msgstr0"; |
al@865 | 593 if ($1 == "msgstr[1]") mode = "msgstr1"; |
al@865 | 594 if ($1 == "msgstr[2]") mode = "msgstr2"; |
al@865 | 595 if ($1 == "msgstr[3]") mode = "msgstr3"; |
al@865 | 596 if ($1 == "msgstr[4]") mode = "msgstr4"; |
al@865 | 597 if ($1 == "msgstr[5]") mode = "msgstr5"; |
al@865 | 598 |
al@865 | 599 if (mode == "msgctxt") msgctxt = msgctxt getstring(); |
al@865 | 600 if (mode == "msgid") msgid = msgid getstring(); |
al@865 | 601 if (mode == "msgstr") msgstr = msgstr getstring(); |
al@865 | 602 if (mode == "msgid_plural") msgid_plural = msgid_plural getstring(); |
al@865 | 603 if (mode == "msgstr0") msgstr0 = msgstr0 getstring(); |
al@865 | 604 if (mode == "msgstr1") msgstr1 = msgstr1 getstring(); |
al@865 | 605 if (mode == "msgstr2") msgstr2 = msgstr2 getstring(); |
al@865 | 606 if (mode == "msgstr3") msgstr3 = msgstr3 getstring(); |
al@865 | 607 if (mode == "msgstr4") msgstr4 = msgstr4 getstring(); |
al@865 | 608 if (mode == "msgstr5") msgstr5 = msgstr5 getstring(); |
al@865 | 609 |
al@865 | 610 if (! $0) { |
al@865 | 611 if (msgid != msgstr) { |
al@865 | 612 if (msgctxt) printf("msgctxt \"%s\"\n", msgctxt); |
al@865 | 613 printf("msgid \"%s\"\n", msgid); |
al@865 | 614 if (msgid_plural) printf("msgid_plural \"%s\"\n", msgid_plural); |
al@865 | 615 if (msgstr) printf("msgstr \"%s\"\n", msgstr); |
al@865 | 616 if (msgstr0) printf("msgstr[0] \"%s\"\n", msgstr0); |
al@865 | 617 if (msgstr1) printf("msgstr[1] \"%s\"\n", msgstr1); |
al@865 | 618 if (msgstr2) printf("msgstr[2] \"%s\"\n", msgstr2); |
al@865 | 619 if (msgstr3) printf("msgstr[3] \"%s\"\n", msgstr3); |
al@865 | 620 if (msgstr4) printf("msgstr[4] \"%s\"\n", msgstr4); |
al@865 | 621 if (msgstr5) printf("msgstr[5] \"%s\"\n", msgstr5); |
al@865 | 622 printf("\n"); |
al@865 | 623 } |
al@865 | 624 clean(); |
al@865 | 625 } |
al@865 | 626 } |
al@865 | 627 if ($0 == "") skip = ""; |
al@865 | 628 } |
al@865 | 629 ' "$tmpfile" > "$tmpfile.awk" |
al@865 | 630 |
al@998 | 631 out="$(msgfmt "$tmpfile.awk" -o "$tmpfile.mo" 2>&1)" |
al@998 | 632 if [ -n "$out" ]; then |
al@1024 | 633 echo "$mo (msgfmt):"$'\n'"$out"$'\n' >> "$the_log" |
al@998 | 634 continue # proceed to next file |
al@998 | 635 fi |
al@865 | 636 |
al@865 | 637 if [ -s "$tmpfile.mo" ]; then |
al@865 | 638 rm "$mo"; mv "$tmpfile.mo" "$mo" |
al@865 | 639 else |
al@1024 | 640 _ 'Error processing %s' "$mo" >> "$the_log" |
al@1024 | 641 echo >> "$the_log" |
al@865 | 642 [ -e "$tmpfile.mo" ] && rm "$tmpfile.mo" |
al@865 | 643 fi |
al@865 | 644 |
al@865 | 645 # Clean |
al@865 | 646 rm "$tmpfile" "$tmpfile.pf" "$tmpfile.awk" |
al@865 | 647 done |
al@865 | 648 |
al@1024 | 649 comp_summary "$time0" "$size0" "$(sizes mo1)" "$the_log" |
al@865 | 650 } |
al@865 | 651 |
al@865 | 652 |
al@865 | 653 # Strip locale definitions: normalize whitespace and remove comments |
al@865 | 654 # Stripping can be disabled with COOKOPTS="!locdef" |
al@865 | 655 |
al@865 | 656 strip_locale_def() { |
al@865 | 657 [ "${COOKOPTS/!locdef/}" != "$COOKOPTS" ] && return |
al@865 | 658 [ ! -d "$install/usr/share/i18n/locales" ] && return |
al@865 | 659 [ -z "$(find $install/usr/share/i18n/locales -type f)" ] && return |
al@865 | 660 |
al@865 | 661 action 'Stripping locale definitions...' |
al@865 | 662 size0=$(sizes loc) |
al@881 | 663 time0=$(get_time) |
al@865 | 664 |
al@865 | 665 for i in $(find $install/usr/share/i18n/locales -type f); do |
al@865 | 666 sed -i 's| | |g; s| *| |g; s|^ ||; /^%/d' $i |
al@865 | 667 done |
al@865 | 668 |
al@865 | 669 comp_summary "$time0" "$size0" "$(sizes loc)" |
al@865 | 670 } |
al@865 | 671 |
al@865 | 672 |
al@865 | 673 # Find and strip: --strip-all (-s) or --strip-debug on static libs as well |
al@865 | 674 # as removing unneeded files like in Python packages. Cross compiled binaries |
al@865 | 675 # must be stripped with cross-tools aka $ARCH-slitaz-*-strip |
al@865 | 676 # Stripping can be disabled with COOKOPTS="!strip" |
al@865 | 677 |
al@865 | 678 strip_package() { |
al@865 | 679 [ "${COOKOPTS/!strip/}" != "$COOKOPTS" ] && return |
al@865 | 680 |
al@1020 | 681 local i ifs="$IFS" |
al@1020 | 682 IFS=$'\n' |
al@1020 | 683 |
al@865 | 684 case "$ARCH" in |
al@865 | 685 arm*|x86_64) export STRIP="$HOST_SYSTEM-strip" ;; |
al@865 | 686 *) export STRIP='strip' ;; |
al@865 | 687 esac |
al@865 | 688 action 'Executing strip on all files...' |
al@865 | 689 size0=0 |
al@865 | 690 size1=0 |
al@881 | 691 time0=$(get_time) |
al@1020 | 692 oldsize=$(sizes strip) |
al@865 | 693 |
al@1027 | 694 |
al@1027 | 695 # GNU strip (GNU Binutils) |
al@1027 | 696 # -p --preserve-dates Copy modified/access timestamps to the output |
al@1027 | 697 # -s --strip-all Remove all symbol and relocation information |
al@1027 | 698 # --strip-unneeded Remove all symbols not needed by relocations |
al@1027 | 699 # -D --enable-deterministic-archives Produce deterministic output when stripping archives |
al@1027 | 700 # -g -S -d --strip-debug Remove all debugging symbols & sections |
al@1027 | 701 |
al@1027 | 702 # Strip executable files |
al@1020 | 703 while read i; do |
al@1027 | 704 $STRIP -ps "$i" 2>/dev/null |
al@1020 | 705 done <<EOT |
al@1027 | 706 $(find_elf EXEC) |
al@1027 | 707 EOT |
al@1027 | 708 |
al@1027 | 709 # Strip shared libraries |
al@1027 | 710 while read i; do |
al@1027 | 711 case $i in |
al@1027 | 712 *.dbg) ;; # skip library.so.*.dbg debugging symbols |
al@1027 | 713 *) $STRIP -p --strip-unneeded "$i" 2>/dev/null;; |
al@1027 | 714 esac |
al@1027 | 715 done <<EOT |
al@1027 | 716 $(find_elf DYN) |
al@1020 | 717 EOT |
al@865 | 718 |
al@1020 | 719 # Strip static libraries |
al@1027 | 720 # See also: https://wiki.debian.org/ReproducibleBuilds/TimestampsInStaticLibraries |
al@1027 | 721 find $fs -name '*.a' -exec $STRIP -pdD '{}' 2>/dev/null \; |
al@955 | 722 |
al@899 | 723 |
al@899 | 724 # Remove Python *.pyc and *.pyo |
al@865 | 725 find $fs -type f \( -name '*.pyc' -o -name '*.pyo' \) -delete 2>/dev/null |
al@865 | 726 |
al@899 | 727 # Remove both with the empty subfolders: |
al@899 | 728 # 1. Perl perllocal.pod and .packlist (unconditionally) |
al@899 | 729 local perlfiles="$(find $fs -type f \( -name 'perllocal.pod' -o -name '.packlist' \))" |
al@899 | 730 # 2. Perl *.pod (if not disabled) |
al@899 | 731 [ "${COOKOPTS/!rmpod/}" == "$COOKOPTS" ] && |
al@899 | 732 perlfiles="$perlfiles $(find $fs -type f -name '*.pod')" |
al@899 | 733 echo "$perlfiles" | xargs rm -f 2>/dev/null |
al@899 | 734 echo "$perlfiles" | awk 'BEGIN{FS=OFS="/"}{$NF="";print}' | xargs rmdir -p 2>/dev/null |
al@899 | 735 |
al@1020 | 736 # Strip documentation inside Perl files (*.pm and *.pl) (if not disabled) |
al@899 | 737 [ "${COOKOPTS/!perlz/}" == "$COOKOPTS" ] && |
al@899 | 738 find $fs -type f \( -name '*.pm' -o -name '*.pl' \) -exec sed -i '/^=/,/^=cut/d' '{}' \; |
al@899 | 739 |
al@1020 | 740 newsize=$(sizes strip) |
al@865 | 741 |
al@1020 | 742 comp_summary "$time0" "$oldsize" "$newsize" |
al@1020 | 743 IFS="$ifs" |
al@865 | 744 } |
al@865 | 745 |
al@865 | 746 |
al@865 | 747 # Strip unsupported locales (.mo files) |
al@865 | 748 |
al@865 | 749 strip_mo_i18n() { |
al@865 | 750 [ "${COOKOPTS/!i18nz/}" != "$COOKOPTS" ] && return |
al@865 | 751 |
al@865 | 752 [ ! -d "$fs/usr/share/locale" ] && return |
al@865 | 753 [ -z "$(find $fs/usr/share/locale -type f -name '*.mo')" ] && return |
al@865 | 754 |
al@865 | 755 action 'Thin out translation files...' |
al@865 | 756 size0=$(sizes mo2) |
al@881 | 757 time0=$(get_time) |
al@865 | 758 |
al@865 | 759 # The variable $LOCALE is set in cook.conf and may be overridden in the receipt. |
al@865 | 760 # Default value is "" (empty). That means for us that we'll use the full |
al@865 | 761 # list of supported locales here. |
al@865 | 762 [ -z "$LOCALE" ] && LOCALE=$(get_supported_locales) |
al@865 | 763 |
al@865 | 764 # Existing locales |
al@865 | 765 elocales=" $(ls -1 "$fs/usr/share/locale" | tr '\n' ' ') " |
al@865 | 766 |
al@865 | 767 # Thin out the list of existing locales. At the end there will be only locales that need |
al@865 | 768 # deleting (and the garbage like '_AU', _US', '_BR' leaving from 'en_AU', 'en_US', 'pt_BR'...) |
al@865 | 769 for keep_locale in $LOCALE; do |
al@865 | 770 elocales=${elocales//$keep_locale} |
al@865 | 771 done |
al@865 | 772 |
al@865 | 773 # Remove the unsupported locales |
al@865 | 774 for rem_locale in $elocales; do |
al@998 | 775 [ ! -d "$fs/usr/share/locale/$rem_locale" ] || |
al@865 | 776 rm -r "$fs/usr/share/locale/$rem_locale" |
al@865 | 777 done |
al@865 | 778 |
al@865 | 779 comp_summary "$time0" "$size0" "$(sizes mo2)" |
al@865 | 780 } |
al@865 | 781 |
al@865 | 782 |
al@865 | 783 |
al@865 | 784 |
al@865 | 785 case $1 in |
al@865 | 786 install) |
al@865 | 787 # Compressors working in the $install |
al@865 | 788 case "$ARCH" in |
al@865 | 789 arm*) ;; |
al@865 | 790 *) |
al@872 | 791 recompress_gz |
al@954 | 792 recompress_zip |
al@865 | 793 compress_manpages |
al@865 | 794 compress_png |
al@865 | 795 compress_svg |
al@865 | 796 compress_ui |
al@865 | 797 ;; |
al@865 | 798 esac |
al@865 | 799 |
al@865 | 800 fix_desktop_files |
al@865 | 801 normalize_mo |
al@865 | 802 strip_locale_def |
al@865 | 803 ;; |
al@865 | 804 fs) |
al@865 | 805 # Compressors working in the $fs |
al@865 | 806 strip_package |
al@865 | 807 strip_mo_i18n |
al@865 | 808 ;; |
al@865 | 809 esac |
al@868 | 810 |
al@868 | 811 # Clean |
al@868 | 812 rm "$cache_stat" |
al@869 | 813 find $comp_cache_root -type f -links -2 -delete |