cookutils annotate modules/deps @ rev 944
modules/deps: accelerate files list creation significantly.
On my machine time has decreased from 60 to 2 seconds!
On my machine time has decreased from 60 to 2 seconds!
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Mon Jun 26 13:57:14 2017 +0300 (2017-06-26) |
parents | 81f33cdb5a66 |
children | 79b65c3a4e40 |
rev | line source |
---|---|
al@932 | 1 #!/bin/sh |
al@932 | 2 # |
al@932 | 3 # deps - module of the SliTaz Cook |
al@932 | 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3 |
al@932 | 5 # |
al@932 | 6 |
al@932 | 7 . /usr/lib/slitaz/libcook.sh |
al@932 | 8 |
al@932 | 9 |
al@932 | 10 # Maintain databases |
al@932 | 11 |
al@932 | 12 fl="$cache/tp.files.db" # combined list of all files |
al@932 | 13 db_so="$cache/tp.so.db" # database with *.so files |
al@932 | 14 db_a="$cache/tp.a.db" # database with *.a files |
al@932 | 15 db_la="$cache/tp.la.db" # database with *.la files |
al@932 | 16 db_pc="$cache/tp.pc.db" # database with *.pc files |
al@932 | 17 |
al@932 | 18 fl_mirrorz='/var/lib/tazpkg/files.list.lzma' # mirror files list |
al@932 | 19 fl_local='/home/slitaz/cache/files.list' # local files list |
al@932 | 20 |
al@932 | 21 # recreate "combined list of all files" in the cases: |
paul@937 | 22 # * if absent |
al@932 | 23 # * mirror files list has been updated |
al@932 | 24 # * local files list has been updated |
al@932 | 25 |
al@938 | 26 case $(hostname) in |
al@938 | 27 # Do we need to use mirror files list? |
paul@939 | 28 # It's useful on casual development on local host, but useless on cooker server |
paul@939 | 29 # (and slows down list creation a lot). |
al@938 | 30 tank) |
al@940 | 31 [ ! -s $fl -o $fl_local -nt $fl ] && cp $fl_local $fl |
al@940 | 32 ;; |
al@940 | 33 *) |
al@938 | 34 if [ ! -s $fl -o $fl_mirrorz -nt $fl -o $fl_local -nt $fl ]; then |
al@940 | 35 action 'Updating %s...' "$(basename $fl)" |
al@938 | 36 # unpack mirror files list |
al@938 | 37 fl_mirror="$(mktemp)" |
al@944 | 38 lzcat $fl_mirrorz | sed '/\.\(so.*\|a\|la\|pc\)$/!d' > $fl_mirror |
al@932 | 39 |
al@938 | 40 # remove packages that exist in local list |
al@938 | 41 cut -d: -f1 $fl_local | uniq | \ |
al@938 | 42 while read package; do |
al@938 | 43 sed -i "/^$package: /d" $fl_mirror |
al@938 | 44 done |
al@932 | 45 |
al@938 | 46 # combine lists |
al@938 | 47 cat $fl_mirror $fl_local > $fl |
al@932 | 48 |
al@938 | 49 # clean |
al@938 | 50 rm $fl_mirror |
al@940 | 51 status |
al@938 | 52 fi |
al@938 | 53 ;; |
al@938 | 54 esac |
al@932 | 55 |
al@932 | 56 # recreate "database with *.so files" in the cases: |
paul@937 | 57 # * if absent |
al@932 | 58 # * combined list of all files has been updated |
al@932 | 59 |
al@932 | 60 if [ ! -s $db_so -o $fl -nt $db_so ]; then |
al@940 | 61 action 'Updating %s...' "$(basename $db_so)" |
al@932 | 62 fgrep '/lib/' $fl | fgrep '.so' | \ |
al@932 | 63 sed 's|^\([^:]*\):.*/\([^/]*\)$|\2\t\1|' | \ |
al@932 | 64 awk -F$'\t' '{if ($2 !~ "uclibc") print}' | \ |
al@932 | 65 sort > $db_so |
al@940 | 66 status |
al@932 | 67 fi |
al@932 | 68 |
al@932 | 69 # recreate "database with *.a files" in the cases: |
paul@937 | 70 # * if absent |
al@932 | 71 # * combined list of all files has been updated |
al@932 | 72 |
al@932 | 73 if [ ! -s $db_a -o $fl -nt $db_a ]; then |
al@940 | 74 action 'Updating %s...' "$(basename $db_a)" |
al@932 | 75 fgrep '/usr/lib/lib' $fl | fgrep '.a' | \ |
al@932 | 76 sed 's|^\([^:]*\):.*/\([^/]*\)$|\2\t\1|' | \ |
al@932 | 77 sort > $db_a |
al@940 | 78 status |
al@932 | 79 fi |
al@932 | 80 |
al@932 | 81 # recreate "database with *.la files" in the cases: |
paul@937 | 82 # * if absent |
al@932 | 83 # * combined list of all files has been updated |
al@932 | 84 |
al@932 | 85 if [ ! -s $db_la -o $fl -nt $db_la ]; then |
al@940 | 86 action 'Updating %s...' "$(basename $db_la)" |
al@932 | 87 fgrep '/usr/lib/' $fl | fgrep '.la' | \ |
al@932 | 88 sed 's|^\([^:]*\): \(.*\)$|\2\t\1|' | \ |
al@932 | 89 sort > $db_la |
al@940 | 90 status |
al@932 | 91 fi |
al@932 | 92 |
al@932 | 93 # recreate "database with *.pc files" in the cases: |
paul@937 | 94 # * if absent |
al@932 | 95 # * combined list of all files has been updated |
al@932 | 96 |
al@932 | 97 if [ ! -s $db_pc -o $fl -nt $db_pc ]; then |
al@940 | 98 action 'Updating %s...' "$(basename $db_pc)" |
al@932 | 99 grep '\.pc$' $fl | \ |
al@932 | 100 sed -e 's|^\([^:]*\):.*/\([^/]*\)$|\2\t\1|' -e '/\tbuildroot$/d' | \ |
al@932 | 101 sort > $db_pc |
al@940 | 102 status |
al@932 | 103 fi |
al@932 | 104 |
al@932 | 105 |
al@932 | 106 |
al@932 | 107 |
paul@937 | 108 # Auxiliary function that deals with "not found" packages as well as with "repeatedly found" packages |
al@932 | 109 |
al@932 | 110 outpkg() { |
al@932 | 111 pkgs="$1" |
al@932 | 112 case $pkgs in |
al@932 | 113 *ld-linux.so*);; |
al@932 | 114 *linux-gate.so*);; |
al@932 | 115 *) |
al@932 | 116 echo "$pkgs" | awk ' |
al@932 | 117 # if both packages exist in list, return the first one only |
al@932 | 118 function s(pkg1, pkg2) { |
al@932 | 119 if (index(" "$0" ", " "pkg1" ") && index(" "$0" ", " " pkg2 " ")) |
al@932 | 120 $0 = pkg1; |
al@932 | 121 } |
al@932 | 122 { |
al@932 | 123 s("wine", "wine-rt"); |
al@932 | 124 s("samba", "samba-pam"); |
al@932 | 125 s("mesa", "mesa-wayland"); |
al@932 | 126 s("mysql", "mariadb"); |
al@932 | 127 s("perl", "perl-thread"); |
al@932 | 128 s("xorg-server", "xorg-server-light"); |
al@932 | 129 s("cairo", "cairo-gl"); |
al@932 | 130 s("freetype", "freetype-infinality"); |
al@932 | 131 s("freetype", "freetype-without-harfbuzz"); |
al@932 | 132 s("harfbuzz", "harfbuzz-icu"); |
al@932 | 133 s("openbox", "openbox-imlib2"); |
al@932 | 134 s("gcc-lib-base", "gcc49-lib-base"); # also gcc54-lib-base and gcc61-lib-base |
al@932 | 135 s("polkit", "polkit-pam"); |
al@932 | 136 s("libgudev", "eudev"); # also systemd |
al@932 | 137 s("xz-dev", "liblzma-dev"); |
al@932 | 138 |
al@932 | 139 if (! index($0, "glibc-base") && |
al@932 | 140 ! index($0, "gcc-lib-base") && |
al@932 | 141 ! index($0, "glibc-dev") && |
al@932 | 142 $0 != "gcc") |
al@932 | 143 print gensub(" ", "|", "g"); |
al@932 | 144 }';; |
al@932 | 145 esac >>$tmptmp |
al@932 | 146 } |
al@932 | 147 |
al@932 | 148 |
al@932 | 149 # Search for item $1 in db $2 |
al@932 | 150 |
al@932 | 151 indb() { |
al@932 | 152 local res="$(awk -vi="$1" ' |
al@932 | 153 { |
al@932 | 154 if ($1 == i) |
al@932 | 155 { print $2; found = 1; } |
al@932 | 156 } |
al@932 | 157 END { |
al@932 | 158 if (found != 1) { |
al@932 | 159 if (index(i, "statically linked")) |
al@932 | 160 print gensub(" ", "_", "g", i); |
al@932 | 161 else |
al@932 | 162 printf("[%s]\n", i); |
al@932 | 163 } |
al@932 | 164 } |
al@932 | 165 ' $2 | tr '\n' ' ')" |
al@932 | 166 outpkg "${res% }" |
al@932 | 167 } |
al@932 | 168 |
al@932 | 169 |
al@932 | 170 # Like `ldd` function but returns packages names where dependency exists. |
al@932 | 171 # Also can process some development files |
al@932 | 172 |
al@932 | 173 tp_ldd() { |
al@932 | 174 unset IFS |
al@932 | 175 tmptmp=$(mktemp) |
al@932 | 176 |
al@932 | 177 case $1 in |
al@932 | 178 *.la) |
al@932 | 179 # found dependencies in the *.la files |
al@932 | 180 libs=$(. $1; echo $dependency_libs) |
al@932 | 181 for i in $libs; do |
al@932 | 182 case $i in |
al@932 | 183 -l*) indb "lib${i#-l}.so" $db_so;; # FIXME: I'm not sure it's about a *.so, but *.a often absent |
al@932 | 184 *.la) indb "$i" $db_la;; |
al@932 | 185 esac |
al@932 | 186 done |
al@932 | 187 ;; |
al@932 | 188 *.pc) |
al@932 | 189 # found dependencies in the *.pc files |
al@932 | 190 # Syntax examples: |
al@932 | 191 # Requires: glib-2.0 gobject-2.0 |
al@932 | 192 # Requires.private: gmodule-no-export-2.0 |
al@932 | 193 # Requires: gobject-2.0,gio-2.0 |
al@938 | 194 # Requires.private: nspr >= 4.9.2 |
al@932 | 195 pcs=$(grep '^Requires' $1 | cut -d: -f2 | tr ',' ' ' | tr '\n' ' ') |
al@932 | 196 for i in $pcs; do |
al@938 | 197 isitlib=$(echo $i | tr -d '<=>0-9.') |
al@938 | 198 # if it contains only comparisons, numbers, dot - it is not lib, skip |
al@938 | 199 [ -n "$isitlib" ] || continue |
al@932 | 200 indb "$i.pc" $db_pc |
al@932 | 201 done |
al@932 | 202 # Syntax examples: |
al@932 | 203 # Libs: -L${libdir} -lgio-2.0 |
al@932 | 204 # Libs.private: -lz -lresolv |
al@932 | 205 libs=$(grep '^Libs' $1 | cut -d: -f2 | tr '\n' ' ') |
al@932 | 206 for i in $libs; do |
al@932 | 207 case $i in |
al@932 | 208 -l*) indb "lib${i#-l}.so" $db_so;; |
al@932 | 209 esac |
al@932 | 210 done |
al@932 | 211 ;; |
al@932 | 212 */lib/modules/*) |
al@932 | 213 echo 'linux' |
al@932 | 214 ;; |
al@932 | 215 *.pl|*.pm) |
al@932 | 216 echo 'perl' |
al@932 | 217 ;; |
al@932 | 218 *.py) |
al@932 | 219 echo 'python' |
al@932 | 220 ;; |
al@932 | 221 *) |
al@932 | 222 LD_PRELOAD= LD_TRACE_LOADED_OBJECTS=1 /lib/ld-linux* "$1" 2>/dev/null | \ |
al@932 | 223 sed 's| =>.*||; s| (.*||; s|\t||' | \ |
al@932 | 224 while read i; do |
al@932 | 225 indb "$i" $db_so |
al@932 | 226 done |
al@932 | 227 ;; |
al@932 | 228 esac |
al@932 | 229 |
al@932 | 230 sort -u $tmptmp |
al@932 | 231 rm $tmptmp |
al@932 | 232 } |
al@932 | 233 |
al@932 | 234 |
al@932 | 235 # Return all the names of packages bundled in this receipt |
al@932 | 236 |
al@932 | 237 all_names() { |
al@932 | 238 local split=" $SPLIT " |
al@932 | 239 if [ "${split/ $PACKAGE /}" != "$split" ]; then |
al@932 | 240 # $PACKAGE included somewhere in $SPLIT (probably in the end). |
al@932 | 241 # We should build packages in the order defined in the $SPLIT. |
al@932 | 242 echo $SPLIT |
al@932 | 243 else |
al@932 | 244 # We'll build the $PACKAGE, then all defined in the $SPLIT. |
al@932 | 245 echo $PACKAGE $SPLIT |
al@932 | 246 fi |
al@932 | 247 } |
al@932 | 248 |
al@932 | 249 |
al@932 | 250 |
al@932 | 251 |
al@932 | 252 unset IFS |
al@932 | 253 . $WOK/$1/receipt |
al@932 | 254 |
al@932 | 255 for pkg in $(all_names); do |
al@932 | 256 title 'Dependencies for "%s"' "$pkg" |
al@932 | 257 IFS=$'\n' |
al@932 | 258 while read file; do |
al@932 | 259 tp_ldd "$WOK/$1/taz/$pkg-$VERSION/fs$file" |
al@932 | 260 done < $WOK/$1/taz/$pkg-$VERSION/files.list | sort -u | grep -v "^$pkg$" |
al@932 | 261 done |
al@932 | 262 |
al@932 | 263 newline |