tazwok rev 553 4.3
Speed optimization of md5sum & build-depends scan core loops.
author | Antoine Bodin <gokhlayeh@slitaz.org> |
---|---|
date | Tue Jan 03 23:34:29 2012 +0100 (2012-01-03) |
parents | 176f3c91b68a |
children | 50581bf79fd4 |
files | tazwok |
line diff
1.1 --- a/tazwok Sat Dec 10 03:42:32 2011 +0100 1.2 +++ b/tazwok Tue Jan 03 23:34:29 2012 +0100 1.3 @@ -377,7 +377,7 @@ 1.4 # it with new build_depends to know what to install and what to 1.5 # what to remove. 1.6 plan_remove=" $MISSING_PACKAGE $remove_later " 1.7 - [ ! "${plan_remove// }" ] && unset plan_remove 1.8 + [ ! "`echo "$plan_remove" | tr -d ' '`" ] && unset plan_remove 1.9 unset MISSING_PACKAGE remove_later 1.10 rwanted=$(look_for_rwanted) 1.11 1.12 @@ -390,13 +390,15 @@ 1.13 if [ ! -d "$INSTALLED/$pkg" ] ; then 1.14 MISSING_PACKAGE="$MISSING_PACKAGE $pkg" 1.15 fi 1.16 - if [ "$plan_remove" != "${plan_remove/ $pkg }" ]; then 1.17 + if echo "$plan_remove" \ 1.18 + | fgrep -q " $pkg " 1.19 + then 1.20 plan_remove="${plan_remove/ $pkg / }" 1.21 remove_later="$remove_later $pkg" 1.22 fi 1.23 grep -q ^$pkg$ $broken && broken_pkg="$broken_pkg$pkg " 1.24 done 1.25 - 1.26 + 1.27 # Don't cook if a depend is broken. 1.28 if [ "$broken_pkg" ]; then 1.29 MISSING_PACKAGE=$plan_remove 1.30 @@ -1077,18 +1079,33 @@ 1.31 return 0 1.32 } 1.33 1.34 +get_cook_files() 1.35 +{ 1.36 + if [ -d stuff ]; then 1.37 + cook_files=`find receipt description.txt stuff 2>/dev/null` 1.38 + elif [ -f description.txt ]; then 1.39 + cook_files="receipt 1.40 +description.txt" 1.41 + else 1.42 + cook_files="receipt" 1.43 + fi 1.44 +} 1.45 + 1.46 gen_cookmd5() 1.47 { 1.48 # md5sum of cooking stuff make tazwok able to check for changes 1.49 # without hg. 1.50 cd $WOK/$PACKAGE 1.51 - md5sum receipt > md5 1.52 - [ -f description.txt ] && md5sum description.txt >> md5 1.53 - if [ -d stuff ]; then 1.54 - find stuff | while read file; do 1.55 - md5sum $file >> md5 1.56 - done 1.57 + local IFS=" 1.58 +" 1.59 + 1.60 + if [ ! "$cook_files" ]; then 1.61 + get_cook_files 1.62 fi 1.63 + 1.64 + md5sum $cook_files > md5 1.65 + 1.66 + unset cook_files 1.67 } 1.68 1.69 set_pkg_broken() 1.70 @@ -1189,12 +1206,12 @@ 1.71 cp description.txt taz/$PACKAGE-$VERSION 1.72 report end-step 1.73 fi 1.74 - 1.75 + 1.76 # Generate md5 of cooking stuff to look for commit later. 1.77 gen_cookmd5 1.78 echo -e "\n# md5sum of cooking stuff :" >> taz/$PACKAGE-$VERSION/receipt 1.79 cat md5 | sed 's/^/# /' >> taz/$PACKAGE-$VERSION/receipt 1.80 - 1.81 + 1.82 # Create the files.list by redirecting find output. 1.83 report step "Creating the list of files" 1.84 cd taz/$PACKAGE-$VERSION 1.85 @@ -2011,6 +2028,73 @@ 1.86 done 1.87 } 1.88 1.89 +# We use md5 of cooking stuff in the packaged receipt to check 1.90 +# commit. We look consecutively in 3 different locations : 1.91 +# - in the wok/PACKAGE/taz/* folder 1.92 +# - in the receipt in the package in incoming repository 1.93 +# - in the receipt in the package in packages repository 1.94 +# If md5sums match, there's no commit. 1.95 +check_for_commit_using_md5sum() 1.96 +{ 1.97 + if [ ! -f $WOK/$PACKAGE/md5 ]; then 1.98 + sed -n '/# md5sum of cooking stuff :/,$p' receipt | \ 1.99 + sed -e 1d -e 's/^# //' > $WOK/$PACKAGE/md5 1.100 + cd $WOK/$PACKAGE 1.101 + fi 1.102 + 1.103 + if [ "$1" ]; then 1.104 + check_for_commit_using_time && return 1.105 + fi 1.106 + 1.107 + if [ -s md5 ]; then 1.108 + if md5sum -cs md5; then 1.109 + 1.110 + # If md5sum check if ok, check for new/missing files in 1.111 + # cooking stuff. 1.112 + for file in $([ -f receipt ] && echo receipt; \ 1.113 + [ -f description.txt ] && echo description.txt; \ 1.114 + [ -d stuff ] && find stuff); do 1.115 + if ! fgrep -q " $file" md5; then 1.116 + set_commited 1.117 + fi 1.118 + done 1.119 + else 1.120 + set_commited 1.121 + fi 1.122 + else 1.123 + set_commited 1.124 + fi 1.125 +} 1.126 + 1.127 +set_commited() 1.128 +{ 1.129 + ! grep -q ^$PACKAGE$ $PACKAGES_REPOSITORY/commit && 1.130 + echo $PACKAGE >> $PACKAGES_REPOSITORY/commit 1.131 + gen_cookmd5 1.132 + update_dep_db 1.133 +} 1.134 + 1.135 +check_for_commit_using_time() 1.136 +{ 1.137 + local count=0 file cook_files 1.138 + 1.139 + get_cook_files 1.140 + 1.141 + for file in $cook_files; do 1.142 + if [ $file -nt md5 ]; then 1.143 + return 1 1.144 + else 1.145 + count=$((count+1)) 1.146 + fi 1.147 + done 1.148 + 1.149 + if [ "$cook_files" ] \ 1.150 + && [ ! "$count md5" = "`wc -l md5`" ] 1.151 + then 1.152 + return 1 1.153 + fi 1.154 +} 1.155 + 1.156 check_for_commit() 1.157 { 1.158 if ! check_for_pkg_in_wok; then 1.159 @@ -2021,52 +2105,15 @@ 1.160 RECEIPT=$WOK/$PACKAGE/receipt 1.161 source_receipt 1.162 1.163 - # We use md5 of cooking stuff in the packaged receipt to check 1.164 - # commit. We look consecutively in 3 different locations : 1.165 - # - in the wok/PACKAGE/taz/* folder 1.166 - # - in the receipt in the package in incoming repository 1.167 - # - in the receipt in the package in packages repository 1.168 - # If md5sums match, there's no commit. 1.169 - check_for_commit_using_md5sum() 1.170 - { 1.171 - if [ ! -f $WOK/$PACKAGE/md5 ]; then 1.172 - sed -n '/# md5sum of cooking stuff :/,$p' receipt | \ 1.173 - sed -e 1d -e 's/^# //' > $WOK/$PACKAGE/md5 1.174 - cd $WOK/$PACKAGE 1.175 - fi 1.176 - 1.177 - if [ -s md5 ]; then 1.178 - if md5sum -cs md5; then 1.179 - 1.180 - # If md5sum check if ok, check for new/missing files in 1.181 - # cooking stuff. 1.182 - for file in $([ -f receipt ] && echo receipt; \ 1.183 - [ -f description.txt ] && echo description.txt; \ 1.184 - [ -d stuff ] && find stuff); do 1.185 - if ! fgrep -q " $file" md5; then 1.186 - set_commited 1.187 - fi 1.188 - done 1.189 - else 1.190 - set_commited 1.191 - fi 1.192 - else 1.193 - set_commited 1.194 - fi 1.195 - } 1.196 - set_commited() 1.197 - { 1.198 - grep -q ^$PACKAGE$ $commit || echo $PACKAGE >> $commit 1.199 - gen_cookmd5 1.200 - update_dep_db 1.201 - } 1.202 - taz_dir=$(echo $WOK/$PACKAGE/taz/$PACKAGE-* | fgrep -v '*') 1.203 if [ -f $WOK/$PACKAGE/md5 ]; then 1.204 cd $WOK/$PACKAGE 1.205 - check_for_commit_using_md5sum 1.206 - elif [ "$taz_dir" ]; then 1.207 - cd $taz_dir 1.208 - check_for_commit_using_md5sum 1.209 + check_for_commit_using_md5sum or time 1.210 + elif [ -d "$WOK/$PACKAGE/taz" ] \ 1.211 + && taz_dir=$(echo $WOK/$PACKAGE/taz/$PACKAGE-* | fgrep -v '*') \ 1.212 + && [ -d "$taz_dir" ] 1.213 + then 1.214 + cd "$taz_dir" 1.215 + check_for_commit_using_md5sum or time 1.216 else 1.217 pkg=$(echo $INCOMING_REPOSITORY/$PACKAGE-$VERSION*.tazpkg | fgrep -v '*') 1.218 [ "$pkg" ] || pkg=$(echo $PACKAGES_REPOSITORY/$PACKAGE-$VERSION*.tazpkg | fgrep -v '*') 1.219 @@ -2080,7 +2127,6 @@ 1.220 fi 1.221 [ "$forced" ] || echo $PACKAGE >> $tmp/checked 1.222 done 1.223 - return 1.224 } 1.225 1.226 gen_cook_list()