tazwok rev 553 4.3

Speed optimization of md5sum & build-depends scan core loops.
author Antoine Bodin <gokhlayeh@slitaz.org>
date Tue Jan 03 23:34:29 2012 +0100 (2012-01-03)
parents 176f3c91b68a
children 50581bf79fd4
files tazwok
line diff
     1.1 --- a/tazwok	Sat Dec 10 03:42:32 2011 +0100
     1.2 +++ b/tazwok	Tue Jan 03 23:34:29 2012 +0100
     1.3 @@ -377,7 +377,7 @@
     1.4  	# it with new build_depends to know what to install and what to
     1.5  	# what to remove.
     1.6  	plan_remove=" $MISSING_PACKAGE $remove_later "
     1.7 -	[ ! "${plan_remove// }" ] && unset plan_remove
     1.8 +	[ ! "`echo "$plan_remove" | tr -d ' '`" ] && unset plan_remove
     1.9  	unset MISSING_PACKAGE remove_later
    1.10  	rwanted=$(look_for_rwanted)
    1.11  
    1.12 @@ -390,13 +390,15 @@
    1.13  		if [ ! -d "$INSTALLED/$pkg" ] ; then
    1.14  			MISSING_PACKAGE="$MISSING_PACKAGE $pkg"
    1.15  		fi
    1.16 -		if [ "$plan_remove" != "${plan_remove/ $pkg }" ]; then
    1.17 +		if echo "$plan_remove" \
    1.18 +			| fgrep -q " $pkg "
    1.19 +		then
    1.20  			plan_remove="${plan_remove/ $pkg / }"
    1.21  			remove_later="$remove_later $pkg"
    1.22  		fi
    1.23  		grep -q ^$pkg$ $broken && broken_pkg="$broken_pkg$pkg "
    1.24  	done
    1.25 -	
    1.26 +
    1.27  	# Don't cook if a depend is broken.
    1.28  	if [ "$broken_pkg" ]; then
    1.29  		MISSING_PACKAGE=$plan_remove
    1.30 @@ -1077,18 +1079,33 @@
    1.31  	return 0
    1.32  }
    1.33  
    1.34 +get_cook_files()
    1.35 +{
    1.36 +	if [ -d stuff ]; then
    1.37 +		cook_files=`find receipt description.txt stuff 2>/dev/null`
    1.38 +	elif [ -f description.txt ]; then
    1.39 +		cook_files="receipt
    1.40 +description.txt"
    1.41 +	else
    1.42 +		cook_files="receipt"
    1.43 +	fi
    1.44 +}
    1.45 +
    1.46  gen_cookmd5()
    1.47  {
    1.48  	# md5sum of cooking stuff make tazwok able to check for changes
    1.49  	# without hg.
    1.50  	cd $WOK/$PACKAGE
    1.51 -	md5sum receipt > md5
    1.52 -	[ -f description.txt ] && md5sum description.txt >> md5
    1.53 -	if [ -d stuff ]; then
    1.54 -		find stuff | while read file; do
    1.55 -			md5sum $file >> md5
    1.56 -		done
    1.57 +	local IFS="
    1.58 +"
    1.59 +
    1.60 +	if [ ! "$cook_files" ]; then
    1.61 +		get_cook_files
    1.62  	fi
    1.63 +
    1.64 +	md5sum $cook_files > md5
    1.65 +
    1.66 +	unset cook_files
    1.67  }
    1.68  
    1.69  set_pkg_broken()
    1.70 @@ -1189,12 +1206,12 @@
    1.71  		cp description.txt taz/$PACKAGE-$VERSION
    1.72  		report end-step
    1.73  	fi
    1.74 -	
    1.75 +
    1.76  	# Generate md5 of cooking stuff to look for commit later.
    1.77  	gen_cookmd5
    1.78  	echo -e "\n# md5sum of cooking stuff :" >> taz/$PACKAGE-$VERSION/receipt
    1.79  	cat md5 | sed 's/^/# /' >> taz/$PACKAGE-$VERSION/receipt
    1.80 -	
    1.81 +
    1.82  	# Create the files.list by redirecting find output.
    1.83  	report step "Creating the list of files"
    1.84  	cd taz/$PACKAGE-$VERSION
    1.85 @@ -2011,6 +2028,73 @@
    1.86  	done
    1.87  }
    1.88  
    1.89 +# We use md5 of cooking stuff in the packaged receipt to check
    1.90 +# commit. We look consecutively in 3 different locations :
    1.91 +# - in the wok/PACKAGE/taz/* folder
    1.92 +# - in the receipt in the package in incoming repository
    1.93 +# - in the receipt in the package in packages repository
    1.94 +# If md5sums match, there's no commit.
    1.95 +check_for_commit_using_md5sum()
    1.96 +{
    1.97 +	if [ ! -f $WOK/$PACKAGE/md5 ]; then
    1.98 +		sed -n '/# md5sum of cooking stuff :/,$p' receipt | \
    1.99 +			sed -e 1d -e 's/^# //' > $WOK/$PACKAGE/md5
   1.100 +		cd $WOK/$PACKAGE
   1.101 +	fi
   1.102 +
   1.103 +	if [ "$1" ]; then
   1.104 +		check_for_commit_using_time && return
   1.105 +	fi
   1.106 +
   1.107 +	if [ -s md5 ]; then					
   1.108 +		if md5sum -cs md5; then
   1.109 +		
   1.110 +		# If md5sum check if ok, check for new/missing files in
   1.111 +		# cooking stuff.
   1.112 +			for file in $([ -f receipt ] && echo receipt; \
   1.113 +			[ -f description.txt ] && echo description.txt; \
   1.114 +			[ -d stuff ] && find stuff); do
   1.115 +				if ! fgrep -q "  $file" md5; then
   1.116 +					set_commited
   1.117 +				fi
   1.118 +			done
   1.119 +		else
   1.120 +			set_commited
   1.121 +		fi
   1.122 +	else
   1.123 +		set_commited
   1.124 +	fi
   1.125 +}
   1.126 +
   1.127 +set_commited()
   1.128 +{
   1.129 +	! grep -q ^$PACKAGE$ $PACKAGES_REPOSITORY/commit &&
   1.130 +	echo $PACKAGE >> $PACKAGES_REPOSITORY/commit
   1.131 +	gen_cookmd5
   1.132 +	update_dep_db
   1.133 +}
   1.134 +
   1.135 +check_for_commit_using_time()
   1.136 +{
   1.137 +	local count=0 file cook_files
   1.138 +	
   1.139 +	get_cook_files
   1.140 +	
   1.141 +	for file in $cook_files; do
   1.142 +		if [ $file -nt md5 ]; then
   1.143 +			return 1
   1.144 +		else
   1.145 +			count=$((count+1))
   1.146 +		fi
   1.147 +	done
   1.148 +	
   1.149 +	if [ "$cook_files" ] \
   1.150 +		&& [ ! "$count md5" = "`wc -l md5`" ]
   1.151 +	then
   1.152 +		return 1
   1.153 +	fi
   1.154 +}
   1.155 +
   1.156  check_for_commit()
   1.157  {
   1.158  	if ! check_for_pkg_in_wok; then
   1.159 @@ -2021,52 +2105,15 @@
   1.160  		RECEIPT=$WOK/$PACKAGE/receipt
   1.161  		source_receipt
   1.162  
   1.163 -		# We use md5 of cooking stuff in the packaged receipt to check
   1.164 -		# commit. We look consecutively in 3 different locations :
   1.165 -		# - in the wok/PACKAGE/taz/* folder
   1.166 -		# - in the receipt in the package in incoming repository
   1.167 -		# - in the receipt in the package in packages repository
   1.168 -		# If md5sums match, there's no commit.
   1.169 -		check_for_commit_using_md5sum()
   1.170 -		{
   1.171 -			if [ ! -f $WOK/$PACKAGE/md5 ]; then
   1.172 -				sed -n '/# md5sum of cooking stuff :/,$p' receipt | \
   1.173 -					sed -e 1d -e 's/^# //' > $WOK/$PACKAGE/md5
   1.174 -				cd $WOK/$PACKAGE
   1.175 -			fi
   1.176 -		
   1.177 -			if [ -s md5 ]; then					
   1.178 -				if md5sum -cs md5; then
   1.179 -		
   1.180 -				# If md5sum check if ok, check for new/missing files in
   1.181 -				# cooking stuff.
   1.182 -				for file in $([ -f receipt ] && echo receipt; \
   1.183 -						[ -f description.txt ] && echo description.txt; \
   1.184 -						[ -d stuff ] && find stuff); do
   1.185 -						if ! fgrep -q "  $file" md5; then
   1.186 -							set_commited
   1.187 -						fi
   1.188 -					done
   1.189 -				else
   1.190 -					set_commited
   1.191 -				fi
   1.192 -			else
   1.193 -				set_commited
   1.194 -			fi
   1.195 -		}
   1.196 -		set_commited()
   1.197 -		{
   1.198 -			grep -q ^$PACKAGE$ $commit || echo $PACKAGE >> $commit
   1.199 -			gen_cookmd5
   1.200 -			update_dep_db
   1.201 -		}
   1.202 -		taz_dir=$(echo $WOK/$PACKAGE/taz/$PACKAGE-* | fgrep -v '*')
   1.203  		if [ -f $WOK/$PACKAGE/md5 ]; then
   1.204  			cd $WOK/$PACKAGE
   1.205 -			check_for_commit_using_md5sum
   1.206 -		elif [ "$taz_dir" ]; then
   1.207 -			cd $taz_dir
   1.208 -			check_for_commit_using_md5sum
   1.209 +			check_for_commit_using_md5sum or time
   1.210 +		elif [ -d "$WOK/$PACKAGE/taz" ] \
   1.211 +			&& taz_dir=$(echo $WOK/$PACKAGE/taz/$PACKAGE-* | fgrep -v '*') \
   1.212 +			&& [ -d "$taz_dir" ]
   1.213 +		then
   1.214 +			cd "$taz_dir"
   1.215 +			check_for_commit_using_md5sum or time
   1.216  		else
   1.217  			pkg=$(echo $INCOMING_REPOSITORY/$PACKAGE-$VERSION*.tazpkg | fgrep -v '*')
   1.218  			[ "$pkg" ] || pkg=$(echo $PACKAGES_REPOSITORY/$PACKAGE-$VERSION*.tazpkg | fgrep -v '*')
   1.219 @@ -2080,7 +2127,6 @@
   1.220  		fi
   1.221  	[ "$forced" ] || echo $PACKAGE >> $tmp/checked
   1.222  	done
   1.223 -	return
   1.224  }
   1.225  
   1.226  gen_cook_list()