cookutils diff cooker @ rev 745
cook pkgdb: small fix for repeated action
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon Jun 22 12:29:02 2015 +0300 (2015-06-22) |
| parents | 38b417993ec2 |
| children | 75fe6eb7e7bd |
line diff
1.1 --- a/cooker Fri Oct 10 17:56:59 2014 +0200 1.2 +++ b/cooker Mon Jun 22 12:29:02 2015 +0300 1.3 @@ -6,6 +6,7 @@ 1.4 # here since it's not an end user tool and it's not useful. All devs should 1.5 # at least understand basic English. 1.6 # 1.7 + 1.8 . /usr/lib/slitaz/libcook.sh 1.9 1.10 # Set pkg name and use same wok as cook. 1.11 @@ -20,7 +21,7 @@ 1.12 # 1.13 1.14 usage() { 1.15 - cat << EOT 1.16 + cat <<EOT 1.17 1.18 Usage: cooker [command] [pkg|list|note|hours] 1.19 1.20 @@ -44,19 +45,25 @@ 1.21 exit 0 1.22 } 1.23 1.24 + 1.25 # Some messages occur in activity but log verbose output when checking for commits 1.26 # into a log file. 1.27 + 1.28 log_commits() { 1.29 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log 1.30 } 1.31 1.32 + 1.33 # Clean up before exit when check and cook commits finish. 1.34 + 1.35 clean_exit() { 1.36 - rm -f $command && touch $command 1.37 + rm -f $command; touch $command 1.38 rm -f $pidfile 1.39 } 1.40 1.41 + 1.42 # Summary for commits. 1.43 + 1.44 commits_summary() { 1.45 msg="from revision $cur to $new" 1.46 [ "$new" == "$cur" ] && msg="revision $new" 1.47 @@ -66,21 +73,21 @@ 1.48 separator 1.49 echo "Hg wok revision : $cur" 1.50 echo "Pulled revision : $new" 1.51 - echo "Check date : $(date '+%Y-%m-%d %H:%M:%S')" 1.52 + echo "Check date : $(date '+%F %T')" 1.53 } 1.54 1.55 + 1.56 # Scan packages build deps and fill up cookorder list. 1.57 + 1.58 cook_order_scan() { 1.59 rm -f $cookorder 1.60 touch $cookorder 1.61 - for pkg in $(cat $cooklist) 1.62 - do 1.63 + for pkg in $(cat $cooklist); do 1.64 unset WANTED BUILD_DEPENDS 1.65 . $wok/$pkg/receipt 1.66 # The :: is for web interface color. 1.67 [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS" 1.68 - for dep in $WANTED $BUILD_DEPENDS 1.69 - do 1.70 + for dep in $WANTED $BUILD_DEPENDS; do 1.71 if grep -q "^$dep$" $cooklist; then 1.72 if ! grep -q "^$dep$" $cookorder; then 1.73 echo "$dep" >> $cookorder 1.74 @@ -90,30 +97,30 @@ 1.75 done 1.76 1.77 # Append unordered packages to cookorder. 1.78 - for pkg in $(cat $cooklist) 1.79 - do 1.80 + for pkg in $(cat $cooklist); do 1.81 if ! grep -q "^$pkg$" $cookorder; then 1.82 echo "$pkg" >> $cookorder 1.83 fi 1.84 done 1.85 } 1.86 1.87 + 1.88 # Scan and rescan until the cooklist is ordered then handle WANTED. 1.89 + 1.90 cook_order() { 1.91 time=$(date +%s) 1.92 scan=0 1.93 1.94 # Keep an original cooklist so we do a diff when ordering is finished. 1.95 cp -f $cooklist $cooklist.0 1.96 - echo "cookorder" > $command 1.97 - echo -e "\nInitial Cooker order scan" 1.98 + echo 'cookorder' > $command 1.99 + echo -e '\nInitial Cooker order scan' 1.100 separator 1.101 cook_order_scan 1.102 1.103 # Diff between the cooklist and new ordered list ? So copy the last 1.104 # cookorder to cooklist and rescan it. 1.105 - while /bin/true 1.106 - do 1.107 + while /bin/true; do 1.108 diff $cooklist $cookorder > $cookorder.diff 1.109 if [ -s "$cookorder.diff" ]; then 1.110 scan=$(($scan + 1)) 1.111 @@ -132,14 +139,12 @@ 1.112 1.113 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist. 1.114 # TODO: grep the line number to get pkg position and keep it higher. 1.115 - echo -e "\nHandle WANTED package" 1.116 + echo -e '\nHandle WANTED package' 1.117 separator 1.118 - for pkg in $(cat $cooklist) 1.119 - do 1.120 + for pkg in $(cat $cooklist); do 1.121 unset WANTED 1.122 . $wok/$pkg/receipt 1.123 - for wanted in $WANTED 1.124 - do 1.125 + for wanted in $WANTED; do 1.126 echo "$pkg :: $wanted" 1.127 if grep -q ^${wanted}$ $cooklist; then 1.128 sed -i -e "/^$pkg$/"d \ 1.129 @@ -149,35 +154,41 @@ 1.130 done 1.131 1.132 # Show ordered cooklist 1.133 - echo -e "\nCooklist order" 1.134 + echo -e '\nCooklist order' 1.135 separator 1.136 cat $cooklist 1.137 separator 1.138 + 1.139 time=$(($(date +%s) - $time)) 1.140 pkgs=$(cat $cooklist | wc -l) 1.141 - echo -e "\nSummary for cookorder" 1.142 + echo -e '\nSummary for cookorder' 1.143 separator 1.144 - cat << EOT 1.145 + cat <<EOT 1.146 Ordered packages : $pkgs 1.147 Scans executed : $scan 1.148 Scan duration : ${time}s 1.149 EOT 1.150 - separator && rm -f $command 1.151 + separator 1.152 + 1.153 + rm -f $command 1.154 } 1.155 1.156 + 1.157 # Remove blocked (faster this way than grepping before). 1.158 + 1.159 strip_blocked() { 1.160 - for pkg in $(cat $blocked) 1.161 - do 1.162 + for pkg in $(cat $blocked); do 1.163 sed -i /^${pkg}$/d $cooklist 1.164 - done && sed -i /^$/d $cooklist 1.165 + done 1.166 + sed -i /^$/d $cooklist 1.167 } 1.168 1.169 + 1.170 # Use in default mode and with all cmd. 1.171 + 1.172 cook_commits() { 1.173 if [ -s "$commits" ]; then 1.174 - for pkg in $(cat $commits) 1.175 - do 1.176 + for pkg in $(cat $commits); do 1.177 echo "cook:$pkg" > $command 1.178 cook $pkg || broken 1.179 sed -i /^${pkg}$/d $commits 1.180 @@ -185,15 +196,17 @@ 1.181 fi 1.182 } 1.183 1.184 + 1.185 # Cook all packages in a cooklist. 1.186 + 1.187 cook_list() { 1.188 - for pkg in $(cat $cooklist) 1.189 - do 1.190 + for pkg in $(cat $cooklist); do 1.191 cook $pkg || broken 1.192 sed -i /^${pkg}$/d $cooklist 1.193 done 1.194 } 1.195 1.196 + 1.197 # Create a arch.$ARCH file for each package cooked for the target host 1.198 # architecture 1.199 # 1.200 @@ -202,13 +215,13 @@ 1.201 # packages to provide one full featured desktop, servers and goodies useful 1.202 # for the host system. 1.203 # 1.204 + 1.205 arch_db() { 1.206 count=0 1.207 echo "Cleaning packages DB : arch.$ARCH" 1.208 rm -f $wok/*/arch.$ARCH && cd $wok 1.209 echo "Creating $ARCH packages DB..." 1.210 - for pkg in * 1.211 - do 1.212 + for pkg in *; do 1.213 HOST_ARCH= 1.214 . $wok/$pkg/receipt 1.215 if [ -n "$HOST_ARCH" ]; then 1.216 @@ -220,7 +233,7 @@ 1.217 unset HOST_ARCH 1.218 else 1.219 # HOST_ARCH not set --> in i486 1.220 - if [ "$ARCH" == "i486" ]; then 1.221 + if [ "$ARCH" == 'i486' ]; then 1.222 count=$(($count + 1)) 1.223 echo "Adding: $pkg" 1.224 touch $pkg/arch.$ARCH 1.225 @@ -230,40 +243,49 @@ 1.226 echo "Packages for $ARCH : $count" 1.227 } 1.228 1.229 + 1.230 # 1.231 # Commands 1.232 # 1.233 + 1.234 case "$1" in 1.235 usage|help|-u|-h) 1.236 usage ;; 1.237 + 1.238 setup|-s) 1.239 # Setup the Cooker environment. 1.240 - echo -e "\nSetting up the Cooker" 1.241 + echo -e '\nSetting up the Cooker' 1.242 mkdir -p $CACHE 1.243 echo "Cooker setup using: $SLITAZ" | log 1.244 separator 1.245 - for pkg in $SETUP_PKGS mercurial rsync tazlito 1.246 - do 1.247 + for pkg in $SETUP_PKGS mercurial rsync tazlito; do 1.248 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg 1.249 done 1.250 mkdir -p $SLITAZ && cd $SLITAZ 1.251 - [ -d "${wok}-hg" ] && echo -e "Hg wok already exists.\n" && exit 1 1.252 - [ -d "$wok" ] && echo -e "Build wok already exists.\n" && exit 1 1.253 + if [ -d "${wok}-hg" ]; then 1.254 + echo -e 'Hg wok already exists.\n' 1.255 + exit 1 1.256 + fi 1.257 + if [ -d "$wok" ]; then 1.258 + echo -e 'Build wok already exists.\n' 1.259 + exit 1 1.260 + fi 1.261 1.262 # Directories and files 1.263 echo "mkdir's and touch files in: $SLITAZ" 1.264 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC 1.265 - for f in $activity $blocked $broken $commits $cooklist $command 1.266 - do 1.267 + for f in $activity $blocked $broken $commits $cooklist $command; do 1.268 touch $f 1.269 done 1.270 hg clone $WOK_URL ${wok}-hg || exit 1 1.271 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors 1.272 cp -a ${wok}-hg $wok 1.273 - separator && newline ;; 1.274 + separator; newline ;; 1.275 + 1.276 arch-db) 1.277 # Manually create arch packages DB. 1.278 arch_db ;; 1.279 + 1.280 setup-cron) 1.281 # Create cron job for the cooker. 1.282 [ "$2" ] || hours=2 1.283 @@ -275,132 +297,158 @@ 1.284 echo "0 */$hours * * * /usr/bin/cooker --output=html" >> $crontabs 1.285 killall crond 2>/dev/null && /etc/init.d/crond start 1.286 fi ;; 1.287 + 1.288 check-cron) 1.289 - [ -f "$crontabs" ] || \ 1.290 - echo "There is no $crontabs here. Use setup-cron option." && exit 1 1.291 + if [ ! -f "$crontabs" ]; then 1.292 + echo "There is no $crontabs here. Use setup-cron option." 1.293 + exit 1 1.294 + fi 1.295 fgrep /usr/bin/cooker $crontabs ;; 1.296 + 1.297 note|-n) 1.298 - # Blocked a pkg and want others to know why ? Post a note! 1.299 - note="$2" 1.300 - date=$(date "+%Y-%m-%d %H:%M") 1.301 - [ "$note" ] && echo "$date : $note" >> $cooknotes ;; 1.302 + # Blocked a pkg and want others to know why? Post a note! 1.303 + [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;; 1.304 + 1.305 notes|-ns) 1.306 # View cooknotes. 1.307 - echo -e "\nCooknotes" 1.308 + echo -e '\nCooknotes' 1.309 separator 1.310 cat $cooknotes 1.311 - separator && newline ;; 1.312 + separator; newline ;; 1.313 + 1.314 block|-b) 1.315 # Block a package. 1.316 [ "$pkg" ] && cook $pkg --block ;; 1.317 + 1.318 unblock|-ub) 1.319 # Unblock a package. 1.320 [ "$pkg" ] && cook $pkg --unblock ;; 1.321 + 1.322 reverse|-r) 1.323 # Cook all reverse dependencies for a package. This command lets us 1.324 # control the Cooker manually for commits that will cook a lot of packages. 1.325 # 1.326 - # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse" 1.327 + # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse" 1.328 # 1.329 - [ ! -d "$wok/$pkg" ] && echo -e "\nNo package $2 found.\n" && exit 0 1.330 - rm -f $cooklist && touch $cooklist && cd $wok 1.331 + if [ ! -d "$wok/$pkg" ]; then 1.332 + echo -e "\nNo package $2 found.\n" 1.333 + exit 0 1.334 + fi 1.335 + rm -f $cooklist; touch $cooklist 1.336 echo -e "\nReverse cooklist for: $pkg" 1.337 - separator && cd $wok 1.338 - for rev in * 1.339 - do 1.340 - unset WANTED DEPENDS BUILD_DEPENDS && . $wok/$rev/receipt 1.341 + separator 1.342 + 1.343 + cd $wok 1.344 + for rev in *; do 1.345 + unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt 1.346 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then 1.347 echo "$rev" | tee -a $cooklist 1.348 fi 1.349 - done && separator 1.350 + done 1.351 + separator 1.352 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n" 1.353 strip_blocked 1.354 cook_order | tee $LOGS/cookorder.log 1.355 cook_list ;; 1.356 + 1.357 pkg|-p) 1.358 # Same as 'cook pkg' but with log for web interface. 1.359 cook $pkg || broken 1.360 clean_exit ;; 1.361 + 1.362 cat|-c) 1.363 # Cook all packages of a category. 1.364 cat="$2" 1.365 - rm -f $cooklist && touch $cooklist && cd $wok 1.366 - for pkg in * 1.367 - do 1.368 - unset CATEGORY && . $pkg/receipt 1.369 + rm -f $cooklist; touch $cooklist 1.370 + 1.371 + cd $wok 1.372 + for pkg in *; do 1.373 + unset CATEGORY; . $pkg/receipt 1.374 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist 1.375 done 1.376 strip_blocked 1.377 cook_order | tee $LOGS/cookorder.log 1.378 cook_list ;; 1.379 + 1.380 flavor|-f) 1.381 # Cook all packages of a flavor. 1.382 name="$2" 1.383 - [ ! -d "$flavors/$name" ] && \ 1.384 - echo -e "\nSpecified flavor does not exist: $name\n" && exit 1 1.385 - [ -d "$flavors/.hg" ] && cd $flavors && hg pull -u 1.386 - list=$flavors/$name/packages.list 1.387 + if [ ! -d "$flavors/$name" ]; then 1.388 + echo -e "\nSpecified flavor does not exist: $name\n" 1.389 + exit 1 1.390 + fi 1.391 + if [ -d "$flavors/.hg" ]; then 1.392 + cd $flavors; hg pull -u 1.393 + fi 1.394 + list="$flavors/$name/packages.list" 1.395 cp -a $list $cooklist 1.396 strip_blocked 1.397 cook_order | tee $LOGS/cookorder.log 1.398 cook_list ;; 1.399 + 1.400 list|-l) 1.401 # Cook a list of packages given in argument. 1.402 list="$2" 1.403 - [ ! -f "$list" ] && \ 1.404 - echo -e "\nSpecified list does not exist: $list\n" && exit 1 1.405 + if [ ! -f "$list" ]; then 1.406 + echo -e "\nSpecified list does not exist: $list\n" 1.407 + exit 1 1.408 + fi 1.409 cp -a $list $cooklist 1.410 strip_blocked 1.411 cook_order | tee $LOGS/cookorder.log 1.412 cook_list ;; 1.413 + 1.414 rev|-r) 1.415 # Cook or recook a specific Hg revision. 1.416 rev="$2" 1.417 [ "$rev" ] || exit 0 1.418 + rm -f $cooklist; touch $cooklist 1.419 + 1.420 cd $wok 1.421 - rm -f $cooklist && touch $cooklist 1.422 - for pkg in $(hg log --rev=$rev --template "{files}") 1.423 - do 1.424 - echo "$pkg" | cut -d "/" -f 1 >> $cooklist 1.425 + for pkg in $(hg log --rev=$rev --template "{files}"); do 1.426 + echo "$pkg" | cut -d/ -f1 >> $cooklist 1.427 done 1.428 strip_blocked 1.429 cook_order | tee $LOGS/cookorder.log 1.430 cook_list ;; 1.431 + 1.432 all|-a) 1.433 # Try to build all unbuilt packages except blocked's. 1.434 - echo "cooker:all" > $command 1.435 - rm -f $cooklist && touch $cooklist 1.436 - newline && cd $wok 1.437 - echo "Cooker cooklist" 1.438 + echo 'cooker:all' > $command 1.439 + rm -f $cooklist; touch $cooklist 1.440 + newline 1.441 + echo 'Cooker cooklist' 1.442 separator 1.443 1.444 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt 1.445 # if it exists since extra version is added when packing the package. 1.446 - echo "Searching for all unbuilt packages" | log 1.447 - for pkg in * 1.448 - do 1.449 + echo 'Searching for all unbuilt packages' | log 1.450 + 1.451 + cd $wok 1.452 + for pkg in *; do 1.453 unset EXTRAVERSION 1.454 . $pkg/receipt 1.455 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \ 1.456 . $pkg/taz/$PACKAGE-$VERSION/receipt 1.457 - if [ ! -f "$PKGS/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg" ] 1.458 - then 1.459 - echo $pkg && echo $pkg >> $cooklist 1.460 + if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then 1.461 + echo $pkg; echo $pkg >> $cooklist 1.462 fi 1.463 done 1.464 strip_blocked 1.465 cook_order | tee $LOGS/cookorder.log 1.466 echo "Packages to cook: $(cat $cooklist | wc -l)" | log 1.467 cook_list ;; 1.468 + 1.469 *) 1.470 # Default is to cook all commits if not yet running. 1.471 - [ "$1" ] && usage 1.472 + [ -n "$1" ] && usage 1.473 cooklist=$commits 1.474 if [ -f "$pidfile" ]; then 1.475 pid=$(cat $pidfile) 1.476 if [ -s /proc/$pid/status ]; then 1.477 - gettext -e "\nStill cooking latest commits with pid:" 1.478 - echo -e " $pid\n" && exit 0 1.479 + echo -e "\nStill cooking latest commits with pid:" 1.480 + echo -e " $pid\n" 1.481 + exit 0 1.482 fi 1.483 rm -f "$pidfile" 1.484 fi 1.485 @@ -408,7 +456,7 @@ 1.486 # Start and get a PID file. 1.487 rm -f $LOGS/commits.log 1.488 newline 1.489 - echo "Checking for commits" | log_commits 1.490 + echo 'Checking for commits' | log_commits 1.491 separator | tee -a $LOGS/commits.log 1.492 1.493 echo $$ > $pidfile 1.494 @@ -416,7 +464,7 @@ 1.495 rm -f $pidfile $command && exit 1' INT TERM 1.496 1.497 echo "Cooker PID : $$" | log_commits 1.498 - echo "Cooker date : $(date '+%Y-%m-%d %H:%M:%S')" | log_commits 1.499 + echo "Cooker date : $(date '+%F %T')" | log_commits 1.500 1.501 # Get revisions. Here we have 2 echoes since we want a msg on screen, 1.502 # in commits log and activity DB without a space before. 1.503 @@ -424,8 +472,8 @@ 1.504 cur=$(hg head --template '{rev}\n') 1.505 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits 1.506 echo "Updating wok: ${wok}-hg" | log 1.507 - echo "hg:pull" > $command 1.508 - cd ${wok}-hg && hg pull -u | log_commits 1.509 + echo 'hg:pull' > $command 1.510 + cd $wok-hg; hg pull -u | log_commits 1.511 new=$(hg head --template '{rev}\n') 1.512 # Store last rev to be used by CGI so it doesn't need to call hg head 1.513 # on each load. 1.514 @@ -435,25 +483,24 @@ 1.515 # files as before. 1.516 if [ "$new" -gt "$cur" ]; then 1.517 echo "Changes found from: $cur to $new" | log 1.518 - echo "Syncing build wok with Hg wok..." | log_commits 1.519 - rsync -r -t -c -l -u -v -D -E ${wok}-hg/ $wok/ | \ 1.520 + echo 'Syncing build wok with Hg wok...' | log_commits 1.521 + rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \ 1.522 sed '/^$/'d | log_commits 1.523 else 1.524 echo "No revision changes: $cur vs $new" | log 1.525 separator | log_commits 1.526 - clean_exit && newline && exit 0 1.527 + clean_exit; newline 1.528 + exit 0 1.529 fi 1.530 1.531 # Get and display modifications. 1.532 - cd ${wok}-hg 1.533 + cd $wok-hg 1.534 commits_summary | log_commits 1.535 cur=$(($cur + 1)) 1.536 - rm -f $commits.tmp && touch $commits.tmp 1.537 - for rev in $(seq $cur $new) 1.538 - do 1.539 - for file in $(hg log --rev=$rev --template "{files}") 1.540 - do 1.541 - pkg=$(echo $file | cut -d "/" -f 1) 1.542 + rm -f $commits.tmp; touch $commits.tmp 1.543 + for rev in $(seq $cur $new); do 1.544 + for file in $(hg log --rev=$rev --template "{files}"); do 1.545 + pkg=$(echo $file | cut -d/ -f1) 1.546 desc=$(hg log --rev=$rev --template "{desc}" $file) 1.547 echo "Committed package : $pkg - $desc" | log_commits 1.548 echo $pkg >> $commits.tmp 1.549 @@ -463,8 +510,7 @@ 1.550 # We may have deleted packages and files in stuff/. Remove it and 1.551 # clean DB as well as log file. 1.552 cd $wok 1.553 - for pkg in * 1.554 - do 1.555 + for pkg in *; do 1.556 if [ ! -d "${wok}-hg/$pkg" ]; then 1.557 echo "Removing package: $pkg" | log_commits 1.558 . $wok/$pkg/receipt 1.559 @@ -475,13 +521,12 @@ 1.560 1.561 # Keep previous commit and discard duplicate lines 1.562 cat $commits $commits.tmp | sed /"^$"/d > $commits.new 1.563 - uniq $commits.new > $commits && rm $commits.* 1.564 + uniq $commits.new > $commits; rm $commits.* 1.565 1.566 # Handle cross compilation. Create arch packages DB and remove pkgs 1.567 # not cooked for this arch from the commits list. 1.568 arch_db 1.569 - for pkg in $(cat $commits) 1.570 - do 1.571 + for pkg in $(cat $commits); do 1.572 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then 1.573 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \ 1.574 log_commits 1.575 @@ -490,7 +535,7 @@ 1.576 echo "Cooker arch : $ARCH" | log_commits 1.577 fi 1.578 done 1.579 - 1.580 + 1.581 # Stats 1.582 pkgs=$(cat $commits | wc -l) 1.583 echo "Packages to cook: $pkgs" | log