wok view tazbb/stuff/tazbb @ rev 4185
tazbb: fix sort cook list
| author | Pascal Bellard <pascal.bellard@slitaz.org> |
|---|---|
| date | Wed Sep 23 08:28:30 2009 +0200 (2009-09-23) |
| parents | 81109f81ca28 |
| children | a44ae688ff17 |
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
39 usage()
40 {
41 echo -e "\nSliTaz developers and build host tool\n
42 \033[1mUsage: \033[0m `basename $0` [command] [--option]
43 \033[1mCommands: \033[0m\n
44 usage Print this short usage and command list.
45 list-pkgs List last cooked packages with date.
46 report Run in report mode and dont cook anything [--verbose].
47 cook Cook, install and log a single package build.
48 cook-all Cook all missing, modified or unbuilt packages.
49 cook-commit Cook all packages affected by a commit in the last update.
50 test-pkgs Execute a test suite on all packages [--verbose].
51 [un]block Block or unblock a package to skip or enable building.
52 mail Send mail to package maintainer with tazbbmail.
53 clean-up Remove old packages [--verbose|--dry-run].
54 clean-log Remove all generated build log files.\n"
55 }
57 status()
58 {
59 local CHECK=$?
60 echo -en "\033[70G"
61 if [ $CHECK = 0 ]; then
62 echo "Done"
63 else
64 echo "Failed"
65 fi
66 return $CHECK
67 }
69 top_summary()
70 {
71 cat > $DB_DIR/summary << _EOT_
72 Update : `date`
73 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
74 _EOT_
75 }
77 packages_summary()
78 {
79 if ! grep -q "^Packages" $DB_DIR/summary; then
80 cat >> $DB_DIR/summary << _EOT_
81 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
82 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
83 _EOT_
84 fi
85 }
87 packages_summary_update()
88 {
89 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
90 $DB_DIR/summary
91 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
92 $DB_DIR/summary
93 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
94 $DB_DIR/summary
95 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
96 $DB_DIR/summary
97 }
99 list_packages()
100 {
101 cd $PACKAGES_REPOSITORY
102 ls -1t *.tazpkg | head -20 | \
103 while read file
104 do
105 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
106 echo " $file"
107 done
108 }
110 show_report()
111 {
112 echo "Cooklist"
113 echo "================================================================================"
114 cat $DB_DIR/cooklist && echo ""
115 echo "Blocked"
116 echo "================================================================================"
117 cat $DB_DIR/blocked && echo ""
118 echo ""
119 }
121 # URL encoding
122 escape()
123 {
124 echo $1 | sed -e 's/+/%2B/g' -e 's|/|%2F|g' -e 's/:/%3A/g'
125 }
127 update_wok()
128 {
129 echo ""
130 echo "(updating wok)" > $DB_DIR/running
131 cd $HG_WOK
132 LAST_REV=`hg head --template '{rev}\n'`
133 hg pull && hg update
134 NEW_REV=`hg head --template '{rev}\n'`
135 # Gen a new summary and link last revision for the web interface.
136 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
137 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
138 top_summary
139 # Copy Hg wok if new revision or exit to stop process since nothing
140 # have change (--forced can be used).
141 if [ "$NEW_REV" != "$LAST_REV" ]; then
142 size=`du -sh $HG_WOK | awk '{ print $1 }'`
143 echo -n "Copying Hg wok to the build wok ($size)... "
144 #rsync -r -n -t $HG_WOK/ $BUILD_WOK/
145 cp -a $HG_WOK/* $BUILD_WOK
146 cp -a $HG_WOK/.hg $BUILD_WOK
147 echo -e "Done\n"
148 else
149 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
150 if [ "$2" != "--forced" ]; then
151 echo -e "Nothing to cook...\n"
152 packages_summary
153 rm -f $LOCK_FILE && exit 0
154 fi
155 fi
156 fi
157 }
159 # Running 'tazbb report' should not cook anything and --verbose option
160 # can be used to display more messages.
161 check_wok()
162 {
163 # Clean up last results.
164 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
165 rm -f $DB_DIR/report && touch $DB_DIR/report
166 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
167 echo "Checking all files in: $HG_WOK"
168 echo "================================================================================"
169 echo "(checking wok)" > $DB_DIR/running
170 for pkg in $HG_WOK/*
171 do
172 EXTRAVERSION=""
173 WANTED=""
174 . $pkg/receipt
175 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
176 # Skip blocked packages.
177 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
178 echo "Blocked : $PACKAGE ($VERSION)" && continue
179 fi
181 # Bristuff hack until the receipt are improved...
182 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
183 if [ "$VERSION" = "bristuff" ]; then
184 . $BUILD_WOK/$PACKAGE/taz/*/receipt
185 fi
187 # First check if package exit. Package naming _must_ be in the form of:
188 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
189 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
190 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
191 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
192 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
193 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
194 echo "$PACKAGE" >> $DB_DIR/cooklist
195 fi
196 else
197 # Check if package is up-to-date.
198 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
199 for file in `find $pkg -type f`
200 do
201 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
202 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
203 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
204 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
205 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
206 echo "$PACKAGE" >> $DB_DIR/cooklist
207 fi
208 done
209 fi
210 # Now check if package is built and not already in the list.
211 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
212 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
213 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
214 echo "$PACKAGE" >> $DB_DIR/cooklist
215 fi
216 # Rebuild unbuilt packages list with link to log file. This list
217 # is also generated by cook_inslall to have real time stats.
218 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
219 echo "<a href=\"log.php?package=$(escape $PACKAGE)\">$PACKAGE</a>" \
220 >> $DB_DIR/unbuilt
221 fi
222 done
223 packages_summary
224 }
226 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
227 # can cook last changes.
228 check_commit()
229 {
230 echo "(checking commit)" > $DB_DIR/running
231 cd $HG_WOK
232 # Clean up last results.
233 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
234 # Get the name of modified packages by the revision range. +1 last
235 # commit was build by the previous build.
236 LAST_REV=$(($LAST_REV+1))
237 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
238 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
239 do
240 pkg=`echo $file | cut -d "/" -f 1`
241 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
242 . $pkg/receipt
243 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
244 echo "$PACKAGE" >> $DB_DIR/cooklist
245 fi
246 done
247 packages_summary
248 }
250 # Cook one package
251 cook_package()
252 {
253 EXTRAVERSION=""
254 DEPENDS=""
255 BUILD_DEPENDS=""
256 SOURCE=""
257 WANTED=""
258 echo "(cooking <a href=\"log.php?package=$(escape $pkg)\">$pkg</a>)" > $DB_DIR/running
259 tazwok clean $pkg
260 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
261 # Install new package (important for new shared libs). Note
262 # that tests are done separatly with 'test_packages' and should
263 # be done by tazwok.
264 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
265 . $BUILD_WOK/$pkg/taz/*/receipt
266 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
267 > $DB_DIR/running
268 yes | tazpkg install \
269 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
270 --forced
271 return 0
272 fi
273 return 1
274 }
276 # Sort list according WANTED and BUILD_DEPENDS
277 sort_cook_list()
278 {
279 sort | while read pkg; do
280 echo -n "$pkg"
281 WANTED=""
282 BUILD_DEPENDS=""
283 . $BUILD_WOK/$pkg/receipt
284 MISSING=""
285 for i in $WANTED $BUILD_DEPENDS ; do
286 if [ ! -f $BUILD_WOK/$i/taz/*/receipt ]; then
287 case " $MISSING " in
288 *\ $i\ *);;
289 *) echo -n " $i";;
290 esac
291 MISSING="$MISSING $i"
292 fi
293 done
294 echo ""
295 done | awk '
296 function show(name)
297 {
298 print name;
299 got[name]++;
300 if (revdepcnt[name] > 0)
301 for (i = split(revdep[name], pkg, " "); i > 0; i--)
302 if (--depcnt[pkg[i]] == 0) show(pkg[i]);
303 }
305 {
306 if ($2 == "") show($1);
307 else {
308 depcnt[$1] = NF - 1;
309 unres = unres " " $1;
310 for (i = 2; i <= NF; i++) {
311 if (got[$i] > 0) continue;
312 revdepcnt[$i]++;
313 revdep[$i] = revdep[$i] " " $1;
314 }
315 }
316 }
317 END {
318 for (i = split(unres, pkg, " "); i > 0; i--)
319 if (depcnt[pkg[i]] > 0) print pkg[i];
320 }
321 '
322 }
324 # Here we cook all packages found in the cooklist.
325 cook_install()
326 {
327 echo "" > $DB_DIR/unbuilt
328 for pkg in `cat $DB_DIR/cooklist | sort_cook_list`
329 do
330 if ! cook_package $pkg; then
331 # Link to build log.
332 echo "<a href=\"log.php?package=$(escape $pkg)\">$pkg</a>" >> \
333 $DB_DIR/unbuilt
334 fi
335 # Remove package from the cooklist and empty lines for HTML <pre>.
336 sed -i /"^$pkg$"/d $DB_DIR/cooklist
337 sed -i '/^$/d' $DB_DIR/cooklist
338 packages_summary_update
339 done
340 }
342 # Remove old packages in the build wok and clean pkgs repository. The
343 # Hg wok is copied into the build wok so packages removed by hg must be
344 # removed. To remove old packages in the repository we look into the
345 # build wok and dont remove unbuilt packages. Clean-up will also remove
346 # all corrupted packages.
347 clean_up()
348 {
349 touch $DB_DIR/removed
350 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
351 echo "(cleaning)" > $DB_DIR/running
352 for pkg in `ls $HG_WOK`
353 do
354 if [ ! -d $BUILD_WOK/$pkg ]; then
355 case $2 in
356 --dry-run)
357 echo "Removing directory : $pkg" ;;
358 --verbose)
359 echo "Removing directory : $pkg"
360 rm -rf $BUILD_WOK/$pkg ;;
361 *)
362 rm -rf $BUILD_WOK/$pkg ;;
363 esac
364 fi
365 done
366 # Build a packages list with EXTRAVERSION so we can grep into it.
367 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
368 for receipt in $BUILD_WOK/*/taz/*/receipt
369 do
370 EXTRAVERSION=""
371 . $receipt
372 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
373 done
374 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
375 do
376 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
377 case $2 in
378 --dry-run)
379 echo "Removing package : $pkg" ;;
380 --verbose)
381 echo "Removing package : $pkg"
382 echo "$pkg" >> $DB_DIR/removed
383 rm -f $PACKAGES_REPOSITORY/$pkg ;;
384 *)
385 echo "$pkg" >> $DB_DIR/removed
386 rm -f $PACKAGES_REPOSITORY/$pkg ;;
387 esac
388 fi
389 done
390 # Remove all corrupted packages
391 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
392 do
393 case $2 in
394 --dry-run)
395 echo "Removing corrupted: $pkg" ;;
396 --verbose)
397 echo "Removing corrupted: $pkg"
398 echo "$pkg" >> $DB_DIR/removed
399 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
400 *)
401 echo "$pkg" >> $DB_DIR/removed
402 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
403 esac
404 done
405 echo ""
406 # Keep the 20 last removed packages list.
407 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
408 mv -f /tmp/removed.tail $DB_DIR/removed
409 }
411 blocked_urls()
412 {
413 rm -f $DB_DIR/blocked.urls
414 for pkg in `cat $DB_DIR/blocked`
415 do
416 if [ -f $LOG_DIR/$pkg.log ]; then
417 echo "<a href=\"log.php?package=$(escape $pkg)\">$pkg</a>" >> \
418 $DB_DIR/blocked.urls
419 else
420 echo "$pkg" >> $DB_DIR/blocked.urls
421 fi
422 done
423 }
425 # 4k, not a meta or a get-* package and no files = buggy package
426 test_packages()
427 {
428 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
429 echo "================================================================================"
430 echo "(testing packages)" > $DB_DIR/running
431 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
432 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
433 do
434 tmp=/tmp/bb-test.$$
435 CATEGORY=""
436 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
437 mkdir -p $tmp && cd $tmp
438 cpio -i receipt 2>/dev/null < $pkg
439 . ./receipt
440 if [ "$CATEGORY" != "meta" ]; then
441 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
442 cpio -i fs.cpio.gz 2>/dev/null < $pkg
443 if [ ! -f fs.cpio.gz ]; then
444 echo "Missing filesystem `basename $pkg`"
445 if [ -f $LOG_DIR/$PACKAGE.log ];then
446 echo "Missing filesystem `basename $pkg` <a href=\"log.php?package=$(escape $PACKAGE)\">Log</a>" \
447 >> $DB_DIR/corrupted
448 else
449 echo "Missing filesystem `basename $pkg`" \
450 >> $DB_DIR/corrupted
451 fi
452 else
453 zcat fs.cpio.gz | cpio -id 2>/dev/null
454 files=`find fs -type f`
455 if [ -z "$files" ]; then
456 echo "Empty filesystem `basename $pkg`"
457 if [ -f $LOG_DIR/$PACKAGE.log ]; then
458 echo "Empty filesystem `basename $pkg` <a href=\"log.php?package=$(escape $PACKAGE)\">Log</a>" \
459 >> $DB_DIR/corrupted
460 else
461 echo "Empty filesystem `basename $pkg`" \
462 >> $DB_DIR/corrupted
463 fi
464 fi
465 fi
466 fi
467 cd .. && rm -rf $tmp
468 fi
469 done
470 packages_summary_update
471 echo ""
472 }
474 case "$1" in
475 list-pkgs)
476 # List last cooked packages.
477 list_packages ;;
478 report)
479 # Run in report mode. If an update is done we must cook-all to
480 # rebuild all updated packages.
481 [ "$2" == "--update" ] && update_wok $@ || echo ""
482 check_wok $@
483 test_packages $@
484 show_report ;;
485 cook)
486 # Cook, install and log a single package build.
487 if [ -z $2 ]; then
488 echo "Please specify a package on the command line."
489 rm -f $LOCK_FILE && exit 0
490 fi
491 pkg=$2
492 echo "Starting to cook and install: $pkg"
493 if ! cook_package $pkg; then
494 echo "Unable to install: $pkg"
495 fi ;;
496 cook-all)
497 # Update wok, gen report (with cooklist), cook all packages, test,
498 # clean, gen new report and lists.
499 update_wok $@
500 check_wok $@
501 cook_install
502 test_packages $@
503 clean_up $@
504 check_wok $@
505 echo "(generating lists)" > $DB_DIR/running
506 tazwok gen-list --text
507 echo "" ;;
508 cook-commit)
509 # Cook all packages affected by the last commits in the wok.
510 # Clean up is done only by cook-all to avoid rebuild of corrupted
511 # packages on each commit.
512 update_wok $@
513 check_commit
514 cook_install
515 test_packages $@
516 check_wok $@
517 echo "(generating lists)" > $DB_DIR/running
518 tazwok gen-list --text
519 echo "" ;;
520 block)
521 # Add a pkg name to the list of blocked packages.
522 echo ""
523 if grep -qs "^$2$" $DB_DIR/blocked; then
524 echo -e "$2 is already in the blocked packages list."
525 else
526 echo -n "Adding $2 to : $DB_DIR/blocked... "
527 echo "$2" >> $DB_DIR/blocked && echo "Done"
528 if grep -q "^$2$" $DB_DIR/cooklist; then
529 echo -n "Removing $2 from : $DB_DIR/cooklist... "
530 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
531 packages_summary_update
532 fi
533 fi
534 blocked_urls
535 echo "" ;;
536 unblock)
537 # Remove a pkg name from the list of blocked packages.
538 echo ""
539 if grep -qs "^$2$" $DB_DIR/blocked; then
540 echo -n "Removing $2 from : $DB_DIR/blocked... "
541 sed -i /"^$2$"/d $DB_DIR/blocked
542 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
543 echo -n "Adding $2 to : $DB_DIR/cooklist... "
544 echo "$2" >> $DB_DIR/cooklist && echo "Done"
545 packages_summary_update
546 else
547 echo -e "$2 is not in the blocked packages list."
548 fi
549 blocked_urls
550 echo "" ;;
551 test-pkgs)
552 # Start a test suite on all builded packages.
553 test_packages $@ ;;
554 test-suite)
555 # Start a test suite on all builded package and the wok using
556 # the great 'tazwok check'.
557 #
558 # test_packages > $LOG_DIR/test-suite.log
559 # tazwok check >> $LOG_DIR/test-suite.log
560 #
561 test_packages $@
562 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
563 mail)
564 # Tazbbmail Pythom script wrapper.
565 PACKAGE=$2
566 tazbbmail $PACKAGE ;;
567 clean-up)
568 # Remove old packages and generate new packages lists.
569 update_wok $@
570 clean_up $@
571 packages_summary_update
572 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
573 clean-log)
574 logs=`ls $LOG_DIR | wc -l`
575 echo -n "Cleaning: $LOG_DIR... "
576 rm -rf $LOG_DIR/*
577 echo "$logs log removed" ;;
578 *)
579 usage ;;
580 esac
582 echo "" > $DB_DIR/running
583 rm -f $LOCK_FILE
585 exit 0