wok view tazbb/stuff/tazbb @ rev 3512
Add Tazbb (SliTaz Build Bot)
| author | Christophe Lincoln <pankso@slitaz.org> |
|---|---|
| date | Sat Jun 20 04:41:53 2009 +0200 (2009-06-20) |
| parents | |
| children | 1f7365e120bb |
line source
1 #!/bin/sh
2 # Tazbb - SliTaz Build Bot.
3 # System wide config file: /etc/slitaz/tazbb.conf
4 #
5 # Tazbb is a tool to automate package building, it can be run manually
6 # or via a cron job. On SliTaz build host, tazbb is run in a chroot env.
7 #
8 # (c) 2009 SliTaz GNU/Linux project - GNU gpl v3
9 #
11 # Include config file or exit if no file found.
12 if [ -f "./tazbb.conf" ]; then
13 . ./tazbb.conf
14 elif [ -f "/etc/slitaz/tazbb.conf" ]; then
15 . /etc/slitaz/tazbb.conf
16 else
17 echo -e "\nNo config file found: tazbb.conf...\n" && exit 0
18 fi
20 # Tazbb is only for root.
21 if test $(id -u) != 0 ; then
22 echo -e "\nYou must be root to run: `basename $0`.\n" && exit 0
23 fi
25 # Let tazbb finish is work and make sure needed files exist.
26 if [ -f $LOCK_FILE ]; then
27 case $1 in
28 usage|list-*|*block)
29 continue ;;
30 *)
31 echo -e "\nTazbb is already running and locked...\n"
32 exit 0 ;;
33 esac
34 else
35 mkdir -p $DB_DIR $LOG_DIR
36 touch $LOCK_FILE $DB_DIR/blocked
37 fi
39 usage()
40 {
41 echo -e "\nSliTaz developers and build host tool\n
42 \033[1mUsage: \033[0m `basename $0` [command] [--option]
43 \033[1mCommands: \033[0m\n
44 usage Print this short usage and command list.
45 list-pkgs List last cooked packages with date.
46 report Run in report mode and dont cook anything [--verbose].
47 cook-all Cook all missing, modified or unbuilt packages.
48 cook-commit Cook all packages affected by a commit in the last update.
49 test-pkgs Execute a test suite on all packages [--verbose].
50 [un]block Block or unblock a package to skip or enable building.
51 clean-up Remove old packages [--verbose|--dry-run].
52 clean-log Remove all generated build log files.\n"
53 }
55 status()
56 {
57 local CHECK=$?
58 echo -en "\033[70G"
59 if [ $CHECK = 0 ]; then
60 echo "Done"
61 else
62 echo "Failed"
63 fi
64 return $CHECK
65 }
67 top_summary()
68 {
69 cat > $DB_DIR/summary << _EOT_
70 Update : `date`
71 Revision : $NEW_REV (<a href="$HG_URL/log/$NEW_REV">changelog</a>)
72 _EOT_
73 }
75 packages_summary()
76 {
77 if ! grep -q "^Packages" $DB_DIR/summary; then
78 cat >> $DB_DIR/summary << _EOT_
79 Packages : `ls $BUILD_WOK | wc -l` in the wok, `cat $DB_DIR/cooklist | wc -l` to cook, \
80 `cat $DB_DIR/blocked | wc -l` blocked, `cat $DB_DIR/corrupted | wc -l` corrupted
81 _EOT_
82 fi
83 }
85 packages_summary_update()
86 {
87 sed -i s/"[0-9]* in the wok"/"`ls $BUILD_WOK | wc -l` in the wok"/ \
88 $DB_DIR/summary
89 sed -i s/"[0-9]* to cook"/"`cat $DB_DIR/cooklist | wc -l` to cook"/ \
90 $DB_DIR/summary
91 sed -i s/"[0-9]* blocked"/"`cat $DB_DIR/blocked | wc -l` blocked"/ \
92 $DB_DIR/summary
93 sed -i s/"[0-9]* corrupted"/"`cat $DB_DIR/corrupted | wc -l` corrupted"/ \
94 $DB_DIR/summary
95 }
97 list_packages()
98 {
99 cd $PACKAGES_REPOSITORY
100 ls -1t *.tazpkg | head -20 | \
101 while read file
102 do
103 echo -n $(stat -c '%y' $PACKAGES_REPOSITORY/$file | cut -d. -f1)
104 echo " $file"
105 done
106 }
108 show_report()
109 {
110 echo "Cooklist"
111 echo "================================================================================"
112 cat $DB_DIR/cooklist && echo ""
113 echo "Blocked"
114 echo "================================================================================"
115 cat $DB_DIR/blocked && echo ""
116 echo "Corrupted"
117 echo ""
118 }
120 update_wok()
121 {
122 echo ""
123 echo "(updating wok)" > $DB_DIR/running
124 cd $HG_WOK
125 LAST_REV=`hg head --template '{rev}\n'`
126 hg pull && hg update
127 NEW_REV=`hg head --template '{rev}\n'`
128 # Gen a new summary and link last revision for the web interface.
129 echo -e "\nHg wok : $HG_WOK ($NEW_REV)"
130 echo -e "Build wok : $BUILD_WOK ($LAST_REV)\n"
131 top_summary
132 # Copy Hg wok if new revision or exit to stop process since nothing
133 # have change (--forced can be used).
134 if [ "$NEW_REV" != "$LAST_REV" ]; then
135 size=`du -sh $HG_WOK | awk '{ print $1 }'`
136 echo -n "Copying Hg wok to the build wok ($size)... "
137 cp -a $HG_WOK/* $BUILD_WOK
138 cp -a $HG_WOK/.hg $BUILD_WOK
139 echo -e "Done\n"
140 else
141 if [ "$1" = "cook-all" ] || [ "$1" = "cook-commit" ]; then
142 if [ "$2" != "--forced" ]; then
143 echo -e "Nothing to cook...\n"
144 packages_summary
145 rm -f $LOCK_FILE && exit 0
146 fi
147 fi
148 fi
149 }
151 # Running 'tazbb report' should not cook anything and --verbose option
152 # can be used to display more messages.
153 check_wok()
154 {
155 # Clean up last results.
156 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
157 rm -f $DB_DIR/report && touch $DB_DIR/report
158 rm -f $DB_DIR/unbuilt && touch $DB_DIR/unbuilt
159 echo "Checking all files in: $HG_WOK"
160 echo "================================================================================"
161 echo "(checking wok)" > $DB_DIR/running
162 for pkg in $HG_WOK/*
163 do
164 EXTRAVERSION=""
165 WANTED=""
166 . $pkg/receipt
167 [ "$2" = "--verbose" ] && echo "Package : $PACKAGE"
168 # Skip blocked packages.
169 if grep -qs "^$PACKAGE$" $DB_DIR/blocked; then
170 echo "Blocked : $PACKAGE ($VERSION)" && continue
171 fi
173 # Bristuff hack until the receipt are improved...
174 #[ "$VERSION" = "bristuff" ] && VERSION=`get_version`
175 if [ "$VERSION" = "bristuff" ]; then
176 . $BUILD_WOK/$PACKAGE/taz/*/receipt
177 fi
179 # First check if package exit. Package naming _must_ be in the form of:
180 # $PACKAGE-$VERSION or $PACKAGE-${VERSION}$EXTRAVERSION (Kernel string).
181 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-$VERSION.tazpkg ]; then
182 [ -z "$EXTRAVERSION" ] && EXTRAVERSION="_$KERNEL"
183 if [ ! -f $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg ]; then
184 [ "$1" = "report" ] && echo "Missing : $PACKAGE ($VERSION)"
185 echo "Missing : $PACKAGE ($VERSION)" >> $DB_DIR/report
186 echo "$PACKAGE" >> $DB_DIR/cooklist
187 fi
188 else
189 # Check if package is up-to-date.
190 PKG_DATE=`date -u -r $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg '+%m%d%H%M%Y'`
191 for file in `find $pkg -type f`
192 do
193 FILE_DATE=`date -u -r $file '+%m%d%H%M%Y'`
194 [ "$2" = "--verbose" ] && echo " -> Checking: $file"
195 if [ "$FILE_DATE" -gt "$PKG_DATE" ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
196 [ "$1" = "report" ] && echo "Refresh : $PACKAGE ($VERSION)"
197 echo "Refresh : $PACKAGE ($VERSION)" >> $DB_DIR/report
198 echo "$PACKAGE" >> $DB_DIR/cooklist
199 fi
200 done
201 fi
202 # Now check if package is built and not already in the list.
203 if [ ! -d $BUILD_WOK/$PACKAGE/taz ] && ! grep -q $PACKAGE $DB_DIR/cooklist; then
204 [ "$1" = "report" ] && echo "Unbuilt : $PACKAGE ($VERSION)"
205 echo "Unbuilt : $PACKAGE ($VERSION)" >> $DB_DIR/report
206 echo "$PACKAGE" >> $DB_DIR/cooklist
207 fi
208 # Rebuild unbuilt packages list with link to log file. This list
209 # is also generated by cook_inslall to have real time stats.
210 if [ ! -d $BUILD_WOK/$PACKAGE/taz ]; then
211 echo "<a href=\"log.php?package=$PACKAGE\">$PACKAGE</a>" \
212 >> $DB_DIR/unbuilt
213 fi
214 done
215 packages_summary
216 }
218 # Create a new cooklist and summary (dont modify report) so 'tazbb cook-commit'
219 # can cook last changes.
220 check_commit()
221 {
222 echo "(checking commit)" > $DB_DIR/running
223 cd $HG_WOK
224 # Clean up last results.
225 rm -f $DB_DIR/cooklist && touch $DB_DIR/cooklist
226 # Get the name of modified packages by the revision range. +1 last
227 # commit was build by the previous build.
228 LAST_REV=$(($LAST_REV+1))
229 echo -e "Will cook from revision $LAST_REV to $NEW_REV\n"
230 for file in `hg log --rev=$LAST_REV:$NEW_REV --template '{files}\n'`
231 do
232 pkg=`echo $file | cut -d "/" -f 1`
233 if ! grep -q ^$pkg$ $DB_DIR/cooklist; then
234 . $pkg/receipt
235 echo "Commit : $PACKAGE ($VERSION)" >> $DB_DIR/report
236 echo "$PACKAGE" >> $DB_DIR/cooklist
237 fi
238 done
239 packages_summary
240 }
242 # Here we cook all packages found in the cooklist.
243 cook_install()
244 {
245 echo "" > $DB_DIR/unbuilt
246 for pkg in `cat $DB_DIR/cooklist | sort`
247 do
248 EXTRAVERSION=""
249 DEPENDS=""
250 BUILD_DEPENDS=""
251 SOURCE=""
252 WANTED=""
253 echo "(cooking <a href=\"log.php?package=$pkg\">$pkg</a>)" > $DB_DIR/running
254 tazwok clean $pkg
255 script -c "echo 'install' | tazwok cook $pkg" $LOG_DIR/$pkg.log
256 # Install new package (important for new shared libs). Note
257 # that tests are done separatly with 'test_packages' and should
258 # be done by tazwok.
259 if [ -f $BUILD_WOK/$pkg/taz/*/receipt ]; then
260 . $BUILD_WOK/$pkg/taz/*/receipt
261 echo "(installing $PACKAGE-${VERSION}$EXTRAVERSION.tazpkg)" \
262 > $DB_DIR/running
263 yes | tazpkg install \
264 $PACKAGES_REPOSITORY/$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg \
265 --forced
266 else
267 # Link to build log.
268 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
269 $DB_DIR/unbuilt
270 fi
271 # Remove package from the cooklist and empty lines for HTML <pre>.
272 sed -i /"^$pkg$"/d $DB_DIR/cooklist
273 sed -i '/^$/d' $DB_DIR/cooklist
274 done
275 }
277 # Remove old packages in the build wok and clean pkgs repository. The
278 # Hg wok is copied into the build wok so packages removed by hg must be
279 # removed. To remove old packages in the repository we look into the
280 # build wok and dont remove unbuilt packages. Clean-up will also remove
281 # all corrupted packages.
282 clean_up()
283 {
284 touch $DB_DIR/removed
285 echo -e "\nCleaning the build wok, old and corrupted packages...\n"
286 echo "(cleaning)" > $DB_DIR/running
287 for pkg in `ls $HG_WOK`
288 do
289 if [ ! -d $BUILD_WOK/$pkg ]; then
290 case $2 in
291 --dry-run)
292 echo "Removing directory : $pkg" ;;
293 --verbose)
294 echo "Removing directory : $pkg"
295 rm -rf $BUILD_WOK/$pkg ;;
296 *)
297 rm -rf $BUILD_WOK/$pkg ;;
298 esac
299 fi
300 done
301 # Build a packages list with EXTRAVERSION so we can grep into it.
302 rm -f $DB_DIR/packaged && touch $DB_DIR/packaged
303 for receipt in $BUILD_WOK/*/taz/*/receipt
304 do
305 EXTRAVERSION=""
306 . $receipt
307 echo "$PACKAGE-${VERSION}$EXTRAVERSION.tazpkg" >> $DB_DIR/packaged
308 done
309 for pkg in `cd $PACKAGES_REPOSITORY && ls *.tazpkg`
310 do
311 if ! grep -q "^$pkg$" $DB_DIR/packaged; then
312 case $2 in
313 --dry-run)
314 echo "Removing package : $pkg" ;;
315 --verbose)
316 echo "Removing package : $pkg"
317 echo "$pkg" >> $DB_DIR/removed
318 rm -f $PACKAGES_REPOSITORY/$pkg ;;
319 *)
320 echo "$pkg" >> $DB_DIR/removed
321 rm -f $PACKAGES_REPOSITORY/$pkg ;;
322 esac
323 fi
324 done
325 # Remove all corrupted packages
326 for pkg in `cat $DB_DIR/corrupted | awk '{ print $3 }'`
327 do
328 case $2 in
329 --dry-run)
330 echo "Removing corrupted: $pkg" ;;
331 --verbose)
332 echo "Removing corrupted: $pkg"
333 echo "$pkg" >> $DB_DIR/removed
334 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
335 *)
336 echo "$pkg" >> $DB_DIR/removed
337 rm -rf $PACKAGES_REPOSITORY/$pkg ;;
338 esac
339 done
340 echo ""
341 # Keep the 20 last removed packages list.
342 cat $DB_DIR/removed | tail -n 20 > /tmp/removed.tail
343 mv -f /tmp/removed.tail $DB_DIR/removed
344 }
346 blocked_urls()
347 {
348 rm -f $DB_DIR/blocked.urls
349 for pkg in `cat $DB_DIR/blocked`
350 do
351 if [ -f $LOG_DIR/$pkg.log ]; then
352 echo "<a href=\"log.php?package=$pkg\">$pkg</a>" >> \
353 $DB_DIR/blocked.urls
354 else
355 echo "$pkg" >> $DB_DIR/blocked.urls
356 fi
357 done
358 }
360 # 4k, not a meta or a get-* package and no files = buggy package
361 test_packages()
362 {
363 echo -e "\nTesting all packages in: $PACKAGES_REPOSITORY"
364 echo "================================================================================"
365 echo "(testing packages)" > $DB_DIR/running
366 rm -f $DB_DIR/corrupted && touch $DB_DIR/corrupted
367 for pkg in $PACKAGES_REPOSITORY/*.tazpkg
368 do
369 tmp=/tmp/bb-test.$$
370 CATEGORY=""
371 if du $pkg | grep -qw '^4' && ! echo `basename $pkg` | grep -q '^get-'; then
372 mkdir -p $tmp && cd $tmp
373 cpio -i receipt 2>/dev/null < $pkg
374 . ./receipt
375 if [ "$CATEGORY" != "meta" ]; then
376 [ "$2" = "--verbose" ] && echo "Testing: $PACKAGE"
377 cpio -i fs.cpio.gz 2>/dev/null < $pkg
378 if [ ! -f fs.cpio.gz ]; then
379 echo "No fs : `basename $pkg`"
380 if [ -f $LOG_DIR/$PACKAGE.log ];then
381 echo "No fs : `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
382 >> $DB_DIR/corrupted
383 else
384 echo "No fs : `basename $pkg`" \
385 >> $DB_DIR/corrupted
386 fi
387 else
388 zcat fs.cpio.gz | cpio -id 2>/dev/null
389 files=`find fs -type f`
390 if [ -z "$files" ]; then
391 echo "Empty : `basename $pkg`"
392 if [ -f $LOG_DIR/$PACKAGE.log ]; then
393 echo "Empty : `basename $pkg` <a href=\"log.php?package=$PACKAGE\">Log</a>" \
394 >> $DB_DIR/corrupted
395 else
396 echo "No fs : `basename $pkg`" \
397 >> $DB_DIR/corrupted
398 fi
399 fi
400 fi
401 fi
402 cd .. && rm -rf $tmp
403 fi
404 done
405 packages_summary_update
406 echo ""
407 }
409 case "$1" in
410 list-pkgs)
411 # List last cooked packages.
412 list_packages ;;
413 report)
414 # Run in report mode. If an update is done we must cook-all to
415 # rebuild all updated packages.
416 [ "$2" == "--update" ] && update_wok $@ || echo ""
417 check_wok $@
418 test_packages $@
419 show_report ;;
420 cook-all)
421 # Update wok, gen report (with cooklist), cook all packages, test,
422 # clean, gen new report and lists.
423 update_wok $@
424 check_wok $@
425 cook_install
426 test_packages $@
427 clean_up $@
428 check_wok $@
429 echo "(generating lists)" > $DB_DIR/running
430 tazwok gen-list --text
431 echo "" ;;
432 cook-commit)
433 # Cook all packages affected by the last commits in the wok.
434 update_wok $@
435 check_commit
436 cook_install
437 test_packages $@
438 clean_up $@
439 check_wok $@
440 echo "(generating lists)" > $DB_DIR/running
441 tazwok gen-list --text
442 echo "" ;;
443 block)
444 # Add a pkg name to the list of blocked packages.
445 echo ""
446 if grep -qs "^$2$" $DB_DIR/blocked; then
447 echo -e "$2 is already in the blocked packages list."
448 else
449 echo -n "Adding $2 to : $DB_DIR/blocked... "
450 echo "$2" >> $DB_DIR/blocked && echo "Done"
451 if grep -q "^$2$" $DB_DIR/cooklist; then
452 echo -n "Removing $2 from : $DB_DIR/cooklist... "
453 sed -i /"^$2$"/d $DB_DIR/cooklist && echo "Done"
454 packages_summary_update
455 fi
456 fi
457 blocked_urls
458 echo "" ;;
459 unblock)
460 # Remove a pkg name from the list of blocked packages.
461 echo ""
462 if grep -qs "^$2$" $DB_DIR/blocked; then
463 echo -n "Removing $2 from : $DB_DIR/blocked... "
464 sed -i /"^$2$"/d $DB_DIR/blocked
465 sed -i '/^$/d' $DB_DIR/blocked && echo "Done"
466 echo -n "Adding $2 to : $DB_DIR/cooklist... "
467 echo "$2" >> $DB_DIR/cooklist && echo "Done"
468 packages_summary_update
469 else
470 echo -e "$2 is not in the blocked packages list."
471 fi
472 blocked_urls
473 echo "" ;;
474 test-pkgs)
475 # Start a test suite on all builded packages.
476 test_packages $@ ;;
477 test-suite)
478 # Start a test suite on all builded package and the wok using
479 # the great 'tazwok check'.
480 #
481 # test_packages > $LOG_DIR/test-suite.log
482 # tazwok check >> $LOG_DIR/test-suite.log
483 #
484 test_packages $@
485 script -c "tazwok check" $LOG_DIR/test-suite.log ;;
486 clean-up)
487 # Remove old packages and generate new packages lists.
488 update_wok $@
489 clean_up $@
490 packages_summary_update
491 [ "$2" != "--dry-run" ] && tazwok gen-list --text ;;
492 clean-log)
493 logs=`ls $LOG_DIR | wc -l`
494 echo -n "Cleaning: $LOG_DIR... "
495 rm -rf $LOG_DIR/*
496 echo "$logs log removed" ;;
497 *)
498 usage ;;
499 esac
501 echo "" > $DB_DIR/running
502 rm -f $LOCK_FILE
504 exit 0