cookutils view cooker @ rev 584

Update cooker usage and add better error detection.
author Christopher Rogers <slaxemulator@gmail.com>
date Fri Feb 01 23:46:34 2013 +0000 (2013-02-01)
parents 4b1f207c4058
children
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
9 . /usr/lib/slitaz/libcook.sh
10 . /usr/lib/slitaz/libcookorder.sh
11
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14
15 # PID file.
16 pidfile='/var/run/cooker.pid'
17 COMMAND="$1"
18
19 #
20 # Functions
21 #
22
23 usage() {
24 cat << EOT
25
26 Usage: cooker [command] [pkg|list|note|hours]
27
28 Options:
29 usage|-u $(gettext "Display this short usage.")
30 setup|-s $(gettext "Setup the Cooker environment.")
31 setup-cron $(gettext "Setup a cron job for the Cooker.")
32 arch-db $(gettext "Create host arch packages DB.")
33 note|-n $(gettext "Add a note to the cooknotes.")
34 notes|-ns $(gettext "Display all the cooknotes.")
35 block|-b $(gettext "Block a package so cook will skip it.")
36 unblock|-ub $(gettext "Unblock a blocked package.")
37 pkg|-p $(gettext "Same as 'cook pkg' but with cooker log.")
38 flavor|-f $(gettext "Cook all packages of a flavor.")
39 list|-l $(gettext "Cook all packages in the given list.")
40 cat|-c $(gettext "Cook all packages of a category.")
41 rev|-r $(gettext "Cook packages of a specific revision.")
42 all|-a $(gettext "Find and cook all unbuilt packages.")
43 cook commands:
44 upwok $(gettext "Update wok.")
45 new $(gettext "Create a new package with a receipt.")
46 gen-wok-db $(gettext "Build cook order files.")
47 gen-src $(gettext "Build source list.")
48 check-incoming $(gettext "Move incoming packages to packages folder.")
49 pkgdb $(gettext "Create packages DB lists and flavors.")
50
51
52 EOT
53 exit 0
54 }
55
56 # Some messages occur in activity but log verbose output when checking for commits
57 # into a log file.
58 log_commits() {
59 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
60 }
61
62 # Clean up before exit when check and cook commits finish.
63 clean_exit() {
64 rm -f $command && touch $command
65 rm -f $pidfile
66 }
67
68 # Summary for commits.
69 commits_summary() {
70 msg="from revision $cur to $new"
71 [ "$new" == "$cur" ] && msg="revision $new"
72 echo "Will cook $msg"
73 separator
74 echo -e "\nSummary for commits"
75 separator
76 echo "Hg wok revision : $cur"
77 echo "Pulled revision : $new"
78 echo "Check date : $(date '+%Y-%m-%d %H:%M:%S')"
79 }
80
81 # Scan packages build deps and fill up cookorder list.
82 cook_order_scan() {
83 rm -f $cookorder
84 touch $cookorder
85 for pkg in $(cat $cooklist)
86 do
87 unset WANTED BUILD_DEPENDS BDEPS
88 . $WOK/$pkg/receipt
89 # The :: is for web interface color.
90 if [ "$AUTO_DEP" -a ! "$WANTED" ]; then
91 BDEPS=$(scan $PACKAGE --look_for=bdep --with_dev | \
92 grep -v $(for i in $(look_for_rwanted) $PACKAGE; do echo " -e ^$i$"; done))
93 else
94 BDEPS="$BUILD_DEPENDS"
95 fi
96 [ "$WANTED$BDEPS" ] && echo "$pkg :: $WANTED $BDEPS"
97 for dep in $BDEPS
98 do
99 if grep -q "^$dep$" $cooklist; then
100 if ! grep -q "^$dep$" $cookorder; then
101 echo "$dep" >> $cookorder
102 fi
103 fi
104 done
105 done
106
107 # Append unordered packages to cookorder.
108 for pkg in $(cat $cooklist)
109 do
110 if ! grep -q "^$pkg$" $cookorder; then
111 echo "$pkg" >> $cookorder
112 fi
113 done
114 }
115
116 # Scan and rescan until the cooklist is ordered then handle WANTED.
117 cook_order() {
118 time=$(date +%s)
119 scan=0
120
121 # Keep an original cooklist so we do a diff when ordering is finished.
122 cp -f $cooklist $cooklist.0
123 echo "cookorder" > $command
124 echo -e "\nInitial Cooker order scan"
125 separator
126
127 if [ "$COMMAND" = "pkg" ]; then
128 if ! grep -q "^$pkg$" $cooklist; then
129 if [ -f "$WOK/$pkg/receipt" ]; then
130 echo "$pkg" > $cooklist
131 fi
132 fi
133 fi
134
135 cook_order_scan
136
137 # Diff between the cooklist and new ordered list ? So copy the last
138 # cookorder to cooklist and rescan it.
139 while /bin/true
140 do
141 diff $cooklist $cookorder > $cookorder.diff
142 if [ -s "$cookorder.diff" ]; then
143 scan=$(($scan + 1))
144 echo -e "\nDiff scan: $scan"
145 separator
146 mv -f $cookorder $cooklist
147 cook_order_scan
148 else
149 break
150 fi
151 done
152
153 if [ "$COOK_WANTED" ]; then
154 PACKAGE=$pkg
155 for i in $PACKAGE $(look_for_rwanted); do
156 if ! grep -q "^${i}$" $cooklist; then
157 echo $i >> $cooklist
158 fi
159 done
160 fi
161
162 # Keep a diff between submited cooklist and the ordered.
163 diff $cooklist.0 $cooklist > $cooklist.diff
164 rm -f $cookorder $cookorder.diff $cooklist.0
165
166 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
167 # TODO: grep the line number to get pkg position and keep it higher.
168 echo -e "\nHandle WANTED package"
169 separator
170 for pkg in $(cat $cooklist)
171 do
172 unset WANTED
173 . $WOK/$pkg/receipt
174 if [ "$WANTED" ]; then
175 echo "$pkg :: $WANTED"
176 if grep -q ^${WANTED}$ $cooklist; then
177 sed -i -e "/^$pkg$/"d \
178 -e "/^$WANTED$/ a $pkg" $cooklist
179 fi
180 fi
181 done
182
183 # Show ordered cooklist
184 echo -e "\nCooklist order"
185 separator
186 cat $cooklist
187 separator
188 time=$(($(date +%s) - $time))
189 pkgs=$(cat $cooklist | wc -l)
190 echo -e "\nSummary for cookorder"
191 separator
192 cat << EOT
193 Ordered packages : $pkgs
194 Scans executed : $scan
195 Scan duration : ${time}s
196 EOT
197 separator && rm -f $command
198 }
199
200 # Use in default mode and with all cmd.
201 cook_commits() {
202 if [ -s "$commits" ]; then
203 for pkg in $(cat $commits)
204 do
205 echo "cook:$pkg" > $command
206 cook $pkg || broken
207 sed -i /^${pkg}$/d $commits
208 if [ "$COOK_WANTED" ]; then
209 PACKAGE=$pkg
210 for pkg in $(look_for_rwanted); do
211 if ! grep -q "^$pkg$" $commits; then
212 [ -f "$WOK/$pkg/receipt" ] || continue
213 echo "cook:$pkg" > $command
214 cook $pkg
215 fi
216 done
217 fi
218 done
219 fi
220 }
221
222 # Cook all packages in a cooklist.
223 cook_list() {
224 for pkg in $(cat $cooklist)
225 do
226 cook $pkg || broken
227 sed -i /^${pkg}$/d $cooklist
228 if [ "$COOK_WANTED" ]; then
229 PACKAGE=$pkg
230 for wpkg in $(look_for_rwanted); do
231 if ! grep -q "^$wpkg$" $cooklist; then
232 if [ -f "$WOK/$wpkg/receipt" ]; then
233 cook $wpkg
234 sed -i /^${wpkg}$/d $cooklist
235 fi
236 fi
237 done
238 fi
239 if [ "$1" = "--list" ]; then
240 pkgdb incoming
241 fi
242 done
243 }
244
245 # reverse pkg build when doing a big update to
246 # rebuild all packages that depends on it
247 reverse_cook_list() {
248 local pkg="$1"
249 [ ! -f "$WOK/$pkg/receipt" ] && echo -e "\nNo package $pkg found.\n" && exit 0
250 cp -a $cooklist $cooklist.bak
251 rm -f $cooklist && touch $cooklist
252 echo -e "\nReverse cooklist for: $pkg"
253 separator && cd $WOK
254 for rev in *
255 do
256 [ -f $WOK/$rev/receipt ] || continue
257 unset DEPENDS BUILD_DEPENDS && . $WOK/$rev/receipt
258 if echo "$DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
259 echo "$rev" | tee -a $cooklist
260 fi
261 done && separator
262 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
263 strip_blocked
264 cook_order | tee $LOGS/cookorder.log
265 cook_list
266 }
267
268 # Create a arch.$ARCH file for each package cooked for the target host
269 # architecture
270 #
271 # The deal: we dont want all packages handled by cooker commands and stats,
272 # since we dont cross compile all packages for each arch but only a set of
273 # packages to provide one full featured desktop, servers and goodies useful
274 # for the host system.
275 #
276 arch_db() {
277 count=0
278 echo "Cleaning packages DB : arch.$ARCH"
279 rm -f $wok/*/arch.$ARCH && cd $wok
280 echo "Creating $ARCH packages DB..."
281 for pkg in *
282 do
283 unset HOST_ARCH
284 . $wok/$pkg/receipt
285 #: ${HOST_ARCH=i486}
286 if $(echo "$HOST_ARCH" | fgrep -q $ARCH); then
287 count=$(($count + 1))
288 echo "Adding: $pkg"
289 touch $pkg/arch.$ARCH
290 fi
291 done
292 echo "Packages for $ARCH : $count"
293 }
294
295
296 #
297 # Commands
298 #
299 case "$COMMAND" in
300 usage|help|-u|-h)
301 usage ;;
302 setup|-s)
303 # Setup the Cooker environment.
304 echo -e "\nSetting up the Cooker"
305 echo "Cooker setup using: $SLITAZ" | log
306 separator
307 echo "$PKGS" > $LOCALSTATE/mirror
308 mkdir -p $LOCALSTATE/undigest/incoming
309 echo "$INCOMING" > $LOCALSTATE/undigest/incoming/mirror
310 echo "incoming" > $LOCALSTATE/priority
311 tazpkg recharge
312 for pkg in $INSTALL_PKGS
313 do
314 [ ! -f "$INSTALLED/$pkg/receipt" ] && tazpkg get-install $pkg
315 done
316 ls -1 "$INSTALLED" > $CACHE/chroot-pkgs
317 mkdir -p $SLITAZ && cd $SLITAZ
318 [ -d "$WOKHG" ] || hg clone $WOK_URL $WOKHG
319 rsync_wok
320 init_db_files
321 cook gen-wok-db
322 echo "Make unbuild list"
323 cook unbuild --full
324 [ "$AUTO_PURGE_SRC" ] && cook gen-src $SRC
325 #[ -d "$FLAVORS" ] || hg clone $FLAVORS_URL flavors
326 separator && newline ;;
327 arch-db)
328 # Manually create arch packages DB.
329 arch_db ;;
330 setup-cron)
331 # Create cron job for the cooker.
332 [ "$2" ] || hours=2
333 if [ ! -f "$crontabs" ]; then
334 mkdir -p /var/spool/cron/crontabs
335 fi
336 if ! fgrep -q /usr/bin/cooker $crontabs; then
337 echo "# Run SliTaz Cooker every $hours hours" > $crontabs
338 echo "0 */$hours * * * /usr/bin/cooker --output=html" >> $crontabs
339 killall crond 2>/dev/null && /etc/init.d/crond start
340 fi ;;
341 check-cron)
342 [ -f "$crontabs" ] || \
343 echo "There is no $crontabs here. Use setup-cron option." && exit 1
344 fgrep /usr/bin/cooker $crontabs ;;
345 note|-n)
346 # Blocked a pkg and want others to know why ? Post a note!
347 note="$2"
348 date=$(date "+%Y-%m-%d %H:%M")
349 [ "$note" ] && echo "$date : $note" >> $cooknotes ;;
350 notes|-ns)
351 # View cooknotes.
352 echo -e "\nCooknotes"
353 separator
354 cat $cooknotes
355 separator && newline ;;
356 block|-b)
357 # Block a package.
358 [ "$pkg" ] && cook $pkg --block ;;
359 unblock|-ub)
360 # Unblock a package.
361 [ "$pkg" ] && cook $pkg --unblock ;;
362 reverse|-r)
363 # Cook all reverse dependencies for a package. This command lets us
364 # control the Cooker manually for commits that will cook a lot of packages.
365 #
366 # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse"
367 #
368 reverse_cook_list $pkg ;;
369 chroot)
370 [ -d $SLITAZ/chroot ] || tazdev gen-chroot $2
371 [ -d $SLITAZ/chroot ] && tazdev chroot $2
372 ;;
373 toolchain)
374 toolchain=$CACHE/toolchain
375 [ -f $PKGS/fullco.txt ] || cook gen-wok-db
376 cook gen-cooklist toolchain > $toolchain
377 newline > $cooklist
378 echo "cooker:toolchain" > $command
379 echo "First build slitaz-toolchain"
380 [ -f $blocked ] && rm -f $blocked
381 touch $blocked
382 cook slitaz-toolchain
383 SKIP="glibc gcc binutils linux-api-headers slitaz-toolchain cookutils"
384 for i in $SKIP; do
385 echo "$i" >> $blocked
386 done
387 for pkg in $(cat $blocked)
388 do
389 sed -i /^${pkg}$/d $toolchain
390 done && sed -i /^$/d $toolchain
391 for pkg in $(cat $toolchain)
392 do
393 [ -f "$WOK/$pkg/receipt" ] || continue
394 cook $pkg
395 sed -i /^${pkg}$/d $toolchain
396 if [ "$COOK_WANTED" ]; then
397 PACKAGE=$pkg
398 for pkg in $(look_for_rwanted); do
399 if ! grep -q "^$pkg$" $toolchain; then
400 [ -f "$WOK/$pkg/receipt" ] || continue
401 cook $pkg
402 fi
403 done
404 fi
405 cook pkgdb incoming
406 done
407 [ -f $toolchain ] && rm -f $toolchain
408 clean_exit ;;
409 pkg|-p)
410 # Same as 'cook pkg' but with log for web interface.
411 [ ! -f "$WOK/$pkg/receipt" ] && echo "$pkg doesn't exist in wok" && exit 1
412 if [ "$COOK_WANTED" ]; then
413 PACKAGE=$pkg
414 strip_blocked
415 cook_order | tee $LOGS/cookorder.log
416 cook_list
417 if [ "$AUTO_UPDATE_DB" ]; then
418 cook pkgdb $INCOMING
419 fi
420 else
421 cook $pkg
422 fi
423 clean_exit ;;
424 cat|-c)
425 # Cook all packages of a category.
426 cat="$2"
427 cp -a $cooklist $cooklist.bak
428 rm -f $cooklist && touch $cooklist && cd $WOK
429 for pkg in *
430 do
431 unset CATEGORY && . $pkg/receipt
432 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
433 done
434 strip_blocked
435 cook_order | tee $LOGS/cookorder.log
436 cook_list --list ;;
437 flavor|-f)
438 # Cook all packages of a flavor.
439 name="$2"
440 [ ! -d "$FLAVORS/$name" ] && \
441 echo -e "\nSpecified flavor does not exist: $name\n" && exit 1
442 [ -d "$FLAVORS/.hg" ] && cd $FLAVORS && hg pull -u
443 list=$FLAVORS/$name/packages.list
444 cp -a $list $cooklist
445 strip_blocked
446 cook_order | tee $LOGS/cookorder.log
447 cook_list ;;
448 list|-l)
449 # Cook a list of packages given in argument.
450 list="$cooklist"
451 [ "$2" ] && list="$2"
452 [ -f "$list" ] || $(gettext -e "$list doesn't exist." && exit 1)
453 if [ "$list" != "$cooklist" ]; then
454 cp -a $list $cooklist
455 fi
456 unset pkg
457 strip_blocked
458 if [ "$UPCOOKLIST" ]; then
459 gen_cook_list
460 else
461 cook_order | tee $LOGS/cookorder.log
462 fi
463 cook_list --list
464 cook pkgdb $INCOMING ;;
465 rev|-r)
466 # Cook or recook a specific Hg revision.
467 rev="$2"
468 [ "$rev" ] || exit 0
469 cd $WOK
470 cp -a $cooklist $cooklist.bak
471 rm -f $cooklist && touch $cooklist
472 for pkg in $(hg log --rev=$rev --template "{files}")
473 do
474 echo "$pkg" | cut -d "/" -f 1 >> $cooklist
475 done
476 strip_blocked
477 cook_order | tee $LOGS/cookorder.log
478 cook_list ;;
479 all|-a)
480 # Try to build all unbuilt packages except blocked's.
481 echo "cooker:all" > $command
482 cp -a $cooklist $cooklist.bak
483 rm -f $cooklist && touch $cooklist
484 newline && cd $WOK
485 echo "Cooker cooklist"
486 separator
487
488 cook unbuild
489 cp -a $unbuild $cooklist
490 strip_blocked
491 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
492 cook_list ;;
493 new|pkgdb|get-src|check-incoming|gen-wok-db|upwok)
494 cook "$COMMAND" "$2" "$3" "$4" "$5" ;;
495 *)
496 # Default is to cook all commits if not yet running.
497 [ "$COMMAND" ] && usage
498 cooklist=$commits
499 if [ -f "$pidfile" ]; then
500 pid=$(cat $pidfile)
501 if [ -s /proc/$pid/status ]; then
502 gettext -e "\nStill cooking latest commits with pid:"
503 echo -e " $pid\n" && exit 0
504 fi
505 rm -f "$pidfile"
506 fi
507
508 # Start and get a PID file.
509 rm -f $LOGS/commits.log
510 newline
511 echo "Checking for commits" | log_commits
512 separator | tee -a $LOGS/commits.log
513
514 echo $$ > $pidfile
515 trap 'echo -e "\nCooker stopped: PID $$\n" && \
516 rm -f $pidfile $command && exit 1' INT TERM
517
518 echo "Cooker PID : $$" | log_commits
519 echo "Cooker date : $(date '+%Y-%m-%d %H:%M:%S')" | log_commits
520
521 # Get revisions. Here we have 2 echoes since we want a msg on screen,
522 # in commits log and activity DB without a space before.
523 cd $WOK || exit 1
524 branch=$(cat /etc/slitaz-release)
525 cur=$(hg head $branch --template '{rev}\n')
526 echo "Updating wok : $WOKHG (rev $cur)" | log_commits
527 echo "Updating wok: $WOKHG" | log
528 echo "hg:pull" > $command
529 cd $WOKHG && hg pull -u | log_commits
530 new=$(hg head $branch --template '{rev}\n')
531 # Store last rev to be used by CGI so it doesn't need to call hg head
532 # on each load.
533 echo "$new" > $wokrev
534
535 # Sync build wok with rsync so we don't take care about removing old
536 # files as before.
537 if [ "$new" -gt "$cur" ]; then
538 echo "Changes found from: $cur to $new" | log
539 echo "Syncing build wok with Hg wok..." | log_commits
540 rsync -r -t -c -l -u -v -D -E $WOKHG/ $WOK/ | \
541 sed '/^$/'d | log_commits
542 else
543 echo "No revision changes: $cur vs $new" | log
544 separator | log_commits
545 clean_exit && newline && exit 0
546 fi
547
548 # Get and display modifications.
549 cd $WOKHG
550 commits_summary | log_commits
551 cur=$(($cur + 1))
552 rm -f $commits.tmp && touch $commits.tmp
553 for rev in $(seq $cur $new)
554 do
555 for file in $(hg log --rev=$rev --template "{files}")
556 do
557 pkg=$(echo $file | cut -d "/" -f 1)
558 desc=$(hg log --rev=$rev --template "{desc}" $file)
559 echo "Commited package : $pkg - $desc" | log_commits
560 echo $pkg >> $commits.tmp
561 done
562 done
563
564 # We may have deleted packages and files in stuff/. Remove it and
565 # clean DB as well as log file.
566 cd $WOK
567 for pkg in *
568 do
569 if [ ! -d "$WOKHG/$pkg" ]; then
570 echo "Removing package: $pkg" | log_commits
571 . $WOK/$pkg/receipt
572 if [ -f "$INCOMING/$PACKAGE-$VERSION*" ]; then
573 rm -rf $INCOMING/$PACKAGE-$VERSION*
574 fi
575 if [ -f "$PKGS/$PACKAGE-$VERSION*" ]; then
576 rm -rf $PKGS/$PACKAGE-$VERSION*
577 fi
578 rm -rf $WOK/$pkg $LOGS/$pkg.log
579 sed -i "/^${pkg}$/"d $blocked $broken $commits.tmp
580 fi
581 done
582
583 # Keep previous commit and discard duplicate lines
584 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
585 uniq $commits.new > $commits && rm $commits.*
586
587 # Handle cross compilation. Create arch packages DB and remove pkgs
588 # not cooked for this arch from the commits list.
589 case "$ARCH" in
590 arm)
591 arch_db | log_commits
592 for pkg in $(cat $commits)
593 do
594 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
595 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
596 log_commits
597 sed -i "/^${pkg}$/"d $commits
598 fi
599 done ;;
600 i486)
601 echo "Cooker arch : $ARCH (default)" | log_commits ;;
602 *)
603 echo "Cooker arch : $ARCH" | log_commits ;;
604 esac
605
606 # Stats
607 pkgs=$(cat $commits | wc -l)
608 echo "Packages to cook: $pkgs" | log
609 echo "Packages to cook : $pkgs" | log_commits
610 separator | log_commits
611 newline
612 strip_blocked
613 cook_order | tee $LOGS/cookorder.log
614 cook_commits
615 pkgdb $INCOMING
616 clean_exit ;;
617 esac
618
619 exit 0