cookutils view cooker @ rev 761
cooker: crontab for the poke button (again)
| author | Pascal Bellard <pascal.bellard@slitaz.org> |
|---|---|
| date | Sun Sep 27 23:26:29 2015 +0200 (2015-09-27) |
| parents | 75fe6eb7e7bd |
| children | 91cb129f0b7b |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [command] [pkg|list|note|hours]
28 Options:
29 usage|-u Display this short usage.
30 setup|-s Setup the Cooker environment.
31 setup-cron Setup a cron job for the Cooker.
32 arch-db Create host arch packages DB.
33 note|-n Add a note to the cooknotes.
34 notes|-ns Display all the cooknotes.
35 block|-b Block a package so cook will skip it.
36 unblock|-ub Unblock a blocked package.
37 pkg|-p Same as 'cook pkg' but with cooker log.
38 flavor|-f Cook all packages of a flavor.
39 list|-l Cook all packages in the given list.
40 cat|-c Cook all packages of a category.
41 rev|-r Cook packages of a specific revision.
42 all|-a Find and cook all unbuilt packages.
44 EOT
45 exit 0
46 }
49 # Some messages occur in activity but log verbose output when checking for commits
50 # into a log file.
52 log_commits() {
53 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
54 }
57 # Clean up before exit when check and cook commits finish.
59 clean_exit() {
60 rm -f $command; touch $command
61 rm -f $pidfile
62 }
65 # Summary for commits.
67 commits_summary() {
68 msg="from revision $cur to $new"
69 [ "$new" == "$cur" ] && msg="revision $new"
70 echo "Will cook $msg"
71 separator
72 echo -e "\nSummary for commits"
73 separator
74 echo "Hg wok revision : $cur"
75 echo "Pulled revision : $new"
76 echo "Check date : $(date '+%F %T')"
77 }
80 # Scan packages build deps and fill up cookorder list.
82 cook_order_scan() {
83 rm -f $cookorder
84 touch $cookorder
85 for pkg in $(cat $cooklist); do
86 unset WANTED BUILD_DEPENDS
87 . $wok/$pkg/receipt
88 # The :: is for web interface color.
89 [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS"
90 for dep in $WANTED $BUILD_DEPENDS; do
91 if grep -q "^$dep$" $cooklist; then
92 if ! grep -q "^$dep$" $cookorder; then
93 echo "$dep" >> $cookorder
94 fi
95 fi
96 done
97 done
99 # Append unordered packages to cookorder.
100 for pkg in $(cat $cooklist); do
101 if ! grep -q "^$pkg$" $cookorder; then
102 echo "$pkg" >> $cookorder
103 fi
104 done
105 }
108 # Scan and rescan until the cooklist is ordered then handle WANTED.
110 cook_order() {
111 time=$(date +%s)
112 scan=0
114 # Keep an original cooklist so we do a diff when ordering is finished.
115 cp -f $cooklist $cooklist.0
116 echo 'cookorder' > $command
117 echo -e '\nInitial Cooker order scan'
118 separator
119 cook_order_scan
121 # Diff between the cooklist and new ordered list ? So copy the last
122 # cookorder to cooklist and rescan it.
123 while /bin/true; do
124 diff $cooklist $cookorder > $cookorder.diff
125 if [ -s "$cookorder.diff" ]; then
126 scan=$(($scan + 1))
127 echo -e "\nDiff scan: $scan"
128 separator
129 mv -f $cookorder $cooklist
130 cook_order_scan
131 else
132 break
133 fi
134 done
136 # Keep a diff between submitted cooklist and the ordered.
137 diff $cooklist.0 $cooklist > $cooklist.diff
138 rm -f $cookorder $cookorder.diff $cooklist.0
140 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
141 # TODO: grep the line number to get pkg position and keep it higher.
142 echo -e '\nHandle WANTED package'
143 separator
144 for pkg in $(cat $cooklist); do
145 unset WANTED
146 . $wok/$pkg/receipt
147 for wanted in $WANTED; do
148 echo "$pkg :: $wanted"
149 if grep -q ^${wanted}$ $cooklist; then
150 sed -i -e "/^$pkg$/"d \
151 -e "/^$wanted$/ a $pkg" $cooklist
152 fi
153 done
154 done
156 # Show ordered cooklist
157 echo -e '\nCooklist order'
158 separator
159 cat $cooklist
160 separator
162 time=$(($(date +%s) - $time))
163 pkgs=$(cat $cooklist | wc -l)
164 echo -e '\nSummary for cookorder'
165 separator
166 cat <<EOT
167 Ordered packages : $pkgs
168 Scans executed : $scan
169 Scan duration : ${time}s
170 EOT
171 separator
173 rm -f $command
174 }
177 # Remove blocked (faster this way than grepping before).
179 strip_blocked() {
180 for pkg in $(cat $blocked); do
181 sed -i /^${pkg}$/d $cooklist
182 done
183 sed -i /^$/d $cooklist
184 }
187 # Use in default mode and with all cmd.
189 cook_commits() {
190 if [ -s "$commits" ]; then
191 for pkg in $(cat $commits); do
192 echo "cook:$pkg" > $command
193 cook $pkg || broken
194 sed -i /^${pkg}$/d $commits
195 done
196 fi
197 }
200 # Cook all packages in a cooklist.
202 cook_list() {
203 for pkg in $(cat $cooklist); do
204 cook $pkg || broken
205 sed -i /^${pkg}$/d $cooklist
206 done
207 }
210 # Create a arch.$ARCH file for each package cooked for the target host
211 # architecture
212 #
213 # The deal: we dont want all packages handled by cooker commands and stats,
214 # since we dont cross compile all packages for each arch but only a set of
215 # packages to provide one full featured desktop, servers and goodies useful
216 # for the host system.
217 #
219 arch_db() {
220 count=0
221 echo "Cleaning packages DB : arch.$ARCH"
222 rm -f $wok/*/arch.$ARCH && cd $wok
223 echo "Creating $ARCH packages DB..."
224 for pkg in *; do
225 HOST_ARCH=
226 . $wok/$pkg/receipt
227 if [ -n "$HOST_ARCH" ]; then
228 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
229 count=$(($count + 1))
230 echo "Adding: $pkg"
231 touch $pkg/arch.$ARCH
232 fi
233 unset HOST_ARCH
234 else
235 # HOST_ARCH not set --> in i486
236 if [ "$ARCH" == 'i486' ]; then
237 count=$(($count + 1))
238 echo "Adding: $pkg"
239 touch $pkg/arch.$ARCH
240 fi
241 fi
242 done
243 echo "Packages for $ARCH : $count"
244 }
247 #
248 # Commands
249 #
251 case "$1" in
252 usage|help|-u|-h)
253 usage ;;
255 setup|-s)
256 # Setup the Cooker environment.
257 echo -e '\nSetting up the Cooker'
258 mkdir -p $CACHE
259 echo "Cooker setup using: $SLITAZ" | log
260 separator
261 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
262 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
263 done
264 mkdir -p $SLITAZ && cd $SLITAZ
265 if [ -d "${wok}-hg" ]; then
266 echo -e 'Hg wok already exists.\n'
267 exit 1
268 fi
269 if [ -d "$wok" ]; then
270 echo -e 'Build wok already exists.\n'
271 exit 1
272 fi
274 # Directories and files
275 echo "mkdir's and touch files in: $SLITAZ"
276 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
277 for f in $activity $blocked $broken $commits $cooklist $command; do
278 touch $f
279 done
280 hg clone $WOK_URL ${wok}-hg || exit 1
281 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
282 cp -a ${wok}-hg $wok
283 separator; newline ;;
285 arch-db)
286 # Manually create arch packages DB.
287 arch_db ;;
289 setup-cron)
290 # Create cron job for the cooker.
291 [ "$2" ] || hours=2
292 if [ ! -f "$crontabs" ]; then
293 mkdir -p /var/spool/cron/crontabs
294 fi
295 if ! fgrep -q /usr/bin/cooker $crontabs; then
296 cat > $crontabs << EOT
297 # Run SliTaz Cooker every $hours hours
298 0 */$hours * * * /usr/bin/cooker --output=html
299 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
300 EOT
301 touch $CACHE/cooker-request
302 chmod 666 $CACHE/cooker-request
303 killall crond 2>/dev/null && /etc/init.d/crond start
304 fi ;;
306 check-cron)
307 if [ ! -f "$crontabs" ]; then
308 echo "There is no $crontabs here. Use setup-cron option."
309 exit 1
310 fi
311 fgrep /usr/bin/cooker $crontabs ;;
313 note|-n)
314 # Blocked a pkg and want others to know why? Post a note!
315 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
317 notes|-ns)
318 # View cooknotes.
319 echo -e '\nCooknotes'
320 separator
321 cat $cooknotes
322 separator; newline ;;
324 block|-b)
325 # Block a package.
326 [ "$pkg" ] && cook $pkg --block ;;
328 unblock|-ub)
329 # Unblock a package.
330 [ "$pkg" ] && cook $pkg --unblock ;;
332 reverse|-r)
333 # Cook all reverse dependencies for a package. This command lets us
334 # control the Cooker manually for commits that will cook a lot of packages.
335 #
336 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
337 #
338 if [ ! -d "$wok/$pkg" ]; then
339 echo -e "\nNo package $2 found.\n"
340 exit 0
341 fi
342 rm -f $cooklist; touch $cooklist
343 echo -e "\nReverse cooklist for: $pkg"
344 separator
346 cd $wok
347 for rev in *; do
348 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
349 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
350 echo "$rev" | tee -a $cooklist
351 fi
352 done
353 separator
354 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
355 strip_blocked
356 cook_order | tee $LOGS/cookorder.log
357 cook_list ;;
359 pkg|-p)
360 # Same as 'cook pkg' but with log for web interface.
361 cook $pkg || broken
362 clean_exit ;;
364 cat|-c)
365 # Cook all packages of a category.
366 cat="$2"
367 rm -f $cooklist; touch $cooklist
369 cd $wok
370 for pkg in *; do
371 unset CATEGORY; . $pkg/receipt
372 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
373 done
374 strip_blocked
375 cook_order | tee $LOGS/cookorder.log
376 cook_list ;;
378 flavor|-f)
379 # Cook all packages of a flavor.
380 name="$2"
381 if [ ! -d "$flavors/$name" ]; then
382 echo -e "\nSpecified flavor does not exist: $name\n"
383 exit 1
384 fi
385 if [ -d "$flavors/.hg" ]; then
386 cd $flavors; hg pull -u
387 fi
388 list="$flavors/$name/packages.list"
389 cp -a $list $cooklist
390 strip_blocked
391 cook_order | tee $LOGS/cookorder.log
392 cook_list ;;
394 list|-l)
395 # Cook a list of packages given in argument.
396 list="$2"
397 if [ ! -f "$list" ]; then
398 echo -e "\nSpecified list does not exist: $list\n"
399 exit 1
400 fi
401 cp -a $list $cooklist
402 strip_blocked
403 cook_order | tee $LOGS/cookorder.log
404 cook_list ;;
406 rev|-r)
407 # Cook or recook a specific Hg revision.
408 rev="$2"
409 [ "$rev" ] || exit 0
410 rm -f $cooklist; touch $cooklist
412 cd $wok
413 for pkg in $(hg log --rev=$rev --template "{files}"); do
414 echo "$pkg" | cut -d/ -f1 >> $cooklist
415 done
416 strip_blocked
417 cook_order | tee $LOGS/cookorder.log
418 cook_list ;;
420 all|-a)
421 # Try to build all unbuilt packages except blocked's.
422 echo 'cooker:all' > $command
423 rm -f $cooklist; touch $cooklist
424 newline
425 echo 'Cooker cooklist'
426 separator
428 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
429 # if it exists since extra version is added when packing the package.
430 echo 'Searching for all unbuilt packages' | log
432 cd $wok
433 for pkg in *; do
434 unset EXTRAVERSION
435 . $pkg/receipt
436 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
437 . $pkg/taz/$PACKAGE-$VERSION/receipt
438 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
439 echo $pkg; echo $pkg >> $cooklist
440 fi
441 done
442 strip_blocked
443 cook_order | tee $LOGS/cookorder.log
444 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
445 cook_list ;;
447 *)
448 # Default is to cook all commits if not yet running.
449 [ -n "$1" ] && usage
450 cooklist=$commits
451 if [ -f "$pidfile" ]; then
452 pid=$(cat $pidfile)
453 if [ -s /proc/$pid/status ]; then
454 echo -e "\nStill cooking latest commits with pid:"
455 echo -e " $pid\n"
456 exit 0
457 fi
458 rm -f "$pidfile"
459 fi
461 # Start and get a PID file.
462 rm -f $LOGS/commits.log
463 newline
464 echo 'Checking for commits' | log_commits
465 separator | tee -a $LOGS/commits.log
467 echo $$ > $pidfile
468 trap 'echo -e "\nCooker stopped: PID $$\n" && \
469 rm -f $pidfile $command && exit 1' INT TERM
471 echo "Cooker PID : $$" | log_commits
472 echo "Cooker date : $(date '+%F %T')" | log_commits
474 # Get revisions. Here we have 2 echoes since we want a msg on screen,
475 # in commits log and activity DB without a space before.
476 cd $wok || exit 1
477 cur=$(hg head --template '{rev}\n')
478 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
479 echo "Updating wok: ${wok}-hg" | log
480 echo 'hg:pull' > $command
481 cd $wok-hg; hg pull -u | log_commits
482 new=$(hg head --template '{rev}\n')
483 # Store last rev to be used by CGI so it doesn't need to call hg head
484 # on each load.
485 echo "$new" > $wokrev
487 # Sync build wok with rsync so we don't take care about removing old
488 # files as before.
489 if [ "$new" -gt "$cur" ]; then
490 echo "Changes found from: $cur to $new" | log
491 echo 'Syncing build wok with Hg wok...' | log_commits
492 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
493 sed '/^$/'d | log_commits
494 else
495 echo "No revision changes: $cur vs $new" | log
496 separator | log_commits
497 clean_exit; newline
498 exit 0
499 fi
501 # Get and display modifications.
502 cd $wok-hg
503 commits_summary | log_commits
504 cur=$(($cur + 1))
505 rm -f $commits.tmp; touch $commits.tmp
506 for rev in $(seq $cur $new); do
507 for file in $(hg log --rev=$rev --template "{files}"); do
508 pkg=$(echo $file | cut -d/ -f1)
509 desc=$(hg log --rev=$rev --template "{desc}" $file)
510 echo "Committed package : $pkg - $desc" | log_commits
511 echo $pkg >> $commits.tmp
512 done
513 done
515 # We may have deleted packages and files in stuff/. Remove it and
516 # clean DB as well as log file.
517 cd $wok
518 for pkg in *; do
519 if [ ! -d "${wok}-hg/$pkg" ]; then
520 echo "Removing package: $pkg" | log_commits
521 . $wok/$pkg/receipt
522 rm -rf $PKGS/$PACKAGE-$VERSION* $wok/$pkg $LOGS/$pkg.log
523 sed -i "/^${pkg}$/"d $blocked $broken $commits.tmp
524 fi
525 done
527 # Keep previous commit and discard duplicate lines
528 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
529 uniq $commits.new > $commits; rm $commits.*
531 # Handle cross compilation. Create arch packages DB and remove pkgs
532 # not cooked for this arch from the commits list.
533 arch_db
534 for pkg in $(cat $commits); do
535 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
536 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
537 log_commits
538 sed -i "/^${pkg}$/"d $commits
539 else
540 echo "Cooker arch : $ARCH" | log_commits
541 fi
542 done
544 # Stats
545 pkgs=$(cat $commits | wc -l)
546 echo "Packages to cook: $pkgs" | log
547 echo "Packages to cook : $pkgs" | log_commits
548 separator | log_commits
549 newline
550 strip_blocked
551 cook_order | tee $LOGS/cookorder.log
552 cook_commits
553 clean_exit ;;
554 esac
556 exit 0