cookutils view cooker @ rev 974
german translation completed
author | Hans-G?nter Theisgen |
---|---|
date | Mon Oct 02 16:24:14 2017 +0100 (2017-10-02) |
parents | 48b237fbb760 |
children | 9c835beabf65 |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [<command>] [<options>]
28 Commands with <options>:
29 -u | usage Display this short usage.
30 -s | setup Setup the Cooker environment.
31 setup-cron [<hours>] Setup a cron job for the Cooker.
32 check-cron Check Cooker cron job.
33 arch-db Create host arch packages DB.
34 -n | note <note_text> Add a note to the cooknotes.
35 -ns | notes Display all the cooknotes.
36 -b | block <package> Block a package so cook will skip it.
37 -ub | unblock <package> Unblock a blocked package.
38 -R | reverse <package> Cook all reverse dependencies for a package.
39 -p | pkg <package> Same as 'cook pkg' but with cooker log.
40 -f | flavor <flavor_name> Cook all packages of a flavor.
41 -l | list <list_file> Cook all packages in the given list.
42 -c | cat <category> Cook all packages of a category.
43 -r | rev <rev_number> Cook packages of a specific revision.
44 -a | all Find and cook all unbuilt packages.
45 -T | tasks List existing cooker tasks.
46 -t | task <task> Executing specified task.
47 -o | outgoing Find changes in wok that we can move to wok-hg.
49 EOT
50 exit 0
51 }
54 # Some messages occur in activity but log verbose output when checking for commits
55 # into a log file.
57 log_commits() {
58 sed '/^.\//d' | sed '/^.hg/d' | tee -a $LOGS/commits.log
59 }
62 # Clean up before exit when check and cook commits finish.
64 clean_exit() {
65 rm -f $command; touch $command
66 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
67 echo -n "$previous_command" > $command
68 rm -f $pidfile
69 }
72 # Summary for commits.
74 commits_summary() {
75 msg="from revision $cur to $new"
76 [ "$new" == "$cur" ] && msg="revision $new"
77 echo "Will cook $msg"
78 separator
79 title "Summary for commits"
80 echo "Hg wok revision : $cur"
81 echo "Pulled revision : $new"
82 echo "Check date : $(date '+%F %T')"
83 }
86 # Return all the names of packages bundled in this receipt
88 all_names() {
89 local split=" $SPLIT "
90 unset SPLIT
91 . $wok/$pkg/receipt
93 if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then
94 # For receipts v1: $SPLIT may present in the $WANTED package,
95 # but split packages have their own receipts
96 echo $PACKAGE
97 elif [ "${split/ $PACKAGE /}" != "$split" ]; then
98 echo $SPLIT
99 else
100 echo $PACKAGE $SPLIT
101 fi
102 }
105 # Scan packages build deps and fill up cookorder list.
107 cook_order_scan() {
108 rm -f $cookorder $cookorder.split
109 touch $cookorder $cookorder.split
111 # Make combined split table: beginning from actual information with fresh
112 # commits. Example:
113 # freetype freetype freetype-dev
114 # harfbuzz harfbuzz harfbuzz-apps harfbuzz-dev
115 while read pkg; do
116 echo "$pkg $(all_names)" >> $cookorder.split
117 done < $cooklist
118 cat $cache/split.db >> $cookorder.split
120 maxlen=$(wc -L < $cooklist)
122 while read pkg; do
123 unset WANTED BUILD_DEPENDS
124 . $wok/$pkg/receipt
125 bdeps=$(
126 # Substitite each package of BUILD_DEPENDS list by the "main"
127 # receipt which builds this package. Example:
128 # BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz"
129 for i in $BUILD_DEPENDS; do
130 main="$(awk -F$'\t' -vi="$i" '{
131 if (index(" " $2 " ", i)) {print $1; exit}
132 }' $cookorder.split)"
133 echo ${main:-$i}
134 done
135 )
136 # The :: is for web interface color.
137 bdeps=$(echo $WANTED $bdeps | tr '\n' ' ')
138 printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps"
139 for dep in $bdeps; do
140 if grep -q "^$dep$" $cooklist; then
141 if ! grep -q "^$dep$" $cookorder; then
142 echo "$dep" >> $cookorder
143 fi
144 fi
145 done
146 done < $cooklist
148 # Append unordered packages to cookorder.
149 while read pkg; do
150 if ! grep -q "^$pkg$" $cookorder; then
151 echo "$pkg" >> $cookorder
152 fi
153 done < $cooklist
154 }
157 # Scan and rescan until the cooklist is ordered then handle WANTED.
159 cook_order() {
160 time=$(date +%s)
161 scan=0
162 rm -rf $cache/cookorder.d
163 mkdir -p $cache/cookorder.d
165 # Keep an original cooklist so we do a diff when ordering is finished.
166 cp -f $cooklist $cooklist.0
167 echo 'cookorder' > $command
168 title 'Initial Cooker order scan'
169 cook_order_scan
171 # Diff between the cooklist and new ordered list ? So copy the last
172 # cookorder to cooklist and rescan it.
173 while /bin/true; do
174 if ! cmp -s $cooklist $cookorder; then
175 scan=$(($scan + 1))
176 title "Diff scan: $scan"
178 md5stamp=$(md5sum $cookorder | cut -d' ' -f1)
179 if [ -e "$cache/cookorder.d/$md5stamp" ]; then
180 newline
181 echo 'A dependency loop was detected. Interrupting the cookorder.'
182 break
183 fi
184 touch $cache/cookorder.d/$md5stamp
186 mv -f $cookorder $cooklist
187 cook_order_scan
189 else
190 break
191 fi
192 done
193 # Clean
194 rm -rf $cache/cookorder.d; rm $cookorder.split
196 # Keep a diff between submitted cooklist and the ordered.
197 diff $cooklist.0 $cooklist > $cooklist.diff
198 rm -f $cookorder $cooklist.0
200 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
201 # TODO: grep the line number to get pkg position and keep it higher.
202 title 'Handle WANTED package'
203 while read pkg; do
204 unset WANTED
205 . $wok/$pkg/receipt
206 for wanted in $WANTED; do
207 echo "$pkg :: $wanted"
208 if grep -q ^${wanted}$ $cooklist; then
209 sed -i -e "/^$pkg$/d" \
210 -e "/^$wanted$/ a $pkg" $cooklist
211 fi
212 done
213 done < $cooklist
215 # Show ordered cooklist
216 title 'Cooklist order'
217 cat $cooklist
218 separator
220 time=$(($(date +%s) - $time))
221 pkgs=$(wc -l < $cooklist)
222 title 'Summary for cookorder'
223 cat <<EOT
224 Ordered packages : $pkgs
225 Scans executed : $scan
226 Scan duration : ${time}s
227 EOT
228 separator
230 rm -f $command
231 }
234 # Remove blocked (faster this way than grepping before).
236 strip_blocked() {
237 while read pkg; do
238 sed -i "/^${pkg}$/d" $cooklist
239 done < $blocked
240 sed -i '/^$/d' $cooklist
241 }
244 # Use in default mode and with all cmd.
246 cook_commits() {
247 if [ -s "$commits" ]; then
248 while read pkg; do
249 ps | grep -q "cook $pkg$" && continue
250 echo "cook:$pkg" > $command
251 cook $pkg || broken
252 sed -i "/^${pkg}$/d" $commits
253 done < $commits
254 fi
255 }
258 # Cook all packages in a cooklist.
260 cook_list() {
261 while read pkg; do
262 ps | grep -q "cook $pkg$" && continue
263 cook $pkg || broken
264 sed -i "/^${pkg}$/d" $cooklist
265 done < $cooklist
266 }
269 # Create a arch.$ARCH file for each package cooked for the target host
270 # architecture
271 #
272 # The deal: we don't want all packages handled by cooker commands and stats,
273 # since we don't cross compile all packages for each arch but only a set of
274 # packages to provide one full featured desktop, servers and goodies useful
275 # for the host system.
276 #
278 arch_db() {
279 count=0
280 echo "Cleaning packages DB : arch.$ARCH"
281 rm -f $wok/*/arch.$ARCH && cd $wok
282 echo "Creating $ARCH packages DB..."
283 for pkg in *; do
284 [ -s $wok/$pkg/receipt ] || continue
285 HOST_ARCH=
286 . $wok/$pkg/receipt
287 if [ -n "$HOST_ARCH" ]; then
288 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
289 count=$(($count + 1))
290 echo "Adding: $pkg"
291 touch $pkg/arch.$ARCH
292 fi
293 unset HOST_ARCH
294 else
295 # HOST_ARCH not set --> in i486
296 if [ "$ARCH" == 'i486' ]; then
297 count=$(($count + 1))
298 echo "Adding: $pkg"
299 touch $pkg/arch.$ARCH
300 fi
301 fi
302 done
303 echo "Packages for $ARCH : $count"
304 }
307 # Compare wok and wok-hg file $1, display signs:
308 # '+' file added, '-' file removed, '~' file changed, '=' file not changed
310 compare_wok_file() {
311 local f1='n' f2='n' # n: not exists, e: exists
312 [ -e "$wok/$1" ] && f1='e'
313 [ -e "$wok-hg/$1" ] && f2='e'
314 case "$f1$f2" in
315 en) echo "+ $1";;
316 ne) [ -n "$del" ] && echo "- $1";;
317 ee)
318 if cmp -s "$wok/$1" "$wok-hg/$1"; then
319 [ -n "$eq" ] && echo "= $1"
320 else
321 echo "~ $1"
322 fi
323 ;;
324 esac
325 }
328 # Compare wok and wok-hg folder $1; process only:
329 # receipt, description.*txt, all files in the stuff folder
331 compare_wok_folder() {
332 IFS=$'\n'
333 {
334 for i in $wok $wok-hg; do
335 ls $i/$1/receipt 2>/dev/null
336 ls $i/$1/description.*txt 2>/dev/null
337 [ -d $i/$1/stuff ] && find $i/$1/stuff -type f
338 done
339 } | sed "s|$wok/$1/||; s|$wok-hg/$1/||" | sort -u | \
340 while read file; do
341 compare_wok_file "$1/$file"
342 done
343 }
346 # Compare entire wok
348 compare_wok() {
349 {
350 cd $wok; ls
351 cd $wok-hg; ls
352 } | sort -u | \
353 while read folder; do
354 result="$(compare_wok_folder $folder)"
355 [ -n "$result" ] && echo -e "$result\n"
356 done
357 }
360 #
361 # Commands
362 #
364 previous_command="$(cat $command 2>/dev/null)"
365 case "$1" in
366 usage|help|-u|-h)
367 usage ;;
369 setup|-s)
370 # Setup the Cooker environment.
371 title 'Setting up the Cooker'
372 mkdir -p $CACHE
373 echo "Cooker setup using: $SLITAZ" | log
374 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
375 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
376 done
377 mkdir -p $SLITAZ && cd $SLITAZ
378 if [ -d "${wok}-hg" ]; then
379 echo -e 'Hg wok already exists.\n'
380 exit 1
381 fi
382 if [ -d "$wok" ]; then
383 echo -e 'Build wok already exists.\n'
384 exit 1
385 fi
387 # Directories and files
388 echo "mkdir's and touch files in: $SLITAZ"
389 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
390 for f in $activity $blocked $broken $commits $cooklist $command; do
391 touch $f
392 done
393 hg clone $WOK_URL ${wok}-hg || exit 1
394 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
395 cp -a ${wok}-hg $wok
396 footer ;;
398 arch-db)
399 # Manually create arch packages DB.
400 arch_db ;;
402 setup-cron)
403 # Create cron job for the cooker.
404 [ "$2" ] || hours=2
405 if [ ! -f "$crontabs" ]; then
406 mkdir -p /var/spool/cron/crontabs
407 fi
408 if ! fgrep -q /usr/bin/cooker $crontabs; then
409 cat > $crontabs <<EOT
410 # Run SliTaz Cooker every $hours hours
411 59 */$hours * * * touch $CACHE/cooker-request
412 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
413 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
414 EOT
415 touch $CACHE/cooker-request $CACHE/recook-packages
416 chmod 666 $CACHE/cooker-request $CACHE/recook-packages
417 killall crond 2>/dev/null && /etc/init.d/crond start
418 fi ;;
420 check-cron)
421 if [ ! -f "$crontabs" ]; then
422 echo "There is no $crontabs here. Use setup-cron option."
423 exit 1
424 fi
425 fgrep /usr/bin/cooker $crontabs ;;
427 note|-n)
428 # Blocked a pkg and want others to know why? Post a note!
429 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
431 notes|-ns)
432 # View cooknotes.
433 title 'Cooknotes'
434 cat $cooknotes
435 footer ;;
437 block|-b)
438 # Block a package.
439 [ "$pkg" ] && cook $pkg --block ;;
441 unblock|-ub)
442 # Unblock a package.
443 [ "$pkg" ] && cook $pkg --unblock ;;
445 reverse|-r)
446 # Cook all reverse dependencies for a package. This command lets us
447 # control the Cooker manually for commits that will cook a lot of packages.
448 #
449 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
450 #
451 if [ ! -d "$wok/$pkg" ]; then
452 echo -e "\nNo package $2 found.\n"
453 exit 0
454 fi
455 rm -f $cooklist; touch $cooklist
456 title "Reverse cooklist for: $pkg"
458 cd $wok
459 for rev in *; do
460 [ -s $wok/$rev/receipt ] || continue
461 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
462 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
463 echo "$rev" | tee -a $cooklist
464 fi
465 done
466 footer "Reverse dependencies found: $(wc -l < $cooklist)"
467 strip_blocked
468 cook_order | tee $LOGS/cookorder.log
469 cook_list ;;
471 pkg|-p)
472 # Same as 'cook pkg' but with log for web interface.
473 ps | grep -q "cook $pkg$" && echo 'Already running' && continue
474 cook $pkg || broken
475 clean_exit ;;
477 cat|-c)
478 # Cook all packages of a category.
479 cat="$2"
480 rm -f $cooklist; touch $cooklist
482 cd $wok
483 for pkg in *; do
484 [ -s $pkg/receipt ] || continue
485 unset CATEGORY; . $pkg/receipt
486 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
487 done
488 strip_blocked
489 cook_order | tee $LOGS/cookorder.log
490 cook_list ;;
492 flavor|-f)
493 # Cook all packages of a flavor.
494 name="$2"
495 if [ ! -d "$flavors/$name" ]; then
496 echo -e "\nSpecified flavor does not exist: $name\n"
497 exit 1
498 fi
499 if [ -d "$flavors/.hg" ]; then
500 cd $flavors; hg pull -u
501 fi
502 list="$flavors/$name/packages.list"
503 cp -a $list $cooklist
504 strip_blocked
505 cook_order | tee $LOGS/cookorder.log
506 cook_list ;;
508 list|-l)
509 # Cook a list of packages given in argument.
510 list="$2"
511 if [ ! -f "$list" ]; then
512 echo -e "\nSpecified list does not exist: $list\n"
513 exit 1
514 fi
515 cat $list >> $cooklist
516 echo -n > $list
517 strip_blocked
518 cook_order | tee $LOGS/cookorder.log
519 cook_list ;;
521 rev|-r)
522 # Cook or recook a specific Hg revision.
523 rev="$2"
524 [ "$rev" ] || exit 0
525 rm -f $cooklist; touch $cooklist
527 cd $wok
528 for pkg in $(hg log --rev=$rev --template "{files}"); do
529 echo "$pkg" | cut -d/ -f1 >> $cooklist
530 done
531 strip_blocked
532 cook_order | tee $LOGS/cookorder.log
533 cook_list ;;
535 all|-a)
536 # Try to build all unbuilt packages except blocked's.
537 echo 'cooker:all' > $command
538 rm -f $cooklist; touch $cooklist
539 title 'Cooker cooklist'
541 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
542 # if it exists since extra version is added when packing the package.
543 echo 'Searching for all unbuilt packages' | log
545 cd $wok
546 for pkg in *; do
547 [ -s $pkg/receipt ] || continue
548 unset EXTRAVERSION
549 . $pkg/receipt
550 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
551 . $pkg/taz/$PACKAGE-$VERSION/receipt
552 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
553 echo $pkg; echo $pkg >> $cooklist
554 fi
555 done
556 strip_blocked
557 cook_order | tee $LOGS/cookorder.log
558 echo "Packages to cook: $(wc -l < $cooklist)" | log
559 cook_list ;;
561 tasks|-T)
562 # List existing cooker tasks
563 [ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0
564 title 'Cooker tasks list'
565 last=$(ls $tasks | tail -n1)
566 for task in $(ls $tasks); do
567 . $tasks/$task
568 echo "Task name : $task"
569 echo "Description : $DESC"
570 separator $([ $task != $last ] && echo '-')
571 done
572 newline ;;
574 task|-t)
575 # Executing specified task
576 task="$2"
577 title "Executing cooker task: $task"
578 . $tasks/$task; task
579 footer "Task $task finished" ;;
581 outgoing|-o)
582 # Find changes in wok that we can move to wok-hg
583 compare_wok
584 ;;
586 *)
587 # Default is to cook all commits if not yet running.
588 [ -n "$1" ] && usage
589 cooklist=$commits
590 if [ -f "$pidfile" ]; then
591 pid=$(cat $pidfile)
592 if [ -s /proc/$pid/status ]; then
593 echo -e "\nStill cooking latest commits with pid:"
594 echo -e " $pid\n"
595 exit 0
596 fi
597 rm -f "$pidfile"
598 fi
600 # Start and get a PID file.
601 rm -f $LOGS/commits.log
602 newline
603 echo 'Checking for commits' | log_commits
604 separator | tee -a $LOGS/commits.log
606 echo $$ > $pidfile
607 trap 'echo -e "\nCooker stopped: PID $$\n" && \
608 rm -f $pidfile $command && exit 1' INT TERM
610 echo "Cooker PID : $$" | log_commits
611 echo "Cooker date : $(date '+%F %T')" | log_commits
613 # Get revisions. Here we have 2 echoes since we want a msg on screen,
614 # in commits log and activity DB without a space before.
615 cd $wok || exit 1
616 cur=$(hg head --template '{rev}\n')
617 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
618 echo "Updating wok: ${wok}-hg" | log
619 echo 'hg:pull' > $command
620 cd $wok-hg; hg pull -u | log_commits
621 new=$(hg head --template '{rev}\n')
622 # Store last rev to be used by CGI so it doesn't need to call hg head
623 # on each load.
624 echo "$new" > $wokrev
626 # Sync build wok with rsync so we don't take care about removing old
627 # files as before.
628 if [ "$new" -gt "$cur" ]; then
629 echo "Changes found from: $cur to $new" | log
630 echo 'Syncing build wok with Hg wok...' | log_commits
631 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
632 sed '/^$/d' | log_commits
633 else
634 echo "No revision changes: $cur vs $new" | log
635 separator | log_commits
636 clean_exit; newline
637 exit 0
638 fi
640 # Get and display modifications.
641 cd $wok-hg
642 commits_summary | log_commits
643 cur=$(($cur + 1))
644 rm -f $commits.tmp; touch $commits.tmp
645 for rev in $(seq $cur $new); do
646 for file in $(hg log --rev=$rev --template "{files}"); do
647 pkg=$(echo $file | cut -d/ -f1)
648 desc=$(hg log --rev=$rev --template "{desc}" $file)
649 echo "Committed package : $pkg - $desc" | log_commits
650 echo $pkg >> $commits.tmp
651 done
652 done
654 # We may have deleted packages and files in stuff/. Remove it and
655 # clean DB as well as log file.
656 cd $wok
657 for pkg in *; do
658 if [ ! -d "$wok-hg/$pkg" ]; then
659 echo "Removing package: $pkg" | log_commits
660 if [ -s $wok/$pkg/receipt ]; then
661 . $wok/$pkg/receipt
662 rm -f $PKGS/$PACKAGE-$VERSION*
663 fi
664 rm -rf $wok/$pkg $LOGS/$pkg.log
665 sed -i "/^${pkg}$/d" $blocked $broken $commits.tmp
666 sed -i "/^$pkg\t/d" $PKGS/packages.info
667 fi
668 if [ -d "$wok/$pkg/stuff" ]; then
669 if [ ! -d "$wok-hg/$pkg/stuff" ]; then
670 echo "Removing stuff: $pkg/stuff" | log_commits
671 rm -rf $wok/$pkg/stuff
672 else
673 for stuff_file in $(cd $wok/$pkg/stuff; find \( -type f -o -type l \) | sed 's|^\./||'); do
674 if [ ! -f "$wok-hg/$pkg/stuff/$stuff_file" -a \
675 ! -h "$wok-hg/$pkg/stuff/$stuff_file" ]; then
676 echo "Removing file from stuff: $wok/$pkg/stuff/$stuff_file" | log_commits
677 rm -f $wok/$pkg/stuff/$stuff_file
678 rmdir --parents --ignore-fail-on-non-empty $(dirname "$wok/$pkg/stuff/$stuff_file")
679 fi
680 done
681 fi
682 fi
683 done
685 # Keep previous commit and discard duplicate lines
686 cat $commits $commits.tmp | sed '/^$/d' > $commits.new
687 uniq $commits.new > $commits; rm $commits.*
689 # Handle cross compilation. Create arch packages DB and remove pkgs
690 # not cooked for this arch from the commits list.
691 arch_db
692 while read pkg; do
693 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
694 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
695 log_commits
696 sed -i "/^${pkg}$/d" $commits
697 else
698 echo "Cooker arch : $ARCH" | log_commits
699 fi
700 done < $commits
702 # Stats
703 pkgs=$(wc -l < $commits)
704 echo "Packages to cook: $pkgs" | log
705 echo "Packages to cook : $pkgs" | log_commits
706 separator | log_commits
707 newline
708 strip_blocked
709 cook_order | tee $LOGS/cookorder.log
710 cook_commits
711 clean_exit ;;
712 esac
714 exit 0