cookutils view cooker @ rev 898
Add lighttpd/*: this is for cook.slitaz.org Lighttpd-driven web interface.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Thu May 11 00:17:06 2017 +0300 (2017-05-11) |
| parents | 2850833e8670 |
| children | 71118129435e |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [command] [pkg|list|note|hours]
28 Options:
29 usage|-u Display this short usage.
30 setup|-s Setup the Cooker environment.
31 setup-cron Setup a cron job for the Cooker.
32 arch-db Create host arch packages DB.
33 note|-n Add a note to the cooknotes.
34 notes|-ns Display all the cooknotes.
35 block|-b Block a package so cook will skip it.
36 unblock|-ub Unblock a blocked package.
37 pkg|-p Same as 'cook pkg' but with cooker log.
38 flavor|-f Cook all packages of a flavor.
39 list|-l Cook all packages in the given list.
40 cat|-c Cook all packages of a category.
41 rev|-r Cook packages of a specific revision.
42 all|-a Find and cook all unbuilt packages.
44 EOT
45 exit 0
46 }
49 # Some messages occur in activity but log verbose output when checking for commits
50 # into a log file.
52 log_commits() {
53 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
54 }
57 # Clean up before exit when check and cook commits finish.
59 clean_exit() {
60 rm -f $command; touch $command
61 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
62 echo -n "$previous_command" > $command
63 rm -f $pidfile
64 }
67 # Summary for commits.
69 commits_summary() {
70 msg="from revision $cur to $new"
71 [ "$new" == "$cur" ] && msg="revision $new"
72 echo "Will cook $msg"
73 separator
74 echo -e "\nSummary for commits"
75 separator
76 echo "Hg wok revision : $cur"
77 echo "Pulled revision : $new"
78 echo "Check date : $(date '+%F %T')"
79 }
82 # Scan packages build deps and fill up cookorder list.
84 cook_order_scan() {
85 rm -f $cookorder
86 touch $cookorder
87 for pkg in $(cat $cooklist); do
88 unset WANTED BUILD_DEPENDS
89 . $wok/$pkg/receipt
90 # The :: is for web interface color.
91 [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS"
92 for dep in $WANTED $BUILD_DEPENDS; do
93 if grep -q "^$dep$" $cooklist; then
94 if ! grep -q "^$dep$" $cookorder; then
95 echo "$dep" >> $cookorder
96 fi
97 fi
98 done
99 done
101 # Append unordered packages to cookorder.
102 for pkg in $(cat $cooklist); do
103 if ! grep -q "^$pkg$" $cookorder; then
104 echo "$pkg" >> $cookorder
105 fi
106 done
107 }
110 # Scan and rescan until the cooklist is ordered then handle WANTED.
112 cook_order() {
113 time=$(date +%s)
114 scan=0
116 # Keep an original cooklist so we do a diff when ordering is finished.
117 cp -f $cooklist $cooklist.0
118 echo 'cookorder' > $command
119 echo -e '\nInitial Cooker order scan'
120 separator
121 cook_order_scan
123 # Diff between the cooklist and new ordered list ? So copy the last
124 # cookorder to cooklist and rescan it.
125 while /bin/true; do
126 diff $cooklist $cookorder > $cookorder.diff
127 if [ -s "$cookorder.diff" ]; then
128 scan=$(($scan + 1))
129 echo -e "\nDiff scan: $scan"
130 separator
131 mv -f $cookorder $cooklist
132 cook_order_scan
133 else
134 break
135 fi
136 done
138 # Keep a diff between submitted cooklist and the ordered.
139 diff $cooklist.0 $cooklist > $cooklist.diff
140 rm -f $cookorder $cookorder.diff $cooklist.0
142 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
143 # TODO: grep the line number to get pkg position and keep it higher.
144 echo -e '\nHandle WANTED package'
145 separator
146 for pkg in $(cat $cooklist); do
147 unset WANTED
148 . $wok/$pkg/receipt
149 for wanted in $WANTED; do
150 echo "$pkg :: $wanted"
151 if grep -q ^${wanted}$ $cooklist; then
152 sed -i -e "/^$pkg$/"d \
153 -e "/^$wanted$/ a $pkg" $cooklist
154 fi
155 done
156 done
158 # Show ordered cooklist
159 echo -e '\nCooklist order'
160 separator
161 cat $cooklist
162 separator
164 time=$(($(date +%s) - $time))
165 pkgs=$(cat $cooklist | wc -l)
166 echo -e '\nSummary for cookorder'
167 separator
168 cat <<EOT
169 Ordered packages : $pkgs
170 Scans executed : $scan
171 Scan duration : ${time}s
172 EOT
173 separator
175 rm -f $command
176 }
179 # Remove blocked (faster this way than grepping before).
181 strip_blocked() {
182 for pkg in $(cat $blocked); do
183 sed -i /^${pkg}$/d $cooklist
184 done
185 sed -i /^$/d $cooklist
186 }
189 # Use in default mode and with all cmd.
191 cook_commits() {
192 if [ -s "$commits" ]; then
193 for pkg in $(cat $commits); do
194 ps | grep -q "cook $pkg$" && continue
195 echo "cook:$pkg" > $command
196 cook $pkg || broken
197 sed -i /^${pkg}$/d $commits
198 done
199 fi
200 }
203 # Cook all packages in a cooklist.
205 cook_list() {
206 for pkg in $(cat $cooklist); do
207 ps | grep -q "cook $pkg$" && continue
208 cook $pkg || broken
209 sed -i /^${pkg}$/d $cooklist
210 done
211 }
214 # Create a arch.$ARCH file for each package cooked for the target host
215 # architecture
216 #
217 # The deal: we dont want all packages handled by cooker commands and stats,
218 # since we dont cross compile all packages for each arch but only a set of
219 # packages to provide one full featured desktop, servers and goodies useful
220 # for the host system.
221 #
223 arch_db() {
224 count=0
225 echo "Cleaning packages DB : arch.$ARCH"
226 rm -f $wok/*/arch.$ARCH && cd $wok
227 echo "Creating $ARCH packages DB..."
228 for pkg in *; do
229 [ -s $wok/$pkg/receipt ] || continue
230 HOST_ARCH=
231 . $wok/$pkg/receipt
232 if [ -n "$HOST_ARCH" ]; then
233 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
234 count=$(($count + 1))
235 echo "Adding: $pkg"
236 touch $pkg/arch.$ARCH
237 fi
238 unset HOST_ARCH
239 else
240 # HOST_ARCH not set --> in i486
241 if [ "$ARCH" == 'i486' ]; then
242 count=$(($count + 1))
243 echo "Adding: $pkg"
244 touch $pkg/arch.$ARCH
245 fi
246 fi
247 done
248 echo "Packages for $ARCH : $count"
249 }
252 #
253 # Commands
254 #
256 previous_command="$(cat $command)"
257 case "$1" in
258 usage|help|-u|-h)
259 usage ;;
261 setup|-s)
262 # Setup the Cooker environment.
263 echo -e '\nSetting up the Cooker'
264 mkdir -p $CACHE
265 echo "Cooker setup using: $SLITAZ" | log
266 separator
267 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
268 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
269 done
270 mkdir -p $SLITAZ && cd $SLITAZ
271 if [ -d "${wok}-hg" ]; then
272 echo -e 'Hg wok already exists.\n'
273 exit 1
274 fi
275 if [ -d "$wok" ]; then
276 echo -e 'Build wok already exists.\n'
277 exit 1
278 fi
280 # Directories and files
281 echo "mkdir's and touch files in: $SLITAZ"
282 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
283 for f in $activity $blocked $broken $commits $cooklist $command; do
284 touch $f
285 done
286 hg clone $WOK_URL ${wok}-hg || exit 1
287 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
288 cp -a ${wok}-hg $wok
289 separator; newline ;;
291 arch-db)
292 # Manually create arch packages DB.
293 arch_db ;;
295 setup-cron)
296 # Create cron job for the cooker.
297 [ "$2" ] || hours=2
298 if [ ! -f "$crontabs" ]; then
299 mkdir -p /var/spool/cron/crontabs
300 fi
301 if ! fgrep -q /usr/bin/cooker $crontabs; then
302 cat > $crontabs << EOT
303 # Run SliTaz Cooker every $hours hours
304 59 */$hours * * * touch $CACHE/cooker-request
305 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
306 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
307 EOT
308 touch $CACHE/cooker-request $CACHE/recook-packages
309 chmod 666 $CACHE/cooker-request $CACHE/recook-packages
310 killall crond 2>/dev/null && /etc/init.d/crond start
311 fi ;;
313 check-cron)
314 if [ ! -f "$crontabs" ]; then
315 echo "There is no $crontabs here. Use setup-cron option."
316 exit 1
317 fi
318 fgrep /usr/bin/cooker $crontabs ;;
320 note|-n)
321 # Blocked a pkg and want others to know why? Post a note!
322 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
324 notes|-ns)
325 # View cooknotes.
326 echo -e '\nCooknotes'
327 separator
328 cat $cooknotes
329 separator; newline ;;
331 block|-b)
332 # Block a package.
333 [ "$pkg" ] && cook $pkg --block ;;
335 unblock|-ub)
336 # Unblock a package.
337 [ "$pkg" ] && cook $pkg --unblock ;;
339 reverse|-r)
340 # Cook all reverse dependencies for a package. This command lets us
341 # control the Cooker manually for commits that will cook a lot of packages.
342 #
343 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
344 #
345 if [ ! -d "$wok/$pkg" ]; then
346 echo -e "\nNo package $2 found.\n"
347 exit 0
348 fi
349 rm -f $cooklist; touch $cooklist
350 echo -e "\nReverse cooklist for: $pkg"
351 separator
353 cd $wok
354 for rev in *; do
355 [ -s $wok/$rev/receipt ] || continue
356 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
357 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
358 echo "$rev" | tee -a $cooklist
359 fi
360 done
361 separator
362 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
363 strip_blocked
364 cook_order | tee $LOGS/cookorder.log
365 cook_list ;;
367 pkg|-p)
368 # Same as 'cook pkg' but with log for web interface.
369 ps | grep -q "cook $pkg$" && echo 'Already running' && continue
370 cook $pkg || broken
371 clean_exit ;;
373 cat|-c)
374 # Cook all packages of a category.
375 cat="$2"
376 rm -f $cooklist; touch $cooklist
378 cd $wok
379 for pkg in *; do
380 [ -s $pkg/receipt ] || continue
381 unset CATEGORY; . $pkg/receipt
382 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
383 done
384 strip_blocked
385 cook_order | tee $LOGS/cookorder.log
386 cook_list ;;
388 flavor|-f)
389 # Cook all packages of a flavor.
390 name="$2"
391 if [ ! -d "$flavors/$name" ]; then
392 echo -e "\nSpecified flavor does not exist: $name\n"
393 exit 1
394 fi
395 if [ -d "$flavors/.hg" ]; then
396 cd $flavors; hg pull -u
397 fi
398 list="$flavors/$name/packages.list"
399 cp -a $list $cooklist
400 strip_blocked
401 cook_order | tee $LOGS/cookorder.log
402 cook_list ;;
404 list|-l)
405 # Cook a list of packages given in argument.
406 list="$2"
407 if [ ! -f "$list" ]; then
408 echo -e "\nSpecified list does not exist: $list\n"
409 exit 1
410 fi
411 cat $list >> $cooklist
412 echo -n > $list
413 strip_blocked
414 cook_order | tee $LOGS/cookorder.log
415 cook_list ;;
417 rev|-r)
418 # Cook or recook a specific Hg revision.
419 rev="$2"
420 [ "$rev" ] || exit 0
421 rm -f $cooklist; touch $cooklist
423 cd $wok
424 for pkg in $(hg log --rev=$rev --template "{files}"); do
425 echo "$pkg" | cut -d/ -f1 >> $cooklist
426 done
427 strip_blocked
428 cook_order | tee $LOGS/cookorder.log
429 cook_list ;;
431 all|-a)
432 # Try to build all unbuilt packages except blocked's.
433 echo 'cooker:all' > $command
434 rm -f $cooklist; touch $cooklist
435 newline
436 echo 'Cooker cooklist'
437 separator
439 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
440 # if it exists since extra version is added when packing the package.
441 echo 'Searching for all unbuilt packages' | log
443 cd $wok
444 for pkg in *; do
445 [ -s $pkg/receipt ] || continue
446 unset EXTRAVERSION
447 . $pkg/receipt
448 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
449 . $pkg/taz/$PACKAGE-$VERSION/receipt
450 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
451 echo $pkg; echo $pkg >> $cooklist
452 fi
453 done
454 strip_blocked
455 cook_order | tee $LOGS/cookorder.log
456 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
457 cook_list ;;
459 *)
460 # Default is to cook all commits if not yet running.
461 [ -n "$1" ] && usage
462 cooklist=$commits
463 if [ -f "$pidfile" ]; then
464 pid=$(cat $pidfile)
465 if [ -s /proc/$pid/status ]; then
466 echo -e "\nStill cooking latest commits with pid:"
467 echo -e " $pid\n"
468 exit 0
469 fi
470 rm -f "$pidfile"
471 fi
473 # Start and get a PID file.
474 rm -f $LOGS/commits.log
475 newline
476 echo 'Checking for commits' | log_commits
477 separator | tee -a $LOGS/commits.log
479 echo $$ > $pidfile
480 trap 'echo -e "\nCooker stopped: PID $$\n" && \
481 rm -f $pidfile $command && exit 1' INT TERM
483 echo "Cooker PID : $$" | log_commits
484 echo "Cooker date : $(date '+%F %T')" | log_commits
486 # Get revisions. Here we have 2 echoes since we want a msg on screen,
487 # in commits log and activity DB without a space before.
488 cd $wok || exit 1
489 cur=$(hg head --template '{rev}\n')
490 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
491 echo "Updating wok: ${wok}-hg" | log
492 echo 'hg:pull' > $command
493 cd $wok-hg; hg pull -u | log_commits
494 new=$(hg head --template '{rev}\n')
495 # Store last rev to be used by CGI so it doesn't need to call hg head
496 # on each load.
497 echo "$new" > $wokrev
499 # Sync build wok with rsync so we don't take care about removing old
500 # files as before.
501 if [ "$new" -gt "$cur" ]; then
502 echo "Changes found from: $cur to $new" | log
503 echo 'Syncing build wok with Hg wok...' | log_commits
504 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
505 sed '/^$/'d | log_commits
506 else
507 echo "No revision changes: $cur vs $new" | log
508 separator | log_commits
509 clean_exit; newline
510 exit 0
511 fi
513 # Get and display modifications.
514 cd $wok-hg
515 commits_summary | log_commits
516 cur=$(($cur + 1))
517 rm -f $commits.tmp; touch $commits.tmp
518 for rev in $(seq $cur $new); do
519 for file in $(hg log --rev=$rev --template "{files}"); do
520 pkg=$(echo $file | cut -d/ -f1)
521 desc=$(hg log --rev=$rev --template "{desc}" $file)
522 echo "Committed package : $pkg - $desc" | log_commits
523 echo $pkg >> $commits.tmp
524 done
525 done
527 # We may have deleted packages and files in stuff/. Remove it and
528 # clean DB as well as log file.
529 cd $wok
530 for pkg in *; do
531 if [ ! -d "${wok}-hg/$pkg" ]; then
532 echo "Removing package: $pkg" | log_commits
533 if [ -s $wok/$pkg/receipt ]; then
534 . $wok/$pkg/receipt
535 rm -f $PKGS/$PACKAGE-$VERSION*
536 fi
537 rm -rf $wok/$pkg $LOGS/$pkg.log
538 sed -i "/^${pkg}$/"d $blocked $broken $commits.tmp
539 fi
540 done
542 # Keep previous commit and discard duplicate lines
543 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
544 uniq $commits.new > $commits; rm $commits.*
546 # Handle cross compilation. Create arch packages DB and remove pkgs
547 # not cooked for this arch from the commits list.
548 arch_db
549 for pkg in $(cat $commits); do
550 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
551 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
552 log_commits
553 sed -i "/^${pkg}$/"d $commits
554 else
555 echo "Cooker arch : $ARCH" | log_commits
556 fi
557 done
559 # Stats
560 pkgs=$(cat $commits | wc -l)
561 echo "Packages to cook: $pkgs" | log
562 echo "Packages to cook : $pkgs" | log_commits
563 separator | log_commits
564 newline
565 strip_blocked
566 cook_order | tee $LOGS/cookorder.log
567 cook_commits
568 clean_exit ;;
569 esac
571 exit 0