cookutils view cooker @ rev 893
web/cooker.cgi: work with HTTP headers "Last-Modified", "If-Modified-Since", "HTTP 304 Not Modified", "HTTP 404 Not Found". Other small improvements.
SliTaz Next Cooker is working right now with this version of cooker.cgi.
SliTaz Next Cooker is working right now with this version of cooker.cgi.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Mon Mar 20 05:12:29 2017 +0200 (2017-03-20) |
parents | 2850833e8670 |
children | 71118129435e |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and discuss with the AUTHORS before adding anything here. PS: no translations
6 # here since it's not an end user tool and it's not useful. All devs should
7 # at least understand basic English.
8 #
10 . /usr/lib/slitaz/libcook.sh
12 # Set pkg name and use same wok as cook.
13 pkg="$2"
14 wok="$WOK"
16 # PID file.
17 pidfile='/var/run/cooker.pid'
19 #
20 # Functions
21 #
23 usage() {
24 cat <<EOT
26 Usage: cooker [command] [pkg|list|note|hours]
28 Options:
29 usage|-u Display this short usage.
30 setup|-s Setup the Cooker environment.
31 setup-cron Setup a cron job for the Cooker.
32 arch-db Create host arch packages DB.
33 note|-n Add a note to the cooknotes.
34 notes|-ns Display all the cooknotes.
35 block|-b Block a package so cook will skip it.
36 unblock|-ub Unblock a blocked package.
37 pkg|-p Same as 'cook pkg' but with cooker log.
38 flavor|-f Cook all packages of a flavor.
39 list|-l Cook all packages in the given list.
40 cat|-c Cook all packages of a category.
41 rev|-r Cook packages of a specific revision.
42 all|-a Find and cook all unbuilt packages.
44 EOT
45 exit 0
46 }
49 # Some messages occur in activity but log verbose output when checking for commits
50 # into a log file.
52 log_commits() {
53 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
54 }
57 # Clean up before exit when check and cook commits finish.
59 clean_exit() {
60 rm -f $command; touch $command
61 [ "$previous_command" ] && ps | grep -q "${previous_command/:/ }" &&
62 echo -n "$previous_command" > $command
63 rm -f $pidfile
64 }
67 # Summary for commits.
69 commits_summary() {
70 msg="from revision $cur to $new"
71 [ "$new" == "$cur" ] && msg="revision $new"
72 echo "Will cook $msg"
73 separator
74 echo -e "\nSummary for commits"
75 separator
76 echo "Hg wok revision : $cur"
77 echo "Pulled revision : $new"
78 echo "Check date : $(date '+%F %T')"
79 }
82 # Scan packages build deps and fill up cookorder list.
84 cook_order_scan() {
85 rm -f $cookorder
86 touch $cookorder
87 for pkg in $(cat $cooklist); do
88 unset WANTED BUILD_DEPENDS
89 . $wok/$pkg/receipt
90 # The :: is for web interface color.
91 [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS"
92 for dep in $WANTED $BUILD_DEPENDS; do
93 if grep -q "^$dep$" $cooklist; then
94 if ! grep -q "^$dep$" $cookorder; then
95 echo "$dep" >> $cookorder
96 fi
97 fi
98 done
99 done
101 # Append unordered packages to cookorder.
102 for pkg in $(cat $cooklist); do
103 if ! grep -q "^$pkg$" $cookorder; then
104 echo "$pkg" >> $cookorder
105 fi
106 done
107 }
110 # Scan and rescan until the cooklist is ordered then handle WANTED.
112 cook_order() {
113 time=$(date +%s)
114 scan=0
116 # Keep an original cooklist so we do a diff when ordering is finished.
117 cp -f $cooklist $cooklist.0
118 echo 'cookorder' > $command
119 echo -e '\nInitial Cooker order scan'
120 separator
121 cook_order_scan
123 # Diff between the cooklist and new ordered list ? So copy the last
124 # cookorder to cooklist and rescan it.
125 while /bin/true; do
126 diff $cooklist $cookorder > $cookorder.diff
127 if [ -s "$cookorder.diff" ]; then
128 scan=$(($scan + 1))
129 echo -e "\nDiff scan: $scan"
130 separator
131 mv -f $cookorder $cooklist
132 cook_order_scan
133 else
134 break
135 fi
136 done
138 # Keep a diff between submitted cooklist and the ordered.
139 diff $cooklist.0 $cooklist > $cooklist.diff
140 rm -f $cookorder $cookorder.diff $cooklist.0
142 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist.
143 # TODO: grep the line number to get pkg position and keep it higher.
144 echo -e '\nHandle WANTED package'
145 separator
146 for pkg in $(cat $cooklist); do
147 unset WANTED
148 . $wok/$pkg/receipt
149 for wanted in $WANTED; do
150 echo "$pkg :: $wanted"
151 if grep -q ^${wanted}$ $cooklist; then
152 sed -i -e "/^$pkg$/"d \
153 -e "/^$wanted$/ a $pkg" $cooklist
154 fi
155 done
156 done
158 # Show ordered cooklist
159 echo -e '\nCooklist order'
160 separator
161 cat $cooklist
162 separator
164 time=$(($(date +%s) - $time))
165 pkgs=$(cat $cooklist | wc -l)
166 echo -e '\nSummary for cookorder'
167 separator
168 cat <<EOT
169 Ordered packages : $pkgs
170 Scans executed : $scan
171 Scan duration : ${time}s
172 EOT
173 separator
175 rm -f $command
176 }
179 # Remove blocked (faster this way than grepping before).
181 strip_blocked() {
182 for pkg in $(cat $blocked); do
183 sed -i /^${pkg}$/d $cooklist
184 done
185 sed -i /^$/d $cooklist
186 }
189 # Use in default mode and with all cmd.
191 cook_commits() {
192 if [ -s "$commits" ]; then
193 for pkg in $(cat $commits); do
194 ps | grep -q "cook $pkg$" && continue
195 echo "cook:$pkg" > $command
196 cook $pkg || broken
197 sed -i /^${pkg}$/d $commits
198 done
199 fi
200 }
203 # Cook all packages in a cooklist.
205 cook_list() {
206 for pkg in $(cat $cooklist); do
207 ps | grep -q "cook $pkg$" && continue
208 cook $pkg || broken
209 sed -i /^${pkg}$/d $cooklist
210 done
211 }
214 # Create a arch.$ARCH file for each package cooked for the target host
215 # architecture
216 #
217 # The deal: we dont want all packages handled by cooker commands and stats,
218 # since we dont cross compile all packages for each arch but only a set of
219 # packages to provide one full featured desktop, servers and goodies useful
220 # for the host system.
221 #
223 arch_db() {
224 count=0
225 echo "Cleaning packages DB : arch.$ARCH"
226 rm -f $wok/*/arch.$ARCH && cd $wok
227 echo "Creating $ARCH packages DB..."
228 for pkg in *; do
229 [ -s $wok/$pkg/receipt ] || continue
230 HOST_ARCH=
231 . $wok/$pkg/receipt
232 if [ -n "$HOST_ARCH" ]; then
233 if $(echo "$HOST_ARCH" | egrep -q "$ARCH|any"); then
234 count=$(($count + 1))
235 echo "Adding: $pkg"
236 touch $pkg/arch.$ARCH
237 fi
238 unset HOST_ARCH
239 else
240 # HOST_ARCH not set --> in i486
241 if [ "$ARCH" == 'i486' ]; then
242 count=$(($count + 1))
243 echo "Adding: $pkg"
244 touch $pkg/arch.$ARCH
245 fi
246 fi
247 done
248 echo "Packages for $ARCH : $count"
249 }
252 #
253 # Commands
254 #
256 previous_command="$(cat $command)"
257 case "$1" in
258 usage|help|-u|-h)
259 usage ;;
261 setup|-s)
262 # Setup the Cooker environment.
263 echo -e '\nSetting up the Cooker'
264 mkdir -p $CACHE
265 echo "Cooker setup using: $SLITAZ" | log
266 separator
267 for pkg in $SETUP_PKGS mercurial rsync tazlito; do
268 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
269 done
270 mkdir -p $SLITAZ && cd $SLITAZ
271 if [ -d "${wok}-hg" ]; then
272 echo -e 'Hg wok already exists.\n'
273 exit 1
274 fi
275 if [ -d "$wok" ]; then
276 echo -e 'Build wok already exists.\n'
277 exit 1
278 fi
280 # Directories and files
281 echo "mkdir's and touch files in: $SLITAZ"
282 mkdir -p $PKGS $LOGS $FEEDS $CACHE $SRC
283 for f in $activity $blocked $broken $commits $cooklist $command; do
284 touch $f
285 done
286 hg clone $WOK_URL ${wok}-hg || exit 1
287 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors
288 cp -a ${wok}-hg $wok
289 separator; newline ;;
291 arch-db)
292 # Manually create arch packages DB.
293 arch_db ;;
295 setup-cron)
296 # Create cron job for the cooker.
297 [ "$2" ] || hours=2
298 if [ ! -f "$crontabs" ]; then
299 mkdir -p /var/spool/cron/crontabs
300 fi
301 if ! fgrep -q /usr/bin/cooker $crontabs; then
302 cat > $crontabs << EOT
303 # Run SliTaz Cooker every $hours hours
304 59 */$hours * * * touch $CACHE/cooker-request
305 */5 * * * * [ $CACHE/cooker-request -nt $CACHE/activity ] && /usr/bin/cooker --output=html
306 */5 * * * * [ -z "$(pidof cooker)" ] && [ -s $CACHE/recook-packages ] && /usr/bin/cooker list $CACHE/recook-packages
307 EOT
308 touch $CACHE/cooker-request $CACHE/recook-packages
309 chmod 666 $CACHE/cooker-request $CACHE/recook-packages
310 killall crond 2>/dev/null && /etc/init.d/crond start
311 fi ;;
313 check-cron)
314 if [ ! -f "$crontabs" ]; then
315 echo "There is no $crontabs here. Use setup-cron option."
316 exit 1
317 fi
318 fgrep /usr/bin/cooker $crontabs ;;
320 note|-n)
321 # Blocked a pkg and want others to know why? Post a note!
322 [ -n "$2" ] && echo "$(date '+%F %R') : $2" >> $cooknotes ;;
324 notes|-ns)
325 # View cooknotes.
326 echo -e '\nCooknotes'
327 separator
328 cat $cooknotes
329 separator; newline ;;
331 block|-b)
332 # Block a package.
333 [ "$pkg" ] && cook $pkg --block ;;
335 unblock|-ub)
336 # Unblock a package.
337 [ "$pkg" ] && cook $pkg --unblock ;;
339 reverse|-r)
340 # Cook all reverse dependencies for a package. This command lets us
341 # control the Cooker manually for commits that will cook a lot of packages.
342 #
343 # Use hg commit? Ex: hg commit -m "Message bla bla | cooker:reverse"
344 #
345 if [ ! -d "$wok/$pkg" ]; then
346 echo -e "\nNo package $2 found.\n"
347 exit 0
348 fi
349 rm -f $cooklist; touch $cooklist
350 echo -e "\nReverse cooklist for: $pkg"
351 separator
353 cd $wok
354 for rev in *; do
355 [ -s $wok/$rev/receipt ] || continue
356 unset WANTED DEPENDS BUILD_DEPENDS; . $wok/$rev/receipt
357 if echo "$WANTED $DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
358 echo "$rev" | tee -a $cooklist
359 fi
360 done
361 separator
362 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
363 strip_blocked
364 cook_order | tee $LOGS/cookorder.log
365 cook_list ;;
367 pkg|-p)
368 # Same as 'cook pkg' but with log for web interface.
369 ps | grep -q "cook $pkg$" && echo 'Already running' && continue
370 cook $pkg || broken
371 clean_exit ;;
373 cat|-c)
374 # Cook all packages of a category.
375 cat="$2"
376 rm -f $cooklist; touch $cooklist
378 cd $wok
379 for pkg in *; do
380 [ -s $pkg/receipt ] || continue
381 unset CATEGORY; . $pkg/receipt
382 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
383 done
384 strip_blocked
385 cook_order | tee $LOGS/cookorder.log
386 cook_list ;;
388 flavor|-f)
389 # Cook all packages of a flavor.
390 name="$2"
391 if [ ! -d "$flavors/$name" ]; then
392 echo -e "\nSpecified flavor does not exist: $name\n"
393 exit 1
394 fi
395 if [ -d "$flavors/.hg" ]; then
396 cd $flavors; hg pull -u
397 fi
398 list="$flavors/$name/packages.list"
399 cp -a $list $cooklist
400 strip_blocked
401 cook_order | tee $LOGS/cookorder.log
402 cook_list ;;
404 list|-l)
405 # Cook a list of packages given in argument.
406 list="$2"
407 if [ ! -f "$list" ]; then
408 echo -e "\nSpecified list does not exist: $list\n"
409 exit 1
410 fi
411 cat $list >> $cooklist
412 echo -n > $list
413 strip_blocked
414 cook_order | tee $LOGS/cookorder.log
415 cook_list ;;
417 rev|-r)
418 # Cook or recook a specific Hg revision.
419 rev="$2"
420 [ "$rev" ] || exit 0
421 rm -f $cooklist; touch $cooklist
423 cd $wok
424 for pkg in $(hg log --rev=$rev --template "{files}"); do
425 echo "$pkg" | cut -d/ -f1 >> $cooklist
426 done
427 strip_blocked
428 cook_order | tee $LOGS/cookorder.log
429 cook_list ;;
431 all|-a)
432 # Try to build all unbuilt packages except blocked's.
433 echo 'cooker:all' > $command
434 rm -f $cooklist; touch $cooklist
435 newline
436 echo 'Cooker cooklist'
437 separator
439 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt
440 # if it exists since extra version is added when packing the package.
441 echo 'Searching for all unbuilt packages' | log
443 cd $wok
444 for pkg in *; do
445 [ -s $pkg/receipt ] || continue
446 unset EXTRAVERSION
447 . $pkg/receipt
448 [ -f "$pkg/taz/$PACKAGE-$VERSION/receipt" ] && \
449 . $pkg/taz/$PACKAGE-$VERSION/receipt
450 if [ ! -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ]; then
451 echo $pkg; echo $pkg >> $cooklist
452 fi
453 done
454 strip_blocked
455 cook_order | tee $LOGS/cookorder.log
456 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
457 cook_list ;;
459 *)
460 # Default is to cook all commits if not yet running.
461 [ -n "$1" ] && usage
462 cooklist=$commits
463 if [ -f "$pidfile" ]; then
464 pid=$(cat $pidfile)
465 if [ -s /proc/$pid/status ]; then
466 echo -e "\nStill cooking latest commits with pid:"
467 echo -e " $pid\n"
468 exit 0
469 fi
470 rm -f "$pidfile"
471 fi
473 # Start and get a PID file.
474 rm -f $LOGS/commits.log
475 newline
476 echo 'Checking for commits' | log_commits
477 separator | tee -a $LOGS/commits.log
479 echo $$ > $pidfile
480 trap 'echo -e "\nCooker stopped: PID $$\n" && \
481 rm -f $pidfile $command && exit 1' INT TERM
483 echo "Cooker PID : $$" | log_commits
484 echo "Cooker date : $(date '+%F %T')" | log_commits
486 # Get revisions. Here we have 2 echoes since we want a msg on screen,
487 # in commits log and activity DB without a space before.
488 cd $wok || exit 1
489 cur=$(hg head --template '{rev}\n')
490 echo "Updating wok : ${wok}-hg (rev $cur)" | log_commits
491 echo "Updating wok: ${wok}-hg" | log
492 echo 'hg:pull' > $command
493 cd $wok-hg; hg pull -u | log_commits
494 new=$(hg head --template '{rev}\n')
495 # Store last rev to be used by CGI so it doesn't need to call hg head
496 # on each load.
497 echo "$new" > $wokrev
499 # Sync build wok with rsync so we don't take care about removing old
500 # files as before.
501 if [ "$new" -gt "$cur" ]; then
502 echo "Changes found from: $cur to $new" | log
503 echo 'Syncing build wok with Hg wok...' | log_commits
504 rsync -r -t -c -l -u -v -D -E $wok-hg/ $wok/ | \
505 sed '/^$/'d | log_commits
506 else
507 echo "No revision changes: $cur vs $new" | log
508 separator | log_commits
509 clean_exit; newline
510 exit 0
511 fi
513 # Get and display modifications.
514 cd $wok-hg
515 commits_summary | log_commits
516 cur=$(($cur + 1))
517 rm -f $commits.tmp; touch $commits.tmp
518 for rev in $(seq $cur $new); do
519 for file in $(hg log --rev=$rev --template "{files}"); do
520 pkg=$(echo $file | cut -d/ -f1)
521 desc=$(hg log --rev=$rev --template "{desc}" $file)
522 echo "Committed package : $pkg - $desc" | log_commits
523 echo $pkg >> $commits.tmp
524 done
525 done
527 # We may have deleted packages and files in stuff/. Remove it and
528 # clean DB as well as log file.
529 cd $wok
530 for pkg in *; do
531 if [ ! -d "${wok}-hg/$pkg" ]; then
532 echo "Removing package: $pkg" | log_commits
533 if [ -s $wok/$pkg/receipt ]; then
534 . $wok/$pkg/receipt
535 rm -f $PKGS/$PACKAGE-$VERSION*
536 fi
537 rm -rf $wok/$pkg $LOGS/$pkg.log
538 sed -i "/^${pkg}$/"d $blocked $broken $commits.tmp
539 fi
540 done
542 # Keep previous commit and discard duplicate lines
543 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
544 uniq $commits.new > $commits; rm $commits.*
546 # Handle cross compilation. Create arch packages DB and remove pkgs
547 # not cooked for this arch from the commits list.
548 arch_db
549 for pkg in $(cat $commits); do
550 if [ ! -f "$wok/$pkg/arch.$ARCH" ]; then
551 echo "Cooker arch : skip $pkg (not included in: $ARCH)" | \
552 log_commits
553 sed -i "/^${pkg}$/"d $commits
554 else
555 echo "Cooker arch : $ARCH" | log_commits
556 fi
557 done
559 # Stats
560 pkgs=$(cat $commits | wc -l)
561 echo "Packages to cook: $pkgs" | log
562 echo "Packages to cook : $pkgs" | log_commits
563 separator | log_commits
564 newline
565 strip_blocked
566 cook_order | tee $LOGS/cookorder.log
567 cook_commits
568 clean_exit ;;
569 esac
571 exit 0