cookutils view cooker @ rev 42
Improve extract_source (Thanks Godane)
| author | Christophe Lincoln <pankso@slitaz.org> |
|---|---|
| date | Sat May 07 02:22:37 2011 +0200 (2011-05-07) |
| parents | 302a81b24e6b |
| children | 176385db9cdb |
line source
1 #!/bin/sh
2 #
3 # SliTaz Build Bot. The Cooker is a tool to automate and test SliTaz package
4 # building. Please read the Cookbook documentation for more information
5 # and put talk to AUTHORS before adding anything here. PS: no translation
6 # here since it's not a end user tool and it's not usfull, all devs should
7 # at least understand basic English.
8 #
10 [ -f "/etc/slitaz/cook.conf" ] && . /etc/slitaz/cook.conf
11 [ -f "cook.conf" ] && . ./cook.conf
13 # The same wok as cook.
14 wok="$WOK"
15 flavors="$SLITAZ/flavors"
17 # Cooker DB files.
18 activity="$CACHE/activity"
19 commits="$CACHE/commits"
20 cooklist="$CACHE/cooklist"
21 cookorder="$CACHE/cookorder"
22 command="$CACHE/command"
23 blocked="$CACHE/blocked"
24 broken="$CACHE/broken"
25 cooknotes="$CACHE/cooknotes"
27 #
28 # Functions
29 #
31 usage() {
32 cat << EOT
34 Usage: cooker [command] [pkg|list|note]
36 Options:
37 usage|-u Display this short usage.
38 setup|-s Setup the Cooker environment.
39 note|-n Add a note to the cooknotes.
40 notes|-ns Display all the cooknotes.
41 pkg|-p Same as 'cook pkg' but with cooker log.
42 flavor|-f Cook all packages of a flavor.
43 list|-l Cook all package in the given list.
44 cat|-c Cook all packages of a category.
45 all|-a Find and cook all unbuilt packages.
47 EOT
48 exit 0
49 }
51 separator() {
52 echo "================================================================================"
53 }
55 # Lograte activity.
56 [ -s "$activity" ] && tail -n 20 $activity > /tmp/tail && \
57 mv -f /tmp/tail $activity
59 # Log activities, we want first letter capitalized.
60 log() {
61 grep ^[A-Z] | \
62 sed s"#^[A-Z]\([^']*\)#$(date '+%Y-%m-%d %H:%M') : \0#" >> $activity
63 }
65 # Some message goes in activity but log verbose output when checking for commits
66 # into a log file.
67 log_commits() {
68 sed '/^.\//'d | sed '/^.hg/'d | tee -a $LOGS/commits.log
69 }
71 # Log broken packages
72 broken() {
73 echo "$pkg" >> $broken
74 }
76 # Clean up after cook sucess.
77 empty_command() {
78 rm -f $command && touch $command
79 }
81 # Summary for commits.
82 commits_summary() {
83 msg="from revision $cur to $new"
84 [ "$new" == "$cur" ] && msg="revision $new"
85 echo "Will cook $msg"
86 separator
87 echo -e "\nSummary for commits"
88 separator
89 echo "Hg wok revision : $cur"
90 echo "Pulled revision : $new"
91 echo "Check date : $(date '+%Y-%m-%d %H:%M')"
92 }
94 # Scan packages build deps an fill up cookorder list.
95 cook_order_scan() {
96 touch $cooklist $cookorder
97 for pkg in $(cat $cooklist)
98 do
99 unset BUILD_DEPENDS
100 . $wok/$pkg/receipt
101 # The :: is for web interface color.
102 [ "$BUILD_DEPENDS" ] && echo "$pkg :: $BUILD_DEPENDS"
103 for dep in $BUILD_DEPENDS
104 do
105 if grep -q "^$dep$" $cooklist; then
106 if ! grep -q "^$dep$" $cookorder; then
108 echo "$dep" >> $cookorder
109 fi
110 fi
111 done
112 done
114 # Append unordered packages to cookored.
115 for pkg in $(cat $cooklist)
116 do
117 if ! grep -q "^$pkg$" $cookorder; then
118 echo "$pkg" >> $cookorder
119 fi
120 done
121 }
123 # Scan and rescan untill the cooklist is ordered then handle WANTED.
124 cook_order() {
125 time=$(date +%s)
126 scan=0
128 # Keep an original cooklist so we do a diff when ordering is finished.
129 cp -f $cooklist $cooklist.0
130 echo -e "\nInitial Cooker order scan"
131 separator
132 cook_order_scan
134 # Diff between the cooklist and new ordered list ? So copy the last
135 # cookorder to cooklist and rescan it.
136 while /bin/true
137 do
138 diff $cooklist $cookorder > $cookorder.diff
139 if [ -s "$cookorder.diff" ]; then
140 scan=$(($scan + 1))
141 echo -e "\nDiff scan: $scan"
142 separator
143 mv -f $cookorder $cooklist
144 cook_order_scan
145 else
146 break
147 fi
148 done
150 # Keep a diff between submited cooklist and the ordered.
151 diff $cooklist.0 $cooklist > $cooklist.diff
152 rm -f $cookorder $cookorder.diff $cooklist.0
154 # Scan is finish: append pkg to WANTED
155 echo -e "\nHandle WANTED package"
156 separator
157 for pkg in $(cat $cooklist)
158 do
159 unset WANTED
160 . $wok/$pkg/receipt
161 if [ "$WANTED" ]; then
162 echo "$pkg :: $WANTED"
163 sed -i -e "/^$pkg$/"d \
164 -e "/^$WANTED$/ a $pkg" $cooklist
165 fi
166 done
168 # Show ordered cooklist
169 echo -e "\nCooklist order"
170 separator
171 cat $cooklist
172 separator
173 time=$(($(date +%s) - $time))
174 pkgs=$(cat $cooklist | wc -l)
175 echo -e "\nSummary for cookorder"
176 separator
177 cat << EOT
178 Ordered packages : $pkgs
179 Scans executed : $scan
180 Scan duration : ${time}s
181 EOT
182 separator
183 }
185 # Remove blocked (faster this way than grepping before).
186 strip_blocked() {
187 for pkg in $(cat $blocked)
188 do
189 sed -i /^${pkg}$/d $cooklist
190 done && sed -i /^$/d $cooklist
191 }
193 # Uses in default mode and with all cmd.
194 cook_commits() {
195 if [ -s "$commits" ]; then
196 for pkg in $(cat $commits)
197 do
198 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
199 echo "cook:$pkg" > $command
200 cook $pkg || broken
201 sed -i /^${pkg}$/d $commits
202 done
203 fi
204 }
206 # Cook all package in a cooklist.
207 cook_list() {
208 for pkg in $(cat $cooklist)
209 do
210 if [ ! -d "$wok/$pkg/install" ]; then
211 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
212 cook $pkg || broken
213 sed -i /^${pkg}$/d $cooklist
214 fi
215 done
216 }
218 #
219 # Commands
220 #
221 case "$1" in
222 usage|help|-u|-h)
223 usage ;;
224 setup|-s)
225 # Setup the Cooker environment.
226 echo -e "\nSetting up the Cooker"
227 echo "Cooker setup using: $SLITAZ" | log
228 separator
229 for pkg in $SETUP_PKGS mercurial rsync
230 do
231 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg
232 done
233 mkdir -p $SLITAZ && cd $SLITAZ
234 [ -d "${wok}-hg" ] && echo -e "Hg wok already exist.\n" && exit 1
235 [ -d "$wok" ] && echo -e "Build wok already exist.\n" && exit 1
236 [ -d "$flavors" ] && echo -e "Flavors repo already exist.\n" && exit 1
238 # Directories and files
239 echo "mkdir's and touch files in: $SLITAZ"
240 mkdir -p $PKGS $LOGS $CACHE $SRC
241 for f in $activity $blocked $broken $commits $cooklist $command
242 do
243 touch $f
244 done
245 hg clone $WOK_URL ${wok}-hg || exit 1
246 hg clone $FLAVORS_URL flavors
247 cp -a ${wok}-hg $wok
248 separator && echo "" ;;
249 note|-n)
250 # Blocked a pkg and want other to know why ? Post a note!
251 note="$2"
252 [ "$note" ] && echo "$note" >> $cooknotes ;;
253 notes|-ns)
254 # View cooknotes.
255 echo -e "\nCooknotes"
256 separator
257 cat $cooknotes
258 separator && echo "" ;;
259 reverse|-r)
260 # Cook all reverse dependencies for a packages. This command let us
261 # control the Cooker manually for commit that will cook a lot of packages.
262 #
263 # Use hg commit ? Ex: hg commit -m "Message bla bla | cooker:reverse"
264 #
265 pkg="$2"
266 [ ! -d "$wok/$pkg" ] && echo -e "\nNo package $2 found.\n" && exit 0
267 rm -f $cooklist && touch $cooklist && cd $wok
268 echo -e "\nReverse cooklist for: $pkg"
269 separator && cd $wok
270 for rev in *
271 do
272 unset DEPENDS BUILD_DEPENDS && . $wok/$rev/receipt
273 if echo "$DEPENDS $BUILD_DEPENDS" | fgrep -q $pkg; then
274 echo "$rev" | tee -a $cooklist
275 fi
276 done && separator
277 echo -e "Reverse dependencies found: $(cat $cooklist | wc -l)\n"
278 strip_blocked
279 cook_order | tee $LOGS/cookorder.log
280 cook_list ;;
281 pkg|-p)
282 # Same as 'cook pkg' but with log for web interface.
283 pkg="$2"
284 echo "Cook started for: <a href='cooker.cgi?pkg=$pkg'>$pkg</a>" | log
285 cook $pkg || broken
286 empty_command ;;
287 cat|-c)
288 # Cook all packages of a category.
289 cat="$2"
290 rm -f $cooklist && touch $cooklist && cd $wok
291 for pkg in *
292 do
293 unset CATEGORY && . $pkg/receipt
294 [ "$CATEGORY" == "$cat" ] && echo $pkg >> $cooklist
295 done
296 strip_blocked
297 cook_order | tee $LOGS/cookorder.log
298 cook_list ;;
299 flavor|-f)
300 # Cook all packages of a flavor.
301 name="$2"
302 [ ! -d "$flavors/$name" ] && \
303 echo -e "\nSpecified flavor does not exist: $name\n" && exit 1
304 list=$flavors/$name/packages.list
305 cp -a $list $cooklist
306 strip_blocked
307 cook_order | tee $LOGS/cookorder.log
308 cook_list ;;
309 list|-l)
310 # Cook a list og package given in argument.
311 list="$2"
312 [ ! -f "$list" ] && \
313 echo -e "\nSpecified list does not exist: $list\n" && exit 1
314 cp -a $list $cooklist
315 strip_blocked
316 cook_order | tee $LOGS/cookorder.log
317 cook_list ;;
318 all|-a)
319 # Try to build all unbuilt packages except blocked's.
320 echo "cooker:all" > $command
321 rm -f $cooklist && touch $cooklist
322 echo "" && cd $wok
323 echo "Cooker cooklist"
324 separator
326 # Find all unbuilt packages.
327 echo "Searching for all unbuilt packages" | log
328 for pkg in *
329 do
330 . $pkg/receipt
331 [ ! -f "$PKGS/$PACKAGE-${VERSION}${EXTRAVERSION}.tazpkg" ] && \
332 echo $pkg >> $cooklist
333 done
335 strip_blocked
336 echo "Packages to cook: $(cat $cooklist | wc -l)" | log
337 cook_order | tee $LOGS/cookorder.log
338 cook_list && empty_command;;
339 *)
340 # Default is to cook all commits.
341 [ "$1" ] && usage
342 cooklist=$CACHE/commits
343 rm -f $LOGS/commits.log
344 echo ""
345 echo "Checking for commits" | log_commits
346 separator | tee -a $LOGS/commits.log
348 # Get revisions.
349 cd $wok || ( echo "No wok found: $wok" && exit 1 )
350 cur=$(hg head --template '{rev}\n')
351 echo "Updating Hg wok: ${wok}-hg" | log
352 echo "hg:pull" > $command
353 [ -f "${wok}-hg/.hg/hgrc" ] && cd ${wok}-hg && hg pull -u | log_commits
354 new=$(hg head --template '{rev}\n')
356 # Sync build wok with rsync so we dont take care about removing old
357 # files as before.
358 if [ "$new" -gt "$cur" ]; then
359 echo "Changes found from: $cur to $new" | log
360 echo "Syncing build wok with Hg wok..." | log_commits
361 rsync -r -t -c -l -u -p -v -D -E --delete ${wok}-hg/ $wok/ | \
362 sed '/^$/'d | log_commits
363 else
364 echo "No revision changes: $cur vs $new" | log
365 separator | log_commits
366 empty_command && echo "" && exit 0
367 fi
369 # Get and display modifications.
370 cd ${wok}-hg
371 cur=$(($cur + 1))
372 commits_summary | log_commits
374 rm -f $commits.tmp && touch $commits.tmp
375 for rev in $(seq $cur $new); do
376 #log=$(hg log --rev=$rev --template "{files}\t{desc}\n")
377 log=$(hg log --rev=$rev --template "{files}\n" | cut -d "/" -f 1)
379 for file in $log; do
380 desc=$(hg log --rev=$rev --template "{desc}" $file)
381 echo "Commited : $file - $desc" | log_commits
382 echo $file >> $commits.tmp
383 done
384 done
386 # Keep previews commit and discard duplicate lines
387 cat $commits $commits.tmp | sed /"^$"/d > $commits.new
388 uniq $commits.new > $commits && rm $commits.*
389 echo "Packages to cook : $(cat $commits | wc -l)" | log_commits
390 separator | log_commits
391 echo ""
392 strip_blocked
393 cook_order | tee $LOGS/cookorder.log
394 cook_commits && empty_command ;;
395 esac
397 exit 0