cookutils rev 942
cooker: explain usage, improve cook order to be used with split v1 & v2, add tasks support.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Sun Jun 25 14:14:30 2017 +0300 (2017-06-25) |
parents | adda5217f721 |
children | 5595f56612fe |
files | cooker lib/libcook.sh lighttpd/index.cgi tasks/freetype-harfbuzz tasks/up-toolchain |
line diff
1.1 --- a/cooker Fri Jun 23 12:02:34 2017 +0300 1.2 +++ b/cooker Sun Jun 25 14:14:30 2017 +0300 1.3 @@ -23,23 +23,27 @@ 1.4 usage() { 1.5 cat <<EOT 1.6 1.7 -Usage: cooker [command] [pkg|list|note|hours] 1.8 +Usage: cooker [<command>] [<options>] 1.9 1.10 -Options: 1.11 - usage|-u Display this short usage. 1.12 - setup|-s Setup the Cooker environment. 1.13 - setup-cron Setup a cron job for the Cooker. 1.14 - arch-db Create host arch packages DB. 1.15 - note|-n Add a note to the cooknotes. 1.16 - notes|-ns Display all the cooknotes. 1.17 - block|-b Block a package so cook will skip it. 1.18 - unblock|-ub Unblock a blocked package. 1.19 - pkg|-p Same as 'cook pkg' but with cooker log. 1.20 - flavor|-f Cook all packages of a flavor. 1.21 - list|-l Cook all packages in the given list. 1.22 - cat|-c Cook all packages of a category. 1.23 - rev|-r Cook packages of a specific revision. 1.24 - all|-a Find and cook all unbuilt packages. 1.25 +Commands with <options>: 1.26 + -u | usage Display this short usage. 1.27 + -s | setup Setup the Cooker environment. 1.28 + setup-cron [<hours>] Setup a cron job for the Cooker. 1.29 + check-cron Check Cooker cron job. 1.30 + arch-db Create host arch packages DB. 1.31 + -n | note <note_text> Add a note to the cooknotes. 1.32 + -ns | notes Display all the cooknotes. 1.33 + -b | block <package> Block a package so cook will skip it. 1.34 + -ub | unblock <package> Unblock a blocked package. 1.35 + -R | reverse <package> Cook all reverse dependencies for a package. 1.36 + -p | pkg <package> Same as 'cook pkg' but with cooker log. 1.37 + -f | flavor <flavor_name> Cook all packages of a flavor. 1.38 + -l | list <list_file> Cook all packages in the given list. 1.39 + -c | cat <category> Cook all packages of a category. 1.40 + -r | rev <rev_number> Cook packages of a specific revision. 1.41 + -a | all Find and cook all unbuilt packages. 1.42 + -T | tasks List existing cooker tasks. 1.43 + -t | task <task> Executing specified task. 1.44 1.45 EOT 1.46 exit 0 1.47 @@ -71,25 +75,67 @@ 1.48 [ "$new" == "$cur" ] && msg="revision $new" 1.49 echo "Will cook $msg" 1.50 separator 1.51 - echo -e "\nSummary for commits" 1.52 - separator 1.53 + title "Summary for commits" 1.54 echo "Hg wok revision : $cur" 1.55 echo "Pulled revision : $new" 1.56 echo "Check date : $(date '+%F %T')" 1.57 } 1.58 1.59 1.60 +# Return all the names of packages bundled in this receipt 1.61 + 1.62 +all_names() { 1.63 + local split=" $SPLIT " 1.64 + unset SPLIT 1.65 + . $wok/$pkg/receipt 1.66 + 1.67 + if ! head -n1 $WOK/$pkg/receipt | fgrep -q 'v2'; then 1.68 + # For receipts v1: $SPLIT may present in the $WANTED package, 1.69 + # but split packages have their own receipts 1.70 + echo $PACKAGE 1.71 + elif [ "${split/ $PACKAGE /}" != "$split" ]; then 1.72 + echo $SPLIT 1.73 + else 1.74 + echo $PACKAGE $SPLIT 1.75 + fi 1.76 +} 1.77 + 1.78 + 1.79 # Scan packages build deps and fill up cookorder list. 1.80 1.81 cook_order_scan() { 1.82 - rm -f $cookorder 1.83 - touch $cookorder 1.84 + rm -f $cookorder $cookorder.split 1.85 + touch $cookorder $cookorder.split 1.86 + 1.87 + # Make combined split table: beginning from actual information with fresh 1.88 + # commits. Example: 1.89 + # freetype freetype freetype-dev 1.90 + # harfbuzz harfbuzz harfbuzz-apps harfbuzz-dev 1.91 + while read pkg; do 1.92 + echo "$pkg $(all_names)" >> $cookorder.split 1.93 + done < $cooklist 1.94 + cat $cache/split.db >> $cookorder.split 1.95 + 1.96 + maxlen=$(wc -L < $cooklist) 1.97 + 1.98 while read pkg; do 1.99 unset WANTED BUILD_DEPENDS 1.100 . $wok/$pkg/receipt 1.101 + bdeps=$( 1.102 + # Substitite each package of BUILD_DEPENDS list by the "main" 1.103 + # receipt which builds this package. Example: 1.104 + # BUILD_DEPENDS="freetype-dev harfbuzz-dev" -> bdeps="freetype harfbuzz" 1.105 + for i in $BUILD_DEPENDS; do 1.106 + main="$(awk -F$'\t' -vi="$i" '{ 1.107 + if (index(" " $2 " ", i)) {print $1; exit} 1.108 + }' $cookorder.split)" 1.109 + echo ${main:-$i} 1.110 + done 1.111 + ) 1.112 # The :: is for web interface color. 1.113 - [ "$WANTED$BUILD_DEPENDS" ] && echo "$pkg :: $WANTED $BUILD_DEPENDS" 1.114 - for dep in $WANTED $BUILD_DEPENDS; do 1.115 + bdeps=$(echo $WANTED $bdeps | tr '\n' ' ') 1.116 + printf "%-${maxlen}s :: %s\n" "$pkg" "$bdeps" 1.117 + for dep in $bdeps; do 1.118 if grep -q "^$dep$" $cooklist; then 1.119 if ! grep -q "^$dep$" $cookorder; then 1.120 echo "$dep" >> $cookorder 1.121 @@ -112,37 +158,47 @@ 1.122 cook_order() { 1.123 time=$(date +%s) 1.124 scan=0 1.125 + rm -rf $cache/cookorder.d 1.126 + mkdir -p $cache/cookorder.d 1.127 1.128 # Keep an original cooklist so we do a diff when ordering is finished. 1.129 cp -f $cooklist $cooklist.0 1.130 echo 'cookorder' > $command 1.131 - echo -e '\nInitial Cooker order scan' 1.132 - separator 1.133 + title 'Initial Cooker order scan' 1.134 cook_order_scan 1.135 1.136 # Diff between the cooklist and new ordered list ? So copy the last 1.137 # cookorder to cooklist and rescan it. 1.138 while /bin/true; do 1.139 - diff $cooklist $cookorder > $cookorder.diff 1.140 - if [ -s "$cookorder.diff" ]; then 1.141 + if ! cmp -s $cooklist $cookorder; then 1.142 scan=$(($scan + 1)) 1.143 - echo -e "\nDiff scan: $scan" 1.144 - separator 1.145 + title "Diff scan: $scan" 1.146 + 1.147 + md5stamp=$(md5sum $cookorder | cut -d' ' -f1) 1.148 + if [ -e "$cache/cookorder.d/$md5stamp" ]; then 1.149 + newline 1.150 + echo 'A dependency loop was detected. Interrupting the cookorder.' 1.151 + break 1.152 + fi 1.153 + touch $cache/cookorder.d/$md5stamp 1.154 + 1.155 mv -f $cookorder $cooklist 1.156 cook_order_scan 1.157 + 1.158 else 1.159 break 1.160 fi 1.161 done 1.162 + # Clean 1.163 + rm -rf $cache/cookorder.d; rm $cookorder.split 1.164 1.165 # Keep a diff between submitted cooklist and the ordered. 1.166 diff $cooklist.0 $cooklist > $cooklist.diff 1.167 - rm -f $cookorder $cookorder.diff $cooklist.0 1.168 + rm -f $cookorder $cooklist.0 1.169 1.170 # Scan finished: append pkg to WANTED or leave it in the ordered cooklist. 1.171 # TODO: grep the line number to get pkg position and keep it higher. 1.172 - echo -e '\nHandle WANTED package' 1.173 - separator 1.174 + title 'Handle WANTED package' 1.175 while read pkg; do 1.176 unset WANTED 1.177 . $wok/$pkg/receipt 1.178 @@ -156,15 +212,13 @@ 1.179 done < $cooklist 1.180 1.181 # Show ordered cooklist 1.182 - echo -e '\nCooklist order' 1.183 - separator 1.184 + title 'Cooklist order' 1.185 cat $cooklist 1.186 separator 1.187 1.188 time=$(($(date +%s) - $time)) 1.189 pkgs=$(wc -l < $cooklist) 1.190 - echo -e '\nSummary for cookorder' 1.191 - separator 1.192 + title 'Summary for cookorder' 1.193 cat <<EOT 1.194 Ordered packages : $pkgs 1.195 Scans executed : $scan 1.196 @@ -260,10 +314,9 @@ 1.197 1.198 setup|-s) 1.199 # Setup the Cooker environment. 1.200 - echo -e '\nSetting up the Cooker' 1.201 + title 'Setting up the Cooker' 1.202 mkdir -p $CACHE 1.203 echo "Cooker setup using: $SLITAZ" | log 1.204 - separator 1.205 for pkg in $SETUP_PKGS mercurial rsync tazlito; do 1.206 [ ! -d "$INSTALLED/$pkg" ] && tazpkg get-install $pkg 1.207 done 1.208 @@ -286,7 +339,7 @@ 1.209 hg clone $WOK_URL ${wok}-hg || exit 1 1.210 [ -d "$flavors" ] || hg clone $FLAVORS_URL flavors 1.211 cp -a ${wok}-hg $wok 1.212 - separator; newline ;; 1.213 + footer ;; 1.214 1.215 arch-db) 1.216 # Manually create arch packages DB. 1.217 @@ -323,10 +376,9 @@ 1.218 1.219 notes|-ns) 1.220 # View cooknotes. 1.221 - echo -e '\nCooknotes' 1.222 - separator 1.223 + title 'Cooknotes' 1.224 cat $cooknotes 1.225 - separator; newline ;; 1.226 + footer ;; 1.227 1.228 block|-b) 1.229 # Block a package. 1.230 @@ -347,8 +399,7 @@ 1.231 exit 0 1.232 fi 1.233 rm -f $cooklist; touch $cooklist 1.234 - echo -e "\nReverse cooklist for: $pkg" 1.235 - separator 1.236 + title "Reverse cooklist for: $pkg" 1.237 1.238 cd $wok 1.239 for rev in *; do 1.240 @@ -358,8 +409,7 @@ 1.241 echo "$rev" | tee -a $cooklist 1.242 fi 1.243 done 1.244 - separator 1.245 - echo -e "Reverse dependencies found: $(wc -l < $cooklist)\n" 1.246 + footer "Reverse dependencies found: $(wc -l < $cooklist)" 1.247 strip_blocked 1.248 cook_order | tee $LOGS/cookorder.log 1.249 cook_list ;; 1.250 @@ -432,9 +482,7 @@ 1.251 # Try to build all unbuilt packages except blocked's. 1.252 echo 'cooker:all' > $command 1.253 rm -f $cooklist; touch $cooklist 1.254 - newline 1.255 - echo 'Cooker cooklist' 1.256 - separator 1.257 + title 'Cooker cooklist' 1.258 1.259 # Find all unbuilt packages. Get EXTRAVERSION from packed receipt 1.260 # if it exists since extra version is added when packing the package. 1.261 @@ -456,6 +504,26 @@ 1.262 echo "Packages to cook: $(wc -l < $cooklist)" | log 1.263 cook_list ;; 1.264 1.265 + tasks|-T) 1.266 + # List existing cooker tasks 1.267 + [ ! -d "$tasks" ] && echo 'There are no tasks.' && exit 0 1.268 + title 'Cooker tasks list' 1.269 + last=$(ls $tasks | tail -n1) 1.270 + for task in $(ls $tasks); do 1.271 + . $tasks/$task 1.272 + echo "Task name : $task" 1.273 + echo "Description : $DESC" 1.274 + separator $([ $task != $last ] && echo '-') 1.275 + done 1.276 + newline ;; 1.277 + 1.278 + task|-t) 1.279 + # Executing specified task 1.280 + task="$2" 1.281 + title "Executing cooker task: $task" 1.282 + . $tasks/$task; task 1.283 + footer "Task $task finished" ;; 1.284 + 1.285 *) 1.286 # Default is to cook all commits if not yet running. 1.287 [ -n "$1" ] && usage
2.1 --- a/lib/libcook.sh Fri Jun 23 12:02:34 2017 +0300 2.2 +++ b/lib/libcook.sh Sun Jun 25 14:14:30 2017 +0300 2.3 @@ -43,6 +43,7 @@ 2.4 cooknotes="$cache/cooknotes" 2.5 cooktime="$cache/cooktime" 2.6 crontabs="/var/spool/cron/crontabs/root" 2.7 +tasks="$SLITAZ/tasks" 2.8 2.9 2.10 # Lograte activity.
3.1 --- a/lighttpd/index.cgi Fri Jun 23 12:02:34 2017 +0300 3.2 +++ b/lighttpd/index.cgi Sun Jun 25 14:14:30 2017 +0300 3.3 @@ -523,7 +523,8 @@ 3.4 s|$_fs|<var>\${fs}</var>|g; 3.5 s|$_stuff|<var>\${stuff}</var>|g" \ 3.6 -e "s|\[9\([1-6]\)m|<span class='c\10'>|; 3.7 - s|\[39m|</span>|;" 3.8 + s|\[39m|</span>|; 3.9 + s|\[1m|<strong>|g; s|\[0m|</strong>|g;" 3.10 ;; 3.11 3.12 files) 3.13 @@ -815,7 +816,7 @@ 3.14 echo "<h2>Log for: ${name%.log}</h2>" 3.15 if fgrep -q "Summary" $log; then 3.16 echo '<pre class="log">' 3.17 - grep -A 20 '^Summary' $log | syntax_highlighter log 3.18 + grep -A 20 'Summary' $log | syntax_highlighter log 3.19 echo '</pre>' 3.20 fi 3.21 echo '<pre class="log">'
4.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 4.2 +++ b/tasks/freetype-harfbuzz Sun Jun 25 14:14:30 2017 +0300 4.3 @@ -0,0 +1,10 @@ 4.4 +# SliTaz Cooker task. 4.5 + 4.6 +DESC="Resolve circular dependency between freetype and harfbuzz" 4.7 + 4.8 +task() { 4.9 + cook freetype 4.10 + cook harfbuzz 4.11 + cook freetype --harfbuzz 4.12 + cook harfbuzz 4.13 +}
5.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 5.2 +++ b/tasks/up-toolchain Sun Jun 25 14:14:30 2017 +0300 5.3 @@ -0,0 +1,73 @@ 5.4 +# SliTaz Cooker task. 5.5 + 5.6 +DESC="Update SliTaz toolchain" 5.7 + 5.8 +ver() { 5.9 + grep ^VERSION $WOK/$1/receipt | cut -d '"' -f2 5.10 +} 5.11 + 5.12 +task() { 5.13 + . $WOK/slitaz-toolchain/receipt 5.14 + 5.15 + tmplog=$LOGS/$PACKAGE.tmplog 5.16 + 5.17 + cat > $tmplog <<EOT 5.18 +Cook: $PACKAGE $VERSION 5.19 +$(separator) 5.20 +Cook toolchain : started $(date '+%F %R') 5.21 +Architecture : $ARCH 5.22 +Build system : $BUILD_SYSTEM 5.23 +Host system : $HOST_SYSTEM 5.24 +$(separator -) 5.25 +EOT 5.26 + 5.27 + echo "cook: Binutils first pass : $(date '+%F %R')" >> $tmplog 5.28 + cook binutils 5.29 + echo "cook: GCC first pass : $(date '+%F %R')" >> $tmplog 5.30 + cook gcc --first-pass 5.31 + echo "cook: Linux API headers : $(date '+%F %R')" >> $tmplog 5.32 + cook linux-api-headers 5.33 + echo "cook: Glibc : $(date '+%F %R')" >> $tmplog 5.34 + cook glibc 5.35 + echo "cook: Binutils final : $(date '+%F %R')" >> $tmplog 5.36 + cook binutils 5.37 + echo "cook: GCC final : $(date '+%F %R')" >> $tmplog 5.38 + cook gcc 5.39 + 5.40 + cat >> $tmplog <<EOT 5.41 +$(separator) 5.42 + 5.43 +GCC compiler information 5.44 +$(separator) 5.45 +$(gcc -v 2>&1 | sed 's|--|\n --|g') 5.46 +$(separator) 5.47 + 5.48 +EOT 5.49 + 5.50 + # All packages cooked got ther own log so we don't keep them. 5.51 + mv -f $tmplog $LOGS/$PACKAGE.log 5.52 + 5.53 + install=$WOK/$PACKAGE/install 5.54 + mkdir -p $install/usr/share/doc/slitaz 5.55 + cat > $install/usr/share/doc/slitaz/toolchain.txt <<EOT 5.56 +SliTaz GNU/Linux toolchain 5.57 +================================================================================ 5.58 + 5.59 +Build date : $(date "+%F") 5.60 +Architecture : $ARCH 5.61 +Build system : $BUILD_SYSTEM 5.62 +Host system : $HOST_SYSTEM 5.63 + 5.64 +Packages: 5.65 + 5.66 + * Binutils : $(ver binutils) 5.67 + * Linux API headers : $(ver linux-api-headers) 5.68 + * GCC : $(ver gcc) 5.69 + * Glibc : $(ver glibc) 5.70 + 5.71 +Toolchain documentation: http://doc.slitaz.org/en:cookbook:toolchain 5.72 + 5.73 +================================================================================ 5.74 + 5.75 +EOT 5.76 +}