cookutils rev 742

Log pkgdb process; show it using web interface
author Aleksej Bobylev <al.bobylev@gmail.com>
date Sun Jun 07 11:22:35 2015 +0300 (2015-06-07)
parents aaba45fd138a
children 5a795aafa52d
files cook web/cooker.cgi
line diff
     1.1 --- a/cook	Sun Jun 07 00:46:16 2015 +0300
     1.2 +++ b/cook	Sun Jun 07 11:22:35 2015 +0300
     1.3 @@ -1124,6 +1124,10 @@
     1.4  	status
     1.5  }
     1.6  
     1.7 +dblog() { tee -a $LOGS/pkgdb.log; }
     1.8 +
     1.9 +
    1.10 +
    1.11  
    1.12  #
    1.13  # Commands
    1.14 @@ -1430,12 +1434,15 @@
    1.15  		# Create suitable packages list for TazPKG and only for built packages
    1.16  		# as well as flavors files for TazLiTo. We dont need logs since we do it
    1.17  		# manually to ensure everything is fine before syncing the mirror.
    1.18 +
    1.19 +		rm $LOGS/pkgdb.log 2>/dev/null
    1.20 +
    1.21  		case "$2" in
    1.22  			--flavors) ;;
    1.23  			*)
    1.24  				[ -n "$2" ] && PKGS="$2"
    1.25  				if [ ! -d "$PKGS" ]; then
    1.26 -					newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline
    1.27 +					{ newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
    1.28  					exit 1
    1.29  				fi ;;
    1.30  		esac
    1.31 @@ -1446,27 +1453,27 @@
    1.32  
    1.33  		echo 'cook:pkgdb' > $command
    1.34  		_ 'Cook pkgdb: Creating all packages lists' | log
    1.35 -		newline; _ 'Creating lists for "%s"' "$PKGS"; separator
    1.36 +		newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
    1.37  
    1.38 -		_ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"
    1.39 +		{ _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
    1.40  
    1.41  		cd $PKGS
    1.42  		rm -f packages.* extra.list
    1.43  		touch packages.equiv
    1.44  
    1.45 -		_n 'Creating file "%s"' 'packages.list'
    1.46 +		_n 'Creating file "%s"' 'packages.list' | dblog
    1.47  		ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list
    1.48 -		echo " ($(filesize $PKGS/packages.list))"
    1.49 +		echo " ($(filesize $PKGS/packages.list))" | dblog
    1.50  
    1.51 -		_n 'Creating file "%s"' 'packages.md5'
    1.52 +		_n 'Creating file "%s"' 'packages.md5' | dblog
    1.53  		md5sum *.tazpkg > $PKGS/packages.md5
    1.54 -		echo " ($(filesize $PKGS/packages.md5))"
    1.55 +		echo " ($(filesize $PKGS/packages.md5))" | dblog
    1.56  
    1.57  		md5sum packages.md5 | cut -d' ' -f1 > ID
    1.58  		( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs	# md5 and timestamp
    1.59  
    1.60 -		_n 'Creating file "%s"' 'descriptions.txt'
    1.61 -		rm $PKGS/descriptions.txt
    1.62 +		_n 'Creating file "%s"' 'descriptions.txt' | dblog
    1.63 +		rm $PKGS/descriptions.txt 2>/dev/null
    1.64  		for i in $(ls $WOK | sort); do
    1.65  			if [ -e "$WOK/$i/description.txt" ]; then
    1.66  				echo "$i" >> descriptions.txt
    1.67 @@ -1474,10 +1481,10 @@
    1.68  				echo >> descriptions.txt
    1.69  			fi
    1.70  		done
    1.71 -		echo " ($(filesize $PKGS/descriptions.txt))"
    1.72 +		echo " ($(filesize $PKGS/descriptions.txt))" | dblog
    1.73  
    1.74  
    1.75 -		_ 'Creating lists from "%s"' "$WOK"
    1.76 +		_ 'Creating lists from "%s"' "$WOK" | dblog
    1.77  		cd $WOK
    1.78  		for pkg in *; do
    1.79  			unset_receipt
    1.80 @@ -1531,39 +1538,39 @@
    1.81  		done
    1.82  
    1.83  		# Display list size.
    1.84 -		_ 'Done: %s (%s)' 'packages.desc'  "$(filesize $PKGS/packages.desc)"
    1.85 -		_ 'Done: %s (%s)' 'packages.txt'   "$(filesize $PKGS/packages.txt)"
    1.86 -		_ 'Done: %s (%s)' 'packages.info'  "$(filesize $PKGS/packages.info)"
    1.87 -		_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)"
    1.88 +		_ 'Done: %s (%s)' 'packages.desc'  "$(filesize $PKGS/packages.desc)"  | dblog
    1.89 +		_ 'Done: %s (%s)' 'packages.txt'   "$(filesize $PKGS/packages.txt)"   | dblog
    1.90 +		_ 'Done: %s (%s)' 'packages.info'  "$(filesize $PKGS/packages.info)"  | dblog
    1.91 +		_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
    1.92  
    1.93  		cd $PKGS
    1.94  
    1.95  		# files.list.lzma
    1.96 -		_n 'Creating file "%s"' 'files.list.lzma'
    1.97 +		_n 'Creating file "%s"' 'files.list.lzma' | dblog
    1.98  		touch files.list
    1.99  		# pkgs.slitaz.org strongly depends on list sorted by packages names
   1.100  		lzma e files.list files.list.lzma
   1.101 -		echo " ($(filesize $PKGS/files.list.lzma))"
   1.102 +		echo " ($(filesize $PKGS/files.list.lzma))" | dblog
   1.103  
   1.104  		# Pre-sorting filenames causes 10% smaller resulting lzma file
   1.105 -		_n 'Creating file "%s"' 'files-list.lzma'
   1.106 +		_n 'Creating file "%s"' 'files-list.lzma' | dblog
   1.107  		cat files.list | sort -k2 -o files.list.sorted
   1.108  		lzma e files.list.sorted files-list.lzma
   1.109  		rm -f files.list files.list.sorted
   1.110 -		echo " ($(filesize $PKGS/files-list.lzma))"
   1.111 +		echo " ($(filesize $PKGS/files-list.lzma))" | dblog
   1.112  
   1.113  		[ -e files.list.md5 ] && rm files.list.md5
   1.114  		md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
   1.115  
   1.116  		# packages.info.lzma
   1.117  		PI=packages.info
   1.118 -		_n 'Creating file "%s"' 'packages.info.lzma'
   1.119 +		_n 'Creating file "%s"' 'packages.info.lzma' | dblog
   1.120  		touch $PI
   1.121  		lzma e $PI $PI.lzma
   1.122 -		echo " ($(filesize $PKGS/packages.info.lzma))"
   1.123 +		echo " ($(filesize $PKGS/packages.info.lzma))" | dblog
   1.124  
   1.125  		# Make bundle to fast recharge
   1.126 -		_n 'Creating file "%s"' 'bundle.tar.lzma'
   1.127 +		_n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
   1.128  		[ -f bundle.tar.lzma ] && rm bundle.tar.lzma
   1.129  		wget -q http://mirror1.slitaz.org/mirrors
   1.130  		wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
   1.131 @@ -1571,14 +1578,14 @@
   1.132  			mirrors extra.list files-list.md5 packages.info descriptions.txt \
   1.133  			packages.desc packages.md5 packages.txt packages.list packages.equiv
   1.134  		rm ./mirrors
   1.135 -		echo " ($(filesize $PKGS/bundle.tar.lzma))"
   1.136 +		echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
   1.137  
   1.138  		# Display some info.
   1.139 -		separator
   1.140 +		separator | dblog
   1.141  		nb=$(ls $PKGS/*.tazpkg | wc -l)
   1.142  		time=$(($(date +%s) - $time))
   1.143  		# L10n: 's' is for seconds (cooking time)
   1.144 -		_ 'Packages: %s - Time: %ss' "$nb" "$time"; newline
   1.145 +		{ _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
   1.146  
   1.147  
   1.148  		# Create all flavors files at once. Do we really need code to monitor
   1.149 @@ -1587,21 +1594,21 @@
   1.150  		[ "$2" != '--flavors' ] && exit 1
   1.151  
   1.152  		if [ ! -d "$flavors" ]; then
   1.153 -			_ 'Missing flavors folder "%s"' "$flavors"; newline
   1.154 +			{ _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
   1.155  			exit 1
   1.156  		fi
   1.157  
   1.158  		[ ! -d "$live" ] && mkdir -p $live
   1.159 -		_ 'Creating flavors files in "%s"' "$live"
   1.160 +		_ 'Creating flavors files in "%s"' "$live" | dblog
   1.161  		_ 'Cook pkgdb: Creating all flavors' | log
   1.162 -		separator
   1.163 +		separator | dblog
   1.164  
   1.165 -		_ 'Recharging lists to use latest packages...'
   1.166 +		_ 'Recharging lists to use latest packages...' | dblog
   1.167  		tazpkg recharge >/dev/null 2>/dev/null
   1.168  
   1.169  		# We need a custom tazlito config to set working dir to /home/slitaz.
   1.170  		if [ ! -f "$live/tazlito.conf" ]; then
   1.171 -			_ 'Creating configuration file "%s"' 'tazlito.conf'
   1.172 +			_ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
   1.173  			cp /etc/tazlito/tazlito.conf $live
   1.174  			sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
   1.175  				$live/tazlito.conf
   1.176 @@ -1613,20 +1620,21 @@
   1.177  		fi
   1.178  
   1.179  		cd $live
   1.180 -		_ 'Starting to generate flavors...'
   1.181 +		_ 'Starting to generate flavors...' | dblog
   1.182  		rm -f flavors.list *.flavor
   1.183  		for i in $flavors/*; do
   1.184  			fl=$(basename $i)
   1.185 -			_ 'Packing flavor "%s"' "$fl"
   1.186 +			_ 'Packing flavor "%s"' "$fl" | dblog
   1.187  			tazlito pack-flavor $fl >/dev/null || exit 1
   1.188  			tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
   1.189  		done
   1.190  		cp -f $live/*.flavor $live/flavors.list $PKGS
   1.191 -		separator
   1.192 -		_ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline
   1.193 +		separator | dblog
   1.194 +		{ _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
   1.195  		rm -f $command
   1.196 -		separator
   1.197 -		_ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" ;;
   1.198 +		separator | dblog
   1.199 +		_ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
   1.200 +		;;
   1.201  
   1.202  	*)
   1.203  		# Just cook and generate a package.
     2.1 --- a/web/cooker.cgi	Sun Jun 07 00:46:16 2015 +0300
     2.2 +++ b/web/cooker.cgi	Sun Jun 07 11:22:35 2015 +0300
     2.3 @@ -385,6 +385,7 @@
     2.4  	Latest:
     2.5  	<a href="cooker.cgi?file=cookorder.log">cookorder.log</a>
     2.6  	<a href="cooker.cgi?file=commits.log">commits.log</a>
     2.7 +	<a href="cooker.cgi?file=pkgdb.log">pkgdb.log</a>
     2.8  	<a href="cooker.cgi?file=installed.diff">installed.diff</a>
     2.9  	- Architecture $ARCH:
    2.10  	<a href="$toolchain">toolchain</a>