cookutils rev 1059
modules/pkgdb: provide "atomic" update now
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Tue May 29 04:09:44 2018 +0300 (2018-05-29) |
| parents | ea288d4f994f |
| children | 3911a2292888 |
| files | lighttpd/index.cgi modules/pkgdb |
line diff
1.1 --- a/lighttpd/index.cgi Fri May 18 14:12:41 2018 +0300 1.2 +++ b/lighttpd/index.cgi Tue May 29 04:09:44 2018 +0300 1.3 @@ -491,8 +491,10 @@ 1.4 -e 's#Success$#<i>Success</i>#' \ 1.5 -e 's#\([^a-z]\)ok$#\1<i>ok</i>#' \ 1.6 -e 's#\([^a-z]\)yes$#\1<i>yes</i>#' \ 1.7 + -e 's#: \(YES.*\)#: <i>\1</i>#' \ 1.8 -e 's#\([^a-z]\)ON$#\1<i>ON</i>#' \ 1.9 -e 's#\([^a-z]\)no$#\1<u>no</u>#' \ 1.10 + -e 's#: \(NO.*\)#: <u>\1</u>#' \ 1.11 -e 's#\([^a-z]\)none$#\1<u>none</u>#' \ 1.12 -e 's#\([^a-z]\)false$#\1<u>false</u>#' \ 1.13 -e 's#\([^a-z]\)OFF$#\1<u>OFF</u>#' \
2.1 --- a/modules/pkgdb Fri May 18 14:12:41 2018 +0300 2.2 +++ b/modules/pkgdb Tue May 29 04:09:44 2018 +0300 2.3 @@ -24,8 +24,7 @@ 2.4 2.5 2.6 # Create suitable packages list for TazPkg and only for built packages 2.7 -# as well as flavors files for TazLiTo. We don't need logs since we do it 2.8 -# manually to ensure everything is fine before syncing the mirror. 2.9 +# as well as flavors files for TazLiTo. 2.10 2.11 # Find how much time was spent the last time (for web interface) 2.12 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d') 2.13 @@ -63,27 +62,35 @@ 2.14 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog 2.15 2.16 cd $PKGS 2.17 -rm -f packages.* extra.list 2.18 -#touch packages.equiv 2.19 2.20 # Web interface database files should exist on the mirror1 2.21 >.folderlist; >.filelist 2.22 chmod 666 .folderlist .filelist 2.23 2.24 +# Command `cook pkgdb` may be executed by cron. Creating a packages database 2.25 +# takes some time, during which previously pending packages may be continue to 2.26 +# create. This will result in an error due to the lack of a database. This in 2.27 +# turn can lead to errors in the creating of many subsequent packages and will 2.28 +# require manual intervention. 2.29 +# Solution is an atomic update of packages database: first we create new files 2.30 +# (in the separate temp dir), and then instantly (well, almost) replace the old 2.31 +# files by new ones. 2.32 + 2.33 +dbs=$(mktemp -d) 2.34 + 2.35 _n 'Creating file "%s"' 'packages.list' | dblog 2.36 -find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $PKGS/packages.list 2.37 -echo " ($(filesize $PKGS/packages.list))" | dblog 2.38 +find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list 2.39 +echo " ($(filesize $dbs/packages.list))" | dblog 2.40 2.41 _n 'Creating file "%s"' 'packages.md5' | dblog 2.42 -find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $PKGS/packages.md5 2.43 -echo " ($(filesize $PKGS/packages.md5))" | dblog 2.44 -cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages 2.45 +find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5 2.46 +echo " ($(filesize $dbs/packages.md5))" | dblog 2.47 +cp $dbs/packages.md5 $dbs/packages.toremove # list of unnecessary packages 2.48 2.49 -md5sum packages.md5 | cut -d' ' -f1 > ID 2.50 -( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp 2.51 +md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID 2.52 +( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs # md5 and timestamp 2.53 2.54 _n 'Creating file "%s"' 'descriptions.txt' | dblog 2.55 -rm $PKGS/descriptions.txt 2>/dev/null 2.56 for i in $(ls $WOK | sort); do 2.57 [ -d "$WOK/$i/taz" ] || continue 2.58 2.59 @@ -93,25 +100,26 @@ 2.60 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE) 2.61 echo "$pkgname" 2.62 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt" 2.63 + # if description.txt don't ends with \n then add one 2.64 + [ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo 2.65 echo 2.66 - done >> descriptions.txt 2.67 + done >> $dbs/descriptions.txt 2.68 done 2.69 -echo " ($(filesize $PKGS/descriptions.txt))" | dblog 2.70 +echo " ($(filesize $dbs/descriptions.txt))" | dblog 2.71 2.72 _n 'Creating file "%s"' 'bdeps.txt' | dblog 2.73 -rm $PKGS/bdeps.txt 2>/dev/null 2.74 for i in $(ls $WOK | sort); do 2.75 [ -s "$WOK/$i/receipt" ] || continue 2.76 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts 2.77 - echo "$i $bdeps" >> $PKGS/bdeps.txt 2.78 + echo "$i $bdeps" >> $dbs/bdeps.txt 2.79 done 2.80 -echo " ($(filesize $PKGS/bdeps.txt))" | dblog 2.81 +echo " ($(filesize $PKGS/$dbs/bdeps.txt))" | dblog 2.82 2.83 2.84 _ 'Creating lists from "%s"' "$WOK" | dblog 2.85 -[ -e $PKGS/files.list ] && rm $PKGS/files.list 2.86 cd $WOK 2.87 rsumf=$(mktemp) 2.88 +touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv 2.89 for i in *; do 2.90 [ -d "$WOK/$i/taz" ] || continue 2.91 2.92 @@ -124,13 +132,13 @@ 2.93 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then 2.94 2.95 # packages.desc lets us search easily in DB 2.96 - cat >> $PKGS/packages.desc <<EOT 2.97 + cat >> $dbs/packages.desc <<EOT 2.98 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE 2.99 EOT 2.100 2.101 - # packages.txt used by tazpkg and tazpkg-web also to provide 2.102 + # packages.txt used by tazpkg-web also to provide 2.103 # a human readable package list with version and description. 2.104 - cat >> $PKGS/packages.txt <<EOT 2.105 + cat >> $dbs/packages.txt <<EOT 2.106 $PACKAGE 2.107 $VERSION$EXTRAVERSION 2.108 $SHORT_DESC 2.109 @@ -149,7 +157,7 @@ 2.110 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf 2.111 rsum=$(md5sum $rsumf | awk '{print $1}') 2.112 2.113 - cat >> $PKGS/packages.info <<EOT 2.114 + cat >> $dbs/packages.info <<EOT 2.115 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE 2.116 EOT 2.117 2.118 @@ -157,19 +165,19 @@ 2.119 for k in $PROVIDE; do 2.120 DEST='' 2.121 echo $k | fgrep -q : && DEST="${k#*:}:" 2.122 - if grep -qs ^${k%:*}= $PKGS/packages.equiv; then 2.123 + if grep -qs ^${k%:*}= $dbs/packages.equiv; then 2.124 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \ 2.125 - $PKGS/packages.equiv 2.126 + $dbs/packages.equiv 2.127 else 2.128 - echo "${k%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv 2.129 + echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv 2.130 fi 2.131 done 2.132 2.133 # files.list provides a list of all packages files. 2.134 - sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $PKGS/files.list 2.135 + sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list 2.136 2.137 # list of unnecessary packages 2.138 - sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $PKGS/packages.toremove 2.139 + sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $dbs/packages.toremove 2.140 else 2.141 # if receipt variable HOST_ARCH absent/empty or contains ARCH 2.142 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then 2.143 @@ -178,21 +186,20 @@ 2.144 fi 2.145 done 2.146 done 2.147 -touch $PKGS/packages.desc $PKGS/packages.txt $PKGS/packages.info $PKGS/packages.equiv 2.148 rm $rsumf 2.149 2.150 2.151 # Display list size. 2.152 -_ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog 2.153 -_ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog 2.154 -_ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog 2.155 -_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog 2.156 +_ 'Done: %s (%s)' 'packages.desc' "$(filesize $dbs/packages.desc)" | dblog 2.157 +_ 'Done: %s (%s)' 'packages.txt' "$(filesize $dbs/packages.txt)" | dblog 2.158 +_ 'Done: %s (%s)' 'packages.info' "$(filesize $dbs/packages.info)" | dblog 2.159 +_ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog 2.160 2.161 cd $PKGS 2.162 2.163 2.164 # Check for unnecessary packages 2.165 -if [ -s "$PKGS/packages.toremove" ]; then 2.166 +if [ -s "$dbs/packages.toremove" ]; then 2.167 newline | dblog 2.168 case x$rmpkg in 2.169 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;; 2.170 @@ -200,53 +207,54 @@ 2.171 esac 2.172 while read pkgsum pkgfile; do 2.173 echo " - $pkgfile" | dblog 2.174 - sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list 2.175 - sed -i "/ $pkgfile/d" $PKGS/packages.md5 2.176 + sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list 2.177 + sed -i "/ $pkgfile/d" $dbs/packages.md5 2.178 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg 2.179 - done < $PKGS/packages.toremove 2.180 + done < $dbs/packages.toremove 2.181 newline | dblog 2.182 fi 2.183 -rm $PKGS/packages.toremove 2.184 +rm $dbs/packages.toremove 2.185 2.186 2.187 # files.list.lzma 2.188 _n 'Creating file "%s"' 'files.list.lzma' | dblog 2.189 -touch files.list 2.190 +touch $dbs/files.list 2.191 # pkgs.slitaz.org strongly depends on list sorted by packages names 2.192 -#lzma e files.list files.list.lzma 2.193 -/bin/lzma -zeqcT0 files.list > files.list.lzma 2.194 -echo " ($(filesize $PKGS/files.list.lzma))" | dblog 2.195 +#lzma e $dbs/files.list $dbs/files.list.lzma 2.196 +/bin/lzma -zeqcT0 $dbs/files.list > $dbs/files.list.lzma 2.197 +echo " ($(filesize $dbs/files.list.lzma))" | dblog 2.198 2.199 # Pre-sorting filenames causes 10% smaller resulting lzma file 2.200 _n 'Creating file "%s"' 'files-list.lzma' | dblog 2.201 -cat files.list | sort -k2 -o files.list.sorted 2.202 -#lzma e files.list.sorted files-list.lzma 2.203 -/bin/lzma -zeqcT0 files.list.sorted > files-list.lzma 2.204 -rm -f files.list files.list.sorted 2.205 -echo " ($(filesize $PKGS/files-list.lzma))" | dblog 2.206 +cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted 2.207 +#lzma e $dbs/files.list.sorted $dbs/files-list.lzma 2.208 +/bin/lzma -zeqcT0 $dbs/files.list.sorted > $dbs/files-list.lzma 2.209 +rm -f $dbs/files.list $dbs/files.list.sorted 2.210 +echo " ($(filesize $dbs/files-list.lzma))" | dblog 2.211 2.212 -[ -e files.list.md5 ] && rm files.list.md5 2.213 -md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5 2.214 +md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5 2.215 2.216 # Make bundle to fast recharge 2.217 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog 2.218 -[ -f bundle.tar.lzma ] && rm bundle.tar.lzma 2.219 # Make sure to get "mirrors" file 2.220 -until [ -e 'mirrors' ]; do 2.221 - wget -q http://mirror1.slitaz.org/mirrors 2.222 +until [ -e "$dbs/mirrors" ]; do 2.223 + wget -q -O $dbs/mirrors http://mirror1.slitaz.org/mirrors 2.224 echo -n '.' | dblog; sleep 5 2.225 done 2.226 # Make sure to get "extra.list" file 2.227 -until [ -e 'extra.list' ]; do 2.228 - wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list 2.229 +until [ -e "$dbs/extra.list" ]; do 2.230 + wget -q -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list 2.231 echo -n '.' | dblog; sleep 5 2.232 done 2.233 -busybox tar -chf bundle.tar \ 2.234 - mirrors extra.list files-list.md5 packages.info descriptions.txt \ 2.235 - packages.desc packages.md5 packages.txt packages.list packages.equiv 2.236 -/bin/lzma -zeqcT0 bundle.tar > bundle.tar.lzma 2.237 -rm ./bundle.tar ./mirrors 2.238 -echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog 2.239 +( 2.240 + cd $dbs 2.241 + busybox tar -chf bundle.tar \ 2.242 + mirrors extra.list files-list.md5 packages.info descriptions.txt \ 2.243 + packages.desc packages.md5 packages.txt packages.list packages.equiv 2.244 +) 2.245 +/bin/lzma -zeqcT0 $dbs/bundle.tar > $dbs/bundle.tar.lzma 2.246 +rm $dbs/bundle.tar $dbs/mirrors 2.247 +echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog 2.248 2.249 # Display some info. 2.250 separator | dblog 2.251 @@ -255,6 +263,10 @@ 2.252 # L10n: 's' is for seconds (cooking time) 2.253 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog 2.254 2.255 +# "Atomic" update now 2.256 +cp -f $dbs/* $PKGS 2.257 +rm -r $dbs 2.258 + 2.259 2.260 # Create all flavors files at once. Do we really need code to monitor 2.261 # flavors changes? Let's just build them with packages lists before