cookutils rev 725
cook pkgdb: provide new mechanism of recharging: IDs contains "old" ID plus timestamp, bundle.tar.lzma contains all "old" recharge files plus files.list.md5 to determine whether to download massive files.list.lzma or not.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Sun May 03 15:07:29 2015 +0300 (2015-05-03) |
parents | 851a2b8075dc |
children | fce442df7b90 |
files | cook |
line diff
1.1 --- a/cook Fri Apr 17 14:10:59 2015 +0200 1.2 +++ b/cook Sun May 03 15:07:29 2015 +0300 1.3 @@ -1348,20 +1348,26 @@ 1.4 echo "cook:pkgdb" > $command 1.5 _ "Cook pkgdb: Creating all packages lists" | log 1.6 newline; _ "Creating lists for: \$PKGS"; separator 1.7 + 1.8 datenow=$(date "$(_ '+%Y-%m-%d %H:%M')") 1.9 _ "Cook pkgdb started: \$datenow" 1.10 + 1.11 cd $PKGS 1.12 rm -f packages.* extra.list 1.13 ln -s ../get.list extra.list 1.14 + 1.15 _ "Creating: packages.list" 1.16 ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list 1.17 + 1.18 _ "Creating: packages.md5" 1.19 md5sum *.tazpkg > $PKGS/packages.md5 1.20 + 1.21 md5sum packages.md5 | cut -f1 -d' ' > ID 1.22 + ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp 1.23 + 1.24 _ "Creating lists from: \$WOK" 1.25 cd $WOK 1.26 - for pkg in * 1.27 - do 1.28 + for pkg in *; do 1.29 unset_receipt 1.30 . $pkg/receipt 1.31 # PACKED_SIZE and UNPACKED_SIZE are only in built receipt 1.32 @@ -1420,15 +1426,25 @@ 1.33 # files.list.lzma 1.34 _ "Creating: files.list.lzma" 1.35 touch $PKGS/files.list 1.36 - cd $PKGS; lzma e files.list files.list.lzma 1.37 - rm -f files.list 1.38 + # Pre-sorting filenames causes 10% smaller resulting lzma file 1.39 + cat $PKGS/files.list | sort -k2 -o $PKGS/files.list.sorted 1.40 + cd $PKGS; lzma e files.list.sorted files.list.lzma 1.41 + rm -f files.list files.list.sorted 1.42 + 1.43 + md5sum files.list.lzma | cut -d' ' -f1 | tr -d $'\n' > files.list.md5 1.44 1.45 # packages.info.lzma 1.46 PI=packages.info 1.47 _ 'Creating: packages.info.lzma' 1.48 touch $PKGS/$PI 1.49 cd $PKGS; lzma e $PI $PI.lzma 1.50 - rm -f $PI 1.51 + 1.52 + # Make bundle to fast recharge 1.53 + _ 'Creating: bundle.tar.lzma' 1.54 + [ -f bundle.tar.lzma ] && rm bundle.tar.lzma 1.55 + busybox tar -caf bundle.tar.lzma \ 1.56 + extra.list mirrors files.list.md5 packages.info \ 1.57 + packages.desc packages.md5 packages.txt packages.list packages.equiv 1.58 1.59 # Display some info. 1.60 separator