cookutils annotate modules/pkgdb @ rev 949
cook: specify we use busybox fold (don't fail when remove coreutils-file-format or coreutils-multicall containing fold); cooker: remove package from packages.info DB too.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Thu Jul 20 05:07:33 2017 +0300 (2017-07-20) |
parents | 5f0da70605b6 |
children | 4e8c86306ce2 |
rev | line source |
---|---|
al@857 | 1 #!/bin/sh |
al@857 | 2 # |
al@857 | 3 # pkgdb - module of the SliTaz Cook |
al@857 | 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3 |
al@857 | 5 # |
al@857 | 6 |
al@857 | 7 . /usr/lib/slitaz/libcook.sh |
al@857 | 8 |
al@857 | 9 |
al@857 | 10 # |
al@857 | 11 # Functions |
al@857 | 12 # |
al@857 | 13 |
al@857 | 14 dblog() { tee -a $LOGS/pkgdb.log; } |
al@857 | 15 |
al@857 | 16 |
al@857 | 17 # Return size of file in human readible format |
al@857 | 18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases) |
al@857 | 19 |
al@857 | 20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; } |
al@857 | 21 |
al@857 | 22 |
al@857 | 23 |
al@857 | 24 |
al@857 | 25 |
al@857 | 26 # Create suitable packages list for TazPkg and only for built packages |
al@857 | 27 # as well as flavors files for TazLiTo. We don't need logs since we do it |
al@857 | 28 # manually to ensure everything is fine before syncing the mirror. |
al@857 | 29 |
al@914 | 30 # Find how much time was spent the last time (for web interface) |
al@899 | 31 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d') |
al@899 | 32 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime |
al@899 | 33 while read cmd duration start; do |
al@899 | 34 [ $(($start + $duration)) -lt $(date +%s) ] && |
al@899 | 35 echo "sed -i '/^$cmd $duration/d' $cooktime" |
al@899 | 36 done < $cooktime | sh |
al@899 | 37 |
al@857 | 38 rm $LOGS/pkgdb.log 2>/dev/null |
al@857 | 39 |
al@857 | 40 case "$1" in |
al@857 | 41 --flavors|--rmpkg) ;; |
al@857 | 42 *) |
al@857 | 43 [ -n "$1" ] && PKGS="$1" |
al@857 | 44 if [ ! -d "$PKGS" ]; then |
al@857 | 45 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog |
al@857 | 46 exit 1 |
al@857 | 47 fi ;; |
al@857 | 48 esac |
al@857 | 49 |
al@857 | 50 time=$(date +%s) |
al@857 | 51 flavors="$SLITAZ/flavors" |
al@857 | 52 live="$SLITAZ/live" |
al@857 | 53 |
al@857 | 54 echo 'cook:pkgdb' > $command |
al@857 | 55 _ 'Cook pkgdb: Creating all packages lists' | log |
al@857 | 56 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog |
al@857 | 57 |
al@857 | 58 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog |
al@857 | 59 |
al@857 | 60 cd $PKGS |
al@857 | 61 rm -f packages.* extra.list |
al@857 | 62 touch packages.equiv |
al@857 | 63 |
al@857 | 64 _n 'Creating file "%s"' 'packages.list' | dblog |
al@857 | 65 ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list |
al@857 | 66 echo " ($(filesize $PKGS/packages.list))" | dblog |
al@857 | 67 |
al@857 | 68 _n 'Creating file "%s"' 'packages.md5' | dblog |
al@857 | 69 md5sum *.tazpkg > $PKGS/packages.md5 |
al@857 | 70 echo " ($(filesize $PKGS/packages.md5))" | dblog |
al@862 | 71 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages |
al@857 | 72 |
al@857 | 73 md5sum packages.md5 | cut -d' ' -f1 > ID |
al@857 | 74 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp |
al@857 | 75 |
al@857 | 76 _n 'Creating file "%s"' 'descriptions.txt' | dblog |
al@857 | 77 rm $PKGS/descriptions.txt 2>/dev/null |
al@857 | 78 for i in $(ls $WOK | sort); do |
al@914 | 79 [ -d "$WOK/$i/taz" ] || continue |
al@914 | 80 |
al@914 | 81 for j in $(ls $WOK/$i/taz | sort); do |
al@914 | 82 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue |
al@914 | 83 |
al@914 | 84 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE) |
al@914 | 85 echo "$pkgname" >> descriptions.txt |
al@914 | 86 cat "$WOK/$i/taz/$j/description.txt" | sed 's|^$| |' >> descriptions.txt |
al@857 | 87 echo >> descriptions.txt |
al@914 | 88 done |
al@857 | 89 done |
al@857 | 90 echo " ($(filesize $PKGS/descriptions.txt))" | dblog |
al@857 | 91 |
al@857 | 92 |
al@857 | 93 _ 'Creating lists from "%s"' "$WOK" | dblog |
al@857 | 94 cd $WOK |
al@945 | 95 rsumf=$(mktemp) |
al@914 | 96 for i in *; do |
al@914 | 97 [ -d "$WOK/$i/taz" ] || continue |
al@857 | 98 |
al@914 | 99 for j in $(ls $WOK/$i/taz | sort); do |
al@945 | 100 pack="$i/taz/$j" |
al@945 | 101 [ -f "$WOK/$pack/receipt" ] || continue |
al@914 | 102 unset_receipt |
al@945 | 103 . $pack/receipt |
al@857 | 104 |
al@914 | 105 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" -o \ |
al@914 | 106 -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" ]; then |
al@914 | 107 |
al@914 | 108 # packages.desc lets us search easily in DB |
al@914 | 109 cat >> $PKGS/packages.desc <<EOT |
al@857 | 110 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE |
al@857 | 111 EOT |
al@857 | 112 |
al@914 | 113 # packages.txt used by tazpkg and tazpkg-web also to provide |
al@914 | 114 # a human readable package list with version and description. |
al@914 | 115 cat >> $PKGS/packages.txt <<EOT |
al@857 | 116 $PACKAGE |
al@857 | 117 $VERSION$EXTRAVERSION |
al@857 | 118 $SHORT_DESC |
al@857 | 119 $PACKED_SIZE ($UNPACKED_SIZE installed) |
al@857 | 120 |
al@857 | 121 EOT |
al@857 | 122 |
al@914 | 123 # packages.info combines TazPkg separate files |
al@914 | 124 # and will substitute them all |
al@914 | 125 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g') |
al@914 | 126 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts |
al@945 | 127 |
al@945 | 128 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')" |
al@945 | 129 cp $pack/md5sum $rsumf |
al@945 | 130 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf |
al@945 | 131 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf |
al@945 | 132 rsum=$(md5sum $rsumf | awk '{print $1}') |
al@945 | 133 |
al@914 | 134 cat >> $PKGS/packages.info <<EOT |
al@945 | 135 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE |
al@857 | 136 EOT |
al@857 | 137 |
al@914 | 138 # packages.equiv is used by tazpkg install to check depends. |
al@914 | 139 for k in $PROVIDE; do |
al@914 | 140 DEST='' |
al@914 | 141 echo $k | fgrep -q : && DEST="${k#*:}:" |
al@914 | 142 if grep -qs ^${k%:*}= $PKGS/packages.equiv; then |
al@914 | 143 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \ |
al@914 | 144 $PKGS/packages.equiv |
al@914 | 145 else |
al@914 | 146 echo "${k%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv |
al@914 | 147 fi |
al@914 | 148 done |
al@914 | 149 |
al@914 | 150 # files.list provides a list of all packages files. |
al@930 | 151 cat $i/taz/$j/files.list | sed s/^/"$PACKAGE: \0"/ >> $PKGS/files.list |
al@914 | 152 |
al@914 | 153 # list of unnecessary packages |
al@914 | 154 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION.tazpkg/d" $PKGS/packages.toremove |
al@914 | 155 else |
al@914 | 156 # if receipt variable HOST_ARCH absent/empty or contains ARCH |
al@914 | 157 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then |
al@914 | 158 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog |
al@857 | 159 fi |
al@857 | 160 fi |
al@914 | 161 done |
al@857 | 162 done |
al@945 | 163 rm $rsumf |
al@945 | 164 |
al@857 | 165 |
al@857 | 166 # Display list size. |
al@857 | 167 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog |
al@857 | 168 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog |
al@857 | 169 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog |
al@857 | 170 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog |
al@857 | 171 |
al@857 | 172 cd $PKGS |
al@857 | 173 |
al@857 | 174 |
al@862 | 175 # Check for unnecessary packages |
al@857 | 176 if [ -s "$PKGS/packages.toremove" ]; then |
al@857 | 177 newline | dblog |
al@862 | 178 case x$rmpkg in |
al@862 | 179 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;; |
al@862 | 180 *) _ 'Removing unnecessary packages:' | dblog;; |
al@862 | 181 esac |
al@857 | 182 while read pkgsum pkgfile; do |
al@857 | 183 echo " - $pkgfile" | dblog |
al@857 | 184 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list |
al@857 | 185 sed -i "/ $pkgfile/d" $PKGS/packages.md5 |
al@857 | 186 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg |
al@857 | 187 done < $PKGS/packages.toremove |
al@857 | 188 newline | dblog |
al@857 | 189 fi |
al@857 | 190 rm $PKGS/packages.toremove |
al@857 | 191 |
al@857 | 192 |
al@857 | 193 # files.list.lzma |
al@857 | 194 _n 'Creating file "%s"' 'files.list.lzma' | dblog |
al@857 | 195 touch files.list |
al@857 | 196 # pkgs.slitaz.org strongly depends on list sorted by packages names |
al@857 | 197 lzma e files.list files.list.lzma |
al@857 | 198 echo " ($(filesize $PKGS/files.list.lzma))" | dblog |
al@857 | 199 |
al@857 | 200 # Pre-sorting filenames causes 10% smaller resulting lzma file |
al@857 | 201 _n 'Creating file "%s"' 'files-list.lzma' | dblog |
al@857 | 202 cat files.list | sort -k2 -o files.list.sorted |
al@857 | 203 lzma e files.list.sorted files-list.lzma |
al@857 | 204 rm -f files.list files.list.sorted |
al@857 | 205 echo " ($(filesize $PKGS/files-list.lzma))" | dblog |
al@857 | 206 |
al@857 | 207 [ -e files.list.md5 ] && rm files.list.md5 |
al@857 | 208 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5 |
al@857 | 209 |
al@857 | 210 # Make bundle to fast recharge |
al@857 | 211 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog |
al@857 | 212 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma |
al@857 | 213 # Make sure to get "mirrors" file |
al@857 | 214 until [ -e 'mirrors' ]; do |
al@857 | 215 wget -q http://mirror1.slitaz.org/mirrors |
al@857 | 216 echo -n '.' | dblog; sleep 5 |
al@857 | 217 done |
al@857 | 218 # Make sure to get "extra.list" file |
al@857 | 219 until [ -e 'extra.list' ]; do |
al@857 | 220 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list |
al@857 | 221 echo -n '.' | dblog; sleep 5 |
al@857 | 222 done |
al@857 | 223 busybox tar -chaf bundle.tar.lzma \ |
al@857 | 224 mirrors extra.list files-list.md5 packages.info descriptions.txt \ |
al@857 | 225 packages.desc packages.md5 packages.txt packages.list packages.equiv |
al@857 | 226 rm ./mirrors |
al@857 | 227 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog |
al@857 | 228 |
al@857 | 229 # Display some info. |
al@857 | 230 separator | dblog |
al@857 | 231 nb=$(ls $PKGS/*.tazpkg | wc -l) |
al@857 | 232 time=$(($(date +%s) - $time)) |
al@857 | 233 # L10n: 's' is for seconds (cooking time) |
al@857 | 234 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog |
al@857 | 235 |
al@857 | 236 |
al@857 | 237 # Create all flavors files at once. Do we really need code to monitor |
al@857 | 238 # flavors changes? Let's just build them with packages lists before |
al@857 | 239 # syncing the mirror. |
al@899 | 240 [ "$1" != '--flavors' ] && rm $command && exit 1 |
al@857 | 241 |
al@857 | 242 if [ ! -d "$flavors" ]; then |
al@857 | 243 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog |
al@899 | 244 rm $command |
al@857 | 245 exit 1 |
al@857 | 246 fi |
al@857 | 247 |
al@857 | 248 [ ! -d "$live" ] && mkdir -p $live |
al@857 | 249 _ 'Creating flavors files in "%s"' "$live" | dblog |
al@857 | 250 _ 'Cook pkgdb: Creating all flavors' | log |
al@857 | 251 separator | dblog |
al@857 | 252 |
al@857 | 253 _ 'Recharging lists to use latest packages...' | dblog |
al@857 | 254 tazpkg recharge >/dev/null 2>/dev/null |
al@857 | 255 |
al@857 | 256 # We need a custom tazlito config to set working dir to /home/slitaz. |
al@857 | 257 if [ ! -f "$live/tazlito.conf" ]; then |
al@857 | 258 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog |
al@857 | 259 cp /etc/tazlito/tazlito.conf $live |
al@857 | 260 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \ |
al@857 | 261 $live/tazlito.conf |
al@857 | 262 fi |
al@857 | 263 |
al@857 | 264 # Update Hg flavors repo and pack. |
al@857 | 265 if [ -d "$flavors/.hg" ]; then |
al@857 | 266 cd $flavors; hg pull -u |
al@857 | 267 fi |
al@857 | 268 |
al@857 | 269 cd $live |
al@857 | 270 _ 'Starting to generate flavors...' | dblog |
al@857 | 271 rm -f flavors.list *.flavor |
al@857 | 272 for i in $flavors/*; do |
al@857 | 273 fl=$(basename $i) |
al@857 | 274 _ 'Packing flavor "%s"' "$fl" | dblog |
al@857 | 275 tazlito pack-flavor $fl >/dev/null || exit 1 |
al@857 | 276 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list |
al@857 | 277 done |
al@857 | 278 cp -f $live/*.flavor $live/flavors.list $PKGS |
al@857 | 279 separator | dblog |
al@857 | 280 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog |
al@857 | 281 separator | dblog |
al@857 | 282 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog |
al@857 | 283 |
al@899 | 284 rm $command |
al@876 | 285 |
al@857 | 286 |
al@857 | 287 exit 0 |