cookutils annotate modules/pkgdb @ rev 890

Small edits. Move rare code from cook -> modules/cdeps.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Fri Mar 17 00:44:10 2017 +0200 (2017-03-17)
parents c43c8640ab19
children ea11d3c96873
rev   line source
al@857 1 #!/bin/sh
al@857 2 #
al@857 3 # pkgdb - module of the SliTaz Cook
al@857 4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
al@857 5 #
al@857 6
al@857 7 . /usr/lib/slitaz/libcook.sh
al@857 8
al@857 9
al@857 10 #
al@857 11 # Functions
al@857 12 #
al@857 13
al@857 14 dblog() { tee -a $LOGS/pkgdb.log; }
al@857 15
al@857 16
al@857 17 # Return size of file in human readible format
al@857 18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
al@857 19
al@857 20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
al@857 21
al@857 22
al@857 23
al@857 24
al@857 25
al@857 26 # Create suitable packages list for TazPkg and only for built packages
al@857 27 # as well as flavors files for TazLiTo. We don't need logs since we do it
al@857 28 # manually to ensure everything is fine before syncing the mirror.
al@857 29
al@857 30 rm $LOGS/pkgdb.log 2>/dev/null
al@857 31
al@857 32 case "$1" in
al@857 33 --flavors|--rmpkg) ;;
al@857 34 *)
al@857 35 [ -n "$1" ] && PKGS="$1"
al@857 36 if [ ! -d "$PKGS" ]; then
al@857 37 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
al@857 38 exit 1
al@857 39 fi ;;
al@857 40 esac
al@857 41
al@857 42 time=$(date +%s)
al@857 43 flavors="$SLITAZ/flavors"
al@857 44 live="$SLITAZ/live"
al@857 45
al@857 46 echo 'cook:pkgdb' > $command
al@857 47 _ 'Cook pkgdb: Creating all packages lists' | log
al@857 48 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
al@857 49
al@857 50 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
al@857 51
al@857 52 cd $PKGS
al@857 53 rm -f packages.* extra.list
al@857 54 touch packages.equiv
al@857 55
al@857 56 _n 'Creating file "%s"' 'packages.list' | dblog
al@857 57 ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list
al@857 58 echo " ($(filesize $PKGS/packages.list))" | dblog
al@857 59
al@857 60 _n 'Creating file "%s"' 'packages.md5' | dblog
al@857 61 md5sum *.tazpkg > $PKGS/packages.md5
al@857 62 echo " ($(filesize $PKGS/packages.md5))" | dblog
al@862 63 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages
al@857 64
al@857 65 md5sum packages.md5 | cut -d' ' -f1 > ID
al@857 66 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp
al@857 67
al@857 68 _n 'Creating file "%s"' 'descriptions.txt' | dblog
al@857 69 rm $PKGS/descriptions.txt 2>/dev/null
al@857 70 for i in $(ls $WOK | sort); do
al@857 71 if [ -e "$WOK/$i/description.txt" ]; then
al@857 72 echo "$i" >> descriptions.txt
al@857 73 cat "$WOK/$i/description.txt" | sed 's|^$| |' >> descriptions.txt
al@857 74 echo >> descriptions.txt
al@857 75 fi
al@857 76 done
al@857 77 echo " ($(filesize $PKGS/descriptions.txt))" | dblog
al@857 78
al@857 79
al@857 80 _ 'Creating lists from "%s"' "$WOK" | dblog
al@857 81 cd $WOK
al@857 82 for pkg in *; do
al@857 83 unset_receipt
al@857 84 . $pkg/receipt
al@857 85 # PACKED_SIZE and UNPACKED_SIZE are only in built receipt
al@857 86 [ -s $pkg/taz/*/receipt ] && . $pkg/taz/*/receipt
al@857 87
al@857 88 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ] || \
al@857 89 [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" ]; then
al@857 90
al@857 91 # packages.desc lets us search easily in DB
al@857 92 cat >> $PKGS/packages.desc <<EOT
al@857 93 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
al@857 94 EOT
al@857 95
al@857 96 # packages.txt used by tazpkg and tazpkg-web also to provide
al@857 97 # a human readable package list with version and description.
al@857 98 cat >> $PKGS/packages.txt <<EOT
al@857 99 $PACKAGE
al@857 100 $VERSION$EXTRAVERSION
al@857 101 $SHORT_DESC
al@857 102 $PACKED_SIZE ($UNPACKED_SIZE installed)
al@857 103
al@857 104 EOT
al@857 105
al@857 106 # packages.info combines TazPkg separate files
al@857 107 # and will substitute them all
al@857 108 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
al@857 109 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
al@857 110 MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
al@857 111 cat >> $PKGS/packages.info <<EOT
al@890 112 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $MD5 $PROVIDE
al@857 113 EOT
al@857 114
al@857 115 # packages.equiv is used by tazpkg install to check depends.
al@857 116 for i in $PROVIDE; do
al@857 117 DEST=''
al@857 118 echo $i | fgrep -q : && DEST="${i#*:}:"
al@857 119 if grep -qs ^${i%:*}= $PKGS/packages.equiv; then
al@857 120 sed -i "s/^${i%:*}=/${i%:*}=$DEST$PACKAGE /" \
al@857 121 $PKGS/packages.equiv
al@857 122 else
al@857 123 echo "${i%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv
al@857 124 fi
al@857 125 done
al@857 126
al@857 127 # files.list provides a list of all packages files.
al@890 128 [ -d $pkg/taz ] && cat $pkg/taz/*/files.list | sed s/^/"$pkg: \0"/ >> $PKGS/files.list
al@857 129
al@862 130 # list of unnecessary packages
al@857 131 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION.tazpkg/d" $PKGS/packages.toremove
al@857 132 else
al@857 133 # if receipt variable HOST_ARCH absent/empty or contains ARCH
al@857 134 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
al@857 135 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
al@857 136 fi
al@857 137 fi
al@857 138 done
al@857 139
al@857 140 # Display list size.
al@857 141 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog
al@857 142 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog
al@857 143 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog
al@857 144 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
al@857 145
al@857 146 cd $PKGS
al@857 147
al@857 148
al@862 149 # Check for unnecessary packages
al@857 150 if [ -s "$PKGS/packages.toremove" ]; then
al@857 151 newline | dblog
al@862 152 case x$rmpkg in
al@862 153 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
al@862 154 *) _ 'Removing unnecessary packages:' | dblog;;
al@862 155 esac
al@857 156 while read pkgsum pkgfile; do
al@857 157 echo " - $pkgfile" | dblog
al@857 158 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list
al@857 159 sed -i "/ $pkgfile/d" $PKGS/packages.md5
al@857 160 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
al@857 161 done < $PKGS/packages.toremove
al@857 162 newline | dblog
al@857 163 fi
al@857 164 rm $PKGS/packages.toremove
al@857 165
al@857 166
al@857 167 # files.list.lzma
al@857 168 _n 'Creating file "%s"' 'files.list.lzma' | dblog
al@857 169 touch files.list
al@857 170 # pkgs.slitaz.org strongly depends on list sorted by packages names
al@857 171 lzma e files.list files.list.lzma
al@857 172 echo " ($(filesize $PKGS/files.list.lzma))" | dblog
al@857 173
al@857 174 # Pre-sorting filenames causes 10% smaller resulting lzma file
al@857 175 _n 'Creating file "%s"' 'files-list.lzma' | dblog
al@857 176 cat files.list | sort -k2 -o files.list.sorted
al@857 177 lzma e files.list.sorted files-list.lzma
al@857 178 rm -f files.list files.list.sorted
al@857 179 echo " ($(filesize $PKGS/files-list.lzma))" | dblog
al@857 180
al@857 181 [ -e files.list.md5 ] && rm files.list.md5
al@857 182 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
al@857 183
al@857 184 # Make bundle to fast recharge
al@857 185 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
al@857 186 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma
al@857 187 # Make sure to get "mirrors" file
al@857 188 until [ -e 'mirrors' ]; do
al@857 189 wget -q http://mirror1.slitaz.org/mirrors
al@857 190 echo -n '.' | dblog; sleep 5
al@857 191 done
al@857 192 # Make sure to get "extra.list" file
al@857 193 until [ -e 'extra.list' ]; do
al@857 194 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
al@857 195 echo -n '.' | dblog; sleep 5
al@857 196 done
al@857 197 busybox tar -chaf bundle.tar.lzma \
al@857 198 mirrors extra.list files-list.md5 packages.info descriptions.txt \
al@857 199 packages.desc packages.md5 packages.txt packages.list packages.equiv
al@857 200 rm ./mirrors
al@857 201 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
al@857 202
al@857 203 # Display some info.
al@857 204 separator | dblog
al@857 205 nb=$(ls $PKGS/*.tazpkg | wc -l)
al@857 206 time=$(($(date +%s) - $time))
al@857 207 # L10n: 's' is for seconds (cooking time)
al@857 208 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
al@857 209
al@857 210
al@857 211 # Create all flavors files at once. Do we really need code to monitor
al@857 212 # flavors changes? Let's just build them with packages lists before
al@857 213 # syncing the mirror.
al@857 214 [ "$1" != '--flavors' ] && exit 1
al@857 215
al@857 216 if [ ! -d "$flavors" ]; then
al@857 217 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
al@857 218 exit 1
al@857 219 fi
al@857 220
al@857 221 [ ! -d "$live" ] && mkdir -p $live
al@857 222 _ 'Creating flavors files in "%s"' "$live" | dblog
al@857 223 _ 'Cook pkgdb: Creating all flavors' | log
al@857 224 separator | dblog
al@857 225
al@857 226 _ 'Recharging lists to use latest packages...' | dblog
al@857 227 tazpkg recharge >/dev/null 2>/dev/null
al@857 228
al@857 229 # We need a custom tazlito config to set working dir to /home/slitaz.
al@857 230 if [ ! -f "$live/tazlito.conf" ]; then
al@857 231 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
al@857 232 cp /etc/tazlito/tazlito.conf $live
al@857 233 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
al@857 234 $live/tazlito.conf
al@857 235 fi
al@857 236
al@857 237 # Update Hg flavors repo and pack.
al@857 238 if [ -d "$flavors/.hg" ]; then
al@857 239 cd $flavors; hg pull -u
al@857 240 fi
al@857 241
al@857 242 cd $live
al@857 243 _ 'Starting to generate flavors...' | dblog
al@857 244 rm -f flavors.list *.flavor
al@857 245 for i in $flavors/*; do
al@857 246 fl=$(basename $i)
al@857 247 _ 'Packing flavor "%s"' "$fl" | dblog
al@857 248 tazlito pack-flavor $fl >/dev/null || exit 1
al@857 249 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
al@857 250 done
al@857 251 cp -f $live/*.flavor $live/flavors.list $PKGS
al@857 252 separator | dblog
al@857 253 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
al@857 254 separator | dblog
al@857 255 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
al@857 256
al@876 257 > $command
al@876 258
al@857 259
al@857 260 exit 0