cookutils view modules/pkgdb @ rev 982
cook: don't waste time uninstalling bdeps while we are in aufs chroot; modules/pkgdb: make bdeps.txt; lighttpd/index.cgi: show table "Dependencies of packages" in the package's info page.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Fri Oct 20 16:54:05 2017 +0300 (2017-10-20) |
| parents | b37ce91a21e0 |
| children | be4a05be1a6f |
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 #
11 # Functions
12 #
14 dblog() { tee -a $LOGS/pkgdb.log; }
17 # Return size of file in human readible format
18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
26 # Create suitable packages list for TazPkg and only for built packages
27 # as well as flavors files for TazLiTo. We don't need logs since we do it
28 # manually to ensure everything is fine before syncing the mirror.
30 # Find how much time was spent the last time (for web interface)
31 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
32 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
33 while read cmd duration start; do
34 [ $(($start + $duration)) -lt $(date +%s) ] &&
35 echo "sed -i '/^$cmd $duration/d' $cooktime"
36 done < $cooktime | sh
38 rm $LOGS/pkgdb.log 2>/dev/null
40 case "$1" in
41 --flavors|--rmpkg) ;;
42 *)
43 [ -n "$1" ] && PKGS="$1"
44 if [ ! -d "$PKGS" ]; then
45 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
46 exit 1
47 fi ;;
48 esac
50 time=$(date +%s)
51 flavors="$SLITAZ/flavors"
52 live="$SLITAZ/live"
54 echo 'cook:pkgdb' > $command
55 _ 'Cook pkgdb: Creating all packages lists' | log
56 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
58 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
60 cd $PKGS
61 rm -f packages.* extra.list
62 #touch packages.equiv
64 # Web interface database files should exist on the mirror1
65 >.folderlist; >.filelist
66 chmod 666 .folderlist .filelist
68 _n 'Creating file "%s"' 'packages.list' | dblog
69 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $PKGS/packages.list
70 echo " ($(filesize $PKGS/packages.list))" | dblog
72 _n 'Creating file "%s"' 'packages.md5' | dblog
73 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $PKGS/packages.md5
74 echo " ($(filesize $PKGS/packages.md5))" | dblog
75 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages
77 md5sum packages.md5 | cut -d' ' -f1 > ID
78 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp
80 _n 'Creating file "%s"' 'descriptions.txt' | dblog
81 rm $PKGS/descriptions.txt 2>/dev/null
82 for i in $(ls $WOK | sort); do
83 [ -d "$WOK/$i/taz" ] || continue
85 for j in $(ls $WOK/$i/taz | sort); do
86 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
88 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
89 echo "$pkgname"
90 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
91 echo
92 done >> descriptions.txt
93 done
94 echo " ($(filesize $PKGS/descriptions.txt))" | dblog
96 _n 'Creating file "%s"' 'bdeps.txt' | dblog
97 rm $PKGS/bdeps.txt 2>/dev/null
98 for i in $(ls $WOK | sort); do
99 [ -s "$WOK/$i/receipt" ] || continue
100 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
101 echo "$i $bdeps" >> $PKGS/bdeps.txt
102 done
103 echo " ($(filesize $PKGS/bdeps.txt))" | dblog
106 _ 'Creating lists from "%s"' "$WOK" | dblog
107 [ -e $PKGS/files.list ] && rm $PKGS/files.list
108 cd $WOK
109 rsumf=$(mktemp)
110 for i in *; do
111 [ -d "$WOK/$i/taz" ] || continue
113 for j in $(ls $WOK/$i/taz | sort); do
114 pack="$i/taz/$j"
115 [ -f "$WOK/$pack/receipt" ] || continue
116 unset_receipt
117 . ./$pack/receipt
119 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" -o \
120 -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" ]; then
122 # packages.desc lets us search easily in DB
123 cat >> $PKGS/packages.desc <<EOT
124 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
125 EOT
127 # packages.txt used by tazpkg and tazpkg-web also to provide
128 # a human readable package list with version and description.
129 cat >> $PKGS/packages.txt <<EOT
130 $PACKAGE
131 $VERSION$EXTRAVERSION
132 $SHORT_DESC
133 $PACKED_SIZE ($UNPACKED_SIZE installed)
135 EOT
137 # packages.info combines TazPkg separate files
138 # and will substitute them all
139 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
140 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
142 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
143 cp $pack/md5sum $rsumf
144 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
145 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
146 rsum=$(md5sum $rsumf | awk '{print $1}')
148 cat >> $PKGS/packages.info <<EOT
149 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE
150 EOT
152 # packages.equiv is used by tazpkg install to check depends.
153 for k in $PROVIDE; do
154 DEST=''
155 echo $k | fgrep -q : && DEST="${k#*:}:"
156 if grep -qs ^${k%:*}= $PKGS/packages.equiv; then
157 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
158 $PKGS/packages.equiv
159 else
160 echo "${k%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv
161 fi
162 done
164 # files.list provides a list of all packages files.
165 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $PKGS/files.list
167 # list of unnecessary packages
168 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION.tazpkg/d" $PKGS/packages.toremove
169 else
170 # if receipt variable HOST_ARCH absent/empty or contains ARCH
171 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
172 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
173 fi
174 fi
175 done
176 done
177 touch $PKGS/packages.desc $PKGS/packages.txt $PKGS/packages.info $PKGS/packages.equiv
178 rm $rsumf
181 # Display list size.
182 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog
183 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog
184 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog
185 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
187 cd $PKGS
190 # Check for unnecessary packages
191 if [ -s "$PKGS/packages.toremove" ]; then
192 newline | dblog
193 case x$rmpkg in
194 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
195 *) _ 'Removing unnecessary packages:' | dblog;;
196 esac
197 while read pkgsum pkgfile; do
198 echo " - $pkgfile" | dblog
199 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list
200 sed -i "/ $pkgfile/d" $PKGS/packages.md5
201 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
202 done < $PKGS/packages.toremove
203 newline | dblog
204 fi
205 rm $PKGS/packages.toremove
208 # files.list.lzma
209 _n 'Creating file "%s"' 'files.list.lzma' | dblog
210 touch files.list
211 # pkgs.slitaz.org strongly depends on list sorted by packages names
212 lzma e files.list files.list.lzma
213 echo " ($(filesize $PKGS/files.list.lzma))" | dblog
215 # Pre-sorting filenames causes 10% smaller resulting lzma file
216 _n 'Creating file "%s"' 'files-list.lzma' | dblog
217 cat files.list | sort -k2 -o files.list.sorted
218 lzma e files.list.sorted files-list.lzma
219 rm -f files.list files.list.sorted
220 echo " ($(filesize $PKGS/files-list.lzma))" | dblog
222 [ -e files.list.md5 ] && rm files.list.md5
223 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
225 # Make bundle to fast recharge
226 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
227 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma
228 # Make sure to get "mirrors" file
229 until [ -e 'mirrors' ]; do
230 wget -q http://mirror1.slitaz.org/mirrors
231 echo -n '.' | dblog; sleep 5
232 done
233 # Make sure to get "extra.list" file
234 until [ -e 'extra.list' ]; do
235 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
236 echo -n '.' | dblog; sleep 5
237 done
238 busybox tar -chaf bundle.tar.lzma \
239 mirrors extra.list files-list.md5 packages.info descriptions.txt \
240 packages.desc packages.md5 packages.txt packages.list packages.equiv
241 rm ./mirrors
242 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
244 # Display some info.
245 separator | dblog
246 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
247 time=$(($(date +%s) - $time))
248 # L10n: 's' is for seconds (cooking time)
249 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
252 # Create all flavors files at once. Do we really need code to monitor
253 # flavors changes? Let's just build them with packages lists before
254 # syncing the mirror.
255 [ "$1" != '--flavors' ] && rm $command && exit 1
257 if [ ! -d "$flavors" ]; then
258 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
259 rm $command
260 exit 1
261 fi
263 [ ! -d "$live" ] && mkdir -p $live
264 _ 'Creating flavors files in "%s"' "$live" | dblog
265 _ 'Cook pkgdb: Creating all flavors' | log
266 separator | dblog
268 _ 'Recharging lists to use latest packages...' | dblog
269 tazpkg recharge >/dev/null 2>/dev/null
271 # We need a custom tazlito config to set working dir to /home/slitaz.
272 if [ ! -f "$live/tazlito.conf" ]; then
273 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
274 cp /etc/tazlito/tazlito.conf $live
275 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
276 $live/tazlito.conf
277 fi
279 # Update Hg flavors repo and pack.
280 if [ -d "$flavors/.hg" ]; then
281 cd $flavors; hg pull -u
282 fi
284 cd $live
285 _ 'Starting to generate flavors...' | dblog
286 rm -f flavors.list *.flavor
287 for i in $flavors/*; do
288 fl=$(basename $i)
289 _ 'Packing flavor "%s"' "$fl" | dblog
290 tazlito pack-flavor $fl >/dev/null || exit 1
291 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
292 done
293 cp -f $live/*.flavor $live/flavors.list $PKGS
294 separator | dblog
295 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
296 separator | dblog
297 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
299 rm $command
302 exit 0