cookutils view modules/pkgdb @ rev 1145
cooker.cgi: fix src= path
| author | Pascal Bellard <pascal.bellard@slitaz.org> |
|---|---|
| date | Wed Feb 05 15:22:15 2020 +0100 (2020-02-05) |
| parents | 799a5c056c55 |
| children |
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 #
11 # Functions
12 #
14 dblog() { tee -a $LOGS/pkgdb.log; }
17 # Return size of file in human readable format
18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
26 # Create suitable packages list for TazPkg and only for built packages
27 # as well as flavors files for TazLiTo.
29 # Find how much time was spent the last time (for web interface)
30 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
31 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
32 while read cmd duration start; do
33 [ $(($start + $duration)) -lt $(date +%s) ] &&
34 echo "sed -i '/^$cmd $duration/d' $cooktime"
35 done < $cooktime | sh
37 rm $LOGS/pkgdb.log 2>/dev/null
39 case "$1" in
40 --flavors|--rmpkg) ;;
41 *)
42 [ -n "$1" ] && PKGS="$1"
43 if [ ! -d "$PKGS" ]; then
44 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
45 exit 1
46 fi ;;
47 esac
49 time=$(date +%s)
50 flavors="$SLITAZ/flavors"
51 live="$SLITAZ/live"
53 arch=''
54 case "$ARCH" in
55 arm*|x86_64) arch="-$ARCH" ;;
56 esac
58 echo 'cook:pkgdb' > $command
59 _ 'Cook pkgdb: Creating all packages lists' | log
60 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
62 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
64 cd $PKGS
66 # Web interface database files should exist on the mirror1
67 >.folderlist; >.filelist
68 chmod 666 .folderlist .filelist
70 # Command `cook pkgdb` may be executed by cron. Creating a packages database
71 # takes some time, during which previously pending packages may continue to
72 # be created. This will result in an error due to the lack of a database. This in
73 # turn can lead to errors in the creating of many subsequent packages and will
74 # require manual intervention.
75 # Solution is an atomic update of the packages database: first we create new files
76 # (in the separate temp dir), and then instantly (well, almost) replace the old
77 # files by new ones.
79 dbs=$(mktemp -d)
81 _n 'Creating file "%s"' 'packages.list' | dblog
82 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $dbs/packages.list
83 echo " ($(filesize $dbs/packages.list))" | dblog
85 _n 'Creating file "%s"' 'packages.md5' | dblog
86 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $dbs/packages.md5
87 echo " ($(filesize $dbs/packages.md5))" | dblog
88 cp $dbs/packages.md5 $dbs/packages.toremove # list of unnecessary packages
90 md5sum $dbs/packages.md5 | cut -d' ' -f1 > $dbs/ID
91 ( cat $dbs/ID | tr $'\n' ' '; date -ur $dbs/ID +%s ) > $dbs/IDs # md5 and timestamp
93 _n 'Creating file "%s"' 'descriptions.txt' | dblog
94 for i in $(ls $WOK | sort); do
95 [ -d "$WOK/$i/taz" ] || continue
97 for j in $(ls $WOK/$i/taz | sort); do
98 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
100 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
101 echo "$pkgname"
102 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
103 # if description.txt doesn't end with \n then add one
104 [ -z "$(tail -c1 $WOK/$i/taz/$j/description.txt)" ] || echo
105 echo
106 done >> $dbs/descriptions.txt
107 done
108 echo " ($(filesize $dbs/descriptions.txt))" | dblog
110 _n 'Creating file "%s"' 'bdeps.txt' | dblog
111 for i in $(ls $WOK | sort); do
112 [ -s "$WOK/$i/receipt" ] || continue
113 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
114 echo "$i $bdeps" >> $dbs/bdeps.txt
115 done
116 echo " ($(filesize $dbs/bdeps.txt))" | dblog
119 _ 'Creating lists from "%s"' "$WOK" | dblog
120 cd $WOK
121 rsumf=$(mktemp)
122 #touch $dbs/packages.desc $dbs/packages.txt $dbs/packages.info $dbs/packages.equiv
123 touch $dbs/packages.info $dbs/packages.equiv
124 for i in *; do
125 [ -d "$WOK/$i/taz" ] || continue
127 for j in $(ls $WOK/$i/taz | sort); do
128 pack="$i/taz/$j"
129 [ -f "$WOK/$pack/receipt" ] || continue
130 unset_receipt
131 unset PACKED_SIZE UNPACKED_SIZE RSUM PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC MAINTAINER LICENSE WEB_SITE DEPENDS TAGS PROVIDE
132 . ./$pack/receipt
134 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" -o -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-any.tazpkg"]; then
136 # # packages.desc lets us search easily in DB
137 # cat >> $dbs/packages.desc <<-EOT
138 # $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
139 # EOT
140 #
141 # # packages.txt used by tazpkg-web also to provide
142 # # a human readable package list with version and description.
143 # cat >> $dbs/packages.txt <<-EOT
144 # $PACKAGE
145 # $VERSION$EXTRAVERSION
146 # $SHORT_DESC
147 # $PACKED_SIZE ($UNPACKED_SIZE installed)
148 #
149 # EOT
151 # packages.info combines TazPkg separate files
152 # and will substitute them all
153 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
154 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
156 # #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
158 # RSUM defined in new "small" receipts, but may be absent in old
159 if [ -z "$RSUM" ]; then
160 cp $pack/md5sum $rsumf
161 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
162 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
163 RSUM=$(md5sum $rsumf | awk '{print $1}')
164 fi
166 cat >> $dbs/packages.info <<-EOT
167 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $RSUM $PROVIDE
168 EOT
170 # packages.equiv is used by tazpkg install to check depends.
171 for k in $PROVIDE; do
172 DEST=''
173 echo $k | fgrep -q : && DEST="${k#*:}:"
174 if grep -qs ^${k%:*}= $dbs/packages.equiv; then
175 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
176 $dbs/packages.equiv
177 else
178 echo "${k%:*}=$DEST$PACKAGE" >> $dbs/packages.equiv
179 fi
180 done
182 # files.list provides a list of all packages files.
183 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $dbs/files.list
185 # list of unnecessary packages
186 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $dbs/packages.toremove
187 else
188 # if receipt variable HOST_ARCH absent/empty or contains ARCH
189 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
190 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
191 fi
192 fi
193 done
194 done
195 rm $rsumf
198 # Display list size.
199 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $dbs/packages.desc)" | dblog
200 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $dbs/packages.txt)" | dblog
201 _ 'Done: %s (%s)' 'packages.info' "$(filesize $dbs/packages.info)" | dblog
202 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $dbs/packages.equiv)" | dblog
204 cd $PKGS
207 # Check for unnecessary packages
208 if [ -s "$dbs/packages.toremove" ]; then
209 newline | dblog
210 case x$rmpkg in
211 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
212 *) _ 'Removing unnecessary packages:' | dblog;;
213 esac
214 while read pkgsum pkgfile; do
215 echo " - $pkgfile" | dblog
216 sed -i "/${pkgfile%.tazpkg}/d" $dbs/packages.list
217 sed -i "/ $pkgfile/d" $dbs/packages.md5
218 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
219 done < $dbs/packages.toremove
220 newline | dblog
221 fi
222 rm $dbs/packages.toremove
225 # files.list.lzma
226 _n 'Creating file "%s"' 'files.list.lzma' | dblog
227 touch $dbs/files.list
228 # pkgs.slitaz.org strongly depends on list sorted by packages names
229 #lzma e $dbs/files.list $dbs/files.list.lzma
230 /bin/lzma -zeqcT0 $dbs/files.list > $dbs/files.list.lzma
231 echo " ($(filesize $dbs/files.list.lzma))" | dblog
233 # Pre-sorting filenames causes 10% smaller resulting lzma file
234 _n 'Creating file "%s"' 'files-list.lzma' | dblog
235 cat $dbs/files.list | sort -k2 -o $dbs/files.list.sorted
236 #lzma e $dbs/files.list.sorted $dbs/files-list.lzma
237 /bin/lzma -zeqcT0 $dbs/files.list.sorted > $dbs/files-list.lzma
238 rm -f $dbs/files.list $dbs/files.list.sorted
239 echo " ($(filesize $dbs/files-list.lzma))" | dblog
241 md5sum $dbs/files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > $dbs/files-list.md5
243 # Make bundle to fast recharge
244 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
245 # Make sure to get "mirrors" file
246 until [ -e "$dbs/mirrors" ]; do
247 wget -q -O $dbs/mirrors http://mirror1.slitaz.org/mirrors
248 echo -n '.' | dblog; sleep 5
249 done
250 # Make sure to get "extra.list" file
251 until [ -e "$dbs/extra.list" ]; do
252 wget -q -O $dbs/extra.list http://mirror1.slitaz.org/packages/get.list
253 echo -n '.' | dblog; sleep 5
254 done
255 (
256 cd $dbs
257 busybox tar -chf bundle.tar \
258 mirrors extra.list files-list.md5 packages.info descriptions.txt \
259 packages.desc packages.md5 packages.txt packages.list packages.equiv
260 )
261 /bin/lzma -zeqcT0 $dbs/bundle.tar > $dbs/bundle.tar.lzma
262 rm $dbs/bundle.tar $dbs/mirrors
263 echo " ($(filesize $dbs/bundle.tar.lzma))" | dblog
265 # Display some info.
266 separator | dblog
267 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
268 time=$(($(date +%s) - $time))
269 # L10n: 's' is for seconds (cooking time)
270 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
272 # "Atomic" update now
273 ls -l $dbs
274 cp -f $dbs/* $PKGS
275 rm -r $dbs
278 # Create all flavors files at once. Do we really need code to monitor
279 # flavors changes? Let's just build them with packages lists before
280 # syncing the mirror.
281 [ "$1" != '--flavors' ] && rm $command && exit 1
283 if [ ! -d "$flavors" ]; then
284 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
285 rm $command
286 exit 1
287 fi
289 [ ! -d "$live" ] && mkdir -p $live
290 _ 'Creating flavors files in "%s"' "$live" | dblog
291 _ 'Cook pkgdb: Creating all flavors' | log
292 separator | dblog
294 _ 'Recharging lists to use latest packages...' | dblog
295 tazpkg recharge >/dev/null 2>/dev/null
297 # We need a custom tazlito config to set working dir to /home/slitaz.
298 if [ ! -f "$live/tazlito.conf" ]; then
299 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
300 cp /etc/tazlito/tazlito.conf $live
301 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
302 $live/tazlito.conf
303 fi
305 # Update Hg flavors repo and pack.
306 if [ -d "$flavors/.hg" ]; then
307 cd $flavors; hg pull -u
308 fi
310 cd $live
311 _ 'Starting to generate flavors...' | dblog
312 rm -f flavors.list *.flavor
313 for i in $flavors/*; do
314 fl=$(basename $i)
315 _ 'Packing flavor "%s"' "$fl" | dblog
316 tazlito pack-flavor $fl >/dev/null || exit 1
317 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
318 done
319 cp -f $live/*.flavor $live/flavors.list $PKGS
320 separator | dblog
321 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
322 separator | dblog
323 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
325 rm $command
328 exit 0