cookutils view modules/pkgdb @ rev 1034
modules/compressor: sizes() strip: allow spaces in the filenames (fix adobe-air-sdk)
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Wed Mar 07 23:32:19 2018 +0200 (2018-03-07) |
| parents | ef2c335c986f |
| children | 815349e0020b |
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
7 . /usr/lib/slitaz/libcook.sh
10 #
11 # Functions
12 #
14 dblog() { tee -a $LOGS/pkgdb.log; }
17 # Return size of file in human readible format
18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
26 # Create suitable packages list for TazPkg and only for built packages
27 # as well as flavors files for TazLiTo. We don't need logs since we do it
28 # manually to ensure everything is fine before syncing the mirror.
30 # Find how much time was spent the last time (for web interface)
31 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
32 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
33 while read cmd duration start; do
34 [ $(($start + $duration)) -lt $(date +%s) ] &&
35 echo "sed -i '/^$cmd $duration/d' $cooktime"
36 done < $cooktime | sh
38 rm $LOGS/pkgdb.log 2>/dev/null
40 case "$1" in
41 --flavors|--rmpkg) ;;
42 *)
43 [ -n "$1" ] && PKGS="$1"
44 if [ ! -d "$PKGS" ]; then
45 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
46 exit 1
47 fi ;;
48 esac
50 time=$(date +%s)
51 flavors="$SLITAZ/flavors"
52 live="$SLITAZ/live"
54 arch=''
55 case "$ARCH" in
56 arm*|x86_64) arch="-$ARCH" ;;
57 esac
59 echo 'cook:pkgdb' > $command
60 _ 'Cook pkgdb: Creating all packages lists' | log
61 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
63 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
65 cd $PKGS
66 rm -f packages.* extra.list
67 #touch packages.equiv
69 # Web interface database files should exist on the mirror1
70 >.folderlist; >.filelist
71 chmod 666 .folderlist .filelist
73 _n 'Creating file "%s"' 'packages.list' | dblog
74 find . -name '*.tazpkg' | sed 's|^./||; s|.tazpkg$||' > $PKGS/packages.list
75 echo " ($(filesize $PKGS/packages.list))" | dblog
77 _n 'Creating file "%s"' 'packages.md5' | dblog
78 find . -name '*.tazpkg' -exec md5sum '{}' \; | sed 's|./||' > $PKGS/packages.md5
79 echo " ($(filesize $PKGS/packages.md5))" | dblog
80 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages
82 md5sum packages.md5 | cut -d' ' -f1 > ID
83 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp
85 _n 'Creating file "%s"' 'descriptions.txt' | dblog
86 rm $PKGS/descriptions.txt 2>/dev/null
87 for i in $(ls $WOK | sort); do
88 [ -d "$WOK/$i/taz" ] || continue
90 for j in $(ls $WOK/$i/taz | sort); do
91 [ -e "$WOK/$i/taz/$j/description.txt" ] || continue
93 pkgname=$(. $WOK/$i/taz/$j/receipt; echo $PACKAGE)
94 echo "$pkgname"
95 sed 's|^$| |' "$WOK/$i/taz/$j/description.txt"
96 echo
97 done >> descriptions.txt
98 done
99 echo " ($(filesize $PKGS/descriptions.txt))" | dblog
101 _n 'Creating file "%s"' 'bdeps.txt' | dblog
102 rm $PKGS/bdeps.txt 2>/dev/null
103 for i in $(ls $WOK | sort); do
104 [ -s "$WOK/$i/receipt" ] || continue
105 bdeps=$(. $WOK/$i/receipt; echo $BUILD_DEPENDS) # remove newlines from some receipts
106 echo "$i $bdeps" >> $PKGS/bdeps.txt
107 done
108 echo " ($(filesize $PKGS/bdeps.txt))" | dblog
111 _ 'Creating lists from "%s"' "$WOK" | dblog
112 [ -e $PKGS/files.list ] && rm $PKGS/files.list
113 cd $WOK
114 rsumf=$(mktemp)
115 for i in *; do
116 [ -d "$WOK/$i/taz" ] || continue
118 for j in $(ls $WOK/$i/taz | sort); do
119 pack="$i/taz/$j"
120 [ -f "$WOK/$pack/receipt" ] || continue
121 unset_receipt
122 . ./$pack/receipt
124 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg" ]; then
126 # packages.desc lets us search easily in DB
127 cat >> $PKGS/packages.desc <<EOT
128 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
129 EOT
131 # packages.txt used by tazpkg and tazpkg-web also to provide
132 # a human readable package list with version and description.
133 cat >> $PKGS/packages.txt <<EOT
134 $PACKAGE
135 $VERSION$EXTRAVERSION
136 $SHORT_DESC
137 $PACKED_SIZE ($UNPACKED_SIZE installed)
139 EOT
141 # packages.info combines TazPkg separate files
142 # and will substitute them all
143 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
144 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
146 #MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
147 cp $pack/md5sum $rsumf
148 md5sum $pack/receipt | sed 's| [^ ]*/| |' >> $rsumf
149 [ -e "$pack/description.txt" ] && md5sum $pack/description.txt | sed 's| [^ ]*/| |' >> $rsumf
150 rsum=$(md5sum $rsumf | awk '{print $1}')
152 cat >> $PKGS/packages.info <<EOT
153 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $rsum $PROVIDE
154 EOT
156 # packages.equiv is used by tazpkg install to check depends.
157 for k in $PROVIDE; do
158 DEST=''
159 echo $k | fgrep -q : && DEST="${k#*:}:"
160 if grep -qs ^${k%:*}= $PKGS/packages.equiv; then
161 sed -i "s/^${k%:*}=/${k%:*}=$DEST$PACKAGE /" \
162 $PKGS/packages.equiv
163 else
164 echo "${k%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv
165 fi
166 done
168 # files.list provides a list of all packages files.
169 sed "s|^|$PACKAGE: \0|" $i/taz/$j/files.list >> $PKGS/files.list
171 # list of unnecessary packages
172 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION$arch.tazpkg/d" $PKGS/packages.toremove
173 else
174 # if receipt variable HOST_ARCH absent/empty or contains ARCH
175 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
176 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
177 fi
178 fi
179 done
180 done
181 touch $PKGS/packages.desc $PKGS/packages.txt $PKGS/packages.info $PKGS/packages.equiv
182 rm $rsumf
185 # Display list size.
186 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog
187 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog
188 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog
189 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
191 cd $PKGS
194 # Check for unnecessary packages
195 if [ -s "$PKGS/packages.toremove" ]; then
196 newline | dblog
197 case x$rmpkg in
198 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
199 *) _ 'Removing unnecessary packages:' | dblog;;
200 esac
201 while read pkgsum pkgfile; do
202 echo " - $pkgfile" | dblog
203 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list
204 sed -i "/ $pkgfile/d" $PKGS/packages.md5
205 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
206 done < $PKGS/packages.toremove
207 newline | dblog
208 fi
209 rm $PKGS/packages.toremove
212 # files.list.lzma
213 _n 'Creating file "%s"' 'files.list.lzma' | dblog
214 touch files.list
215 # pkgs.slitaz.org strongly depends on list sorted by packages names
216 #lzma e files.list files.list.lzma
217 /bin/lzma -zeqcT0 files.list > files.list.lzma
218 echo " ($(filesize $PKGS/files.list.lzma))" | dblog
220 # Pre-sorting filenames causes 10% smaller resulting lzma file
221 _n 'Creating file "%s"' 'files-list.lzma' | dblog
222 cat files.list | sort -k2 -o files.list.sorted
223 #lzma e files.list.sorted files-list.lzma
224 /bin/lzma -zeqcT0 files.list.sorted > files-list.lzma
225 rm -f files.list files.list.sorted
226 echo " ($(filesize $PKGS/files-list.lzma))" | dblog
228 [ -e files.list.md5 ] && rm files.list.md5
229 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
231 # Make bundle to fast recharge
232 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
233 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma
234 # Make sure to get "mirrors" file
235 until [ -e 'mirrors' ]; do
236 wget -q http://mirror1.slitaz.org/mirrors
237 echo -n '.' | dblog; sleep 5
238 done
239 # Make sure to get "extra.list" file
240 until [ -e 'extra.list' ]; do
241 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
242 echo -n '.' | dblog; sleep 5
243 done
244 busybox tar -chf bundle.tar \
245 mirrors extra.list files-list.md5 packages.info descriptions.txt \
246 packages.desc packages.md5 packages.txt packages.list packages.equiv
247 /bin/lzma -zeqcT0 bundle.tar > bundle.tar.lzma
248 rm ./bundle.tar ./mirrors
249 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
251 # Display some info.
252 separator | dblog
253 nb=$(ls $PKGS/*.tazpkg 2>/dev/null | wc -l)
254 time=$(($(date +%s) - $time))
255 # L10n: 's' is for seconds (cooking time)
256 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
259 # Create all flavors files at once. Do we really need code to monitor
260 # flavors changes? Let's just build them with packages lists before
261 # syncing the mirror.
262 [ "$1" != '--flavors' ] && rm $command && exit 1
264 if [ ! -d "$flavors" ]; then
265 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
266 rm $command
267 exit 1
268 fi
270 [ ! -d "$live" ] && mkdir -p $live
271 _ 'Creating flavors files in "%s"' "$live" | dblog
272 _ 'Cook pkgdb: Creating all flavors' | log
273 separator | dblog
275 _ 'Recharging lists to use latest packages...' | dblog
276 tazpkg recharge >/dev/null 2>/dev/null
278 # We need a custom tazlito config to set working dir to /home/slitaz.
279 if [ ! -f "$live/tazlito.conf" ]; then
280 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
281 cp /etc/tazlito/tazlito.conf $live
282 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
283 $live/tazlito.conf
284 fi
286 # Update Hg flavors repo and pack.
287 if [ -d "$flavors/.hg" ]; then
288 cd $flavors; hg pull -u
289 fi
291 cd $live
292 _ 'Starting to generate flavors...' | dblog
293 rm -f flavors.list *.flavor
294 for i in $flavors/*; do
295 fl=$(basename $i)
296 _ 'Packing flavor "%s"' "$fl" | dblog
297 tazlito pack-flavor $fl >/dev/null || exit 1
298 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
299 done
300 cp -f $live/*.flavor $live/flavors.list $PKGS
301 separator | dblog
302 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
303 separator | dblog
304 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
306 rm $command
309 exit 0