cookutils view modules/pkgdb @ rev 899

Some important changes:
* cook: better die handling; cook can now mark packages as broken by itself; less chatty zip extracting; rename internal function copy() to scopy(); new copy() is the main tool to copy files from $install to $fs (docs to come); automatic patching (if patches are in $stuff/patches and patch list is $stuff/patches/series); better local packages handling; update packages.info database after each successful build, virtual packages are accessible immediately after build.
* doc/cookopts.txt: added "!perlz" and "!rmpod" options description.
* modules/compressor: strip Perl files.
* modules/pkgdb: calculate estimated time for cook:pkgdb too (as well as for cook:package).
author Aleksej Bobylev <al.bobylev@gmail.com>
date Thu May 11 00:36:43 2017 +0300 (2017-05-11)
parents 2cb347d1b668
children a608dbb232df
line source
1 #!/bin/sh
2 #
3 # pkgdb - module of the SliTaz Cook
4 # Copyright (C) SliTaz GNU/Linux - GNU GPL v3
5 #
6
7 . /usr/lib/slitaz/libcook.sh
8
9
10 #
11 # Functions
12 #
13
14 dblog() { tee -a $LOGS/pkgdb.log; }
15
16
17 # Return size of file in human readible format
18 # Note, "du" in opposite returns size occupied by file on disk (4KB multiple in most cases)
19
20 filesize() { busybox ls -lh "$1" | awk '{print $5 "B"}'; }
21
22
23
24
25
26 # Create suitable packages list for TazPkg and only for built packages
27 # as well as flavors files for TazLiTo. We don't need logs since we do it
28 # manually to ensure everything is fine before syncing the mirror.
29
30 lastcooktime=$(sed '/Time:/!d; s|.*: *\([0-9]*\)s.*|\1|' $LOGS/pkgdb.log 2>/dev/null | sed '$!d')
31 [ -n "$lastcooktime" ] && echo "cook:pkgdb $lastcooktime $(date +%s)" >> $cooktime
32 while read cmd duration start; do
33 [ $(($start + $duration)) -lt $(date +%s) ] &&
34 echo "sed -i '/^$cmd $duration/d' $cooktime"
35 done < $cooktime | sh
36
37 rm $LOGS/pkgdb.log 2>/dev/null
38
39 case "$1" in
40 --flavors|--rmpkg) ;;
41 *)
42 [ -n "$1" ] && PKGS="$1"
43 if [ ! -d "$PKGS" ]; then
44 { newline; _ "Packages directory \"%s\" doesn't exist" "$PKGS"; newline; } | dblog
45 exit 1
46 fi ;;
47 esac
48
49 time=$(date +%s)
50 flavors="$SLITAZ/flavors"
51 live="$SLITAZ/live"
52
53 echo 'cook:pkgdb' > $command
54 _ 'Cook pkgdb: Creating all packages lists' | log
55 newline; { _ 'Creating lists for "%s"' "$PKGS"; separator; } | dblog
56
57 { _ 'Cook pkgdb started: %s' "$(date "$(_ '+%%F %%R')")"; newline; } | dblog
58
59 cd $PKGS
60 rm -f packages.* extra.list
61 touch packages.equiv
62
63 _n 'Creating file "%s"' 'packages.list' | dblog
64 ls -1 *.tazpkg | sed s'/.tazpkg//' > $PKGS/packages.list
65 echo " ($(filesize $PKGS/packages.list))" | dblog
66
67 _n 'Creating file "%s"' 'packages.md5' | dblog
68 md5sum *.tazpkg > $PKGS/packages.md5
69 echo " ($(filesize $PKGS/packages.md5))" | dblog
70 cp $PKGS/packages.md5 $PKGS/packages.toremove # list of unnecessary packages
71
72 md5sum packages.md5 | cut -d' ' -f1 > ID
73 ( cat ./ID | tr $'\n' ' '; date -ur ./ID +%s ) > IDs # md5 and timestamp
74
75 _n 'Creating file "%s"' 'descriptions.txt' | dblog
76 rm $PKGS/descriptions.txt 2>/dev/null
77 for i in $(ls $WOK | sort); do
78 if [ -e "$WOK/$i/description.txt" ]; then
79 echo "$i" >> descriptions.txt
80 cat "$WOK/$i/description.txt" | sed 's|^$| |' >> descriptions.txt
81 echo >> descriptions.txt
82 fi
83 done
84 echo " ($(filesize $PKGS/descriptions.txt))" | dblog
85
86
87 _ 'Creating lists from "%s"' "$WOK" | dblog
88 cd $WOK
89 for pkg in *; do
90 unset_receipt
91 . $pkg/receipt
92 # PACKED_SIZE and UNPACKED_SIZE are only in built receipt
93 [ -s $pkg/taz/*/receipt ] && . $pkg/taz/*/receipt
94
95 if [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" ] || \
96 [ -f "$PKGS/$PACKAGE-$VERSION$EXTRAVERSION-$ARCH.tazpkg" ]; then
97
98 # packages.desc lets us search easily in DB
99 cat >> $PKGS/packages.desc <<EOT
100 $PACKAGE | $VERSION$EXTRAVERSION | $SHORT_DESC | $CATEGORY | $WEB_SITE
101 EOT
102
103 # packages.txt used by tazpkg and tazpkg-web also to provide
104 # a human readable package list with version and description.
105 cat >> $PKGS/packages.txt <<EOT
106 $PACKAGE
107 $VERSION$EXTRAVERSION
108 $SHORT_DESC
109 $PACKED_SIZE ($UNPACKED_SIZE installed)
110
111 EOT
112
113 # packages.info combines TazPkg separate files
114 # and will substitute them all
115 SIZES=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
116 DEPENDS=$(echo $DEPENDS) # remove newlines from some receipts
117 MD5="$(fgrep " $PACKAGE-$VERSION$EXTRAVERSION.tazpkg" $PKGS/packages.md5 | awk '{print $1}')"
118 cat >> $PKGS/packages.info <<EOT
119 $PACKAGE $VERSION$EXTRAVERSION $CATEGORY $SHORT_DESC $WEB_SITE $TAGS $SIZES $DEPENDS $MD5 $PROVIDE
120 EOT
121
122 # packages.equiv is used by tazpkg install to check depends.
123 for i in $PROVIDE; do
124 DEST=''
125 echo $i | fgrep -q : && DEST="${i#*:}:"
126 if grep -qs ^${i%:*}= $PKGS/packages.equiv; then
127 sed -i "s/^${i%:*}=/${i%:*}=$DEST$PACKAGE /" \
128 $PKGS/packages.equiv
129 else
130 echo "${i%:*}=$DEST$PACKAGE" >> $PKGS/packages.equiv
131 fi
132 done
133
134 # files.list provides a list of all packages files.
135 [ -d $pkg/taz ] && cat $pkg/taz/*/files.list | sed s/^/"$pkg: \0"/ >> $PKGS/files.list
136
137 # list of unnecessary packages
138 sed -i "/ $PACKAGE-$VERSION$EXTRAVERSION.tazpkg/d" $PKGS/packages.toremove
139 else
140 # if receipt variable HOST_ARCH absent/empty or contains ARCH
141 if [ -z "$HOST_ARCH" -o "${HOST_ARCH/$ARCH/}" != "$HOST_ARCH" ]; then
142 _ ' - absent: %s (%s)' "$PACKAGE-$VERSION$EXTRAVERSION.tazpkg" "$ARCH" | dblog
143 fi
144 fi
145 done
146
147 # Display list size.
148 _ 'Done: %s (%s)' 'packages.desc' "$(filesize $PKGS/packages.desc)" | dblog
149 _ 'Done: %s (%s)' 'packages.txt' "$(filesize $PKGS/packages.txt)" | dblog
150 _ 'Done: %s (%s)' 'packages.info' "$(filesize $PKGS/packages.info)" | dblog
151 _ 'Done: %s (%s)' 'packages.equiv' "$(filesize $PKGS/packages.equiv)" | dblog
152
153 cd $PKGS
154
155
156 # Check for unnecessary packages
157 if [ -s "$PKGS/packages.toremove" ]; then
158 newline | dblog
159 case x$rmpkg in
160 x) _ 'Found unnecessary packages (use `cook pkgdb --rmpkg` to remove):' | dblog;;
161 *) _ 'Removing unnecessary packages:' | dblog;;
162 esac
163 while read pkgsum pkgfile; do
164 echo " - $pkgfile" | dblog
165 sed -i "/${pkgfile%.tazpkg}/d" $PKGS/packages.list
166 sed -i "/ $pkgfile/d" $PKGS/packages.md5
167 [ -n "$rmpkg" ] && rm $PKGS/$pkgfile # remove packages only with --rmpkg
168 done < $PKGS/packages.toremove
169 newline | dblog
170 fi
171 rm $PKGS/packages.toremove
172
173
174 # files.list.lzma
175 _n 'Creating file "%s"' 'files.list.lzma' | dblog
176 touch files.list
177 # pkgs.slitaz.org strongly depends on list sorted by packages names
178 lzma e files.list files.list.lzma
179 echo " ($(filesize $PKGS/files.list.lzma))" | dblog
180
181 # Pre-sorting filenames causes 10% smaller resulting lzma file
182 _n 'Creating file "%s"' 'files-list.lzma' | dblog
183 cat files.list | sort -k2 -o files.list.sorted
184 lzma e files.list.sorted files-list.lzma
185 rm -f files.list files.list.sorted
186 echo " ($(filesize $PKGS/files-list.lzma))" | dblog
187
188 [ -e files.list.md5 ] && rm files.list.md5
189 md5sum files-list.lzma | cut -d' ' -f1 | tr -d $'\n' > files-list.md5
190
191 # Make bundle to fast recharge
192 _n 'Creating file "%s"' 'bundle.tar.lzma' | dblog
193 [ -f bundle.tar.lzma ] && rm bundle.tar.lzma
194 # Make sure to get "mirrors" file
195 until [ -e 'mirrors' ]; do
196 wget -q http://mirror1.slitaz.org/mirrors
197 echo -n '.' | dblog; sleep 5
198 done
199 # Make sure to get "extra.list" file
200 until [ -e 'extra.list' ]; do
201 wget -q -O extra.list http://mirror1.slitaz.org/packages/get.list
202 echo -n '.' | dblog; sleep 5
203 done
204 busybox tar -chaf bundle.tar.lzma \
205 mirrors extra.list files-list.md5 packages.info descriptions.txt \
206 packages.desc packages.md5 packages.txt packages.list packages.equiv
207 rm ./mirrors
208 echo " ($(filesize $PKGS/bundle.tar.lzma))" | dblog
209
210 # Display some info.
211 separator | dblog
212 nb=$(ls $PKGS/*.tazpkg | wc -l)
213 time=$(($(date +%s) - $time))
214 # L10n: 's' is for seconds (cooking time)
215 { _ 'Packages: %s - Time: %ss' "$nb" "$time"; newline; } | dblog
216
217
218 # Create all flavors files at once. Do we really need code to monitor
219 # flavors changes? Let's just build them with packages lists before
220 # syncing the mirror.
221 [ "$1" != '--flavors' ] && rm $command && exit 1
222
223 if [ ! -d "$flavors" ]; then
224 { _ 'Missing flavors folder "%s"' "$flavors"; newline; } | dblog
225 rm $command
226 exit 1
227 fi
228
229 [ ! -d "$live" ] && mkdir -p $live
230 _ 'Creating flavors files in "%s"' "$live" | dblog
231 _ 'Cook pkgdb: Creating all flavors' | log
232 separator | dblog
233
234 _ 'Recharging lists to use latest packages...' | dblog
235 tazpkg recharge >/dev/null 2>/dev/null
236
237 # We need a custom tazlito config to set working dir to /home/slitaz.
238 if [ ! -f "$live/tazlito.conf" ]; then
239 _ 'Creating configuration file "%s"' 'tazlito.conf' | dblog
240 cp /etc/tazlito/tazlito.conf $live
241 sed -i s@WORK_DIR=.*@WORK_DIR=\"/home/slitaz\"@ \
242 $live/tazlito.conf
243 fi
244
245 # Update Hg flavors repo and pack.
246 if [ -d "$flavors/.hg" ]; then
247 cd $flavors; hg pull -u
248 fi
249
250 cd $live
251 _ 'Starting to generate flavors...' | dblog
252 rm -f flavors.list *.flavor
253 for i in $flavors/*; do
254 fl=$(basename $i)
255 _ 'Packing flavor "%s"' "$fl" | dblog
256 tazlito pack-flavor $fl >/dev/null || exit 1
257 tazlito show-flavor $fl --brief --noheader 2>/dev/null >> flavors.list
258 done
259 cp -f $live/*.flavor $live/flavors.list $PKGS
260 separator | dblog
261 { _ 'Total flavors size: %s' "$(du -sh $live | awk '{print $1}')"; newline; } | dblog
262 separator | dblog
263 _ 'Cook pkgdb end: %s' "$(date "$(_ '+%%F %%R')")" | dblog
264
265 rm $command
266
267
268 exit 0