tazpkg view modules/get @ rev 971
modules/get: fix plain mode and cookmode again
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon Nov 26 20:27:56 2018 +0200 (2018-11-26) |
| parents | f079c87ba419 |
| children | 9d683c983e2e |
line source
1 #!/bin/sh
2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
3 # get - TazPkg module
4 # Get package to the cache directory
7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts.
8 #
9 # Recently added other type of packages: packages that can't be compiled due to the closed sources
10 # or due to the giant size of that sources. In this case special script: a) downloads package
11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format,
12 # c) installs this package as "plain" package.
13 #
14 # 2. "Get-packages" - package contains only one special script for downloading/converting.
15 # Sometimes get-package can get some specified version of the package, sometimes - different
16 # (latest) version.
17 #
18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/
19 #
20 # 3. "Extra" get-scripts. Next addition - only script, without packaging.
21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/
22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/
23 #
24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are
25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side
26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines.
27 # So, some selected and free packages are converted on the SliTaz server side, packaged and
28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/
29 #
30 #
31 # Algorithm to get package:
32 # - search package in question in the mirrored packages list, download if exists;
33 # - search "get-package" in the mirrored packages list, download if exists;
34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package.
35 #
36 # Note, here doubling. Imagine, you may wanted "palemoon" package.
37 # 1. No "plain package";
38 # 2. Existing "get-palemoon" package;
39 # 3. Existing "palemoon" extra get-script;
40 # 4. Existing "palemoon" extra converted package.
41 # User will get 2. He can specify "--extra" option to get 3.
42 # FIXME: do something with doubling.
47 # Connect function libraries
48 . /lib/libtaz.sh
49 . /usr/lib/slitaz/libpkg.sh
51 # Get TazPkg working environment
52 . @@MODULES@@/getenv
54 . @@MODULES@@/find-depends
59 # Get cache directory path
61 get_cache_path() {
62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*"
63 # $2 (optional): 'extra' for cache for extra-package scripts
64 # output: path to cache directory
66 local cache_dir
68 if [ "$2" == 'extra' ]; then
69 cache_dir="$SAVE_CACHE_DIR/extra/packages"
70 elif [ "$1" == "$PKGS_DB" ]; then
71 # Main repository
72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages"
73 else
74 # Undigest repository
75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages"
76 fi
78 # Make cache directory if absent
79 mkdir -p "$cache_dir"
80 chmod a+w "$cache_dir"
82 echo "$cache_dir"
83 }
86 # Download a file from mirror
88 download_from() {
89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg"
91 debug "\ndownload_from('$1', '$2')"
93 case "$1" in
94 # Mirror URL can have a trailing slash or not.
95 http://* | https://* | ftp://*)
96 debug " wget -c -T 30 -U $UA ${1%/}/$2"
97 q=''; [ -n "$quiet" ] && q='-q'
98 # Display abridged wget status (skip info about connections)
99 wget -c $q -T 30 -U $UA ${1%/}/$2 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2
100 ;;
101 *)
102 debug " cp ${1%/}/$2 ."
103 cp ${1%/}/$2 .;;
104 esac
105 }
108 # This function may be called by a get script.
110 abort_package() {
111 cd "$CUR_DIR"
112 rm -rf "$tmp_dir"
113 [ -n "$1" ] ||
114 set -- 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}"
115 printf "$@"
116 exit 1
117 }
120 # Get extra-package file to the cache
122 get_pkg_extra() {
123 # input: $1 extra-package name (like 'dropbox')
124 # $2 (internal): empty or 'redo' (when recursive calling)
125 # action: download stuff and make package
126 # output: full path to created package
128 debug "\nget_pkg_extra('$1', '$2')"
130 local mirror extra_cache converted tmp_dir script
132 # Check extra.list
133 if [ ! -s "$PKGS_DB/extra.list" ]; then
134 # Empty extra.list
135 if [ "$2" != 'redo' ]; then
136 tazpkg recharge >&2
137 get_pkg_extra "$1" 'redo'; exit 0
138 else
139 # Give up
140 _ 'File "%s" empty.' 'extra.list' >&2
141 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
142 fi
143 fi
145 # Extra.list exists and not empty
146 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then
147 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
148 fi
150 # Extra-package found in extra.list
152 if [ -z "$nocache" ]; then
153 # Go to cache for "get-install" command; don't move for "get" command
154 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')"
155 debug "cd '$extra_cache'"
156 cd "$extra_cache"
158 # Extra-cache should contain packages DB files (packages.info, etc.)
159 # to list extra-packages contained in the extra-cache
160 [ ! -e 'packages.info' -a ! -e "packages-$SLITAZ_ARCH.info" ] &&
161 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
163 pi="packages-$SLITAZ_ARCH.info"
164 [ -e "$pi" ] || pi='packages.info'
166 if [ -e "$pi" ]; then
167 awk -F$'\t' -vp="$1" '$1==p{exit 1}' $pi
168 if [ "$?" -eq 1 ]; then
169 [ -z "$quiet" ] && _ 'Package "%s" already in the cache' "$1" >&2
170 echo -n "$(pwd)/"
171 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' $pi
172 exit 0
173 fi
174 fi
175 fi
177 mirror="$(cat "$PKGS_DB/mirror")"
178 debug "mirror='$mirror'"
181 # Try converted extra-packages first
182 # FIXME: Workaround to get packages filenames (even better to have names and versions separate)
183 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \
184 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)"
185 debug "converted='$converted'"
186 if [ -n "$converted" ]; then
187 case "$mirror" in
188 http://*|https://*|ftp://*)
189 # Default 'http://mirror.slitaz.org/packages/cooking/'
190 # -> 'http://mirror.slitaz.org/packages/extra/'
191 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'"
192 q=''; [ -n "$quiet" ] && q='-q'
193 wget $q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted" \
194 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2;;
195 esac
196 if [ -f "$converted" ]; then
197 echo "$extra_cache/$converted"; exit 0
198 fi
199 fi
202 # Fails to download converted extra-package; now try to download and execute get-script
203 cd ..
204 case "$mirror" in
205 http://*|https://*|ftp://*)
206 # Default 'http://mirror.slitaz.org/packages/cooking/'
207 # -> 'http://mirror.slitaz.org/packages/get/'
208 debug "wget -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'"
209 q=''; [ -n "$quiet" ] && q='-q'
210 wget $q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1" \
211 2>&1 | awk '$0~"%"{printf "%s\r",$0}' >&2;;
212 esac
214 if [ ! -f "$1" ]; then
215 # TODO: extra package or extra package script? :) Too complicated...
216 die 'Unable to download extra package "%s".' "$(boldify $1)"
217 fi
219 # Extra-package script downloaded in the /var/cache/tazpkg/extra/
221 unset_receipt
222 PACKAGE="$1"
223 script="$(realpath $1)"
224 tmp_dir="$(mktemp -d)"; cd "$tmp_dir"
225 # Run script
226 set -e
227 . "$script"
228 set +e
230 [ "$PWD" != "$tmp_dir" ] && mv $PACKAGE* $tmp_dir
231 cd $tmp_dir
232 [ -n "$VERSION" ] || VERSION="$(date +%Y%m%d)"
233 [ -d "$PACKAGE-$VERSION" ] || mv $PACKAGE $PACKAGE-$VERSION || abort_package
234 [ -n "$TARBALL" ] || TARBALL="$(basename $WGET_URL)"
236 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then
237 # Create receipt (if script not created it early) using variables from script
238 cat > "$PACKAGE-$VERSION/receipt" <<EOT
239 # SliTaz package receipt.
241 PACKAGE="$PACKAGE"
242 VERSION="$VERSION"
243 CATEGORY="${CATEGORY:-non-free}"
244 WEB_SITE="${WEB_SITE:-$WEBSITE}"
245 SHORT_DESC="${SHORT_DESC:-The $PACKAGE software package of SliTaz}"
246 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}"
247 LICENSE="${LICENSE:-unknown}"
248 TARBALL="$TARBALL"
249 WGET_URL="${WGET_URL:-$WGETURL}"
250 CONFIG_FILES="$CONFIG_FILES"
251 SUGGESTED="$SUGGESTED"
252 PROVIDE="$PROVIDE"
253 DEPENDS="$DEPENDS"
254 HOST_ARCH="$HOST_ARCH"
255 TAGS="$TAGS"
256 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES"
257 EOT
258 fi
260 # Add dependencies which was found to already defined dependencies
261 DEPENDS="$(unset DEPENDS; . ./$PACKAGE-$VERSION/receipt; echo $DEPENDS)"
262 TMP_DIR="$tmp_dir"
263 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do
264 case " $DEPENDS " in
265 *\ $i\ *) continue;;
266 esac
267 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt"
268 done
270 # Remove empty variables
271 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt"
273 tazpkg pack "$PACKAGE-$VERSION" gzip >&2
275 if [ -z "nocache" ]; then
276 # move package to the extra-cache for "get-install" command
277 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache"
278 # Re-make extra packages database
279 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
280 else
281 # move package to directory where "tazpkg get" command invoked
282 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR"
283 fi
285 # Clean
286 rm -rf "$tmp_dir"
288 # Function output: path to package
289 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg"
290 }
293 # return possible name for a virtual package name
295 virtual_name() {
296 # input: $1 virtual package name
297 # $2 repository db directory
298 local i pi
300 pi="$2/packages-$SLITAZ_ARCH.info"
301 [ -e "$pi" ] || pi="$2/packages.info"
303 unset IFS
304 for i in $(grep -hs "^$1=" "$2/packages.equiv" | sed "s/^$1=//"); do
305 if echo $i | fgrep -q : ; then
306 # format 'alternative:newname'
307 # if alternative is installed then substitute newname
308 if [ -f $INSTALLED/${i%:*}/receipt ]; then
309 # substitute package dependency
310 echo ${i#*:}
311 return
312 fi
313 elif ! grep -q "^$1 " "$pi" || [ -f "$INSTALLED/$i/receipt" ]; then
314 # unconditional substitution
315 echo $i
316 return
317 fi
318 done
319 # the real package name
320 echo $1
321 }
323 virtual_pkg() {
324 # input: $1 virtual package name
325 # $2 repository db directory
326 # output: display possible package name
327 local pi
329 debug "\nvirtual_pkg('$1', '$2')"
331 pi="$2/packages-$SLITAZ_ARCH.info"
332 [ -e "$pi" ] || pi="$2/packages.info"
334 if [ "$tazpkg_command" != 'get-install' ]; then
335 # 'get' command: download any package
336 if [ -z "$(awk -F$'\t' -vp="$1" '{if ($1 == p) print p}' "$pi")" ]; then
337 # This package does not exist in the list, it may be a virtual package
338 virtual_name "$1" "$2"
339 else
340 echo $1
341 fi
342 return
343 fi
345 virtual_name "$1" "$2"
346 }
349 # Download package file to the cache
351 get_pkg() {
352 # input: $1 package name either "name" or "name-version"
353 # $2 (internal): empty or 'redo' (when recursive calling)
354 # action: download package from mirror to the cache directory
355 # output: full path to the downloaded package
357 debug "\nget_pkg('$1', '$2')"
358 local repo line namever pkgsum pkgfile pi
360 IFS=$'\n'
361 for rep in $PRIORITY; do
362 pi="$rep/packages-$SLITAZ_ARCH.info"
363 [ -e "$pi" ] || pi="$rep/packages.info"
364 [ -e "$pi" ] || continue
366 # If found, output string "<name>-<ver>-<arch>.tazpkg"
367 # Field #11 may be:
368 # - empty (old format): treat as arch-dependent, use $SLITAZ_ARCH
369 # - 0: arch-independent (*-any.tazpkg)
370 # - 3: 32-bit \ just use $SLITAZ_ARCH in both cases -
371 # - 6: 64-bit / there's no 32-bit package in 64-bit repo
372 pkgfile=$(awk -F$'\t' -vpkg="$(virtual_pkg "$1" "$rep")" -varch="$SLITAZ_ARCH" '
373 $1==pkg || $1"-"$2==pkg {
374 a = ($11 == "0") ? "any" : arch;
375 printf("%s-%s-%s.tazpkg", $1, $2, a);
376 exit
377 }' "$pi")
379 if [ -n "$pkgfile" ]; then
380 pkgsum=$(awk -vfile="$pkgfile" '{if($2==file)print $1}' $rep/packages.md5)
381 break
382 fi
383 done
384 unset IFS
386 debug " rep='$rep'\n pkgfile='$pkgfile'\n pkgsum='$pkgsum'"
388 if [ -z "$pkgfile" ]; then
389 _ 'Unable to find package "%s" in the mirrored packages list.' "$1" >&2
390 # Retry with "get-package"; prevent looping with 'redo'
391 if [ "$2" != 'redo' ]; then
392 get_pkg "get-$1" 'redo'; exit 0
393 else
394 # Retry with extra-package
395 get_pkg_extra "${1#get-}"
396 exit 0
397 fi
398 fi
400 # We need package "$pkgfile" from "$rep" with "$pkgsum"
402 if [ -z "$nocache" ]; then
403 # Go to cache for "get-install" command; don't move for "get" command
404 debug "cd '$(get_cache_path "$rep")'"
405 cd "$(get_cache_path "$rep")"
406 fi
408 # Check if package already downloaded
409 if [ -f "$pkgfile" ]; then
410 [ -z "$nocache" -a -z "$quiet" ] && _ 'Package "%s" already in the cache' "$pkgfile" >&2
412 # Check if downloading complete, resume if not complete
413 if ! tail -c 2k "$pkgfile" | fgrep -q '00000000TRAILER'; then
414 [ -z "$quiet" ] && _ 'Continuing package "%s" download' "$pkgfile" >&2
415 download_from "$(cat "$rep/mirror")" "$pkgfile"
416 fi
417 else
418 # Package absent in the cache, "cold" downloading
419 download_from "$(cat "$rep/mirror")" "$pkgfile"
420 fi
422 # Make sure we downloaded what we want with checksum
424 if [ "$($CHECKSUM "$pkgfile" | cut -d' ' -f1)" != "$pkgsum" ]; then
425 _ 'Checksum error for "%s"' "$pkgfile" >&2
426 rm "$pkgfile"
428 # Recharge DBs and try again; prevent looping with 'redo'
429 if [ "$2" != 'redo' ]; then
430 tazpkg recharge >&2
431 get_pkg "$1" 'redo'; exit 0
432 else
433 # Give up
434 # TODO: try another mirror?
435 die 'Please wait until the mirror synchronization is complete and try again.'
436 fi
437 fi
439 # Output: path to downloaded package
440 printf '%s/%s\n' "$(pwd)" "$pkgfile"
441 }
444 # Get local package from /home/slitaz/packages to co-work with the cookutils
446 get_pkg_cookmode() {
447 # input: $1 package name (like 'advancecomp')
448 # action: find package in /home/slitaz/packages (or where defined in the cook.conf)
449 # output: full path to the found package
450 # ROOT NOT USED
452 local PKGS='/home/slitaz/packages' found='0' i
453 [ -e "/etc/slitaz/cook.conf" ] && . /etc/slitaz/cook.conf
455 # Find local package
456 pi="$PKGS/packages-$SLITAZ_ARCH.info"
457 [ -e "$pi" ] || pi="$PKGS/packages.info"
459 pkgfile="$(awk -F$'\t' -vpkg="$1" -varch="$SLITAZ_ARCH" '{
460 if ($1 == pkg) {
461 a = ($11 == "0") ? "any" : arch;
462 printf("%s-%s-%s.tazpkg", $1, $2, a);
463 exit;
464 }
465 }' $pi)"
467 # Find local provided package
468 [ -n "$pkgfile" ] ||
469 pkgfile="$(awk -F$'\t' -vpkg="$1" -varch="$SLITAZ_ARCH" '{
470 if (index(" " $10 " ", " " pkg " ")) {
471 a = ($11 == "0") ? "any" : arch;
472 printf("%s-%s-%s.tazpkg", $1, $2, a);
473 exit;
474 }
475 }' $pi)"
477 if [ -e "$PKGS/$pkgfile" ]; then
478 echo "$PKGS/$pkgfile"
479 else
480 # Proceed to get package as usual (non-local)
481 get_pkg "$1"
482 fi
483 }
488 # Command 'get-install' calls 'get', then 'install' modules. Check package presence here, on the
489 # first stage, if '--forced' option not given
490 if [ "$tazpkg_command" == 'get-install' ]; then
491 if grep -qs "^$1$" "$BLOCKED"; then
492 _ 'Package "%s" blocked.' "$1" >&2
493 exit 1
494 fi
496 if [ -z "$forced" ]; then
497 awk -F$'\t' -vpv="$1" '$1==pv { exit 1 }' "$PKGS_DB/installed.info"
498 if [ "$?" -eq 1 ]; then
499 [ -z "$quiet" ] && (
500 newline
501 _ '"%s" package is already installed.' "$(colorize 34 "$1")"
502 longline "$(_ 'You can use the --forced option to force installation.')"
503 newline
504 ) >&2
505 # Prevent execution 'install' stage:
506 exit 1
507 fi
508 fi
509 fi
511 if [ -n "$cookmode" ]; then
512 # When '--cookmode' option given, try to find package in the local cook repository,
513 # then, if fails, try to get the package as usual
514 get_pkg_cookmode "$1"
515 elif [ -n "$extra" ]; then
516 # When '--extra' option given, extra-package has priority over 'regular' packages
517 get_pkg_extra "$1"
518 else
519 # Try to get 'package', then 'get-package', then extra 'package'
520 get_pkg "$1"
521 fi