tazpkg annotate modules/get @ rev 846

Remove "busybox" "prefixes" (thanks llev)
We used "busybox wget", etc. to be sure we called Busybox's "wget", not any other "wget". Workaround already done in "getenv" module.
author Aleksej Bobylev <al.bobylev@gmail.com>
date Fri Oct 09 13:14:01 2015 +0300 (2015-10-09)
parents ce7009ff237b
children 8c6346ff1cef
rev   line source
al@844 1 #!/bin/sh
al@844 2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg
al@844 3 # get - TazPkg module
al@844 4 # Get package to the cache directory
al@844 5
al@844 6
al@844 7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts.
al@844 8 #
al@844 9 # Recently added other type of packages: packages that can't be compiled due to the closed sources
al@844 10 # or due to the giant size of that sources. In this case special script: a) downloads package
al@844 11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format,
al@844 12 # c) installs this package as "plain" package.
al@844 13 #
al@844 14 # 2. "Get-packages" - package contains only one special script for downloading/converting.
al@844 15 # Sometimes get-package can get some specified version of the package, sometimes - different
al@844 16 # (latest) version.
al@844 17 #
al@844 18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/
al@844 19 #
al@844 20 # 3. "Extra" get-scripts. Next addition - only script, without packaging.
al@844 21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/
al@844 22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/
al@844 23 #
al@844 24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are
al@844 25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side
al@844 26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines.
al@844 27 # So, some selected and free packages are converted on the SliTaz server side, packaged and
al@844 28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/
al@844 29 #
al@844 30 #
al@844 31 # Algorithm to get package:
al@844 32 # - search package in question in the mirrored packages list, download if exists;
al@844 33 # - search "get-package" in the mirrored packages list, download if exists;
al@844 34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package.
al@844 35 #
al@844 36 # Note, here doubling. Imagine, you may wanted "palemoon" package.
al@844 37 # 1. No "plain package";
al@844 38 # 2. Existing "get-palemoon" package;
al@844 39 # 3. Existing "palemoon" extra get-script;
al@844 40 # 4. Existing "palemoon" extra converted package.
al@844 41 # User will get 2. He can specify "--extra" option to get 3.
al@844 42 # FIXME: do something with doubling.
al@844 43
al@844 44
al@844 45
al@844 46
al@844 47 # Connect function libraries
al@844 48 . /lib/libtaz.sh
al@844 49 . /usr/lib/slitaz/libpkg.sh
al@844 50
al@844 51 # Get TazPkg working environment
al@844 52 . @@MODULES@@/getenv
al@844 53
al@844 54 . @@MODULES@@/find-depends
al@844 55
al@844 56
al@844 57
al@844 58
al@844 59 # Get cache directory path
al@844 60
al@844 61 get_cache_path() {
al@844 62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*"
al@844 63 # $2 (optional): 'extra' for cache for extra-package scripts
al@844 64 # output: path to cache directory
al@844 65
al@844 66 local cache_dir
al@844 67
al@844 68 if [ "$2" == 'extra' ]; then
al@844 69 cache_dir="$SAVE_CACHE_DIR/extra/packages"
al@844 70 elif [ "$1" == "$PKGS_DB" ]; then
al@844 71 # Main repository
al@844 72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages"
al@844 73 else
al@844 74 # Undigest repository
al@844 75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages"
al@844 76 fi
al@844 77
al@844 78 # Make cache directory if absent
al@844 79 mkdir -p "$cache_dir"
al@844 80 chmod a+w "$cache_dir"
al@844 81
al@844 82 echo "$cache_dir"
al@844 83 }
al@844 84
al@844 85
al@844 86 # Download a file from mirror
al@844 87
al@844 88 download_from() {
al@844 89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg"
al@844 90
al@844 91 debug "\ndownload_from('$1', '$2')"
al@844 92
al@844 93 case "$1" in
al@844 94 # Mirror URL can have a trailing slash or not.
al@844 95 http://* | https://* | ftp://*)
al@846 96 debug " wget -c -q -T 30 -U $UA ${1%/}/$2"
al@844 97 # TODO: display abridged wget status
al@846 98 wget -c -q -T 30 -U $UA ${1%/}/$2
al@844 99 ;;
al@844 100 *)
al@844 101 debug " ln -sf ${1%/}/$2 ."
al@844 102 ln -sf ${1%/}/$2 .;;
al@844 103 esac
al@844 104 }
al@844 105
al@844 106
al@844 107 # This function may be called by a get script.
al@844 108
al@844 109 abort_package() {
al@844 110 cd "$CUR_DIR"
al@844 111 rm -rf "$tmp_dir"
al@844 112 echo "${1:-Abort $PACKAGE.}"
al@844 113 exit 1
al@844 114 }
al@844 115
al@844 116
al@844 117 # Get extra-package file to the cache
al@844 118
al@844 119 get_pkg_extra() {
al@844 120 # input: $1 extra-package name (like 'dropbox')
al@844 121 # $2 (internal): empty or 'redo' (when recursive calling)
al@844 122 # action: download stuff and make package
al@844 123 # output: full path to created package
al@844 124
al@844 125 debug "\nget_pkg_extra('$1', '$2')"
al@844 126
al@844 127 local mirror extra_cache converted tmp_dir script
al@844 128
al@844 129 # Check extra.list
al@844 130 if [ ! -s "$PKGS_DB/extra.list" ]; then
al@844 131 # Empty extra.list
al@844 132 if [ "$2" != 'redo' ]; then
al@844 133 tazpkg recharge >&2
al@844 134 get_pkg_extra "$1" 'redo'; exit 0
al@844 135 else
al@844 136 # Give up
al@844 137 _ 'File "%s" empty.' 'extra.list' >&2
al@844 138 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
al@844 139 fi
al@844 140 fi
al@844 141
al@844 142 # Extra.list exists and not empty
al@844 143 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then
al@844 144 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)"
al@844 145 fi
al@844 146
al@844 147 # Extra-package found in extra.list
al@844 148
al@844 149 if [ -z "$nocache" ]; then
al@844 150 # Go to cache for "get-install" command; don't move for "get" command
al@844 151 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')"
al@844 152 debug "cd '$extra_cache'"
al@844 153 cd "$extra_cache"
al@844 154
al@844 155 # Extra-cache should contain packages DB files (packages.info, etc.)
al@844 156 # to list extra-packages contained in the extra-cache
al@844 157 [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
al@844 158
al@844 159 if [ -f 'packages.info' ]; then
al@844 160 awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info
al@844 161 if [ "$?" -eq 1 ]; then
al@844 162 _ 'Package "%s" already in the cache' "$1" >&2
al@844 163 echo -n "$(pwd)/"
al@844 164 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info
al@844 165 exit 0
al@844 166 fi
al@844 167 fi
al@844 168 fi
al@844 169
al@844 170 mirror="$(cat "$PKGS_DB/mirror")"
al@844 171 debug "mirror='$mirror'"
al@844 172
al@844 173
al@844 174 # Try converted extra-packages first
al@844 175 # FIXME: Workaround to get packages filenames (even better to have names and versions separate)
al@846 176 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \
al@844 177 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)"
al@844 178 debug "converted='$converted'"
al@844 179 if [ -n "$converted" ]; then
al@844 180 case "$mirror" in
al@844 181 http://*|https://*|ftp://*)
al@844 182 # Default 'http://mirror.slitaz.org/packages/cooking/'
al@844 183 # -> 'http://mirror.slitaz.org/packages/extra/'
al@846 184 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'"
al@846 185 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted";;
al@844 186 esac
al@844 187 if [ -f "$converted" ]; then
al@844 188 echo "$extra_cache/$converted"; exit 0
al@844 189 fi
al@844 190 fi
al@844 191
al@844 192
al@844 193 # Fails to download converted extra-package; now try to download and execute get-script
al@844 194 cd ..
al@844 195 case "$mirror" in
al@844 196 http://*|https://*|ftp://*)
al@844 197 # Default 'http://mirror.slitaz.org/packages/cooking/'
al@844 198 # -> 'http://mirror.slitaz.org/packages/get/'
al@846 199 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'"
al@846 200 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1";;
al@844 201 esac
al@844 202
al@844 203 if [ ! -f "$1" ]; then
al@844 204 # TODO: extra package or extra package script? :) Too complicated...
al@844 205 die 'Unable to download extra package "%s".' "$(boldify $1)"
al@844 206 fi
al@844 207
al@844 208 # Extra-package script downloaded in the /var/cache/tazpkg/extra/
al@844 209
al@844 210 unset_receipt
al@844 211 PACKAGE="$1"
al@844 212 script="$(realpath $1)"
al@844 213 tmp_dir="$(mktemp -d)"; cd "$tmp_dir"
al@844 214 # Run script
al@844 215 set -e
al@844 216 . "$script"
al@844 217 set +e
al@844 218
al@844 219 if [ ! -d "$PACKAGE-$VERSION" ]; then
al@844 220 abort_package 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}"
al@844 221 fi
al@844 222
al@844 223 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then
al@844 224 # Create receipt (if script not created it early) using variables from script
al@844 225 cat > "$PACKAGE-$VERSION/receipt" <<EOT
al@844 226 # SliTaz package receipt.
al@844 227
al@844 228 PACKAGE="$PACKAGE"
al@844 229 VERSION="${VERSION:-unknown}"
al@844 230 CATEGORY="${CATEGORY:-non-free}"
al@844 231 WEB_SITE="$WEB_SITE"
al@844 232 SHORT_DESC="${SHORT_DESC:-$PACKAGE}"
al@844 233 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}"
al@844 234 LICENSE="$LICENSE"
al@844 235 TARBALL="$TARBALL"
al@844 236 WGET_URL="$WGET_URL"
al@844 237 CONFIG_FILES="$CONFIG_FILES"
al@844 238 SUGGESTED="$SUGGESTED"
al@844 239 PROVIDE="$PROVIDE"
al@844 240 DEPENDS="$DEPENDS"
al@844 241 HOST_ARCH="$HOST_ARCH"
al@844 242 TAGS="$TAGS"
al@844 243 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES"
al@844 244 EOT
al@844 245 fi
al@844 246
al@844 247 # Add dependencies which was found to already defined dependencies
al@844 248 DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)"
al@844 249 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do
al@844 250 case " $DEPENDS " in
al@844 251 *\ $i\ *) continue;;
al@844 252 esac
al@844 253 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt"
al@844 254 done
al@844 255
al@844 256 # Remove empty variables
al@844 257 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt"
al@844 258
al@844 259 tazpkg pack "$PACKAGE-$VERSION" gzip >&2
al@844 260
al@844 261 if [ -z "nocache" ]; then
al@844 262 # move package to the extra-cache for "get-install" command
al@844 263 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache"
al@844 264 # Re-make extra packages database
al@844 265 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null
al@844 266 else
al@844 267 # move package to directory where "tazpkg get" command invoked
al@844 268 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR"
al@844 269 fi
al@844 270
al@844 271 # Clean
al@844 272 rm -rf "$tmp_dir"
al@844 273
al@844 274 # Function output: path to package
al@844 275 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg"
al@844 276 }
al@844 277
al@844 278
al@844 279 # Download package file to the cache
al@844 280
al@844 281 get_pkg() {
al@844 282 # input: $1 package name either "name" or "name-version"
al@844 283 # $2 (internal): empty or 'redo' (when recursive calling)
al@844 284 # action: download package from mirror to the cache directory
al@844 285 # output: full path to the downloaded package
al@844 286
al@844 287 debug "\nget_pkg('$1', '$2')"
al@844 288 local repo line namever pkgsum
al@844 289
al@844 290 IFS=$'\n'
al@844 291 for rep in $PRIORITY; do
al@844 292 [ ! -f "$rep/packages.info" ] && continue
al@844 293 # If found, output string "<name>-<ver>:<sum>"
al@844 294 line=$(awk -F$'\t' -vpkg="$1" \
al@844 295 '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info")
al@844 296 if [ -n "$line" ]; then
al@844 297 namever=${line%:*}; pkgsum=${line#*:}
al@844 298 break
al@844 299 fi
al@844 300 done
al@844 301 unset IFS
al@844 302
al@844 303 debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'"
al@844 304
al@844 305 if [ -z "$line" ]; then
al@844 306 _ 'Unable to find package "%s" in the mirrored packages list.' "$(boldify $1)" >&2
al@844 307 # Retry with "get-package"; prevent looping with 'redo'
al@844 308 if [ "$2" != 'redo' ]; then
al@844 309 get_pkg "get-$1" 'redo'; exit 0
al@844 310 else
al@844 311 # Retry with extra-package
al@844 312 get_pkg_extra "${1#get-}"
al@844 313 exit 0
al@844 314 fi
al@844 315 fi
al@844 316
al@844 317 # We need package "$namever.tazpkg" from "$rep" with "$pkgsum"
al@844 318
al@844 319 if [ -z "$nocache" ]; then
al@844 320 # Go to cache for "get-install" command; don't move for "get" command
al@844 321 debug "cd '$(get_cache_path "$rep")'"
al@844 322 cd "$(get_cache_path "$rep")"
al@844 323 fi
al@844 324
al@844 325 # Check if package already downloaded
al@844 326 if [ -f "$namever.tazpkg" ]; then
al@844 327 [ -z "$nocache" ] && _ 'Package "%s" already in the cache' "$namever" >&2
al@844 328
al@844 329 # Check if downloading complete, resume it not complete
al@844 330 if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then
al@844 331 _ 'Continuing package "%s" download' "$namever" >&2
al@844 332 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
al@844 333 fi
al@844 334 else
al@844 335 # Package absent in the cache, "cold" downloading
al@844 336 download_from "$(cat "$rep/mirror")" "$namever.tazpkg"
al@844 337 fi
al@844 338
al@844 339 # Make sure we downloaded what we want with checksum
al@844 340
al@844 341 if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then
al@845 342 _ 'Checksum error for "%s"' "$namever.tazpkg" >&2
al@844 343 rm "$namever.tazpkg"
al@844 344
al@844 345 # Recharge DBs and try again; prevent looping with 'redo'
al@844 346 if [ "$2" != 'redo' ]; then
al@844 347 tazpkg recharge >&2
al@844 348 get_pkg "$1" 'redo'; exit 0
al@844 349 else
al@844 350 # Give up
al@844 351 # TODO: try another mirror?
al@844 352 die 'Please wait until the mirror synchronization is complete and try again.'
al@844 353 fi
al@844 354 fi
al@844 355
al@844 356 # Output: path to downloaded package
al@844 357 printf '%s/%s.tazpkg\n' "$(pwd)" "$namever"
al@844 358 }
al@844 359
al@844 360
al@844 361
al@844 362
al@844 363 if [ -n "$extra" ]; then
al@844 364 # When '--extra' option given, extra-package has priority over 'regular' packages
al@844 365 get_pkg_extra "$1"
al@844 366 else
al@844 367 # Try to get 'package', then 'get-package', then extra 'package'
al@844 368 get_pkg "$1"
al@844 369 fi