tazpkg annotate modules/get @ rev 846
Remove "busybox" "prefixes" (thanks llev)
We used "busybox wget", etc. to be sure we called Busybox's "wget", not any other "wget". Workaround already done in "getenv" module.
We used "busybox wget", etc. to be sure we called Busybox's "wget", not any other "wget". Workaround already done in "getenv" module.
author | Aleksej Bobylev <al.bobylev@gmail.com> |
---|---|
date | Fri Oct 09 13:14:01 2015 +0300 (2015-10-09) |
parents | ce7009ff237b |
children | 8c6346ff1cef |
rev | line source |
---|---|
al@844 | 1 #!/bin/sh |
al@844 | 2 # TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg |
al@844 | 3 # get - TazPkg module |
al@844 | 4 # Get package to the cache directory |
al@844 | 5 |
al@844 | 6 |
al@844 | 7 # 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts. |
al@844 | 8 # |
al@844 | 9 # Recently added other type of packages: packages that can't be compiled due to the closed sources |
al@844 | 10 # or due to the giant size of that sources. In this case special script: a) downloads package |
al@844 | 11 # compiled for other Linux from official web site, b) converts this package to the TazPkg format, |
al@844 | 12 # c) installs this package as "plain" package. |
al@844 | 13 # |
al@844 | 14 # 2. "Get-packages" - package contains only one special script for downloading/converting. |
al@844 | 15 # Sometimes get-package can get some specified version of the package, sometimes - different |
al@844 | 16 # (latest) version. |
al@844 | 17 # |
al@844 | 18 # Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/ |
al@844 | 19 # |
al@844 | 20 # 3. "Extra" get-scripts. Next addition - only script, without packaging. |
al@844 | 21 # Extra get-scripts repository: http://hg.slitaz.org/get-scripts/ |
al@844 | 22 # Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/ |
al@844 | 23 # |
al@844 | 24 # 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are |
al@844 | 25 # big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side |
al@844 | 26 # (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines. |
al@844 | 27 # So, some selected and free packages are converted on the SliTaz server side, packaged and |
al@844 | 28 # are ready to download by any user: http://mirror.slitaz.org/packages/extra/ |
al@844 | 29 # |
al@844 | 30 # |
al@844 | 31 # Algorithm to get package: |
al@844 | 32 # - search package in question in the mirrored packages list, download if exists; |
al@844 | 33 # - search "get-package" in the mirrored packages list, download if exists; |
al@844 | 34 # - search package in the "extra" get-scripts list, download if exists and run it, get the package. |
al@844 | 35 # |
al@844 | 36 # Note, here doubling. Imagine, you may wanted "palemoon" package. |
al@844 | 37 # 1. No "plain package"; |
al@844 | 38 # 2. Existing "get-palemoon" package; |
al@844 | 39 # 3. Existing "palemoon" extra get-script; |
al@844 | 40 # 4. Existing "palemoon" extra converted package. |
al@844 | 41 # User will get 2. He can specify "--extra" option to get 3. |
al@844 | 42 # FIXME: do something with doubling. |
al@844 | 43 |
al@844 | 44 |
al@844 | 45 |
al@844 | 46 |
al@844 | 47 # Connect function libraries |
al@844 | 48 . /lib/libtaz.sh |
al@844 | 49 . /usr/lib/slitaz/libpkg.sh |
al@844 | 50 |
al@844 | 51 # Get TazPkg working environment |
al@844 | 52 . @@MODULES@@/getenv |
al@844 | 53 |
al@844 | 54 . @@MODULES@@/find-depends |
al@844 | 55 |
al@844 | 56 |
al@844 | 57 |
al@844 | 58 |
al@844 | 59 # Get cache directory path |
al@844 | 60 |
al@844 | 61 get_cache_path() { |
al@844 | 62 # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*" |
al@844 | 63 # $2 (optional): 'extra' for cache for extra-package scripts |
al@844 | 64 # output: path to cache directory |
al@844 | 65 |
al@844 | 66 local cache_dir |
al@844 | 67 |
al@844 | 68 if [ "$2" == 'extra' ]; then |
al@844 | 69 cache_dir="$SAVE_CACHE_DIR/extra/packages" |
al@844 | 70 elif [ "$1" == "$PKGS_DB" ]; then |
al@844 | 71 # Main repository |
al@844 | 72 cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages" |
al@844 | 73 else |
al@844 | 74 # Undigest repository |
al@844 | 75 cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages" |
al@844 | 76 fi |
al@844 | 77 |
al@844 | 78 # Make cache directory if absent |
al@844 | 79 mkdir -p "$cache_dir" |
al@844 | 80 chmod a+w "$cache_dir" |
al@844 | 81 |
al@844 | 82 echo "$cache_dir" |
al@844 | 83 } |
al@844 | 84 |
al@844 | 85 |
al@844 | 86 # Download a file from mirror |
al@844 | 87 |
al@844 | 88 download_from() { |
al@844 | 89 # input: "<mirror_url>" "<package_name>-<version>.tazpkg" |
al@844 | 90 |
al@844 | 91 debug "\ndownload_from('$1', '$2')" |
al@844 | 92 |
al@844 | 93 case "$1" in |
al@844 | 94 # Mirror URL can have a trailing slash or not. |
al@844 | 95 http://* | https://* | ftp://*) |
al@846 | 96 debug " wget -c -q -T 30 -U $UA ${1%/}/$2" |
al@844 | 97 # TODO: display abridged wget status |
al@846 | 98 wget -c -q -T 30 -U $UA ${1%/}/$2 |
al@844 | 99 ;; |
al@844 | 100 *) |
al@844 | 101 debug " ln -sf ${1%/}/$2 ." |
al@844 | 102 ln -sf ${1%/}/$2 .;; |
al@844 | 103 esac |
al@844 | 104 } |
al@844 | 105 |
al@844 | 106 |
al@844 | 107 # This function may be called by a get script. |
al@844 | 108 |
al@844 | 109 abort_package() { |
al@844 | 110 cd "$CUR_DIR" |
al@844 | 111 rm -rf "$tmp_dir" |
al@844 | 112 echo "${1:-Abort $PACKAGE.}" |
al@844 | 113 exit 1 |
al@844 | 114 } |
al@844 | 115 |
al@844 | 116 |
al@844 | 117 # Get extra-package file to the cache |
al@844 | 118 |
al@844 | 119 get_pkg_extra() { |
al@844 | 120 # input: $1 extra-package name (like 'dropbox') |
al@844 | 121 # $2 (internal): empty or 'redo' (when recursive calling) |
al@844 | 122 # action: download stuff and make package |
al@844 | 123 # output: full path to created package |
al@844 | 124 |
al@844 | 125 debug "\nget_pkg_extra('$1', '$2')" |
al@844 | 126 |
al@844 | 127 local mirror extra_cache converted tmp_dir script |
al@844 | 128 |
al@844 | 129 # Check extra.list |
al@844 | 130 if [ ! -s "$PKGS_DB/extra.list" ]; then |
al@844 | 131 # Empty extra.list |
al@844 | 132 if [ "$2" != 'redo' ]; then |
al@844 | 133 tazpkg recharge >&2 |
al@844 | 134 get_pkg_extra "$1" 'redo'; exit 0 |
al@844 | 135 else |
al@844 | 136 # Give up |
al@844 | 137 _ 'File "%s" empty.' 'extra.list' >&2 |
al@844 | 138 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)" |
al@844 | 139 fi |
al@844 | 140 fi |
al@844 | 141 |
al@844 | 142 # Extra.list exists and not empty |
al@844 | 143 if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then |
al@844 | 144 die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)" |
al@844 | 145 fi |
al@844 | 146 |
al@844 | 147 # Extra-package found in extra.list |
al@844 | 148 |
al@844 | 149 if [ -z "$nocache" ]; then |
al@844 | 150 # Go to cache for "get-install" command; don't move for "get" command |
al@844 | 151 extra_cache="$(get_cache_path "$PKGS_DB" 'extra')" |
al@844 | 152 debug "cd '$extra_cache'" |
al@844 | 153 cd "$extra_cache" |
al@844 | 154 |
al@844 | 155 # Extra-cache should contain packages DB files (packages.info, etc.) |
al@844 | 156 # to list extra-packages contained in the extra-cache |
al@844 | 157 [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null |
al@844 | 158 |
al@844 | 159 if [ -f 'packages.info' ]; then |
al@844 | 160 awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info |
al@844 | 161 if [ "$?" -eq 1 ]; then |
al@844 | 162 _ 'Package "%s" already in the cache' "$1" >&2 |
al@844 | 163 echo -n "$(pwd)/" |
al@844 | 164 awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info |
al@844 | 165 exit 0 |
al@844 | 166 fi |
al@844 | 167 fi |
al@844 | 168 fi |
al@844 | 169 |
al@844 | 170 mirror="$(cat "$PKGS_DB/mirror")" |
al@844 | 171 debug "mirror='$mirror'" |
al@844 | 172 |
al@844 | 173 |
al@844 | 174 # Try converted extra-packages first |
al@844 | 175 # FIXME: Workaround to get packages filenames (even better to have names and versions separate) |
al@846 | 176 converted="$(wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \ |
al@844 | 177 tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)" |
al@844 | 178 debug "converted='$converted'" |
al@844 | 179 if [ -n "$converted" ]; then |
al@844 | 180 case "$mirror" in |
al@844 | 181 http://*|https://*|ftp://*) |
al@844 | 182 # Default 'http://mirror.slitaz.org/packages/cooking/' |
al@844 | 183 # -> 'http://mirror.slitaz.org/packages/extra/' |
al@846 | 184 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'" |
al@846 | 185 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted";; |
al@844 | 186 esac |
al@844 | 187 if [ -f "$converted" ]; then |
al@844 | 188 echo "$extra_cache/$converted"; exit 0 |
al@844 | 189 fi |
al@844 | 190 fi |
al@844 | 191 |
al@844 | 192 |
al@844 | 193 # Fails to download converted extra-package; now try to download and execute get-script |
al@844 | 194 cd .. |
al@844 | 195 case "$mirror" in |
al@844 | 196 http://*|https://*|ftp://*) |
al@844 | 197 # Default 'http://mirror.slitaz.org/packages/cooking/' |
al@844 | 198 # -> 'http://mirror.slitaz.org/packages/get/' |
al@846 | 199 debug "wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'" |
al@846 | 200 wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1";; |
al@844 | 201 esac |
al@844 | 202 |
al@844 | 203 if [ ! -f "$1" ]; then |
al@844 | 204 # TODO: extra package or extra package script? :) Too complicated... |
al@844 | 205 die 'Unable to download extra package "%s".' "$(boldify $1)" |
al@844 | 206 fi |
al@844 | 207 |
al@844 | 208 # Extra-package script downloaded in the /var/cache/tazpkg/extra/ |
al@844 | 209 |
al@844 | 210 unset_receipt |
al@844 | 211 PACKAGE="$1" |
al@844 | 212 script="$(realpath $1)" |
al@844 | 213 tmp_dir="$(mktemp -d)"; cd "$tmp_dir" |
al@844 | 214 # Run script |
al@844 | 215 set -e |
al@844 | 216 . "$script" |
al@844 | 217 set +e |
al@844 | 218 |
al@844 | 219 if [ ! -d "$PACKAGE-$VERSION" ]; then |
al@844 | 220 abort_package 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}" |
al@844 | 221 fi |
al@844 | 222 |
al@844 | 223 if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then |
al@844 | 224 # Create receipt (if script not created it early) using variables from script |
al@844 | 225 cat > "$PACKAGE-$VERSION/receipt" <<EOT |
al@844 | 226 # SliTaz package receipt. |
al@844 | 227 |
al@844 | 228 PACKAGE="$PACKAGE" |
al@844 | 229 VERSION="${VERSION:-unknown}" |
al@844 | 230 CATEGORY="${CATEGORY:-non-free}" |
al@844 | 231 WEB_SITE="$WEB_SITE" |
al@844 | 232 SHORT_DESC="${SHORT_DESC:-$PACKAGE}" |
al@844 | 233 MAINTAINER="${MAINTAINER:-nobody@slitaz.org}" |
al@844 | 234 LICENSE="$LICENSE" |
al@844 | 235 TARBALL="$TARBALL" |
al@844 | 236 WGET_URL="$WGET_URL" |
al@844 | 237 CONFIG_FILES="$CONFIG_FILES" |
al@844 | 238 SUGGESTED="$SUGGESTED" |
al@844 | 239 PROVIDE="$PROVIDE" |
al@844 | 240 DEPENDS="$DEPENDS" |
al@844 | 241 HOST_ARCH="$HOST_ARCH" |
al@844 | 242 TAGS="$TAGS" |
al@844 | 243 EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES" |
al@844 | 244 EOT |
al@844 | 245 fi |
al@844 | 246 |
al@844 | 247 # Add dependencies which was found to already defined dependencies |
al@844 | 248 DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)" |
al@844 | 249 for i in $(find_depends "$PACKAGE-$VERSION/fs"); do |
al@844 | 250 case " $DEPENDS " in |
al@844 | 251 *\ $i\ *) continue;; |
al@844 | 252 esac |
al@844 | 253 sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt" |
al@844 | 254 done |
al@844 | 255 |
al@844 | 256 # Remove empty variables |
al@844 | 257 sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt" |
al@844 | 258 |
al@844 | 259 tazpkg pack "$PACKAGE-$VERSION" gzip >&2 |
al@844 | 260 |
al@844 | 261 if [ -z "nocache" ]; then |
al@844 | 262 # move package to the extra-cache for "get-install" command |
al@844 | 263 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache" |
al@844 | 264 # Re-make extra packages database |
al@844 | 265 tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null |
al@844 | 266 else |
al@844 | 267 # move package to directory where "tazpkg get" command invoked |
al@844 | 268 mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR" |
al@844 | 269 fi |
al@844 | 270 |
al@844 | 271 # Clean |
al@844 | 272 rm -rf "$tmp_dir" |
al@844 | 273 |
al@844 | 274 # Function output: path to package |
al@844 | 275 echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg" |
al@844 | 276 } |
al@844 | 277 |
al@844 | 278 |
al@844 | 279 # Download package file to the cache |
al@844 | 280 |
al@844 | 281 get_pkg() { |
al@844 | 282 # input: $1 package name either "name" or "name-version" |
al@844 | 283 # $2 (internal): empty or 'redo' (when recursive calling) |
al@844 | 284 # action: download package from mirror to the cache directory |
al@844 | 285 # output: full path to the downloaded package |
al@844 | 286 |
al@844 | 287 debug "\nget_pkg('$1', '$2')" |
al@844 | 288 local repo line namever pkgsum |
al@844 | 289 |
al@844 | 290 IFS=$'\n' |
al@844 | 291 for rep in $PRIORITY; do |
al@844 | 292 [ ! -f "$rep/packages.info" ] && continue |
al@844 | 293 # If found, output string "<name>-<ver>:<sum>" |
al@844 | 294 line=$(awk -F$'\t' -vpkg="$1" \ |
al@844 | 295 '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info") |
al@844 | 296 if [ -n "$line" ]; then |
al@844 | 297 namever=${line%:*}; pkgsum=${line#*:} |
al@844 | 298 break |
al@844 | 299 fi |
al@844 | 300 done |
al@844 | 301 unset IFS |
al@844 | 302 |
al@844 | 303 debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'" |
al@844 | 304 |
al@844 | 305 if [ -z "$line" ]; then |
al@844 | 306 _ 'Unable to find package "%s" in the mirrored packages list.' "$(boldify $1)" >&2 |
al@844 | 307 # Retry with "get-package"; prevent looping with 'redo' |
al@844 | 308 if [ "$2" != 'redo' ]; then |
al@844 | 309 get_pkg "get-$1" 'redo'; exit 0 |
al@844 | 310 else |
al@844 | 311 # Retry with extra-package |
al@844 | 312 get_pkg_extra "${1#get-}" |
al@844 | 313 exit 0 |
al@844 | 314 fi |
al@844 | 315 fi |
al@844 | 316 |
al@844 | 317 # We need package "$namever.tazpkg" from "$rep" with "$pkgsum" |
al@844 | 318 |
al@844 | 319 if [ -z "$nocache" ]; then |
al@844 | 320 # Go to cache for "get-install" command; don't move for "get" command |
al@844 | 321 debug "cd '$(get_cache_path "$rep")'" |
al@844 | 322 cd "$(get_cache_path "$rep")" |
al@844 | 323 fi |
al@844 | 324 |
al@844 | 325 # Check if package already downloaded |
al@844 | 326 if [ -f "$namever.tazpkg" ]; then |
al@844 | 327 [ -z "$nocache" ] && _ 'Package "%s" already in the cache' "$namever" >&2 |
al@844 | 328 |
al@844 | 329 # Check if downloading complete, resume it not complete |
al@844 | 330 if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then |
al@844 | 331 _ 'Continuing package "%s" download' "$namever" >&2 |
al@844 | 332 download_from "$(cat "$rep/mirror")" "$namever.tazpkg" |
al@844 | 333 fi |
al@844 | 334 else |
al@844 | 335 # Package absent in the cache, "cold" downloading |
al@844 | 336 download_from "$(cat "$rep/mirror")" "$namever.tazpkg" |
al@844 | 337 fi |
al@844 | 338 |
al@844 | 339 # Make sure we downloaded what we want with checksum |
al@844 | 340 |
al@844 | 341 if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then |
al@845 | 342 _ 'Checksum error for "%s"' "$namever.tazpkg" >&2 |
al@844 | 343 rm "$namever.tazpkg" |
al@844 | 344 |
al@844 | 345 # Recharge DBs and try again; prevent looping with 'redo' |
al@844 | 346 if [ "$2" != 'redo' ]; then |
al@844 | 347 tazpkg recharge >&2 |
al@844 | 348 get_pkg "$1" 'redo'; exit 0 |
al@844 | 349 else |
al@844 | 350 # Give up |
al@844 | 351 # TODO: try another mirror? |
al@844 | 352 die 'Please wait until the mirror synchronization is complete and try again.' |
al@844 | 353 fi |
al@844 | 354 fi |
al@844 | 355 |
al@844 | 356 # Output: path to downloaded package |
al@844 | 357 printf '%s/%s.tazpkg\n' "$(pwd)" "$namever" |
al@844 | 358 } |
al@844 | 359 |
al@844 | 360 |
al@844 | 361 |
al@844 | 362 |
al@844 | 363 if [ -n "$extra" ]; then |
al@844 | 364 # When '--extra' option given, extra-package has priority over 'regular' packages |
al@844 | 365 get_pkg_extra "$1" |
al@844 | 366 else |
al@844 | 367 # Try to get 'package', then 'get-package', then extra 'package' |
al@844 | 368 get_pkg "$1" |
al@844 | 369 fi |