tazpkg diff modules/get @ rev 844
Finish modularization. Beta release: still have few FIXMEs and TODOs.
| author | Aleksej Bobylev <al.bobylev@gmail.com> |
|---|---|
| date | Mon Oct 05 03:53:47 2015 +0300 (2015-10-05) |
| parents | |
| children | ce7009ff237b |
line diff
1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000 1.2 +++ b/modules/get Mon Oct 05 03:53:47 2015 +0300 1.3 @@ -0,0 +1,369 @@ 1.4 +#!/bin/sh 1.5 +# TazPkg - Tiny autonomous zone packages manager, hg.slitaz.org/tazpkg 1.6 +# get - TazPkg module 1.7 +# Get package to the cache directory 1.8 + 1.9 + 1.10 +# 1. "Plain packages" - compiled and packed on cook.slitaz.org using receipts. 1.11 +# 1.12 +# Recently added other type of packages: packages that can't be compiled due to the closed sources 1.13 +# or due to the giant size of that sources. In this case special script: a) downloads package 1.14 +# compiled for other Linux from official web site, b) converts this package to the TazPkg format, 1.15 +# c) installs this package as "plain" package. 1.16 +# 1.17 +# 2. "Get-packages" - package contains only one special script for downloading/converting. 1.18 +# Sometimes get-package can get some specified version of the package, sometimes - different 1.19 +# (latest) version. 1.20 +# 1.21 +# Packages 1. and 2. can be found in the wok repository: http://hg.slitaz.org/wok/ 1.22 +# 1.23 +# 3. "Extra" get-scripts. Next addition - only script, without packaging. 1.24 +# Extra get-scripts repository: http://hg.slitaz.org/get-scripts/ 1.25 +# Extra get-scripts mirror: http://mirror.slitaz.org/packages/get/ 1.26 +# 1.27 +# 4. Converted "extra" packages. Next addition - some packages like LibreOffice / OpenOffice are 1.28 +# big to compile them like _1. "Plain packages"_ as well as big to convert them on the user side 1.29 +# (you need a lot of time, CPU, and RAM) and sometimes it is unable on the weak user machines. 1.30 +# So, some selected and free packages are converted on the SliTaz server side, packaged and 1.31 +# are ready to download by any user: http://mirror.slitaz.org/packages/extra/ 1.32 +# 1.33 +# 1.34 +# Algorithm to get package: 1.35 +# - search package in question in the mirrored packages list, download if exists; 1.36 +# - search "get-package" in the mirrored packages list, download if exists; 1.37 +# - search package in the "extra" get-scripts list, download if exists and run it, get the package. 1.38 +# 1.39 +# Note, here doubling. Imagine, you may wanted "palemoon" package. 1.40 +# 1. No "plain package"; 1.41 +# 2. Existing "get-palemoon" package; 1.42 +# 3. Existing "palemoon" extra get-script; 1.43 +# 4. Existing "palemoon" extra converted package. 1.44 +# User will get 2. He can specify "--extra" option to get 3. 1.45 +# FIXME: do something with doubling. 1.46 + 1.47 + 1.48 + 1.49 + 1.50 +# Connect function libraries 1.51 +. /lib/libtaz.sh 1.52 +. /usr/lib/slitaz/libpkg.sh 1.53 + 1.54 +# Get TazPkg working environment 1.55 +. @@MODULES@@/getenv 1.56 + 1.57 +. @@MODULES@@/find-depends 1.58 + 1.59 + 1.60 + 1.61 + 1.62 +# Get cache directory path 1.63 + 1.64 +get_cache_path() { 1.65 + # input: $1 = DB folder such as "$PKGS_DB" or "$PKGS_DB/undigest/*" 1.66 + # $2 (optional): 'extra' for cache for extra-package scripts 1.67 + # output: path to cache directory 1.68 + 1.69 + local cache_dir 1.70 + 1.71 + if [ "$2" == 'extra' ]; then 1.72 + cache_dir="$SAVE_CACHE_DIR/extra/packages" 1.73 + elif [ "$1" == "$PKGS_DB" ]; then 1.74 + # Main repository 1.75 + cache_dir="$SAVE_CACHE_DIR/$SLITAZ_RELEASE/packages" 1.76 + else 1.77 + # Undigest repository 1.78 + cache_dir="$SAVE_CACHE_DIR/${1##*/}/packages" 1.79 + fi 1.80 + 1.81 + # Make cache directory if absent 1.82 + mkdir -p "$cache_dir" 1.83 + chmod a+w "$cache_dir" 1.84 + 1.85 + echo "$cache_dir" 1.86 +} 1.87 + 1.88 + 1.89 +# Download a file from mirror 1.90 + 1.91 +download_from() { 1.92 + # input: "<mirror_url>" "<package_name>-<version>.tazpkg" 1.93 + 1.94 + debug "\ndownload_from('$1', '$2')" 1.95 + 1.96 + case "$1" in 1.97 + # Mirror URL can have a trailing slash or not. 1.98 + http://* | https://* | ftp://*) 1.99 + debug " busybox wget -c -q -T 30 -U $UA ${1%/}/$2" 1.100 + # TODO: display abridged wget status 1.101 + busybox wget -c -q -T 30 -U $UA ${1%/}/$2 1.102 + ;; 1.103 + *) 1.104 + debug " ln -sf ${1%/}/$2 ." 1.105 + ln -sf ${1%/}/$2 .;; 1.106 + esac 1.107 +} 1.108 + 1.109 + 1.110 +# This function may be called by a get script. 1.111 + 1.112 +abort_package() { 1.113 + cd "$CUR_DIR" 1.114 + rm -rf "$tmp_dir" 1.115 + echo "${1:-Abort $PACKAGE.}" 1.116 + exit 1 1.117 +} 1.118 + 1.119 + 1.120 +# Get extra-package file to the cache 1.121 + 1.122 +get_pkg_extra() { 1.123 + # input: $1 extra-package name (like 'dropbox') 1.124 + # $2 (internal): empty or 'redo' (when recursive calling) 1.125 + # action: download stuff and make package 1.126 + # output: full path to created package 1.127 + 1.128 + debug "\nget_pkg_extra('$1', '$2')" 1.129 + 1.130 + local mirror extra_cache converted tmp_dir script 1.131 + 1.132 + # Check extra.list 1.133 + if [ ! -s "$PKGS_DB/extra.list" ]; then 1.134 + # Empty extra.list 1.135 + if [ "$2" != 'redo' ]; then 1.136 + tazpkg recharge >&2 1.137 + get_pkg_extra "$1" 'redo'; exit 0 1.138 + else 1.139 + # Give up 1.140 + _ 'File "%s" empty.' 'extra.list' >&2 1.141 + die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)" 1.142 + fi 1.143 + fi 1.144 + 1.145 + # Extra.list exists and not empty 1.146 + if ! grep -q "^$1|" "$PKGS_DB/extra.list"; then 1.147 + die 'Unable to find package "%s" in the extra packages list.' "$(boldify $1)" 1.148 + fi 1.149 + 1.150 + # Extra-package found in extra.list 1.151 + 1.152 + if [ -z "$nocache" ]; then 1.153 + # Go to cache for "get-install" command; don't move for "get" command 1.154 + extra_cache="$(get_cache_path "$PKGS_DB" 'extra')" 1.155 + debug "cd '$extra_cache'" 1.156 + cd "$extra_cache" 1.157 + 1.158 + # Extra-cache should contain packages DB files (packages.info, etc.) 1.159 + # to list extra-packages contained in the extra-cache 1.160 + [ ! -f 'packages.info' ] && tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null 1.161 + 1.162 + if [ -f 'packages.info' ]; then 1.163 + awk -F$'\t' -vp="$1" '$1==p{exit 1}' packages.info 1.164 + if [ "$?" -eq 1 ]; then 1.165 + _ 'Package "%s" already in the cache' "$1" >&2 1.166 + echo -n "$(pwd)/" 1.167 + awk -F$'\t' -vp="$1" '$1==p{printf "%s-%s.tazpkg\n", $1, $2; exit}' packages.info 1.168 + exit 0 1.169 + fi 1.170 + fi 1.171 + fi 1.172 + 1.173 + mirror="$(cat "$PKGS_DB/mirror")" 1.174 + debug "mirror='$mirror'" 1.175 + 1.176 + 1.177 + # Try converted extra-packages first 1.178 + # FIXME: Workaround to get packages filenames (even better to have names and versions separate) 1.179 + converted="$(busybox wget -O - http://mirror1.slitaz.org/packages/extra/ 2>/dev/null | \ 1.180 + tr \'\" $'\n' | grep "$1-.*\.tazpkg$" | sort -u)" 1.181 + debug "converted='$converted'" 1.182 + if [ -n "$converted" ]; then 1.183 + case "$mirror" in 1.184 + http://*|https://*|ftp://*) 1.185 + # Default 'http://mirror.slitaz.org/packages/cooking/' 1.186 + # -> 'http://mirror.slitaz.org/packages/extra/' 1.187 + debug "busybox wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/extra/$converted'" 1.188 + busybox wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/extra/$converted";; 1.189 + esac 1.190 + if [ -f "$converted" ]; then 1.191 + echo "$extra_cache/$converted"; exit 0 1.192 + fi 1.193 + fi 1.194 + 1.195 + 1.196 + # Fails to download converted extra-package; now try to download and execute get-script 1.197 + cd .. 1.198 + case "$mirror" in 1.199 + http://*|https://*|ftp://*) 1.200 + # Default 'http://mirror.slitaz.org/packages/cooking/' 1.201 + # -> 'http://mirror.slitaz.org/packages/get/' 1.202 + debug "busybox wget -q -T 30 -U '$UA' '${mirror%packages/*}packages/get/$1'" 1.203 + busybox wget -q -T 30 -U "$UA" "${mirror%packages/*}packages/get/$1";; 1.204 + esac 1.205 + 1.206 + if [ ! -f "$1" ]; then 1.207 + # TODO: extra package or extra package script? :) Too complicated... 1.208 + die 'Unable to download extra package "%s".' "$(boldify $1)" 1.209 + fi 1.210 + 1.211 + # Extra-package script downloaded in the /var/cache/tazpkg/extra/ 1.212 + 1.213 + unset_receipt 1.214 + PACKAGE="$1" 1.215 + script="$(realpath $1)" 1.216 + tmp_dir="$(mktemp -d)"; cd "$tmp_dir" 1.217 + # Run script 1.218 + set -e 1.219 + . "$script" 1.220 + set +e 1.221 + 1.222 + if [ ! -d "$PACKAGE-$VERSION" ]; then 1.223 + abort_package 'Could not download "%s" from "%s". Exiting.' "${TARBALL:-$PACKAGE}" "${WGET_URL:-$WEB_SITE}" 1.224 + fi 1.225 + 1.226 + if [ ! -s "$PACKAGE-$VERSION/receipt" ]; then 1.227 + # Create receipt (if script not created it early) using variables from script 1.228 + cat > "$PACKAGE-$VERSION/receipt" <<EOT 1.229 +# SliTaz package receipt. 1.230 + 1.231 +PACKAGE="$PACKAGE" 1.232 +VERSION="${VERSION:-unknown}" 1.233 +CATEGORY="${CATEGORY:-non-free}" 1.234 +WEB_SITE="$WEB_SITE" 1.235 +SHORT_DESC="${SHORT_DESC:-$PACKAGE}" 1.236 +MAINTAINER="${MAINTAINER:-nobody@slitaz.org}" 1.237 +LICENSE="$LICENSE" 1.238 +TARBALL="$TARBALL" 1.239 +WGET_URL="$WGET_URL" 1.240 +CONFIG_FILES="$CONFIG_FILES" 1.241 +SUGGESTED="$SUGGESTED" 1.242 +PROVIDE="$PROVIDE" 1.243 +DEPENDS="$DEPENDS" 1.244 +HOST_ARCH="$HOST_ARCH" 1.245 +TAGS="$TAGS" 1.246 +EXTRA_SOURCE_FILES="$EXTRA_SOURCE_FILES" 1.247 +EOT 1.248 + fi 1.249 + 1.250 + # Add dependencies which was found to already defined dependencies 1.251 + DEPENDS="$(unset DEPENDS; . "$PACKAGE-$VERSION/receipt"; echo $DEPENDS)" 1.252 + for i in $(find_depends "$PACKAGE-$VERSION/fs"); do 1.253 + case " $DEPENDS " in 1.254 + *\ $i\ *) continue;; 1.255 + esac 1.256 + sed -i "s/^DEPENDS=\"/&$i /" "$PACKAGE-$VERSION/receipt" 1.257 + done 1.258 + 1.259 + # Remove empty variables 1.260 + sed -i '/=""$/d' "$PACKAGE-$VERSION/receipt" 1.261 + 1.262 + tazpkg pack "$PACKAGE-$VERSION" gzip >&2 1.263 + 1.264 + if [ -z "nocache" ]; then 1.265 + # move package to the extra-cache for "get-install" command 1.266 + mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$extra_cache" 1.267 + # Re-make extra packages database 1.268 + tazpkg mkdb "$extra_cache" --root='' --forced >/dev/null 1.269 + else 1.270 + # move package to directory where "tazpkg get" command invoked 1.271 + mv -f "$tmp_dir/$PACKAGE-$VERSION.tazpkg" "$CUR_DIR" 1.272 + fi 1.273 + 1.274 + # Clean 1.275 + rm -rf "$tmp_dir" 1.276 + 1.277 + # Function output: path to package 1.278 + echo "$CUR_DIR/$PACKAGE-$VERSION.tazpkg" 1.279 +} 1.280 + 1.281 + 1.282 +# Download package file to the cache 1.283 + 1.284 +get_pkg() { 1.285 + # input: $1 package name either "name" or "name-version" 1.286 + # $2 (internal): empty or 'redo' (when recursive calling) 1.287 + # action: download package from mirror to the cache directory 1.288 + # output: full path to the downloaded package 1.289 + 1.290 + debug "\nget_pkg('$1', '$2')" 1.291 + local repo line namever pkgsum 1.292 + 1.293 + IFS=$'\n' 1.294 + for rep in $PRIORITY; do 1.295 + [ ! -f "$rep/packages.info" ] && continue 1.296 + # If found, output string "<name>-<ver>:<sum>" 1.297 + line=$(awk -F$'\t' -vpkg="$1" \ 1.298 + '$1==pkg || $1"-"$2==pkg {printf "%s-%s:%s", $1, $2, $9; exit}' "$rep/packages.info") 1.299 + if [ -n "$line" ]; then 1.300 + namever=${line%:*}; pkgsum=${line#*:} 1.301 + break 1.302 + fi 1.303 + done 1.304 + unset IFS 1.305 + 1.306 + debug " rep='$rep'\n namever='$namever'\n pkgsum='$pkgsum'" 1.307 + 1.308 + if [ -z "$line" ]; then 1.309 + _ 'Unable to find package "%s" in the mirrored packages list.' "$(boldify $1)" >&2 1.310 + # Retry with "get-package"; prevent looping with 'redo' 1.311 + if [ "$2" != 'redo' ]; then 1.312 + get_pkg "get-$1" 'redo'; exit 0 1.313 + else 1.314 + # Retry with extra-package 1.315 + get_pkg_extra "${1#get-}" 1.316 + exit 0 1.317 + fi 1.318 + fi 1.319 + 1.320 + # We need package "$namever.tazpkg" from "$rep" with "$pkgsum" 1.321 + 1.322 + if [ -z "$nocache" ]; then 1.323 + # Go to cache for "get-install" command; don't move for "get" command 1.324 + debug "cd '$(get_cache_path "$rep")'" 1.325 + cd "$(get_cache_path "$rep")" 1.326 + fi 1.327 + 1.328 + # Check if package already downloaded 1.329 + if [ -f "$namever.tazpkg" ]; then 1.330 + [ -z "$nocache" ] && _ 'Package "%s" already in the cache' "$namever" >&2 1.331 + 1.332 + # Check if downloading complete, resume it not complete 1.333 + if ! tail -c 2k "$namever.tazpkg" | fgrep -q '00000000TRAILER'; then 1.334 + _ 'Continuing package "%s" download' "$namever" >&2 1.335 + download_from "$(cat "$rep/mirror")" "$namever.tazpkg" 1.336 + fi 1.337 + else 1.338 + # Package absent in the cache, "cold" downloading 1.339 + download_from "$(cat "$rep/mirror")" "$namever.tazpkg" 1.340 + fi 1.341 + 1.342 + # Make sure we downloaded what we want with checksum 1.343 + 1.344 + if [ "$($CHECKSUM "$namever.tazpkg" | cut -d' ' -f1)" != "$pkgsum" ]; then 1.345 + _ 'Checksum error for "%s"' "$namever.tazpkg" 1.346 + rm "$namever.tazpkg" 1.347 + 1.348 + # Recharge DBs and try again; prevent looping with 'redo' 1.349 + if [ "$2" != 'redo' ]; then 1.350 + tazpkg recharge >&2 1.351 + get_pkg "$1" 'redo'; exit 0 1.352 + else 1.353 + # Give up 1.354 + # TODO: try another mirror? 1.355 + die 'Please wait until the mirror synchronization is complete and try again.' 1.356 + fi 1.357 + fi 1.358 + 1.359 + # Output: path to downloaded package 1.360 + printf '%s/%s.tazpkg\n' "$(pwd)" "$namever" 1.361 +} 1.362 + 1.363 + 1.364 + 1.365 + 1.366 +if [ -n "$extra" ]; then 1.367 + # When '--extra' option given, extra-package has priority over 'regular' packages 1.368 + get_pkg_extra "$1" 1.369 +else 1.370 + # Try to get 'package', then 'get-package', then extra 'package' 1.371 + get_pkg "$1" 1.372 +fi