wok annotate cookutils/stuff/repo-cooking @ rev 24981

Up pce (20220220-991c55d9)
author Pascal Bellard <pascal.bellard@slitaz.org>
date Sun May 08 10:57:53 2022 +0000 (2022-05-08)
parents
children
rev   line source
pascal@24727 1 #!/bin/sh
pascal@24727 2
pascal@24727 3 # Make repo.json for Repology using SliTaz Cooking packages
pascal@24727 4 # Aleksey Bobylev <al.bobylev@gmail.com>
pascal@24727 5 #
pascal@24727 6 # 2018-08-11: initial, working on my localhost
pascal@24727 7 # 2018-09-04: fix paths for use at Tank
pascal@24727 8 # 2018-09-08: fix space in src, fix md5sum when EXTRAVERSION used, skip empty src, skip broken, store log
pascal@24727 9 # 2019-01-15: block parallel execution
pascal@24727 10 # 2019-01-23: drop duplicate lines in DB (if any), discard empty repo.json, use absolute paths
pascal@24727 11
pascal@24727 12 mirror='/home/slitaz/packages'
pascal@24727 13 local_repo='/home/slitaz/wok-hg'
pascal@24727 14
pascal@24727 15 my_log="/home/slitaz/log/repo-cooking.$(date '+%Y%m%d_%H%M%S')"
pascal@24727 16 block='/home/slitaz/log/repo-cooking.block'
pascal@24727 17
pascal@24727 18 local_bin=/root/bin
pascal@24727 19
pascal@24727 20 # packages.info
pascal@24727 21 # 1: name
pascal@24727 22 # 2: version
pascal@24727 23 # 3: category
pascal@24727 24 # 4: short description
pascal@24727 25 # 5: home
pascal@24727 26 # 6: tags
pascal@24727 27 # 7: size_packed size_unpacked
pascal@24727 28 # 8: dependencies
pascal@24727 29 # 9: checksum
pascal@24727 30 # 10: maintainer
pascal@24727 31 # 11: license
pascal@24727 32 # 12: src URL
pascal@24727 33 # 13: wanted
pascal@24727 34 # 14: build dependencies
pascal@24727 35 # 15: package md5sum
pascal@24727 36 # 16: version without extraversion
pascal@24727 37
pascal@24727 38 timestamp1=$(date '+%s')
pascal@24727 39 date_for_json=$(date '+%F %T')
pascal@24727 40
pascal@24727 41 if [ -e "$block" ]; then
pascal@24727 42 echo "$0 already executed. Exit" | tee "$my_log"
pascal@24727 43 exit 1
pascal@24727 44 fi
pascal@24727 45 touch "$block"
pascal@24727 46
pascal@24727 47 ls ${my_log%.*}* | sed '1,10d;s|.*|rm -f &|' | sh # keep only 10 log files
pascal@24727 48
pascal@24727 49
pascal@24727 50 tempd=$(mktemp -d)
pascal@24727 51
pascal@24727 52 sort -u $mirror/packages.info > $tempd/packages.info # drop duplicate lines
pascal@24727 53 sort -u $mirror/packages.md5 > $tempd/packages.md5 #
pascal@24727 54
pascal@24727 55 # get base settings
pascal@24727 56 c_version=$(awk -F$'\t' -vp='cookutils' '{if ($1 == p) {print $2; exit}}' $tempd/packages.info) #'
pascal@24727 57 cp $mirror/cookutils-$c_version.tazpkg $tempd/
pascal@24727 58
pascal@24727 59 cd $tempd
pascal@24727 60 tazpkg extract $tempd/cookutils-$c_version.tazpkg 2>/dev/null >&2
pascal@24727 61 cp $tempd/cookutils-$c_version/fs/etc/slitaz/cook.conf $tempd/
pascal@24727 62 rm $tempd/cookutils-$c_version.tazpkg
pascal@24727 63 rm -r $tempd/cookutils-$c_version/
pascal@24727 64
pascal@24727 65 # define some useful variables
pascal@24727 66 . $tempd/cook.conf
pascal@24727 67
pascal@24727 68
pascal@24727 69 ls $local_repo \
pascal@24727 70 | while read pkg_i; do
pascal@24727 71 i=$(( i + 1 ))
pascal@24727 72 [ -f "$local_repo/$pkg_i/receipt" ] || continue
pascal@24727 73
pascal@24727 74 unset MAINTAINER LICENSE WGET_URL WANTED BUILD_DEPENDS VERSION EXTRAVERSION
pascal@24727 75
pascal@24727 76 case $i in
pascal@24727 77 *00) echo $i;;
pascal@24727 78 *0) echo -n $i;;
pascal@24727 79 esac
pascal@24727 80 echo -n '.'
pascal@24727 81
pascal@24727 82 . "$local_repo/$pkg_i/receipt"
pascal@24727 83
pascal@24727 84 MAINTAINER=$(echo $MAINTAINER | sed 's|.*<||; s|>||')
pascal@24727 85
pascal@24727 86 LICENSE=$(echo $LICENSE)
pascal@24727 87
pascal@24727 88 WGET_URL=$(echo $WGET_URL | sed 's#^git|git#git#; s#^git|https*#git#; s#^svn|https*#svn#; s#^subversion|svn#svn#; s#^subversion|https*#svn#; s#^svn|svn#svn#; s#^hg|#hg://#; s#^mercurial|#hg://#; s#^bzr|#bzr://#; s#^cvs|#cvs://#')
pascal@24727 89
pascal@24727 90 WANTED=$(echo $WANTED)
pascal@24727 91 case $pkg_i in
pascal@24727 92 locale-*-extra) WANTED='slitaz-i18n-extra';;
pascal@24727 93 locale-*) WANTED='slitaz-i18n';;
pascal@24727 94 esac
pascal@24727 95
pascal@24727 96 BUILD_DEPENDS=$(echo $BUILD_DEPENDS)
pascal@24727 97
pascal@24727 98 MD5SUM="$(fgrep " $pkg_i-$VERSION" $tempd/packages.md5 | cut -d' ' -f1)"
pascal@24727 99 if [ $(echo "$MD5SUM" | wc -l) -ne 1 ]; then
pascal@24727 100 echo "$pkg_i: error: multiple md5sum" >>$my_log
pascal@24727 101 fi
pascal@24727 102
pascal@24727 103 sed -i "/^$pkg_i / s|$| $MAINTAINER $LICENSE $WGET_URL $WANTED $BUILD_DEPENDS $MD5SUM $VERSION|" $tempd/packages.info
pascal@24727 104 done
pascal@24727 105 echo
pascal@24727 106 echo
pascal@24727 107
pascal@24727 108 timestamp2=$(date '+%s')
pascal@24727 109 echo "Stage 1: $(( timestamp2 - timestamp1 )) s" >>$my_log
pascal@24727 110 echo "Packages: $(wc -l ./packages.info)" >>$my_log
pascal@24727 111 echo >>$my_log
pascal@24727 112
pascal@24727 113
pascal@24727 114
pascal@24727 115 {
pascal@24727 116 echo '{"repo":"SliTaz Cooking","date":"'$date_for_json'","items":['
pascal@24727 117
pascal@24727 118 comma_up=''
pascal@24727 119
pascal@24727 120 { ls $local_repo; echo '@the_end@'; } \
pascal@24727 121 | while read pkg_i; do
pascal@24727 122 echo "$pkg_i" >&2
pascal@24727 123
pascal@24727 124 case $pkg_i in
pascal@24727 125 @the_end@) echo ']}'; break;;
pascal@24727 126 get-*) continue;;
pascal@24727 127 esac
pascal@24727 128
pascal@24727 129 pkg_line=$(sed -n "/^$pkg_i /p" $tempd/packages.info)
pascal@24727 130
pascal@24727 131 wanted=$(echo "$pkg_line" | cut -d$'\t' -f 13)
pascal@24727 132 [ -z "$wanted" ] || continue
pascal@24727 133
pascal@24727 134 echo "$pkg_i" >> $tempd/packages.txt
pascal@24727 135
pascal@24727 136 ver=$( echo "$pkg_line" | cut -d$'\t' -f 16)
pascal@24727 137 home=$( echo "$pkg_line" | cut -d$'\t' -f 5)
pascal@24727 138 maintainer=$(echo "$pkg_line" | cut -d$'\t' -f 10)
pascal@24727 139 license=$( echo "$pkg_line" | cut -d$'\t' -f 11 | sed 's| |","|g')
pascal@24727 140 if [ -z "$license" ]; then
pascal@24727 141 echo "$pkg_i: license is empty" >>$my_log
pascal@24727 142 license='unknown'
pascal@24727 143 fi
pascal@24727 144 src=$( echo "$pkg_line" | cut -d$'\t' -f 12 | sed 's| .*||')
pascal@24727 145 bdeps=$( echo "$pkg_line" | cut -d$'\t' -f 14 | sed 's| |","|g')
pascal@24727 146
pascal@24727 147 # skip
pascal@24727 148 if [ -z "$ver" ]; then
pascal@24727 149 echo "$pkg_i: ver empty, skip" >>$my_log
pascal@24727 150 continue
pascal@24727 151 fi
pascal@24727 152
pascal@24727 153 echo "$comma_up{\"meta\":\"$pkg_i\", \"ver\":\"$ver\", \"maintainer\":\"$maintainer\", \"home\":\"$home\", \"license\":[\"$license\"]"
pascal@24727 154
pascal@24727 155 [ -z "$src" ] || echo ", \"src\":\"$src\""
pascal@24727 156 [ -z "$bdeps" ] || echo ", \"bdeps\":[\"$bdeps\"]"
pascal@24727 157
pascal@24727 158 echo ", \"pkgs\":["
pascal@24727 159
pascal@24727 160 comma_up=','
pascal@24727 161 comma=''
pascal@24727 162
pascal@24727 163 want=$(awk -F$'\t' -vp="$pkg_i" '{if ($13 == p) {print $1}}' $tempd/packages.info) #'
pascal@24727 164 for pkg_j in $pkg_i $want; do
pascal@24727 165 echo " - $pkg_j" >> $tempd/packages.txt
pascal@24727 166
pascal@24727 167 pkg_line=$(sed -n "/^$pkg_j /p" $tempd/packages.info)
pascal@24727 168
pascal@24727 169 echo $comma; comma=','
pascal@24727 170
pascal@24727 171 ver_sub=$(echo "$pkg_line" | cut -d$'\t' -f 16)
pascal@24727 172 if [ "$ver" != "$ver_sub" ]; then
pascal@24727 173 echo "$pkg_i > $pkg_j: version mismatch '$ver' -> '$ver_sub'" >>$my_log
pascal@24727 174 fi
pascal@24727 175
pascal@24727 176 ver_ext=$( echo "$pkg_line" | cut -d$'\t' -f 2)
pascal@24727 177 cat=$( echo "$pkg_line" | cut -d$'\t' -f 3)
pascal@24727 178 desc=$( echo "$pkg_line" | cut -d$'\t' -f 4 | sed 's|"|\\"|g')
pascal@24727 179 tags=$( echo "$pkg_line" | cut -d$'\t' -f 6 | sed 's| |","|g')
pascal@24727 180 pkg_size=$( echo "$pkg_line" | cut -d$'\t' -f 7 | cut -d' ' -f1)
pascal@24727 181 installed_size=$(echo "$pkg_line" | cut -d$'\t' -f 7 | cut -d' ' -f2)
pascal@24727 182 deps=$( echo "$pkg_line" | cut -d$'\t' -f 8 | sed 's| |","|g')
pascal@24727 183 pkg_md5sum=$( echo "$pkg_line" | cut -d$'\t' -f 15)
pascal@24727 184
pascal@24727 185 echo "{\"name\":\"$pkg_j\", \"cat\":\"$cat\", \"desc\":\"$desc\""
pascal@24727 186
pascal@24727 187 [ -z "$tags" ] || echo ",\"tags\":[\"$tags\"]"
pascal@24727 188
pascal@24727 189 [ -z "$deps" ] || echo ",\"deps\":[\"$deps\"]"
pascal@24727 190
pascal@24727 191 echo ",\"pkg_size\":\"$pkg_size\", \"installed_size\":\"$installed_size\", \"pkg_md5sum\":\"$pkg_md5sum\", \"get\":\"http://mirror.slitaz.org/packages/cooking/$pkg_j-$ver_ext.tazpkg\""
pascal@24727 192
pascal@24727 193 echo '}'
pascal@24727 194 done
pascal@24727 195
pascal@24727 196 echo ']}'
pascal@24727 197 done
pascal@24727 198 } > $tempd/repo-dirt.json
pascal@24727 199
pascal@24727 200 cat $tempd/repo-dirt.json | $local_bin/jq '.' > $tempd/repo.json
pascal@24727 201
pascal@24727 202 if [ -s $tempd/repo.json ]; then
pascal@24727 203 gzip $tempd/repo.json
pascal@24727 204 $local_bin/advdef -z4 $tempd/repo.json.gz
pascal@24727 205 mv $tempd/repo.json.gz $mirror
pascal@24727 206 else
pascal@24727 207 echo "ERROR! New repo.json was discarded" >>$my_log
pascal@24727 208 fi
pascal@24727 209
pascal@24727 210 rm -rf $tempd
pascal@24727 211
pascal@24727 212 timestamp3=$(date '+%s')
pascal@24727 213
pascal@24727 214 echo -e "\nStage 2: $(( timestamp3 - timestamp2 )) s" >>$my_log
pascal@24727 215 echo "Finished: $(date '+%F %T')" >>$my_log
pascal@24727 216 rm "$block"