scripts/Download

   1 #!/usr/bin/env bash
   2 #
   3 # --- T2-COPYRIGHT-BEGIN ---
   4 # t2/scripts/Download
   5 # Copyright (C) 2004 - 2025 The T2 SDE Project
   6 # Copyright (C) 1998 - 2003 ROCK Linux Project
   7 # SPDX-License-Identifier: GPL-2.0
   8 # --- T2-COPYRIGHT-END ---
   9 #
  10 #   Run this command from the T2 directory as scripts/Download [ options ]
  11 #
  12 #   It enables you to download source files as described in the package
  13 #   definitions (optionally using a mirroring 'cache' server).
  14 #
  15 #   This script also allows for checksum display/validation.
  16
  17 umask 022
  18
  19 . scripts/functions.in
  20 . misc/output/parse-config
  21
  22 eval "$(egrep '^(sdever)=' scripts/parse-config)"
  23 base=$(pwd -P)
  24
  25 if [ "$1" = '--help' ]; then
  26         { echo
  27         echo "Usage:"
  28         echo
  29         echo " scripts/Download [options] [ Package(s) ]"
  30         echo " scripts/Download [options] [ Desc file(s) ]"
  31         echo " scripts/Download [options] -repository Repositories"
  32         echo " scripts/Download [options] { -all | -required }"
  33         echo
  34         echo " Where [options] is an alias for:"
  35         echo "    [ -cfg <config> ] [ -nock ]  [ -alt-dir <AlternativeDirectory> ]"
  36         echo "    [ -mirror <URL> | -check ]  [ -try-questionable ]  [ -notimeout ]"
  37         echo "    [ -longtimeout ]  [ -curl-opt <curl-option>[:<curl-option>[:..]] ]"
  38         echo "    [ -proxy <server>[:<port>] ] [ -proxy-auth <username>[:<password>] ]"
  39         echo "    [ -copy ] [ -move ]"
  40         echo
  41         echo " On default, this script auto-detects the best T2 SDE mirror."
  42         echo
  43         echo " Mirrors can also be a local directories in the form of: 'file:///<dir>'"
  44         echo
  45         echo " scripts/Download -mk-cksum Filename(s)"
  46         echo " scripts/Download [ -list | -list-unknown | -list-missing | -list-cksums ]"
  47         echo; } >&2
  48         exit 1
  49 fi
  50
  51 # -mk-cksum mode (display T2 type package checksum): it
  52 # displays the checksum T2 validates against.
  53 #
  54 # Commonly known compressed tarballs unpacked.
  55 #
  56 if [ "$1" = -mk-cksum ]; then
  57     shift
  58     for x; do
  59         echo -n "$x: "
  60         ck="sha224"
  61         cksum=${ck}sum
  62
  63         if ! type -p $cksum > /dev/null; then
  64             cksum=${cksum#sha} cksum=${cksum%sum}
  65             cksum="shasum -a $cksum"
  66         fi
  67
  68         compressor="$(get_compressor "$x")"
  69         if [ ! -f "$x" ]; then
  70             echo "No such file."
  71         elif [ "$compressor" ]; then
  72             $compressor < "$x" | $cksum | cut -f1 -d' '
  73         else
  74             $cksum < "$x" | cut -f1 -d' '
  75         fi
  76     done
  77     exit 1
  78 fi
  79
  80 # Handle options passed on the command line
  81 #
  82 mkdir -p src/ download/; config=default
  83 this_is_the_2nd_run=0
  84 mirror='' checkonly=0 altdir=''
  85 tryques=0 nocheck=0 options='-this_is_the_2nd_run '
  86 notimeout=0 curl_options='-A T2-downloader --disable-epsv --location -f'
  87 altcopy=link verbose=1 quietmirror=0
  88 downloaderror=0
  89
  90 # load options from the enviroment T2DOWNOPT
  91 # and then clean it to avoid duplication on children processes
  92 #
  93 set -- $T2DOWNOPT "$@"
  94 export T2DOWNOPT=
  95
  96 #
  97 while [ $# -gt 0 ]; do
  98         case "$1" in
  99             -this_is_the_2nd_run)
 100                 this_is_the_2nd_run=1
 101                 ;;
 102
 103             -cfg)
 104                 options="$options -cfg $2"
 105                 config="$2"; shift ;;
 106
 107             -q)
 108                 options="$options -q"
 109                 verbose=0 ;;
 110
 111             -nock)
 112                 # -nock skips checksum checking (don't use lightly)
 113                 options="$options -nock"
 114                 nocheck=1 ;;
 115
 116             -mirror)
 117                 # -mirror uses a mirror for finding source files
 118                 if [ "$2" = auto ]; then
 119                         rm -f download/Mirror-Cache
 120                 else
 121                         mkdir -p download
 122                         echo "$2 $sdever" > download/Mirror-Cache
 123                         options="$options -mirror $2"
 124                         mirror="$2"
 125                 fi
 126                 shift ;;
 127
 128             -quiet-mirror)
 129                 quietmirror=1 ;;
 130
 131             -check)
 132                 # -check just validates the file using the checksum
 133                 options="$options -check"
 134                 checkonly=1 ;;
 135
 136             -notimeout)
 137                 # don't add timeout curl options
 138                 options="$options -notimeout"
 139                 notimeout=2 ;;
 140
 141             -longtimeout)
 142                 # don't add timeout curl options
 143                 options="$options -longtimeout"
 144                 notimeout=1 ;;
 145
 146             -curl-opt)
 147                 # additional curl options
 148                 options="$options -curl-opt $2"
 149                 curl_options="$curl_options `echo $2 | tr : ' '`"
 150                 shift ;;
 151
 152             -proxy)
 153                 # proxy option for curl
 154                 mkdir -p download
 155                 echo -n "$2" > download/Proxy
 156                 options="$options -proxy $2"
 157                 shift ;;
 158
 159             -proxy-auth)
 160                 # proxy authentication for curl - can be seen with ps!
 161                 mkdir -p download
 162                 echo -n "$2" > download/Proxy-auth
 163                 chmod 600 download/Proxy-auth
 164                 options="$options -proxy-auth $2"
 165                 shift ;;
 166
 167             -alt-dir)
 168                 # check for an alternative directory where to search for
 169                 # package source tarballs
 170                 altdir=$(cd $2; pwd -P)
 171                 options="$options -alt-dir $2"
 172                 shift ;;
 173
 174             -try-questionable)
 175                 # also try to download questionable URLs
 176                 options="$options -try-questionable"
 177                 tryques=1 ;;
 178
 179             -move) altcopy=move ;;
 180             -copy) altcopy=copy ;;
 181
 182             *)  break   ;;
 183         esac
 184         shift
 185 done
 186
 187 # Read some config values
 188
 189 target=`grep '^export SDECFG_TARGET=' config/$config/config 2>/dev/null |
 190         cut -f2 -d= | tr -d "'"`
 191 arch=`grep '^export SDECFG_ARCH=' config/$config/config 2>/dev/null |
 192         cut -f2 -d= | tr -d "'"`
 193 arch="${arch:-none}" target="${target:-none}"
 194
 195
 196 if [ $notimeout -eq 0 ]; then
 197         curl_options="$curl_options -Y 10 -y 20 --connect-timeout 12"
 198 elif [ $notimeout -eq 1 ]; then
 199         curl_options="$curl_options -Y 1 -y 60 --connect-timeout 60"
 200 fi
 201
 202 # Disable checking for certificates on https downloads
 203 curl_options="$curl_options -k"
 204
 205 # cksum_chk filename cksum origfile
 206 #
 207 # This function verifies the checksum. If it fails it renames the file
 208 # to file.chksum-err and returns failure.
 209 #
 210 # It seams like the [ ] command has problems with comparing high numbers.
 211 # That's why I'm using a text comparison here.
 212 #
 213 # Not doing anything if checksum is '0' or a text of 'X'.
 214 #
 215 cksum_chk() {
 216         local y="$2"
 217         [ $nocheck = 1 -o -z "${2//0/}" -o -z "${2//X/}" ] && return 0
 218
 219         # determine cksum type
 220         local ck="${y%\}*}"
 221         if [ "$ck" != "$y" ]; then
 222                 ck="${ck#\{}"
 223         else case "${#ck}" in
 224                 56) ck="sha224" ;;
 225                 64) ck="sha256" ;;
 226                 *) ck="ck" ;;
 227         esac; fi
 228         y="${y#\{$ck\}}"
 229
 230         local cksum=${ck}sum
 231         if ! type -p $cksum > /dev/null; then
 232             cksum=${cksum#sha} cksum=${cksum%sum}
 233             cksum="shasum -a $cksum"
 234         fi
 235
 236         local x="`$cksum "$1" | cut -f1 -d' '`"
 237         if [ "$x" != "$y" ]; then
 238                 # Add .cksum-err extension to filename:
 239                 echo "Cksum ERROR: $3.cksum-err ($x)"
 240                 mv "$3" "$3.cksum-err"; return 1
 241         fi
 242         return 0
 243 }
 244
 245 # output (multiline) message only if we are not in quiet mode
 246 #
 247 echo_info() {
 248         if [ "$verbose" == 1 ]; then
 249                 echo "$@" | sed -e 's,^,INFO: ,'
 250         fi
 251 }
 252
 253 # output (multiline) message always
 254 #
 255 echo_warn() {
 256         echo "$@" | sed -e 's,^,INFO: ,'
 257 }
 258
 259 # Autodetect best Mirror and safe url in $mirror
 260 #
 261 info=
 262 detect_mirror() {
 263     if [ -f download/Mirror-Cache ]; then
 264         read mirror mirrorver < download/Mirror-Cache
 265         mirror=${mirror:-none}
 266         if [ "$mirror" = "none" ]; then
 267                 [ "$quietmirror" != 1 ] &&
 268                 var_append info "
 269 " "Found download/Mirror-Cache: none (using original download locations)"
 270                 return
 271         elif [ "$mirrorver" != "$sdever" -a "$mirrorver" != "any" ]; then
 272                 echo_warn "Cached mirror URL in download/Mirror-Cache is outdated."
 273         else
 274                 [ "$quietmirror" != 1 ] && var_append info "
 275 " "Found cached mirror: $mirror"
 276                 return
 277         fi
 278     fi
 279         echo_warn "Auto-detecting best mirror:"
 280
 281         echo_warn "Downloading mirror-list from: t2sde.org"
 282         curl -s -S $curl_options -o src/Download-Mirror-List \
 283                 "https://t2sde.org/cgi-bin/t2-mirrors.cgi?$sdever"
 284
 285         bestval=0 result='No Mirror Found!'
 286         me=$([ -s download/Me ] && cat download/Me)
 287         while read mirror_name; do
 288           if [ "${mirror_name#=}" != "$mirror_name" ]; then
 289                 mirror_name="${mirror_name#= }"
 290                 mirror_name="${mirror_name% =}"
 291                 read mirror_url
 292
 293                 case "$mirror_name" in ($me) continue ;; esac
 294
 295                 echo -n "INFO: Testing <$mirror_name> ..."
 296                 val="$(curl -s $curl_options -m 20 "${mirror_url%/}/DOWNTEST" \
 297                         -w "ok %{speed_download}" -o /dev/null)"
 298                 if [ "$val" = "${val#ok }" -o "$val" = "ok 0.000" ]; then
 299                         echo " error"
 300                 else
 301                         xval=`echo ${val#ok } | tr -d .,`; echo " $val B/s"
 302                         if [ "$xval" -gt "$bestval" ]; then
 303                                 bestval=$xval mirror="${mirror_url%/}"
 304                                 result="Saving mirror $mirror (src/Mirror-Cache)"
 305                         fi
 306                 fi
 307
 308           fi
 309         done < src/Download-Mirror-List
 310         echo "$mirror $sdever" > download/Mirror-Cache
 311         echo_warn "$result"
 312 }
 313
 314 # download_file local-filename download-location cksum repo pkg
 315 #
 316 # This function decides if download directly or from a mirror,
 317 # validates checksum, etc.
 318 # Calls download_file_now to do the actual download.
 319 #
 320 download_file() {
 321         # Init
 322         #
 323         local gzfile="$1" location="$2" cksum="$3" repo="$4" pkg="$5"
 324         # Make src directory for creating tar balls
 325         mkdir -p src/
 326         # Remove optional '-' prefix from $location
 327         [ "${location:0:1}" == '-' ] && location="${location:1}"
 328         # Lock file name:
 329         lkfile="src/down.lockfile.`echo $gzfile | tr / -`"
 330
 331         # Check if it's already there
 332         #
 333         [ -s "$gzfile" -a $checkonly != 1 ] && return 0
 334
 335         # Make locking
 336         #
 337         if [ -s "$lkfile" ]; then
 338                 echo "Found $lkfile -> skip download."
 339                 return 0
 340         fi
 341         trap 'local ret=$?; rm -f "$lkfile"; return $ret' INT
 342         echo $$ > "$lkfile"
 343
 344         # Check if we only like to test the cksum(s)
 345         #
 346         if [ $checkonly = 1 ]; then
 347                 if [ ! -f "$gzfile" ]; then
 348                         echo "File missing: $gzfile"
 349                         rm -f "$lkfile"; trap INT; downloaderror=1; return
 350                 fi
 351                 if [ -z "${cksum##X*}" ]; then
 352                         echo "No checksum (ignore): $gzfile"
 353                         rm -f "$lkfile"; trap INT; return
 354                 fi
 355                 if [ "$cksum" = 0 ]; then
 356                         echo "No checksum (missing): $gzfile"
 357                         rm -f "$lkfile"; trap INT; return
 358                 fi
 359
 360         elif [ -s "$gzfile" ]; then
 361                 echo; echo "Already downloaded: $pkg:$gzfile"
 362
 363         else
 364                 [ "$info" ] && echo_info "$info" && info=
 365                 echo; echo "Downloading $pkg:$gzfile"
 366
 367                 # Existing *.cksum-err
 368                 #
 369                 if [ -s "$gzfile.cksum-err" ]; then
 370                         # cksum-err file alread exists:
 371                         echo "ERROR: found: $gzfile.cksum-err"
 372                         echo "ERROR: That means that we downloaded the" \
 373                              "file already and it had an"
 374                         echo "ERROR: incorrect checksum. Remove the" \
 375                              "*.cksum-err file to force a"
 376                         echo "ERROR: new download of that file."
 377                         rm -f "$lkfile"; trap INT; downloaderror=1; return 1
 378                 fi
 379
 380                 # Existing *.extck-err
 381                 #
 382                 if [ -s "$gzfile.extck-err" ]; then
 383                         # extck-err file alread exists:
 384                         echo "ERROR: found: $gzfile.extck-err"
 385                         echo "ERROR: That means that we downloaded the" \
 386                              "file already and it's content"
 387                         echo "ERROR: did not match it's filename extension." \
 388                              "Remove the *.extck-err file"
 389                         echo "ERROR: to force a new download of that file."
 390                         rm -f "$lkfile"; trap INT; downloaderror=1; return 1
 391                 fi
 392
 393                 # Questionable URL
 394                 #
 395                 if [ "$location" != "${location#\?}" ]; then
 396                         if [ "$tryques" = 0 ]; then
 397                                 echo "ERROR: URL is marked as questionable." \
 398                                         "Not downloading this file."
 399                                 rm -f "$lkfile"; trap INT; return 1
 400                         else
 401                                 echo "WARNING: URL is marked as questionable." \
 402                                         "Downloading it anyways."
 403                                 location="${location#\?}"
 404                         fi
 405                 fi
 406
 407                 # Make directory (if required)
 408                 #
 409                 if [ ! -d `dirname "$gzfile"` ]; then
 410                         mkdir -p `dirname "$gzfile"`
 411                 fi
 412
 413                 # Alternative Directory
 414                 #
 415                 if [ "$altdir" ]; then
 416                     altfile=$(find $altdir/ -name `basename $gzfile` | head -n 1)
 417                 else
 418                     altfile=""
 419                 fi
 420
 421                 if [ "$altfile" ]; then
 422
 423                         echo "Found `basename $gzfile` as: $altfile"
 424                         if [ "$altcopy" = 'link' ]; then
 425                                 cp -lv $altfile $gzfile
 426                         elif [ "$altcopy" = 'copy' ]; then
 427                                 cp -v $altfile $gzfile
 428                         elif [ "$altcopy" = 'move' ]; then
 429                                 mv -v $altfile $gzfile
 430                         fi
 431                 else
 432                     # Mirroring
 433                     #
 434                     read mirror mirrorver < download/Mirror-Cache
 435
 436                     if [ -n "$mirror" -a "$mirror" != "none" -a -z "${gzfile##download/mirror/*}" ]; then
 437                         # try to use mirror
 438                         if ! download_file_now "!$mirror/${gzfile#download/mirror/}" $gzfile; then
 439                                 echo "INFO: Download from mirror failed, trying original URL."
 440                                 download_file_now "$location" $gzfile || downloaderror=1
 441                         fi
 442                     else
 443                         # don't want to use mirror
 444                         download_file_now "$location" $gzfile || downloaderror=1
 445                     fi
 446                 fi
 447
 448                 if [ ! -s "$gzfile" ]; then
 449                         rm -f "$lkfile"; trap INT; return 1
 450                 fi
 451         fi
 452
 453         # unsign .gpg file
 454         if [[ $gzfile = *.gpg ]]; then
 455                 gzfile=${gzfile%.gpg}
 456                 if [ -f $gzfile.gpg ]; then
 457                         echo "Unsigning GnuPG file: $gzfile.gpg"
 458                         gpg $gzfile.gpg
 459                 fi
 460                 if [ ! -f $gzfile ]; then
 461                         echo "Unsigning failed"
 462                         rm -f "$lkfile"; trap INT; return 1
 463                 fi
 464         fi
 465
 466         echo "Checksum testing: $gzfile"
 467         local compressor="$(get_compressor "$gzfile")"
 468         if [ "$compressor" ]; then
 469                 # TODO: w/o temp file
 470                 $compressor < "$gzfile" > src/down.$$.dat
 471                 cksum_chk src/down.$$.dat $cksum "$gzfile" || downloaderror=1
 472                 rm -f src/down.$$.dat
 473         else
 474                 cksum_chk "$gzfile" $cksum "$gzfile" || downloaderror=1
 475         fi
 476
 477         # Free Lock and finish
 478         #
 479         rm -f "$lkfile"; trap INT; return 0
 480 }
 481
 482 # download_file_now location filename
 483 #
 484 # This function executes the actual download using curl.
 485 #
 486 download_file_now() {
 487         local location="$1" gzfile="$2" curlret=0
 488
 489         # Create URL
 490         #
 491         case "$location" in
 492                 manual://*) url="$location"     ;;
 493                 !*)     url="${location#!}"     ;;
 494                 *)      url="${location%/*}/${gzfile##*/}"      ;;
 495         esac
 496
 497         # Download
 498         #
 499         case "$url" in
 500         manual://*)
 501                 # Determine if the file has already been downloaded
 502                 # manually. For this we first look in $HOME then in
 503                 # download/manual.
 504                 downloadpath=${altdir:-$HOME}
 505                 downloadfile="${gzfile##*/}"
 506                 if [ -e $downloadpath/$downloadfile ]; then
 507                         location="file://$downloadpath/"
 508                 else
 509                         location="http://${url#manual://}"
 510                         # No manual download has taken place yet.
 511                         # So inform the user to do so.
 512                         cat <<-EOT
 513                         The file $downloadfile can not be fetched automatically
 514                         please visit: $location
 515                         and download it manually into $HOME or somewhere else using -alt-dir
 516                         EOT
 517                         return 1
 518                 fi
 519
 520                 # Re-use this function with a modified download location.
 521                 download_file_now "$location" $gzfile
 522                 return $?
 523                 ;;
 524         http://*|https://*|ftp://*|file://*)
 525                 if [ -s "$gzfile.incomplete" ]; then
 526                     echo "INFO: Trying to resume previous download .."
 527                     resume="-C -"
 528                 else
 529                     resume=""
 530                 fi
 531
 532                 [ -s download/Translations ] && trfile=download/Translations || trfile=misc/share/DownloadTranslations
 533                 trurl="$(echo "$url" | sed -f $trfile)"
 534                 if [ -n "$trurl" -a "$trurl" != "$url" ]; then
 535                         echo "INFO: URL translated."
 536                         url="$trurl"
 537                 fi
 538                 unset trurl trfile
 539
 540                 curl --progress-bar $resume $curl_options "$url" -o "$gzfile.incomplete"
 541                 curlret="$?"
 542
 543                 if [ "$resume" ] &&
 544                    [ $curlret -eq 33 -o $curlret -eq 36 ]; then
 545                     echo "INFO: Resuming download not possible. -> Overwriting old file."
 546                     rm -f "$gzfile.incomplete"
 547                     curl --progress-bar $curl_options "$url" -o "$gzfile.incomplete"
 548                     curlret="$?"
 549                 fi
 550
 551                 if [ $curlret -ne 0 ]; then
 552                     case "$curlret" in
 553                       18)
 554                       echo "WARNING: Got only some of the file. A re-run of $0"
 555                       echo "WARNING: is required to complete the download." ;;
 556                       22) : ;; # 404 not found
 557                       130)
 558                       echo -e '\rWARNING: CURL got a SIGINT' \
 559                            "(someone pressed Ctrl-C). A re-run of"
 560                       echo "WARNING: $0 is required to complete the download."; sleep 1 ;;
 561                       *)
 562                       echo "$curlret $gzfile $url" \
 563                                         >> src/Download-Errors
 564                       echo -e "\rERROR: CURL Returned error: $curlret" ;;
 565                     esac
 566                     return 1
 567                 elif [ ! -s "$gzfile.incomplete" ]; then
 568                     echo "0 $gzfile $url" >> src/Download-Errors
 569                     echo "ERROR: CURL returned success but we have no data!"
 570                     curlret=1
 571                 else
 572                     case "$gzfile" in
 573                       *.br|*.tbr)
 574                           typeexpr="data" ;;
 575                       *.gz|*.tgz)
 576                           typeexpr="gzip compressed data" ;;
 577                       *.bz2|*.tbz2|*.tbz)
 578                           typeexpr="bzip2 compressed data" ;;
 579                       *.lz|*.tlz)
 580                           typeexpr="lzip compressed data" ;;
 581                       *.lzma)
 582                           typeexpr="LZMA compressed data" ;;
 583                       *.zst|*.tzst|*.zstd)
 584                           typeexpr="Zstandard compressed data" ;;
 585                       *.Z|*.tZ)
 586                           typeexpr="compress'd data" ;;
 587                       *.zip)
 588                           typeexpr="Zip archive data" ;;
 589                       *.jar)
 590                           typeexpr="Java archive data (JAR)" ;;
 591                       *.tar)
 592                           typeexpr="tar archive" ;;
 593                       *.xz|*.txz)
 594                           typeexpr="[xX][zZ] compressed data" ;;
 595                       *)
 596                           echo "WARNING: Unknown file extension: $gzfile"
 597                           typeexpr="." ;;
 598                     esac
 599                     case $(file "$gzfile.incomplete") in
 600                     *$typeexpr*)
 601                         mv "$gzfile"{.incomplete,}
 602                         ;;
 603                     *)
 604                         echo "ERROR: File type does not match filename ($typeexpr)!"
 605                         mv "$gzfile"{.incomplete,.extck-err}
 606                         ;;
 607                     esac
 608                 fi
 609                 ;;
 610         *)
 611                 protocol="${url%%://*}"
 612
 613                 # we need to use $location - $url is already mangled above -ReneR
 614                 # $protocol://$url $options
 615                 url="`echo "$location" | sed "s,$protocol://\([^ ]*\).*,\1,"`"
 616                 options="`echo "$location" | cut -s -d' ' -f2-`"
 617
 618                 case "$protocol" in
 619                         cvs)
 620                                 # the first option is the module name
 621                                 module="${options%% *}"
 622                                 options="${options#* }"
 623                                 cmdline="cvs -z4 -Q -d $url co -P $options $module"
 624
 625                                 # sometimes cvs wants to read ~/.cvspass just for fun ..
 626                                 touch $HOME/.cvspass
 627                                 ;;
 628                         svn|svn\+*) # allow any svn+ other transport and strip the svn+ part off
 629                                 url="${protocol#svn+}://$url"
 630                                 options="${options## *}"
 631                                 if [ "$options" == "" -o "${options:0:1}" == "-" ]; then
 632                                         # the module is the last dir of $url,
 633                                         # w/ or wo/ trailing slash (/)
 634                                         module="${url%%/}"
 635                                         module="${module##*/}"
 636                                 else
 637                                         # the first option is the module name
 638                                         module="${options%% *}"
 639                                         [ "$module" = "$options" ] &&
 640                                           options= || options="${options#* }"
 641                                 fi
 642                                 cmdline="svn export $options $url $module"
 643                                 ;;
 644                         git|git\+*) # allow any git+ other transport and strip the git+ part off
 645                                 url="${protocol#git+}://$url"
 646
 647                                 module="${url##*/}"
 648                                 cmdline="git clone --recursive $git_options $url $module"
 649                                 options="${options#* }"
 650                                 sub_recurse="git submodule update --init --recursive"
 651                                 [ -n $options ] && cmdpp="(cd $module; git checkout $options; $sub_recurse)"
 652                                 ;;
 653                         hg|hg\+*) # allow any hg+ other transport and strip the hg+ part off
 654                                 url="${protocol#hg+}://$url"
 655
 656                                 module="${url##*/}"
 657                                 cmdline="hg clone $url $module"
 658                                 options="${options#* }"
 659                                 [ -n $options ] && cmdpp="(cd $module; hg clone $options)"
 660                                 ;;
 661                         *)
 662                                 echo "$cmdclient unrecognized!"
 663                                 return 1
 664                                 ;;
 665                 esac
 666
 667                 cvsdir="src/down.${protocol}dir.`echo $gzfile | tr / -`"
 668                 saved_pwd=$PWD; mkdir -p $cvsdir; cd $cvsdir
 669
 670                 echo "$cmdline"
 671                 {
 672                   $cmdline || touch .cvs_error
 673                 } &> .cvs_output &
 674
 675                 while fuser .cvs_output &> /dev/null; do
 676                         echo -ne `nice du -sh 2> /dev/null | \
 677                                 cut -f1` 'downloaded from archive so far ...  \r'
 678                         sleep 3
 679                 done
 680
 681                 if [ -f .cvs_error ]; then
 682                         cd $saved_pwd; rm -rf $cvsdir
 683                         echo -e "\nError during checkout."
 684                         return 1
 685                 fi
 686
 687                 echo `du -sh 2> /dev/null | cut -f1` 'downloaded from archive (download finished).'
 688
 689                 if [ `echo * | wc -w` -gt 1 ]; then
 690                         # multi-module module
 691                         echo "Multi-module package detected, relocating ..."
 692                         mkdir t2-module.$$
 693                         for x in *; do
 694                                 [ "$x" != "t2-module.$$" ] && mv -f $x t2-module.$$/
 695                         done
 696                         mkdir -p "$module"
 697                         mv -f t2-module.$$/* "$module"
 698                         rm -f t2-module.$$
 699                 fi
 700
 701                 cd `dirname $module`
 702                 tarname="`basename $gzfile`"
 703                 echo "Preparing files for final tarball."
 704                 [ -n "$cmdpp" ] && eval "$cmdpp"
 705
 706                 if [ `find -type f | wc -l` -gt 4 ]; then
 707                         local compressor="$(get_compressor $tarname)"
 708                         # explicitly stable sort files
 709                         find $module | LANG=C sort |
 710                                 egrep -v -e '/(CVS|.svn|.git|.hg)$' -e '/(CVS|.svn|.git|.hg)/' |
 711                         TZ=UTC tar -c --owner root --group root --mtime 20000101 \
 712                                 --no-recursion --files-from=- | ${compressor/ -d/} > $tarname
 713                         mv $tarname $saved_pwd/$gzfile
 714                 else
 715                         echo "Too few files - assuming checkout failure."
 716                         curlret=1
 717                 fi
 718
 719                 cd $saved_pwd; rm -rf $cvsdir
 720                 ;;
 721         esac
 722         return $curlret
 723 }
 724
 725 list_dtags() {
 726         {
 727                 grep -aH '^\[D\] ' package/*/*/*.desc
 728                 grep -aH '^\[D\] ' {architecture,target}/*/package/*/*.desc
 729                 grep -aH '^[X0-9a-z]' target/*/download.txt 2> /dev/null |
 730                         sed 's,:,:[D] ,'
 731         } | column_clean
 732 }
 733
 734 list_cksums() {
 735         trap '' INT
 736
 737         # we know we only have single spaces due to list_dtags' column_clean
 738         list_dtags | sed -n \
 739           -e 's,[^ ]* \([X0-9a-z]*\) \(.\)\([^ ]*\) -.*,\1 download/local/\2/\2\3,p' \
 740           -e 's,[^ ]* \([X0-9a-z]*\) \(.\)\([^ ]*\) [^-].*,\1 download/mirror/\2/\2\3,p'
 741
 742         trap INT
 743 }
 744
 745 list() {
 746         trap '' INT
 747         list_cksums | cut -f2- -d' '
 748         trap INT
 749 }
 750
 751 list_unknown() {
 752         trap '' INT
 753         mkdir -p src/; list > src/down.$$.lst
 754         ls download/{Proxy,Proxy-auth,Me,Mirror-Cache} \
 755            download/mirror/{README,DOWNTEST,LAST-UPDATE} \
 756                                                 >> src/down.$$.lst 2> /dev/null
 757         find download/* -type f -o -type l 2> /dev/null |
 758         while read fn; do
 759                 grep -qx "$fn" src/down.$$.lst || echo "Unknown file: $fn"
 760         done
 761         rm -f src/down.$$.lst
 762         trap INT
 763 }
 764
 765 list_missing() {
 766         trap '' INT
 767         list |
 768         while read fn; do
 769                 [ -f "$fn" ] || echo "$fn"
 770         done
 771         trap INT
 772 }
 773
 774 repository() {
 775         for repository; do
 776                 packages `echo package/$repository/*/*.desc`
 777         done
 778 }
 779
 780 required() {
 781     # Choosen config must exist
 782     #
 783     if [ ! -f config/$config/packages ]; then
 784         echo "ERROR: Config $config doesn't exist."
 785         echo "ERROR: try scripts/Config -cfg $config first."
 786         exit 1
 787     fi
 788
 789     while read on a b repo pkg c; do
 790         package $pkg
 791     done < <(grep '^X' config/$config/packages)
 792
 793     targetchain="$target" x="$target"
 794     while [ -f "target/$x/extends" ]; do
 795         x="$(< target/$x/extends)"
 796         targetchain="$targetchain $x"
 797     done
 798
 799     for target in $targetchain; do
 800     if [ -f target/$target/download.txt ]; then
 801         while read cksum file url; do
 802             download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
 803         done < target/$target/download.txt
 804     fi
 805     done
 806 }
 807
 808 all() {
 809         trap '' INT
 810         list_dtags | cut -d ' ' -f 2- | while read cksum file url; do
 811                  download_file "`source_file cksum $file "$url"`" "$url" "$cksum"
 812         done
 813         trap INT
 814 }
 815
 816 package() {
 817         local pkg="$1"
 818         detect_confdir # relies on $pkg being set
 819         if [ ! "$confdir" ]; then
 820                 echo "Error: Package $pkg not found!"
 821                 downloaderror=1
 822                 return 1
 823         fi
 824         parse_desc $pkg # relies on $pkg and $confdir being set
 825         while read cksum file url; do
 826                 download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$repo" "$pkg"
 827         done < <(echo "$desc_D")
 828 }
 829
 830 packages() {
 831         local descfile
 832         for arg; do
 833                 case "$arg" in
 834                 target/*)
 835                         if [ ! -f $arg ]; then
 836                             echo "Skipping \"$arg\" (not found)!"
 837                             continue
 838                         fi
 839
 840                         target="`echo $arg | cut -f2 -d/`"
 841
 842                         while read cksum file url; do
 843                             download_file "`source_file cksum $file "$url"`" "$url" "$cksum" "$target"
 844                         done < <(cat $arg)
 845                         ;;
 846                 *)
 847                         if [ "${arg%.desc}" != "$arg" ]; then
 848                         arg="`echo $arg | cut -f3 -d/`"; fi
 849
 850
 851                         # active extensions
 852                         local extender=
 853
 854                         # pkg_*_{pre,post}.conf is only activated if extender
 855                         # is enabled on $config/packages, so we will only
 856                         # download files of those extenders
 857                         #
 858                         for extender in `ls -1 package/*/*/pkg_${arg}_{pre,post}.conf 2> /dev/null |
 859                                          cut -d/ -f3 | sort -u`; do
 860                                 if grep -q "^X .* $extender " \
 861                                    config/$config/packages; then
 862                                         echo_info "Also downloading $extender ..."
 863                                         package $extender
 864                                 fi
 865                         done
 866                         package $arg
 867                         ;;
 868                 esac
 869         done
 870 }
 871
 872 # Things to do only for downloading
 873 #
 874 if [ "${1:0:5}" != "-list" -a $checkonly = 0 ]; then
 875         # Set proxy information
 876         if [ -f download/Proxy ]; then
 877                 proxy="$(< download/Proxy)"
 878                 if [ "$proxy" ]; then
 879                         curl_options="$curl_options --proxy $proxy"
 880                 else
 881                         echo "INFO: No proxy information, removing: download/Proxy"
 882                         rm download/Proxy
 883                 fi
 884         fi
 885         if [ -f download/Proxy-auth ]; then
 886                 proxyauth="$(< download/Proxy-auth)"
 887                 if [ "$proxyauth" ]; then
 888                         curl_options="$curl_options --proxy-user $proxyauth"
 889                         git_options="-c http.proxy=http://$proxyauth@$proxy"
 890                 else
 891                         echo "INFO: No proxy-auth information, removing: download/Proxy-auth"
 892                         rm download/Proxy-auth
 893                 fi
 894         fi
 895
 896         # We need curl
 897         if [ -z "`type -p curl`" ]; then
 898                 echo "ERROR: we need \`curl\` installed and available in \$PATH to proceed."
 899                 exit 2
 900         fi
 901
 902         # Thing to do only once
 903         #
 904         if [ $this_is_the_2nd_run = 0 ]; then
 905                 # am i using a proxy?
 906                 if [ "$proxy" ]; then
 907                         echo "INFO: Setting proxy to: $proxy"
 908                 fi
 909                 if [ "$proxyauth" ]; then
 910                         echo "INFO: Setting proxy authentication information."
 911                 fi
 912
 913                 # do mirror detection
 914                 detect_mirror
 915         fi
 916 fi
 917
 918 case "$1" in
 919         -list)          list ;;
 920         -list-unknown)  list_unknown ;;
 921         -list-missing)  list_missing ;;
 922         -list-cksums)   list_cksums ;;
 923
 924         -required)      required ;;
 925         -all)           all ;;
 926
 927         -repository)    shift; repository "$@" ;;
 928
 929         -*|"")          exec $0 --help ;;
 930
 931         *)              packages "$@" ;;
 932 esac
 933
 934 exit $downloaderror