lib/aur-sync

   1 #!/bin/bash
   2 # aur-sync - download and build AUR packages automatically
   3 [[ -v AUR_DEBUG ]] && set -o xtrace
   4 set -o errexit
   5 argv0=sync
   6 XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
   7 XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
   8 XDG_STATE_HOME=${XDG_STATE_HOME:-$HOME/.local/state}
   9 XDG_DATA_HOME=${XDG_DATA_HOME:-$HOME/.local/share}
  10 AURDEST=${AURDEST:-$XDG_CACHE_HOME/aurutils/$argv0}
  11 AUR_SYNC_USE_NINJA=${AUR_SYNC_USE_NINJA:-0}
  12 AUR_SYNC_GNUPGHOME=${AUR_SYNC_GNUPGHOME:-$XDG_DATA_HOME/aurutils/$argv0/gnupg}
  13 PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  14
  15 # Avoid CDPATH screwing with cd (#1047)
  16 unset -v CDPATH
  17
  18 # default arguments
  19 build_args=(--syncdeps) build_repo_args=()
  20 depends_args=() view_args=() filter_args=() fetch_args=() graph_args=() reparse_args=()
  21
  22 # default options
  23 build=1 chkver_depth=2 download=1 view=1 provides=1 graph=1 keep_going=1
  24
  25 # default options (disabled)
  26 rotate=0 update=0 repo_targets=0 columns=0 auto_key_retrieve=0
  27
  28 args_csv() {
  29     # shellcheck disable=SC2155
  30     local str=$(printf '%s,' "$@")
  31     printf '%s' "${str%,}"
  32 }
  33
  34 lib32() {
  35     awk -v arch="$(uname -m)" '{
  36         if(arch == "i686") {
  37             gsub(/^lib32-/,"")
  38             gsub(/^gcc-multilib$/,"")
  39         }
  40         print
  41     }' "$@"
  42 }
  43
  44 select_pkgspec() {
  45     awk -F'/' -v needle="$1" '{
  46         if (NF == 2 && $1 == needle) {
  47             print $2
  48         }
  49         else if (NF == 1) {
  50             print $1
  51         }
  52     }' "${@:2}"
  53 }
  54
  55 swap() {
  56     awk '{print $2 "\t" $1}' "$@"
  57 }
  58
  59 complement() {
  60     # empty set should not return 1
  61     grep -Fxvf "$@" || return $(( $? - 1 ))
  62 }
  63
  64 trap_exit() {
  65     if [[ ! -v AUR_DEBUG ]]; then
  66         rm -rf -- "$tmp"
  67     else
  68         printf >&2 'AUR_DEBUG: %s: temporary files at %s\n' "$argv0" "$tmp"
  69     fi
  70 }
  71
  72 usage() {
  73     printf >&2 'usage: %s [-d repo] [--root path] [-cdfoSu] pkgname...\n' "$argv0"
  74     exit 1
  75 }
  76
  77 # Clarify tsort errors (#1039)
  78 diag_depcycle() {
  79         echo >&2 'Error:'
  80
  81 cat <<EOF | pr -to 4 >&2
  82 aur-$argv0 encountered a cycle while resolving dependencies. This may be caused
  83 by an error in the PKGBUILD or by cyclic checkdepends. For the latter, retry
  84 building the package(s) with --no-check.
  85 EOF
  86 }
  87
  88 source /usr/share/makepkg/util/message.sh
  89
  90 if [[ ! -v NO_COLOR ]] && [[ ! -v AUR_DEBUG ]]; then
  91     [[ -t 2 ]] && colorize
  92 fi
  93
  94 # mollyguard for makepkg
  95 if (( UID == 0 )) && [[ ! -v AUR_ASROOT ]]; then
  96     printf >&2 'warning: aur-%s is not meant to be run as root.\n' "$argv0"
  97     printf >&2 'warning: To proceed anyway, set the %s variable.\n' 'AUR_ASROOT'
  98     exit 1
  99 fi
 100
 101 # option parsing
 102 opt_short='d:D:k:U:ACcfKLnorRSTuv'
 103 opt_long=('bind:' 'bind-rw:' 'database:' 'directory:' 'ignore:' 'root:'
 104           'makepkg-conf:' 'pacman-conf:' 'chroot' 'continue' 'force' 'ignore-arch'
 105           'log' 'no-confirm' 'no-ver' 'no-graph' 'no-sync' 'no-ver-argv' 'no-view'
 106           'no-provides' 'no-build' 'rmdeps' 'sign' 'temp' 'upgrades' 'pkgver'
 107           'rebuild' 'rebuild-tree' 'rebuild-all' 'ignore-file:' 'remove'
 108           'provides-from:' 'new' 'prevent-downgrade' 'verify' 'makepkg-args:'
 109           'format:' 'no-check' 'keep-going:' 'user:' 'rebase' 'reset' 'ff' 'exclude:'
 110           'columns' 'prefix' 'save:' 'clean' 'cleanbuild' 'auto-key-retrieve')
 111 opt_hidden=('dump-options' 'allan' 'ignorearch' 'ignorefile:' 'noconfirm'
 112             'nover' 'nograph' 'nosync' 'nover-argv' 'noview' 'noprovides' 'nobuild'
 113             'rebuildall' 'rebuildtree' 'rm-deps' 'gpg-sign' 'margs:' 'nocheck'
 114             'no-checkdepends' 'nocheckdepends' 'optdepends' 'repo:' 'autokeyretrieve')
 115
 116 if opts=$(getopt -o "$opt_short" -l "$(args_csv "${opt_long[@]}" "${opt_hidden[@]}")" -n "$argv0" -- "$@"); then
 117     eval set -- "$opts"
 118 else
 119     usage
 120 fi
 121
 122 unset pkg pkg_i repo repo_p ignore_file stdout_file
 123 while true; do
 124     case "$1" in
 125         # sync options
 126         --allan)
 127             rotate=1 ;;
 128         --continue)
 129             download=0 ;;
 130         --ignore)
 131             shift; IFS=, read -a pkg -r <<< "$1"
 132             pkg_i+=("${pkg[@]}") ;;
 133         --ignorefile|--ignore-file)
 134             shift; ignore_file=$1 ;;
 135         -k|--keep-going)
 136             shift; keep_going=$1 ;;
 137         -o|--nobuild|--no-build)
 138             build=0 ;;
 139         --columns)
 140             build=0; columns=1 ;;
 141         --save)
 142             shift; stdout_file=$1 ;;
 143         --optdepends)
 144             depends_args+=(--optdepends)
 145             graph_args+=(-v OPTDEPENDS=1) ;;
 146         --nocheck|--no-check|--nocheckdepends|--no-checkdepends)
 147             depends_args+=(--no-checkdepends)
 148             build_args+=(--no-check)
 149             graph_args+=(-v CHECKDEPENDS=0) ;;
 150         --nograph|--no-graph)
 151             graph=0 ;;
 152         --nosync|--no-sync)
 153             build_args+=(--no-sync) ;;
 154         --nover|--no-ver)
 155             chkver_depth=0 ;;
 156         --nover-argv|--no-ver-argv)
 157             chkver_depth=1 ;;
 158         --noview|--no-view)
 159             view=0 ;;
 160         --noprovides|--no-provides)
 161             provides=0 ;;
 162         -K|--auto-key-retrieve|--autokeyretrieve)
 163             auto_key_retrieve=1 ;;
 164         --provides-from)
 165             shift; IFS=, read -a repo -r <<< "$1"
 166             repo_p+=("${repo[@]}")
 167             provides=1 ;;
 168         --rebuild)
 169             # Command-line targets are excluded from `repo-filter`
 170             build_args+=(-f); chkver_depth=1 ;;
 171         --rebuildtree|--rebuild-tree)
 172             # Dependencies may be removed by `repo-filter` (#1066)
 173             build_args+=(-f); chkver_depth=0; provides=0 ;;
 174         --rebuildall|--rebuild-all)
 175             build_args+=(-f); chkver_depth=0; repo_targets=1 ;;
 176         -u|--upgrades)
 177             update=1 ;;
 178         # database options
 179         -d|--database|--repo)
 180             shift; build_repo_args+=(-d "$1") ;;
 181         --root)
 182             shift; build_repo_args+=(--root "$1") ;;
 183         # fetch options
 184         --ff)
 185             fetch_args+=(--ff) ;;
 186         --rebase)
 187             fetch_args+=(--rebase) ;;
 188         --reset)
 189             fetch_args+=(--reset) ;;
 190         # view options
 191         --format)
 192             shift; view_args+=(--format "$1") ;;
 193         --exclude)
 194             shift; view_args+=(--exclude "$1") ;;
 195         --prefix)
 196             view_args+=(--prefix) ;;  # experimental
 197         # build options
 198         -c|--chroot)
 199             build_args+=(--chroot) ;;
 200         -f|--force)
 201             build_args+=(--force) ;;
 202         -C|--clean)
 203             build_args+=(--clean) ;;
 204         --cleanbuild)
 205             build_args+=(--cleanbuild) ;;
 206         --makepkg-args|--margs)
 207             shift; build_args+=(--margs "$1") ;;
 208         --makepkg-conf)
 209             shift; build_args+=(--makepkg-conf "$1") ;;
 210         --pacman-conf)
 211             shift; build_args+=(--pacman-conf "$1")
 212             filter_args+=(--config "$1") ;;
 213         --pkgver)
 214             build_args+=(--pkgver) ;;
 215         -S|--sign|--gpg-sign)
 216             build_args+=(--sign) ;;
 217         -U|--user)
 218             shift; build_args+=(--user "$1") ;;
 219         # build options (devtools)
 220         -D|--directory)
 221             shift; build_args+=(--directory "$1") ;;
 222         --bind)
 223             shift; build_args+=(--bind "$1") ;;
 224         --bind-rw)
 225             shift; build_args+=(--bind-rw "$1") ;;
 226         -T|--temp)
 227             build_args+=(-T) ;;
 228         # build options (makepkg)
 229         -A|--ignorearch|--ignore-arch)
 230             build_args+=(--ignorearch) ;;
 231         -L|--log)
 232             build_args+=(--log) ;;
 233         -n|--noconfirm|--no-confirm)
 234             build_args+=(--noconfirm) ;;
 235         -r|--rmdeps|--rm-deps)
 236             build_args+=(--rmdeps) ;;
 237         # build options (repo-add)
 238         -R|--remove)
 239             build_args+=(--remove) ;;
 240         -v|--verify)
 241             build_args+=(--verify) ;;
 242         --prevent-downgrade)
 243             build_args+=(--prevent-downgrade) ;;
 244         --new)
 245             build_args+=(--new) ;;
 246         # other options
 247         --dump-options)
 248             printf -- '--%s\n' "${opt_long[@]}" ${AUR_DEBUG+"${opt_hidden[@]}"}
 249             printf -- '%s' "${opt_short}" | sed 's/.:\?/-&\n/g'
 250             exit ;;
 251         --) shift; break ;;
 252     esac
 253     shift
 254 done
 255
 256 # shellcheck disable=SC2174
 257 mkdir -pm 0700 -- "${TMPDIR:-/tmp}/aurutils-$UID"
 258 tmp=$(mktemp -d --tmpdir "aurutils-$UID/$argv0.XXXXXXXX")
 259 trap 'trap_exit' EXIT
 260
 261 if (( rotate )); then
 262     if { hash caesar && target=$(aur pkglist | shuf -n 1); } 2>/dev/null; then
 263         exec bash -c "{ aur \"$argv0\" -c \"$target\" && repo-elephant | caesar 13; } 2>&1 | caesar 13"
 264     else
 265         echo '?'; exit 16 # EBUSY
 266     fi
 267 fi
 268 mkdir -p -- "$AURDEST"
 269
 270 if (( $# + update + repo_targets == 0 )); then
 271     printf >&2 '%s: no targets specified\n' "$argv0"
 272     exit 1
 273 fi
 274
 275 # Write --no-build / --columns results to a file (#1077)
 276 if [[ -v stdout_file ]]; then
 277     stdout_file=$(realpath -- "$stdout_file")
 278 fi
 279
 280 # Retrieve path to local repo (#448, #700, #1135)
 281 { IFS=: read -r _ db_name
 282   IFS=: read -r _ db_root
 283   IFS=: read -r _ db_path
 284   IFS=: read -r _ _
 285   IFS=: read -r _ _
 286 } < <(aur build "${build_args[@]}" "${build_repo_args[@]}" --status)
 287
 288 # Print an error if `build --status` fails (#1151)
 289 if ! wait "$!"; then
 290     printf '%s: error: failed to read build configuration\n' "$argv0"
 291     exit 1
 292 fi
 293
 294 msg >&2 'Using [%s] repository' "$db_name"
 295
 296 # Ignores file can be the result of process substitution (#880)
 297 : "${ignore_file=$XDG_CONFIG_HOME/aurutils/sync/ignore}"
 298
 299 if [[ -r $ignore_file ]] && [[ ! -d $ignore_file ]]; then
 300     # Append file ignores to command-line ignores
 301     while read -r i; do pkg_i+=("$i"); done < <(select_pkgspec "$db_name" "$ignore_file")
 302 fi
 303
 304 # Diagnostic on which packages are ignored
 305 if (( ${#pkg_i[@]} )); then
 306     # Do not include command-line arguments as ignores (#952)
 307     mapfile -t pkg_i < <(complement <(printf '%s\n' "$@") <(printf '%s\n' "${pkg_i[@]}"))
 308     printf '%s: packages ignored: %s\n' "$argv0" "${pkg_i[*]}"
 309
 310     # Ignore local repository targets (#1146)
 311     reparse_args+=(--ignore "$(args_csv "${pkg_i[@]}")")
 312 fi >&2
 313
 314 # Retrieve list of local repository packages ($1 pkgname $2 pkgver)
 315 aur repo-parse "${reparse_args[@]}" -p "$db_path" --list >"$tmp"/db_info
 316
 317 # Build list of AUR targets
 318 { if (( $# )); then
 319       # append command-line arguments
 320       printf '%s\n' "$@"
 321   fi
 322
 323   if (( repo_targets )); then
 324       # append repository packages (all)
 325       cut -f1 <"$tmp"/db_info
 326
 327   elif (( update )); then
 328       # append repository packages (updated)
 329       aur vercmp --quiet <"$tmp"/db_info
 330   fi
 331 } >"$tmp"/argv
 332
 333 # Build AUR dependency graph
 334 if [[ -s $tmp/argv ]]; then
 335     # shellcheck disable=SC2094
 336     aur depends --jsonl "${depends_args[@]}" - <"$tmp"/argv >"$tmp"/depends.jsonl
 337 else
 338     printf >&2 '%s: there is nothing to do\n' "$argv0"
 339     exit
 340 fi
 341
 342 # pkginfo: $1 pkgname $2 pkgbase $3 pkgver
 343 aur format -f '%n\t%b\t%v\n' "$tmp"/depends.jsonl | sort -u >"$tmp"/pkginfo
 344
 345 { if (( ${#pkg_i[@]} )); then
 346       printf '%s\n' "${pkg_i[@]}"  # Ignored packages
 347   fi
 348
 349   # Packages with equal or newer versions are taken as complement
 350   # for the queue. If chkver_argv is enabled, packages on the
 351   # command-line are excluded from this complement.
 352   if (( chkver_depth )); then
 353       # note: AUR cannot be queried by pkgbase (FS#57230)
 354       cut -f1,3 "$tmp"/pkginfo | aur vercmp -p "$tmp"/db_info -c >"$tmp"/current
 355
 356       # shellcheck disable=SC2002
 357       case $chkver_depth in
 358           1) cat "$tmp"/current | complement "$tmp"/argv ;;
 359           2) cat "$tmp"/current ;;
 360       esac
 361   fi
 362
 363   if (( provides )); then
 364       if (( ${#repo_p[@]} )); then
 365           filter_args+=("${repo_p[@]/#/--repo=}")
 366       else
 367           filter_args+=(--sync)
 368       fi
 369
 370       # Note: this uses pacman's copy of the repo (as used by makepkg -s)
 371       cut -f1 "$tmp"/pkginfo | aur repo-filter "${filter_args[@]}" | complement "$tmp"/argv
 372   fi
 373 } >"$tmp"/filter
 374
 375 # Filter out targets determined in the steps above recursively (#1136, #1140)
 376 aur sync--filter -p "$tmp"/depends.jsonl -f "$tmp"/filter >"$tmp"/graph
 377
 378 # XXX: a flat file is needed for aur-{graph,fetch,view}. `ninja` requires the
 379 # build files to be present before dependency resolution (with `ninja -n`) can
 380 # occur, and `ninja -t targets` sorts in alphabetical order. This implies
 381 # that dependency cycles cannot be resolved before retrieving files with
 382 # aur-fetch with `ninja` alone. `tsort` could either be used in this case (with
 383 # a less nice diagnostic on cycles), or fetches done in an arbitrary order
 384 # (e.g. sort -u) with checks for cycles done at build-time.
 385 if ! tsort <"$tmp"/graph >"$tmp"/queue; then
 386     diag_depcycle
 387     exit 22
 388 fi
 389
 390 if [[ -s $tmp/queue ]]; then
 391     cd -- "$AURDEST"
 392 else
 393     printf >&2 '%s: there is nothing to do\n' "$argv0"
 394     exit
 395 fi
 396
 397 if (( download )); then
 398     msg >&2 "Retrieving package files"
 399     aur fetch -S "${fetch_args[@]}" --discard - < "$tmp"/queue >&2
 400 else
 401     xargs -a "$tmp"/queue stat >/dev/null || exit 2 # ensure all directories are available
 402 fi
 403
 404 if (( auto_key_retrieve )); then
 405     # shellcheck disable=SC2174
 406     mkdir -pm 0700 -- "$AUR_SYNC_GNUPGHOME"
 407     declare -A keys_uniq
 408
 409     # Retrieve unique set of gpg keys to be imported
 410     while IFS= read -r path; do
 411         if [[ -f $path/.SRCINFO ]]; then
 412             mapfile -t keys < <(pacini "$path"/.SRCINFO 'validpgpkeys')
 413
 414             for key in "${keys[@]}"; do
 415                 keys_uniq[$key]=1
 416             done
 417         fi
 418     done < "$tmp"/queue
 419
 420     if (( ${#keys_uniq[@]} )); then
 421         printf >&2 '%s: importing key %s\n' "${!keys_uniq[@]}"
 422         GNUPGHOME="$AUR_SYNC_GNUPGHOME" gpg --recv-keys "${!keys_uniq[@]}" >&2
 423     fi
 424
 425     # Pass on verified keys to makepkg
 426     build_args+=(--makepkg-gnupghome="$AUR_SYNC_GNUPGHOME")
 427 fi
 428
 429 # Verify dependency tree (#20)
 430 if (( graph )); then
 431     if ! { while read -r pkg; do
 432                [[ $pkg ]] && printf '%s\0' "$pkg/.SRCINFO"
 433            done
 434          } | xargs -0 cat -- | aur graph "${graph_args[@]}" REVERSE=1
 435     then
 436         printf >&2 '%s: failed to verify dependency graph\n' "$argv0"
 437         exit 1
 438     fi <"$tmp"/queue >"$tmp"/graph
 439
 440     # Recompute dependencies to include `provides` (#837)
 441     if ! tsort < "$tmp"/graph >"$tmp"/queue; then
 442         diag_depcycle
 443         exit 22
 444     fi
 445 fi
 446
 447 { # Sort dependencies (`aur-sync--ninja`, `--columns`)
 448   swap "$tmp"/graph | sort -k1b,1 -k1 -u >"$tmp"/graph.ninja
 449
 450   # Resolve absolute paths (`--no-build`)
 451   while read -r pkg; do
 452       [[ $pkg ]] && printf '%s\n' "$AURDEST/$pkg"
 453   done <"$tmp"/queue >"$tmp"/queue.realpath
 454 }
 455
 456 # Inspect package files
 457 if (( view )); then
 458     aur view -a "$tmp"/queue "${view_args[@]}"
 459 fi
 460
 461 # `--columns` / `--no-build` output
 462 if (( columns )) && [[ -v stdout_file ]]; then
 463     cat "$tmp"/graph.ninja >"$stdout_file"
 464
 465 elif (( columns )); then
 466     cat "$tmp"/graph.ninja
 467
 468 elif ! (( build )) && [[ -v stdout_file ]]; then
 469     cat "$tmp"/queue.realpath >"$stdout_file"
 470
 471 elif ! (( build )); then
 472     cat "$tmp"/queue.realpath
 473
 474 # Build dependency tree with ninja (#908)
 475 elif (( AUR_SYNC_USE_NINJA )); then
 476     # Apply `--save` (#1091)
 477     if [[ -v stdout_file ]]; then
 478         cat "$tmp"/graph.ninja >"$stdout_file"
 479     fi
 480
 481     # Directory for stamp files (concurrent aur-sync processes)
 482     mkdir -p -- "$XDG_STATE_HOME"/aurutils/$argv0
 483     ninja_dir=$XDG_STATE_HOME/aurutils/$argv0/ninja-$USER-$$
 484     mkdir -- "$ninja_dir"
 485
 486     # Generate build.ninja
 487     # input:  $AURDEST/pkgbase/PKGBUILD
 488     # output: $ninja_dir/pkgbase.stamp
 489     aur sync--ninja "$AURDEST" <"$tmp"/graph.ninja >"$ninja_dir"/build.ninja \
 490         -- aur build "${build_args[@]}" -d "$db_name" --root "$db_root"
 491
 492     if NINJA_STATUS='[%s/%t] ' ninja -C "$ninja_dir" -k "$keep_going"; then
 493         # Remove ninja directory on successful build
 494         rm -rf "$ninja_dir"
 495     else
 496         # Print all targets in dependency order
 497         NINJA_STATUS='[%s/%t] ' ninja -nC /var/empty -f "$ninja_dir"/build.ninja | \
 498             # [\w@\.\-\+]: valid characters for pkgname
 499             # alternative: [^\s]+ from rule `env -C ... > pkgbase.stamp`
 500             pcregrep -o1 -o3 '(\[\d+/\d+\] )(.+?)([\w@\.\-\+]+)(\.stamp)' | while read -r status pkg
 501         do
 502             if [[ -f $ninja_dir/$pkg.stamp ]]; then
 503                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${GREEN}[OK]${ALL_OFF}\n" "$status" "$pkg"
 504             else
 505                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${RED}[FAIL]${ALL_OFF}\n" "$status" "$pkg"
 506             fi
 507         done | column -t
 508
 509         # Preserve ninja directory
 510         printf '%s: build files at %s\n' "$argv0" "$ninja_dir"
 511     fi
 512 else
 513     # Apply `--save` (#1091)
 514     if [[ -v stdout_file ]]; then
 515         cat "$tmp"/queue.realpath >"$stdout_file"
 516     fi
 517     aur build "${build_args[@]}" -a "$tmp"/queue.realpath -d "$db_name" --root "$db_root"
 518 fi
 519
 520 # vim: set et sw=4 sts=4 ft=sh: