lib/aur-sync

   1 #!/bin/bash
   2 # aur-sync - download and build AUR packages automatically
   3 [[ -v AUR_DEBUG ]] && set -o xtrace
   4 set -o errexit
   5 argv0=sync
   6 XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
   7 XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
   8 XDG_STATE_HOME=${XDG_STATE_HOME:-$HOME/.local/state}
   9 AURDEST=${AURDEST:-$XDG_CACHE_HOME/aurutils/$argv0}
  10 AUR_SYNC_USE_NINJA=${AUR_SYNC_USE_NINJA:-0}
  11 PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
  12
  13 # Avoid CDPATH screwing with cd (#1047)
  14 unset -v CDPATH
  15
  16 # default arguments
  17 build_args=(--syncdeps) build_repo_args=()
  18 depends_args=() view_args=() filter_args=() fetch_args=() graph_args=() reparse_args=()
  19
  20 # default options
  21 build=1 chkver_depth=2 download=1 view=1 provides=1 graph=1 keep_going=1
  22
  23 # default options (disabled)
  24 rotate=0 update=0 repo_targets=0 columns=0
  25
  26 args_csv() {
  27     # shellcheck disable=SC2155
  28     local str=$(printf '%s,' "$@")
  29     printf '%s' "${str%,}"
  30 }
  31
  32 lib32() {
  33     awk -v arch="$(uname -m)" '{
  34         if(arch == "i686") {
  35             gsub(/^lib32-/,"")
  36             gsub(/^gcc-multilib$/,"")
  37         }
  38         print
  39     }' "$@"
  40 }
  41
  42 select_pkgspec() {
  43     awk -F'/' -v needle="$1" '{
  44         if (NF == 2 && $1 == needle) {
  45             print $2
  46         }
  47         else if (NF == 1) {
  48             print $1
  49         }
  50     }' "${@:2}"
  51 }
  52
  53 swap() {
  54     awk '{print $2 "\t" $1}' "$@"
  55 }
  56
  57 complement() {
  58     # empty set should not return 1
  59     grep -Fxvf "$@" || return $(( $? - 1 ))
  60 }
  61
  62 trap_exit() {
  63     if [[ ! -v AUR_DEBUG ]]; then
  64         rm -rf -- "$tmp"
  65     else
  66         printf >&2 'AUR_DEBUG: %s: temporary files at %s\n' "$argv0" "$tmp"
  67     fi
  68 }
  69
  70 usage() {
  71     printf >&2 'usage: %s [-d repo] [--root path] [-cdfoSu] pkgname...\n' "$argv0"
  72     exit 1
  73 }
  74
  75 source /usr/share/makepkg/util/message.sh
  76
  77 if [[ ! -v NO_COLOR ]] && [[ ! -v AUR_DEBUG ]]; then
  78     [[ -t 2 ]] && colorize
  79 fi
  80
  81 # mollyguard for makepkg
  82 if (( UID == 0 )) && [[ ! -v AUR_ASROOT ]]; then
  83     printf >&2 'warning: aur-%s is not meant to be run as root.\n' "$argv0"
  84     printf >&2 'warning: To proceed anyway, set the %s variable.\n' 'AUR_ASROOT'
  85     exit 1
  86 fi
  87
  88 # option parsing
  89 opt_short='d:D:k:U:ACcfLnorRSTuv'
  90 opt_long=('bind:' 'bind-rw:' 'database:' 'directory:' 'ignore:' 'root:'
  91           'makepkg-conf:' 'pacman-conf:' 'chroot' 'continue' 'force' 'ignore-arch'
  92           'log' 'no-confirm' 'no-ver' 'no-graph' 'no-sync' 'no-ver-argv' 'no-view'
  93           'no-provides' 'no-build' 'rmdeps' 'sign' 'temp' 'upgrades' 'pkgver'
  94           'rebuild' 'rebuild-tree' 'rebuild-all' 'ignore-file:' 'remove'
  95           'provides-from:' 'new' 'prevent-downgrade' 'verify' 'makepkg-args:'
  96           'format:' 'no-check' 'keep-going:' 'user:' 'rebase' 'reset' 'ff' 'exclude:'
  97           'columns' 'prefix' 'save:' 'clean' 'cleanbuild')
  98 opt_hidden=('dump-options' 'allan' 'ignorearch' 'ignorefile:' 'noconfirm'
  99             'nover' 'nograph' 'nosync' 'nover-argv' 'noview' 'noprovides' 'nobuild'
 100             'rebuildall' 'rebuildtree' 'rm-deps' 'gpg-sign' 'margs:' 'nocheck'
 101             'no-checkdepends' 'nocheckdepends' 'optdepends' 'repo:')
 102
 103 if opts=$(getopt -o "$opt_short" -l "$(args_csv "${opt_long[@]}" "${opt_hidden[@]}")" -n "$argv0" -- "$@"); then
 104     eval set -- "$opts"
 105 else
 106     usage
 107 fi
 108
 109 unset pkg pkg_i repo repo_p ignore_file stdout_file
 110 while true; do
 111     case "$1" in
 112         # sync options
 113         --allan)
 114             rotate=1 ;;
 115         --continue)
 116             download=0 ;;
 117         --ignore)
 118             shift; IFS=, read -a pkg -r <<< "$1"
 119             pkg_i+=("${pkg[@]}") ;;
 120         --ignorefile|--ignore-file)
 121             shift; ignore_file=$1 ;;
 122         -k|--keep-going)
 123             shift; keep_going=$1 ;;
 124         -o|--nobuild|--no-build)
 125             build=0 ;;
 126         --columns)
 127             build=0; columns=1 ;;
 128         --save)
 129             shift; stdout_file=$1 ;;
 130         --optdepends)
 131             depends_args+=(--optdepends)
 132             graph_args+=(-v OPTDEPENDS=1) ;;
 133         --nocheck|--no-check|--nocheckdepends|--no-checkdepends)
 134             depends_args+=(--no-checkdepends)
 135             build_args+=(--no-check)
 136             graph_args+=(-v CHECKDEPENDS=0) ;;
 137         --nograph|--no-graph)
 138             graph=0 ;;
 139         --nosync|--no-sync)
 140             build_args+=(--no-sync) ;;
 141         --nover|--no-ver)
 142             chkver_depth=0 ;;
 143         --nover-argv|--no-ver-argv)
 144             chkver_depth=1 ;;
 145         --noview|--no-view)
 146             view=0 ;;
 147         --noprovides|--no-provides)
 148             provides=0 ;;
 149         --provides-from)
 150             shift; IFS=, read -a repo -r <<< "$1"
 151             repo_p+=("${repo[@]}")
 152             provides=1 ;;
 153         --rebuild)
 154             # Command-line targets are excluded from `repo-filter`
 155             build_args+=(-f); chkver_depth=1 ;;
 156         --rebuildtree|--rebuild-tree)
 157             # Dependencies may be removed by `repo-filter` (#1066)
 158             build_args+=(-f); chkver_depth=0; provides=0 ;;
 159         --rebuildall|--rebuild-all)
 160             build_args+=(-f); chkver_depth=0; repo_targets=1 ;;
 161         -u|--upgrades)
 162             update=1 ;;
 163         # database options
 164         -d|--database|--repo)
 165             shift; build_repo_args+=(-d "$1") ;;
 166         --root)
 167             shift; build_repo_args+=(--root "$1") ;;
 168         # fetch options
 169         --ff)
 170             fetch_args+=(--ff) ;;
 171         --rebase)
 172             fetch_args+=(--rebase) ;;
 173         --reset)
 174             fetch_args+=(--reset) ;;
 175         # view options
 176         --format)
 177             shift; view_args+=(--format "$1") ;;
 178         --exclude)
 179             shift; view_args+=(--exclude "$1") ;;
 180         --prefix)
 181             view_args+=(--prefix) ;;  # experimental
 182         # build options
 183         -c|--chroot)
 184             build_args+=(--chroot) ;;
 185         -f|--force)
 186             build_args+=(--force) ;;
 187         -C|--clean)
 188             build_args+=(--clean) ;;
 189         --cleanbuild)
 190             build_args+=(--cleanbuild) ;;
 191         --makepkg-args|--margs)
 192             shift; build_args+=(--margs "$1") ;;
 193         --makepkg-conf)
 194             shift; build_args+=(--makepkg-conf "$1") ;;
 195         --pacman-conf)
 196             shift; build_args+=(--pacman-conf "$1")
 197             filter_args+=(--config "$1") ;;
 198         --pkgver)
 199             build_args+=(--pkgver) ;;
 200         -S|--sign|--gpg-sign)
 201             build_args+=(--sign) ;;
 202         -U|--user)
 203             shift; build_args+=(--user "$1") ;;
 204         # build options (devtools)
 205         -D|--directory)
 206             shift; build_args+=(--directory "$1") ;;
 207         --bind)
 208             shift; build_args+=(--bind "$1") ;;
 209         --bind-rw)
 210             shift; build_args+=(--bind-rw "$1") ;;
 211         -T|--temp)
 212             build_args+=(-T) ;;
 213         # build options (makepkg)
 214         -A|--ignorearch|--ignore-arch)
 215             build_args+=(--ignorearch) ;;
 216         -L|--log)
 217             build_args+=(--log) ;;
 218         -n|--noconfirm|--no-confirm)
 219             build_args+=(--noconfirm) ;;
 220         -r|--rmdeps|--rm-deps)
 221             build_args+=(--rmdeps) ;;
 222         # build options (repo-add)
 223         -R|--remove)
 224             build_args+=(--remove) ;;
 225         -v|--verify)
 226             build_args+=(--verify) ;;
 227         --prevent-downgrade)
 228             build_args+=(--prevent-downgrade) ;;
 229         --new)
 230             build_args+=(--new) ;;
 231         # other options
 232         --dump-options)
 233             printf -- '--%s\n' "${opt_long[@]}" ${AUR_DEBUG+"${opt_hidden[@]}"}
 234             printf -- '%s' "${opt_short}" | sed 's/.:\?/-&\n/g'
 235             exit ;;
 236         --) shift; break ;;
 237     esac
 238     shift
 239 done
 240
 241 # shellcheck disable=SC2174
 242 mkdir -pm 0700 -- "${TMPDIR:-/tmp}/aurutils-$UID"
 243 tmp=$(mktemp -d --tmpdir "aurutils-$UID/$argv0.XXXXXXXX")
 244 trap 'trap_exit' EXIT
 245
 246 # Default to showing PKGBUILD first in patch. (#399)
 247 orderfile=$XDG_CONFIG_HOME/aurutils/$argv0/orderfile
 248 mkdir -p -- "${orderfile%/*}"
 249
 250 if [[ ! -s $orderfile ]]; then
 251     printf 'PKGBUILD\n' > "$orderfile"
 252 fi
 253
 254 if (( rotate )); then
 255     if { hash caesar && target=$(aur pkglist | shuf -n 1); } 2>/dev/null; then
 256         exec bash -c "{ aur \"$argv0\" -c \"$target\" && repo-elephant | caesar 13; } 2>&1 | caesar 13"
 257     else
 258         echo '?'; exit 16 # EBUSY
 259     fi
 260 fi
 261 mkdir -p -- "$AURDEST"
 262
 263 if (( $# + update + repo_targets == 0 )); then
 264     printf >&2 '%s: no targets specified\n' "$argv0"
 265     exit 1
 266 fi
 267
 268 # Write --no-build / --columns results to a file (#1077)
 269 if [[ -v stdout_file ]]; then
 270     stdout_file=$(realpath -- "$stdout_file")
 271 fi
 272
 273 # Retrieve path to local repo (#448, #700, #1135)
 274 { IFS=: read -r _ db_name
 275   IFS=: read -r _ db_root
 276   IFS=: read -r _ db_path
 277   IFS=: read -r _ _
 278   IFS=: read -r _ _
 279 } < <(aur build "${build_args[@]}" "${build_repo_args[@]}" --status)
 280
 281 # Print an error if `build --status` fails (#1151)
 282 if ! wait "$!"; then
 283     printf '%s: error: failed to read build configuration\n' "$argv0"
 284     exit 1
 285 fi
 286
 287 msg >&2 'Using [%s] repository' "$db_name"
 288
 289 # Ignores file can be the result of process substitution (#880)
 290 : "${ignore_file=$XDG_CONFIG_HOME/aurutils/sync/ignore}"
 291
 292 if [[ -r $ignore_file ]] && [[ ! -d $ignore_file ]]; then
 293     # Append file ignores to command-line ignores
 294     while read -r i; do pkg_i+=("$i"); done < <(select_pkgspec "$db_name" "$ignore_file")
 295 fi
 296
 297 # Diagnostic on which packages are ignored
 298 if (( ${#pkg_i[@]} )); then
 299     # Do not include command-line arguments as ignores (#952)
 300     mapfile -t pkg_i < <(complement <(printf '%s\n' "$@") <(printf '%s\n' "${pkg_i[@]}"))
 301     printf '%s: packages ignored: %s\n' "$argv0" "${pkg_i[*]}"
 302
 303     # Ignore local repository targets (#1146)
 304     reparse_args+=(--ignore "$(args_csv "${pkg_i[@]}")")
 305 fi >&2
 306
 307 # Retrieve list of local repository packages ($1 pkgname $2 pkgver)
 308 aur repo-parse "${reparse_args[@]}" -p "$db_path" --list >"$tmp"/db_info
 309
 310 # Build list of AUR targets
 311 { if (( $# )); then
 312       # append command-line arguments
 313       printf '%s\n' "$@"
 314   fi
 315
 316   if (( repo_targets )); then
 317       # append repository packages (all)
 318       cut -f1 <"$tmp"/db_info
 319
 320   elif (( update )); then
 321       # append repository packages (updated)
 322       aur vercmp --quiet <"$tmp"/db_info
 323   fi
 324 } >"$tmp"/argv
 325
 326 # Build AUR dependency graph
 327 if [[ -s $tmp/argv ]]; then
 328     # shellcheck disable=SC2094
 329     aur depends --jsonl "${depends_args[@]}" - <"$tmp"/argv >"$tmp"/depends.jsonl
 330 else
 331     printf >&2 '%s: there is nothing to do\n' "$argv0"
 332     exit
 333 fi
 334
 335 # pkginfo: $1 pkgname $2 pkgbase $3 pkgver
 336 aur format -f '%n\t%b\t%v\n' "$tmp"/depends.jsonl | sort -u >"$tmp"/pkginfo
 337
 338 { if (( ${#pkg_i[@]} )); then
 339       printf '%s\n' "${pkg_i[@]}"  # Ignored packages
 340   fi
 341
 342   # Packages with equal or newer versions are taken as complement
 343   # for the queue. If chkver_argv is enabled, packages on the
 344   # command-line are excluded from this complement.
 345   if (( chkver_depth )); then
 346       # note: AUR cannot be queried by pkgbase (FS#57230)
 347       cut -f1,3 "$tmp"/pkginfo | aur vercmp -p "$tmp"/db_info -c >"$tmp"/current
 348
 349       # shellcheck disable=SC2002
 350       case $chkver_depth in
 351           1) cat "$tmp"/current | complement "$tmp"/argv ;;
 352           2) cat "$tmp"/current ;;
 353       esac
 354   fi
 355
 356   if (( provides )); then
 357       if (( ${#repo_p[@]} )); then
 358           filter_args+=("${repo_p[@]/#/--repo=}")
 359       else
 360           filter_args+=(--sync)
 361       fi
 362
 363       # Note: this uses pacman's copy of the repo (as used by makepkg -s)
 364       cut -f1 "$tmp"/pkginfo | aur repo-filter "${filter_args[@]}" | complement "$tmp"/argv
 365   fi
 366 } >"$tmp"/filter
 367
 368 # Filter out targets determined in the steps above recursively (#1136, #1140)
 369 aur sync--filter -p "$tmp"/depends.jsonl -f "$tmp"/filter >"$tmp"/graph
 370
 371 # XXX: a flat file is needed for aur-{graph,fetch,view}. `ninja` requires the
 372 # build files to be present before dependency resolution (with `ninja -n`) can
 373 # occur, and `ninja -t targets` sorts in alphabetical order. This implies
 374 # that dependency cycles cannot be resolved before retrieving files with
 375 # aur-fetch with `ninja` alone. `tsort` could either be used in this case (with
 376 # a less nice diagnostic on cycles), or fetches done in an arbitrary order
 377 # (e.g. sort -u) with checks for cycles done at build-time.
 378 if ! tsort <"$tmp"/graph >"$tmp"/queue; then
 379     printf >&2 '%s: dependency cycle detected\n' "$argv0"
 380     exit 22
 381 fi
 382
 383 if [[ -s $tmp/queue ]]; then
 384     cd -- "$AURDEST"
 385 else
 386     printf >&2 '%s: there is nothing to do\n' "$argv0"
 387     exit
 388 fi
 389
 390 if (( download )); then
 391     msg >&2 "Retrieving package files"
 392     aur fetch -S "${fetch_args[@]}" --discard --results "$tmp"/fetch_results - < "$tmp"/queue >&2
 393
 394     # shellcheck disable=SC2034
 395     while IFS=: read -r mode rev_old rev path; do
 396         path=${path#file://} name=${path##*/}
 397
 398         case $mode in
 399             clone)
 400                 git -C "$path" config diff.orderFile "$orderfile" ;;
 401             fetch|merge|rebase|reset)
 402                 ;;
 403         esac
 404     done < "$tmp"/fetch_results
 405 else
 406     xargs -a "$tmp"/queue stat >/dev/null || exit 2 # ensure all directories are available
 407 fi
 408
 409 # Verify dependency tree (#20)
 410 if (( graph )); then
 411     if ! { while read -r pkg; do
 412                [[ $pkg ]] && printf '%s\0' "$pkg/.SRCINFO"
 413            done
 414          } | xargs -0 cat -- | aur graph "${graph_args[@]}" REVERSE=1
 415     then
 416         printf >&2 '%s: failed to verify dependency graph\n' "$argv0"
 417         exit 1
 418     fi <"$tmp"/queue >"$tmp"/graph
 419
 420     # Recompute dependencies to include `provides` (#837)
 421     tsort < "$tmp"/graph >"$tmp"/queue || exit 22
 422 fi
 423
 424 { # Sort dependencies (`aur-sync--ninja`, `--columns`)
 425   swap "$tmp"/graph | sort -k1b,1 -k1 -u >"$tmp"/graph.ninja
 426
 427   # Resolve absolute paths (`--no-build`)
 428   while read -r pkg; do
 429       [[ $pkg ]] && printf '%s\n' "$AURDEST/$pkg"
 430   done <"$tmp"/queue >"$tmp"/queue.realpath
 431 }
 432
 433 # Inspect package files
 434 if (( view )); then
 435     aur view -a "$tmp"/queue "${view_args[@]}"
 436 fi
 437
 438 # `--columns` / `--no-build` output
 439 if (( columns )) && [[ -v stdout_file ]]; then
 440     cat "$tmp"/graph.ninja >"$stdout_file"
 441
 442 elif (( columns )); then
 443     cat "$tmp"/graph.ninja
 444
 445 elif ! (( build )) && [[ -v stdout_file ]]; then
 446     cat "$tmp"/queue.realpath >"$stdout_file"
 447
 448 elif ! (( build )); then
 449     cat "$tmp"/queue.realpath
 450
 451 # Build dependency tree with ninja (#908)
 452 elif (( AUR_SYNC_USE_NINJA )); then
 453     # Apply `--save` (#1091)
 454     if [[ -v stdout_file ]]; then
 455         cat "$tmp"/graph.ninja >"$stdout_file"
 456     fi
 457
 458     # Directory for stamp files (concurrent aur-sync processes)
 459     mkdir -p -- "$XDG_STATE_HOME"/aurutils/$argv0
 460     ninja_dir=$XDG_STATE_HOME/aurutils/$argv0/ninja-$USER-$$
 461     mkdir -- "$ninja_dir"
 462
 463     # Generate build.ninja
 464     # input:  $AURDEST/pkgbase/PKGBUILD
 465     # output: $ninja_dir/pkgbase.stamp
 466     aur sync--ninja "$AURDEST" <"$tmp"/graph.ninja >"$ninja_dir"/build.ninja \
 467         -- aur build "${build_args[@]}" -d "$db_name" --root "$db_root"
 468
 469     if NINJA_STATUS='[%s/%t] ' ninja -C "$ninja_dir" -k "$keep_going"; then
 470         # Remove ninja directory on successful build
 471         rm -rf "$ninja_dir"
 472     else
 473         # Print all targets in dependency order
 474         NINJA_STATUS='[%s/%t] ' ninja -nC /var/empty -f "$ninja_dir"/build.ninja | \
 475             # [\w@\.\-\+]: valid characters for pkgname
 476             # alternative: [^\s]+ from rule `env -C ... > pkgbase.stamp`
 477             pcregrep -o1 -o3 '(\[\d+/\d+\] )(.+?)([\w@\.\-\+]+)(\.stamp)' | while read -r status pkg
 478         do
 479             if [[ -f $ninja_dir/$pkg.stamp ]]; then
 480                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${GREEN}[OK]${ALL_OFF}\n" "$status" "$pkg"
 481             else
 482                 printf "${BOLD}${BLUE}%s${ALL_OFF} %s\t${BOLD}${RED}[FAIL]${ALL_OFF}\n" "$status" "$pkg"
 483             fi
 484         done | column -t
 485
 486         # Preserve ninja directory
 487         printf '%s: build files at %s\n' "$argv0" "$ninja_dir"
 488     fi
 489 else
 490     # Apply `--save` (#1091)
 491     if [[ -v stdout_file ]]; then
 492         cat "$tmp"/queue.realpath >"$stdout_file"
 493     fi
 494     aur build "${build_args[@]}" -a "$tmp"/queue.realpath -d "$db_name" --root "$db_root"
 495 fi
 496
 497 # vim: set et sw=4 sts=4 ft=sh: