usr.sbin/freebsd-update/freebsd-update.sh

   1 #!/bin/sh
   2
   3 #-
   4 # SPDX-License-Identifier: BSD-2-Clause
   5 #
   6 # Copyright 2004-2007 Colin Percival
   7 # All rights reserved
   8 #
   9 # Redistribution and use in source and binary forms, with or without
  10 # modification, are permitted providing that the following conditions
  11 # are met:
  12 # 1. Redistributions of source code must retain the above copyright
  13 #    notice, this list of conditions and the following disclaimer.
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  19 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  21 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
  22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  26 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  27 # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  28 # POSSIBILITY OF SUCH DAMAGE.
  29
  30 #### Usage function -- called from command-line handling code.
  31
  32 # Usage instructions.  Options not listed:
  33 # --debug       -- don't filter output from utilities
  34 # --no-stats    -- don't show progress statistics while fetching files
  35 usage () {
  36         cat <<EOF
  37 usage: `basename $0` [options] command ...
  38
  39 Options:
  40   -b basedir   -- Operate on a system mounted at basedir
  41                   (default: /)
  42   -d workdir   -- Store working files in workdir
  43                   (default: /var/db/freebsd-update/)
  44   -f conffile  -- Read configuration options from conffile
  45                   (default: /etc/freebsd-update.conf)
  46   -F           -- Force a fetch operation to proceed in the
  47                   case of an unfinished upgrade
  48   -j jail      -- Operate on the given jail specified by jid or name
  49   -k KEY       -- Trust an RSA key with SHA256 hash of KEY
  50   -r release   -- Target for upgrade (e.g., 13.2-RELEASE)
  51   -s server    -- Server from which to fetch updates
  52                   (default: update.FreeBSD.org)
  53   -t address   -- Mail output of cron command, if any, to address
  54                   (default: root)
  55   --not-running-from-cron
  56                -- Run without a tty, for use by automated tools
  57   --currently-running release
  58                -- Update as if currently running this release
  59 Commands:
  60   fetch        -- Fetch updates from server
  61   cron         -- Sleep rand(3600) seconds, fetch updates, and send an
  62                   email if updates were found
  63   upgrade      -- Fetch upgrades to FreeBSD version specified via -r option
  64   updatesready -- Check if there are fetched updates ready to install
  65   install      -- Install downloaded updates or upgrades
  66   rollback     -- Uninstall most recently installed updates
  67   IDS          -- Compare the system against an index of "known good" files
  68   showconfig   -- Show configuration
  69 EOF
  70         exit 0
  71 }
  72
  73 #### Configuration processing functions
  74
  75 #-
  76 # Configuration options are set in the following order of priority:
  77 # 1. Command line options
  78 # 2. Configuration file options
  79 # 3. Default options
  80 # In addition, certain options (e.g., IgnorePaths) can be specified multiple
  81 # times and (as long as these are all in the same place, e.g., inside the
  82 # configuration file) they will accumulate.  Finally, because the path to the
  83 # configuration file can be specified at the command line, the entire command
  84 # line must be processed before we start reading the configuration file.
  85 #
  86 # Sound like a mess?  It is.  Here's how we handle this:
  87 # 1. Initialize CONFFILE and all the options to "".
  88 # 2. Process the command line.  Throw an error if a non-accumulating option
  89 #    is specified twice.
  90 # 3. If CONFFILE is "", set CONFFILE to /etc/freebsd-update.conf .
  91 # 4. For all the configuration options X, set X_saved to X.
  92 # 5. Initialize all the options to "".
  93 # 6. Read CONFFILE line by line, parsing options.
  94 # 7. For each configuration option X, set X to X_saved iff X_saved is not "".
  95 # 8. Repeat steps 4-7, except setting options to their default values at (6).
  96
  97 CONFIGOPTIONS="KEYPRINT WORKDIR SERVERNAME MAILTO ALLOWADD ALLOWDELETE
  98     KEEPMODIFIEDMETADATA COMPONENTS IGNOREPATHS UPDATEIFUNMODIFIED
  99     BASEDIR VERBOSELEVEL TARGETRELEASE STRICTCOMPONENTS MERGECHANGES
 100     IDSIGNOREPATHS BACKUPKERNEL BACKUPKERNELDIR BACKUPKERNELSYMBOLFILES"
 101
 102 # Set all the configuration options to "".
 103 nullconfig () {
 104         for X in ${CONFIGOPTIONS}; do
 105                 eval ${X}=""
 106         done
 107 }
 108
 109 # For each configuration option X, set X_saved to X.
 110 saveconfig () {
 111         for X in ${CONFIGOPTIONS}; do
 112                 eval ${X}_saved=\$${X}
 113         done
 114 }
 115
 116 # For each configuration option X, set X to X_saved if X_saved is not "".
 117 mergeconfig () {
 118         for X in ${CONFIGOPTIONS}; do
 119                 eval _=\$${X}_saved
 120                 if ! [ -z "${_}" ]; then
 121                         eval ${X}=\$${X}_saved
 122                 fi
 123         done
 124 }
 125
 126 # Set the trusted keyprint.
 127 config_KeyPrint () {
 128         if [ -z ${KEYPRINT} ]; then
 129                 KEYPRINT=$1
 130         else
 131                 return 1
 132         fi
 133 }
 134
 135 # Set the working directory.
 136 config_WorkDir () {
 137         if [ -z ${WORKDIR} ]; then
 138                 WORKDIR=$1
 139         else
 140                 return 1
 141         fi
 142 }
 143
 144 # Set the name of the server (pool) from which to fetch updates
 145 config_ServerName () {
 146         if [ -z ${SERVERNAME} ]; then
 147                 SERVERNAME=$1
 148         else
 149                 return 1
 150         fi
 151 }
 152
 153 # Set the address to which 'cron' output will be mailed.
 154 config_MailTo () {
 155         if [ -z ${MAILTO} ]; then
 156                 MAILTO=$1
 157         else
 158                 return 1
 159         fi
 160 }
 161
 162 # Set whether FreeBSD Update is allowed to add files (or directories, or
 163 # symlinks) which did not previously exist.
 164 config_AllowAdd () {
 165         if [ -z ${ALLOWADD} ]; then
 166                 case $1 in
 167                 [Yy][Ee][Ss])
 168                         ALLOWADD=yes
 169                         ;;
 170                 [Nn][Oo])
 171                         ALLOWADD=no
 172                         ;;
 173                 *)
 174                         return 1
 175                         ;;
 176                 esac
 177         else
 178                 return 1
 179         fi
 180 }
 181
 182 # Set whether FreeBSD Update is allowed to remove files/directories/symlinks.
 183 config_AllowDelete () {
 184         if [ -z ${ALLOWDELETE} ]; then
 185                 case $1 in
 186                 [Yy][Ee][Ss])
 187                         ALLOWDELETE=yes
 188                         ;;
 189                 [Nn][Oo])
 190                         ALLOWDELETE=no
 191                         ;;
 192                 *)
 193                         return 1
 194                         ;;
 195                 esac
 196         else
 197                 return 1
 198         fi
 199 }
 200
 201 # Set whether FreeBSD Update should keep existing inode ownership,
 202 # permissions, and flags, in the event that they have been modified locally
 203 # after the release.
 204 config_KeepModifiedMetadata () {
 205         if [ -z ${KEEPMODIFIEDMETADATA} ]; then
 206                 case $1 in
 207                 [Yy][Ee][Ss])
 208                         KEEPMODIFIEDMETADATA=yes
 209                         ;;
 210                 [Nn][Oo])
 211                         KEEPMODIFIEDMETADATA=no
 212                         ;;
 213                 *)
 214                         return 1
 215                         ;;
 216                 esac
 217         else
 218                 return 1
 219         fi
 220 }
 221
 222 # Add to the list of components which should be kept updated.
 223 config_Components () {
 224         for C in $@; do
 225                 COMPONENTS="${COMPONENTS} ${C}"
 226         done
 227 }
 228
 229 # Remove src component from list if it isn't installed
 230 finalize_components_config () {
 231         COMPONENTS=""
 232         for C in $@; do
 233                 if [ "$C" = "src" ]; then
 234                         if [ -e "${BASEDIR}/usr/src/COPYRIGHT" ]; then
 235                                 COMPONENTS="${COMPONENTS} ${C}"
 236                         else
 237                                 echo "src component not installed, skipped"
 238                         fi
 239                 else
 240                         COMPONENTS="${COMPONENTS} ${C}"
 241                 fi
 242         done
 243 }
 244
 245 # Add to the list of paths under which updates will be ignored.
 246 config_IgnorePaths () {
 247         for C in $@; do
 248                 IGNOREPATHS="${IGNOREPATHS} ${C}"
 249         done
 250 }
 251
 252 # Add to the list of paths which IDS should ignore.
 253 config_IDSIgnorePaths () {
 254         for C in $@; do
 255                 IDSIGNOREPATHS="${IDSIGNOREPATHS} ${C}"
 256         done
 257 }
 258
 259 # Add to the list of paths within which updates will be performed only if the
 260 # file on disk has not been modified locally.
 261 config_UpdateIfUnmodified () {
 262         for C in $@; do
 263                 UPDATEIFUNMODIFIED="${UPDATEIFUNMODIFIED} ${C}"
 264         done
 265 }
 266
 267 # Add to the list of paths within which updates to text files will be merged
 268 # instead of overwritten.
 269 config_MergeChanges () {
 270         for C in $@; do
 271                 MERGECHANGES="${MERGECHANGES} ${C}"
 272         done
 273 }
 274
 275 # Work on a FreeBSD installation mounted under $1
 276 config_BaseDir () {
 277         if [ -z ${BASEDIR} ]; then
 278                 BASEDIR=$1
 279         else
 280                 return 1
 281         fi
 282 }
 283
 284 # When fetching upgrades, should we assume the user wants exactly the
 285 # components listed in COMPONENTS, rather than trying to guess based on
 286 # what's currently installed?
 287 config_StrictComponents () {
 288         if [ -z ${STRICTCOMPONENTS} ]; then
 289                 case $1 in
 290                 [Yy][Ee][Ss])
 291                         STRICTCOMPONENTS=yes
 292                         ;;
 293                 [Nn][Oo])
 294                         STRICTCOMPONENTS=no
 295                         ;;
 296                 *)
 297                         return 1
 298                         ;;
 299                 esac
 300         else
 301                 return 1
 302         fi
 303 }
 304
 305 # Upgrade to FreeBSD $1
 306 config_TargetRelease () {
 307         if [ -z ${TARGETRELEASE} ]; then
 308                 TARGETRELEASE=$1
 309         else
 310                 return 1
 311         fi
 312         if echo ${TARGETRELEASE} | grep -qE '^[0-9.]+$'; then
 313                 TARGETRELEASE="${TARGETRELEASE}-RELEASE"
 314         fi
 315 }
 316
 317 # Pretend current release is FreeBSD $1
 318 config_SourceRelease () {
 319         UNAME_r=$1
 320         if echo ${UNAME_r} | grep -qE '^[0-9.]+$'; then
 321                 UNAME_r="${UNAME_r}-RELEASE"
 322         fi
 323         export UNAME_r
 324 }
 325
 326 # Get the Jail's path and the version of its installed userland
 327 config_TargetJail () {
 328         JAIL=$1
 329         UNAME_r=$(freebsd-version -j ${JAIL})
 330         BASEDIR=$(jls -j ${JAIL} -h path | awk 'NR == 2 {print}')
 331         if [ -z ${BASEDIR} ] || [ -z ${UNAME_r} ]; then
 332                 echo "The specified jail either doesn't exist or" \
 333                       "does not have freebsd-version."
 334                 exit 1
 335         fi
 336         export UNAME_r
 337 }
 338
 339 # Define what happens to output of utilities
 340 config_VerboseLevel () {
 341         if [ -z ${VERBOSELEVEL} ]; then
 342                 case $1 in
 343                 [Dd][Ee][Bb][Uu][Gg])
 344                         VERBOSELEVEL=debug
 345                         ;;
 346                 [Nn][Oo][Ss][Tt][Aa][Tt][Ss])
 347                         VERBOSELEVEL=nostats
 348                         ;;
 349                 [Ss][Tt][Aa][Tt][Ss])
 350                         VERBOSELEVEL=stats
 351                         ;;
 352                 *)
 353                         return 1
 354                         ;;
 355                 esac
 356         else
 357                 return 1
 358         fi
 359 }
 360
 361 config_BackupKernel () {
 362         if [ -z ${BACKUPKERNEL} ]; then
 363                 case $1 in
 364                 [Yy][Ee][Ss])
 365                         BACKUPKERNEL=yes
 366                         ;;
 367                 [Nn][Oo])
 368                         BACKUPKERNEL=no
 369                         ;;
 370                 *)
 371                         return 1
 372                         ;;
 373                 esac
 374         else
 375                 return 1
 376         fi
 377 }
 378
 379 config_BackupKernelDir () {
 380         if [ -z ${BACKUPKERNELDIR} ]; then
 381                 if [ -z "$1" ]; then
 382                         echo "BackupKernelDir set to empty dir"
 383                         return 1
 384                 fi
 385
 386                 # We check for some paths which would be extremely odd
 387                 # to use, but which could cause a lot of problems if
 388                 # used.
 389                 case $1 in
 390                 /|/bin|/boot|/etc|/lib|/libexec|/sbin|/usr|/var)
 391                         echo "BackupKernelDir set to invalid path $1"
 392                         return 1
 393                         ;;
 394                 /*)
 395                         BACKUPKERNELDIR=$1
 396                         ;;
 397                 *)
 398                         echo "BackupKernelDir ($1) is not an absolute path"
 399                         return 1
 400                         ;;
 401                 esac
 402         else
 403                 return 1
 404         fi
 405 }
 406
 407 config_BackupKernelSymbolFiles () {
 408         if [ -z ${BACKUPKERNELSYMBOLFILES} ]; then
 409                 case $1 in
 410                 [Yy][Ee][Ss])
 411                         BACKUPKERNELSYMBOLFILES=yes
 412                         ;;
 413                 [Nn][Oo])
 414                         BACKUPKERNELSYMBOLFILES=no
 415                         ;;
 416                 *)
 417                         return 1
 418                         ;;
 419                 esac
 420         else
 421                 return 1
 422         fi
 423 }
 424
 425 config_CreateBootEnv () {
 426         if [ -z ${BOOTENV} ]; then
 427                 case $1 in
 428                 [Yy][Ee][Ss])
 429                         BOOTENV=yes
 430                         ;;
 431                 [Nn][Oo])
 432                         BOOTENV=no
 433                         ;;
 434                 *)
 435                         return 1
 436                         ;;
 437                 esac
 438         else
 439                 return 1
 440         fi
 441 }
 442 # Handle one line of configuration
 443 configline () {
 444         if [ $# -eq 0 ]; then
 445                 return
 446         fi
 447
 448         OPT=$1
 449         shift
 450         config_${OPT} $@
 451 }
 452
 453 #### Parameter handling functions.
 454
 455 # Initialize parameters to null, just in case they're
 456 # set in the environment.
 457 init_params () {
 458         # Configration settings
 459         nullconfig
 460
 461         # No configuration file set yet
 462         CONFFILE=""
 463
 464         # No commands specified yet
 465         COMMANDS=""
 466
 467         # Force fetch to proceed
 468         FORCEFETCH=0
 469
 470         # Run without a TTY
 471         NOTTYOK=0
 472
 473         # Fetched first in a chain of commands
 474         ISFETCHED=0
 475 }
 476
 477 # Parse the command line
 478 parse_cmdline () {
 479         while [ $# -gt 0 ]; do
 480                 case "$1" in
 481                 # Location of configuration file
 482                 -f)
 483                         if [ $# -eq 1 ]; then usage; fi
 484                         if [ ! -z "${CONFFILE}" ]; then usage; fi
 485                         shift; CONFFILE="$1"
 486                         ;;
 487                 -F)
 488                         FORCEFETCH=1
 489                         ;;
 490                 --not-running-from-cron)
 491                         NOTTYOK=1
 492                         ;;
 493                 --currently-running)
 494                         shift
 495                         config_SourceRelease $1 || usage
 496                         ;;
 497
 498                 # Configuration file equivalents
 499                 -b)
 500                         if [ $# -eq 1 ]; then usage; fi; shift
 501                         config_BaseDir $1 || usage
 502                         ;;
 503                 -d)
 504                         if [ $# -eq 1 ]; then usage; fi; shift
 505                         config_WorkDir $1 || usage
 506                         ;;
 507                 -j)
 508                         if [ $# -eq 1 ]; then usage; fi; shift
 509                         config_TargetJail $1 || usage
 510                         ;;
 511                 -k)
 512                         if [ $# -eq 1 ]; then usage; fi; shift
 513                         config_KeyPrint $1 || usage
 514                         ;;
 515                 -s)
 516                         if [ $# -eq 1 ]; then usage; fi; shift
 517                         config_ServerName $1 || usage
 518                         ;;
 519                 -r)
 520                         if [ $# -eq 1 ]; then usage; fi; shift
 521                         config_TargetRelease $1 || usage
 522                         ;;
 523                 -t)
 524                         if [ $# -eq 1 ]; then usage; fi; shift
 525                         config_MailTo $1 || usage
 526                         ;;
 527                 -v)
 528                         if [ $# -eq 1 ]; then usage; fi; shift
 529                         config_VerboseLevel $1 || usage
 530                         ;;
 531
 532                 # Aliases for "-v debug" and "-v nostats"
 533                 --debug)
 534                         config_VerboseLevel debug || usage
 535                         ;;
 536                 --no-stats)
 537                         config_VerboseLevel nostats || usage
 538                         ;;
 539
 540                 # Commands
 541                 cron | fetch | upgrade | updatesready | install | rollback |\
 542                 IDS | showconfig)
 543                         COMMANDS="${COMMANDS} $1"
 544                         ;;
 545
 546                 # Anything else is an error
 547                 *)
 548                         usage
 549                         ;;
 550                 esac
 551                 shift
 552         done
 553
 554         # Make sure we have at least one command
 555         if [ -z "${COMMANDS}" ]; then
 556                 usage
 557         fi
 558 }
 559
 560 # Parse the configuration file
 561 parse_conffile () {
 562         # If a configuration file was specified on the command line, check
 563         # that it exists and is readable.
 564         if [ ! -z "${CONFFILE}" ] && [ ! -r "${CONFFILE}" ]; then
 565                 echo -n "File does not exist "
 566                 echo -n "or is not readable: "
 567                 echo ${CONFFILE}
 568                 exit 1
 569         fi
 570
 571         # If a configuration file was not specified on the command line,
 572         # use the default configuration file path.  If that default does
 573         # not exist, give up looking for any configuration.
 574         if [ -z "${CONFFILE}" ]; then
 575                 CONFFILE="/etc/freebsd-update.conf"
 576                 if [ ! -r "${CONFFILE}" ]; then
 577                         return
 578                 fi
 579         fi
 580
 581         # Save the configuration options specified on the command line, and
 582         # clear all the options in preparation for reading the config file.
 583         saveconfig
 584         nullconfig
 585
 586         # Read the configuration file.  Anything after the first '#' is
 587         # ignored, and any blank lines are ignored.
 588         L=0
 589         while read LINE; do
 590                 L=$(($L + 1))
 591                 LINEX=`echo "${LINE}" | cut -f 1 -d '#'`
 592                 if ! configline ${LINEX}; then
 593                         echo "Error processing configuration file, line $L:"
 594                         echo "==> ${LINE}"
 595                         exit 1
 596                 fi
 597         done < ${CONFFILE}
 598
 599         # Merge the settings read from the configuration file with those
 600         # provided at the command line.
 601         mergeconfig
 602 }
 603
 604 # Provide some default parameters
 605 default_params () {
 606         # Save any parameters already configured, and clear the slate
 607         saveconfig
 608         nullconfig
 609
 610         # Default configurations
 611         config_WorkDir /var/db/freebsd-update
 612         config_MailTo root
 613         config_AllowAdd yes
 614         config_AllowDelete yes
 615         config_KeepModifiedMetadata yes
 616         config_BaseDir /
 617         config_VerboseLevel stats
 618         config_StrictComponents no
 619         config_BackupKernel yes
 620         config_BackupKernelDir /boot/kernel.old
 621         config_BackupKernelSymbolFiles no
 622         config_CreateBootEnv yes
 623
 624         # Merge these defaults into the earlier-configured settings
 625         mergeconfig
 626 }
 627
 628 # Set utility output filtering options, based on ${VERBOSELEVEL}
 629 fetch_setup_verboselevel () {
 630         case ${VERBOSELEVEL} in
 631         debug)
 632                 QUIETREDIR="/dev/stderr"
 633                 QUIETFLAG=" "
 634                 STATSREDIR="/dev/stderr"
 635                 DDSTATS=".."
 636                 XARGST="-t"
 637                 NDEBUG=" "
 638                 ;;
 639         nostats)
 640                 QUIETREDIR=""
 641                 QUIETFLAG=""
 642                 STATSREDIR="/dev/null"
 643                 DDSTATS=".."
 644                 XARGST=""
 645                 NDEBUG=""
 646                 ;;
 647         stats)
 648                 QUIETREDIR="/dev/null"
 649                 QUIETFLAG="-q"
 650                 STATSREDIR="/dev/stdout"
 651                 DDSTATS=""
 652                 XARGST=""
 653                 NDEBUG="-n"
 654                 ;;
 655         esac
 656 }
 657
 658 # Check if there are any kernel modules installed from ports.
 659 # In that case warn the user that a rebuild from ports (i.e. not from
 660 # packages) might need necessary for the modules to work in the new release.
 661 upgrade_check_kmod_ports() {
 662         local mod_name
 663         local modules
 664         local pattern
 665         local pkg_name
 666         local port_name
 667         local report
 668         local w
 669
 670         if ! pkg -N 2>/dev/null; then
 671                 echo "Skipping kernel modules check. pkg(8) not present."
 672                 return
 673         fi
 674
 675         # Most modules are in /boot/modules but we should actually look
 676         # in every module_path passed to the kernel:
 677         pattern=$(sysctl -n kern.module_path | tr ";" "|")
 678
 679         if [ -z "${pattern}" ]; then
 680                 echo "Empty kern.module_path sysctl. This should not happen."
 681                 echo "Aborting check of kernel modules installed from ports."
 682                 return
 683         fi
 684
 685         # Check the pkg database for modules installed in those directories
 686         modules=$(pkg query '%Fp' | grep -E "${pattern}")
 687
 688         if [ -z "${modules}" ]; then
 689                 return
 690         fi
 691
 692         echo -e "\n"
 693         echo "The following modules have been installed from packages."
 694         echo "As a consequence they might not work when performing a major or minor upgrade."
 695         echo -e "It is advised to rebuild these ports:\n"
 696
 697
 698         report="Module Package Port\n------ ------- ----\n"
 699         for module in ${modules}; do
 700                 w=$(pkg which "${module}")
 701                 mod_name=$(echo "${w}" | awk '{print $1;}')
 702                 pkg_name=$(echo "${w}" | awk '{print $6;}')
 703                 port_name=$(pkg info -o "${pkg_name}" | awk '{print $2;}')
 704                 report="${report}${mod_name} ${pkg_name} ${port_name}\n"
 705         done
 706
 707         echo -e "${report}" | column -t
 708         echo -e "\n"
 709 }
 710
 711 # Perform sanity checks and set some final parameters
 712 # in preparation for fetching files.  Figure out which
 713 # set of updates should be downloaded: If the user is
 714 # running *-p[0-9]+, strip off the last part; if the
 715 # user is running -SECURITY, call it -RELEASE.  Chdir
 716 # into the working directory.
 717 fetchupgrade_check_params () {
 718         export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
 719
 720         _SERVERNAME_z=\
 721 "SERVERNAME must be given via command line or configuration file."
 722         _KEYPRINT_z="Key must be given via -k option or configuration file."
 723         _KEYPRINT_bad="Invalid key fingerprint: "
 724         _WORKDIR_bad="Directory does not exist or is not writable: "
 725         _WORKDIR_bad2="Directory is not on a persistent filesystem: "
 726
 727         if [ -z "${SERVERNAME}" ]; then
 728                 echo -n "`basename $0`: "
 729                 echo "${_SERVERNAME_z}"
 730                 exit 1
 731         fi
 732         if [ -z "${KEYPRINT}" ]; then
 733                 echo -n "`basename $0`: "
 734                 echo "${_KEYPRINT_z}"
 735                 exit 1
 736         fi
 737         if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
 738                 echo -n "`basename $0`: "
 739                 echo -n "${_KEYPRINT_bad}"
 740                 echo ${KEYPRINT}
 741                 exit 1
 742         fi
 743         if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
 744                 echo -n "`basename $0`: "
 745                 echo -n "${_WORKDIR_bad}"
 746                 echo ${WORKDIR}
 747                 exit 1
 748         fi
 749         case `df -T ${WORKDIR}` in */dev/md[0-9]* | *tmpfs*)
 750                 echo -n "`basename $0`: "
 751                 echo -n "${_WORKDIR_bad2}"
 752                 echo ${WORKDIR}
 753                 exit 1
 754                 ;;
 755         esac
 756         chmod 700 ${WORKDIR}
 757         cd ${WORKDIR} || exit 1
 758
 759         # Generate release number.  The s/SECURITY/RELEASE/ bit exists
 760         # to provide an upgrade path for FreeBSD Update 1.x users, since
 761         # the kernels provided by FreeBSD Update 1.x are always labelled
 762         # as X.Y-SECURITY.
 763         RELNUM=`uname -r |
 764             sed -E 's,-p[0-9]+,,' |
 765             sed -E 's,-SECURITY,-RELEASE,'`
 766         ARCH=`uname -m`
 767         FETCHDIR=${RELNUM}/${ARCH}
 768         PATCHDIR=${RELNUM}/${ARCH}/bp
 769
 770         # Disallow upgrade from a version that is not a release
 771         case ${RELNUM} in
 772         *-RELEASE | *-ALPHA*  | *-BETA* | *-RC*)
 773                 ;;
 774         *)
 775                 echo -n "`basename $0`: "
 776                 cat <<- EOF
 777                         Cannot upgrade from a version that is not a release
 778                         (including alpha, beta and release candidates)
 779                         using `basename $0`. Instead, FreeBSD can be directly
 780                         upgraded by source or upgraded to a RELEASE/RELENG version
 781                         prior to running `basename $0`.
 782                         Currently running: ${RELNUM}
 783                 EOF
 784                 exit 1
 785                 ;;
 786         esac
 787
 788         # Figure out what directory contains the running kernel
 789         BOOTFILE=`sysctl -n kern.bootfile`
 790         KERNELDIR=${BOOTFILE%/kernel}
 791         if ! [ -d ${KERNELDIR} ]; then
 792                 echo "Cannot identify running kernel"
 793                 exit 1
 794         fi
 795
 796         # Figure out what kernel configuration is running.  We start with
 797         # the output of `uname -i`, and then make the following adjustments:
 798         # 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
 799         # file says "ident SMP-GENERIC", I don't know...
 800         # 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
 801         # _and_ `sysctl kern.version` contains a line which ends "/SMP", then
 802         # we're running an SMP kernel.  This mis-identification is a bug
 803         # which was fixed in 6.2-STABLE.
 804         KERNCONF=`uname -i`
 805         if [ ${KERNCONF} = "SMP-GENERIC" ]; then
 806                 KERNCONF=SMP
 807         fi
 808         if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
 809                 if sysctl kern.version | grep -qE '/SMP$'; then
 810                         KERNCONF=SMP
 811                 fi
 812         fi
 813
 814         # Define some paths
 815         BSPATCH=/usr/bin/bspatch
 816         SHA256=/sbin/sha256
 817         PHTTPGET=/usr/libexec/phttpget
 818
 819         # Set up variables relating to VERBOSELEVEL
 820         fetch_setup_verboselevel
 821
 822         # Construct a unique name from ${BASEDIR}
 823         BDHASH=`echo ${BASEDIR} | sha256 -q`
 824 }
 825
 826 # Perform sanity checks etc. before fetching updates.
 827 fetch_check_params () {
 828         fetchupgrade_check_params
 829
 830         if ! [ -z "${TARGETRELEASE}" ]; then
 831                 echo -n "`basename $0`: "
 832                 echo -n "'-r' option is meaningless with 'fetch' command.  "
 833                 echo "(Did you mean 'upgrade' instead?)"
 834                 exit 1
 835         fi
 836
 837         # Check that we have updates ready to install
 838         if [ -f ${BDHASH}-install/kerneldone -a $FORCEFETCH -eq 0 ]; then
 839                 echo "You have a partially completed upgrade pending"
 840                 echo "Run '`basename $0` [options] install' first."
 841                 echo "Run '`basename $0` [options] fetch -F' to proceed anyway."
 842                 exit 1
 843         fi
 844 }
 845
 846 # Perform sanity checks etc. before fetching upgrades.
 847 upgrade_check_params () {
 848         fetchupgrade_check_params
 849
 850         # Unless set otherwise, we're upgrading to the same kernel config.
 851         NKERNCONF=${KERNCONF}
 852
 853         # We need TARGETRELEASE set
 854         _TARGETRELEASE_z="Release target must be specified via '-r' option."
 855         if [ -z "${TARGETRELEASE}" ]; then
 856                 echo -n "`basename $0`: "
 857                 echo "${_TARGETRELEASE_z}"
 858                 exit 1
 859         fi
 860
 861         # The target release should be != the current release.
 862         if [ "${TARGETRELEASE}" = "${RELNUM}" ]; then
 863                 echo -n "`basename $0`: "
 864                 echo "Cannot upgrade from ${RELNUM} to itself"
 865                 exit 1
 866         fi
 867
 868         # Turning off AllowAdd or AllowDelete is a bad idea for upgrades.
 869         if [ "${ALLOWADD}" = "no" ]; then
 870                 echo -n "`basename $0`: "
 871                 echo -n "WARNING: \"AllowAdd no\" is a bad idea "
 872                 echo "when upgrading between releases."
 873                 echo
 874         fi
 875         if [ "${ALLOWDELETE}" = "no" ]; then
 876                 echo -n "`basename $0`: "
 877                 echo -n "WARNING: \"AllowDelete no\" is a bad idea "
 878                 echo "when upgrading between releases."
 879                 echo
 880         fi
 881
 882         # Set EDITOR to /usr/bin/vi if it isn't already set
 883         : ${EDITOR:='/usr/bin/vi'}
 884 }
 885
 886 # Perform sanity checks and set some final parameters in
 887 # preparation for installing updates.
 888 install_check_params () {
 889         # Check that we are root.  All sorts of things won't work otherwise.
 890         if [ `id -u` != 0 ]; then
 891                 echo "You must be root to run this."
 892                 exit 1
 893         fi
 894
 895         # Check that securelevel <= 0.  Otherwise we can't update schg files.
 896         if [ `sysctl -n kern.securelevel` -gt 0 ]; then
 897                 echo "Updates cannot be installed when the system securelevel"
 898                 echo "is greater than zero."
 899                 exit 1
 900         fi
 901
 902         # Check that we have a working directory
 903         _WORKDIR_bad="Directory does not exist or is not writable: "
 904         if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
 905                 echo -n "`basename $0`: "
 906                 echo -n "${_WORKDIR_bad}"
 907                 echo ${WORKDIR}
 908                 exit 1
 909         fi
 910         cd ${WORKDIR} || exit 1
 911
 912         # Construct a unique name from ${BASEDIR}
 913         BDHASH=`echo ${BASEDIR} | sha256 -q`
 914
 915         # Check that we have updates ready to install
 916         if ! [ -L ${BDHASH}-install ]; then
 917                 echo "No updates are available to install."
 918                 if [ $ISFETCHED -eq 0 ]; then
 919                         echo "Run '`basename $0` [options] fetch' first."
 920                         exit 2
 921                 fi
 922                 exit 0
 923         fi
 924         if ! [ -f ${BDHASH}-install/INDEX-OLD ] ||
 925             ! [ -f ${BDHASH}-install/INDEX-NEW ]; then
 926                 echo "Update manifest is corrupt -- this should never happen."
 927                 echo "Re-run '`basename $0` [options] fetch'."
 928                 exit 1
 929         fi
 930
 931         # Figure out what directory contains the running kernel
 932         BOOTFILE=`sysctl -n kern.bootfile`
 933         KERNELDIR=${BOOTFILE%/kernel}
 934         if ! [ -d ${KERNELDIR} ]; then
 935                 echo "Cannot identify running kernel"
 936                 exit 1
 937         fi
 938 }
 939
 940 # Creates a new boot environment
 941 install_create_be () {
 942         # Figure out if we're running in a jail and return if we are
 943         if [ `sysctl -n security.jail.jailed` = 1 ]; then
 944                 return 1
 945         fi
 946         # Operating on roots that aren't located at / will, more often than not,
 947         # not touch the boot environment.
 948         if [ "$BASEDIR" != "/" ]; then
 949                 return 1
 950         fi
 951         # Create a boot environment if enabled
 952         if [ ${BOOTENV} = yes ]; then
 953                 bectl check 2>/dev/null
 954                 case $? in
 955                         0)
 956                                 # Boot environment are supported
 957                                 CREATEBE=yes
 958                                 ;;
 959                         255)
 960                                 # Boot environments are not supported
 961                                 CREATEBE=no
 962                                 ;;
 963                         *)
 964                                 # If bectl returns an unexpected exit code, don't create a BE
 965                                 CREATEBE=no
 966                                 ;;
 967                 esac
 968                 if [ ${CREATEBE} = yes ]; then
 969                         echo -n "Creating snapshot of existing boot environment... "
 970                         VERSION=`freebsd-version -ku | sort -V | tail -n 1`
 971                         TIMESTAMP=`date +"%Y-%m-%d_%H%M%S"`
 972                         bectl create -r ${VERSION}_${TIMESTAMP}
 973                         if [ $? -eq 0 ]; then
 974                                 echo "done.";
 975                         else
 976                                 echo "failed."
 977                                 exit 1
 978                         fi
 979                 fi
 980         fi
 981 }
 982
 983 # Perform sanity checks and set some final parameters in
 984 # preparation for UNinstalling updates.
 985 rollback_check_params () {
 986         # Check that we are root.  All sorts of things won't work otherwise.
 987         if [ `id -u` != 0 ]; then
 988                 echo "You must be root to run this."
 989                 exit 1
 990         fi
 991
 992         # Check that we have a working directory
 993         _WORKDIR_bad="Directory does not exist or is not writable: "
 994         if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
 995                 echo -n "`basename $0`: "
 996                 echo -n "${_WORKDIR_bad}"
 997                 echo ${WORKDIR}
 998                 exit 1
 999         fi
1000         cd ${WORKDIR} || exit 1
1001
1002         # Construct a unique name from ${BASEDIR}
1003         BDHASH=`echo ${BASEDIR} | sha256 -q`
1004
1005         # Check that we have updates ready to rollback
1006         if ! [ -L ${BDHASH}-rollback ]; then
1007                 echo "No rollback directory found."
1008                 exit 1
1009         fi
1010         if ! [ -f ${BDHASH}-rollback/INDEX-OLD ] ||
1011             ! [ -f ${BDHASH}-rollback/INDEX-NEW ]; then
1012                 echo "Update manifest is corrupt -- this should never happen."
1013                 exit 1
1014         fi
1015 }
1016
1017 # Perform sanity checks and set some final parameters
1018 # in preparation for comparing the system against the
1019 # published index.  Figure out which index we should
1020 # compare against: If the user is running *-p[0-9]+,
1021 # strip off the last part; if the user is running
1022 # -SECURITY, call it -RELEASE.  Chdir into the working
1023 # directory.
1024 IDS_check_params () {
1025         export HTTP_USER_AGENT="freebsd-update (${COMMAND}, `uname -r`)"
1026
1027         _SERVERNAME_z=\
1028 "SERVERNAME must be given via command line or configuration file."
1029         _KEYPRINT_z="Key must be given via '-k' option or configuration file."
1030         _KEYPRINT_bad="Invalid key fingerprint: "
1031         _WORKDIR_bad="Directory does not exist or is not writable: "
1032
1033         if [ -z "${SERVERNAME}" ]; then
1034                 echo -n "`basename $0`: "
1035                 echo "${_SERVERNAME_z}"
1036                 exit 1
1037         fi
1038         if [ -z "${KEYPRINT}" ]; then
1039                 echo -n "`basename $0`: "
1040                 echo "${_KEYPRINT_z}"
1041                 exit 1
1042         fi
1043         if ! echo "${KEYPRINT}" | grep -qE "^[0-9a-f]{64}$"; then
1044                 echo -n "`basename $0`: "
1045                 echo -n "${_KEYPRINT_bad}"
1046                 echo ${KEYPRINT}
1047                 exit 1
1048         fi
1049         if ! [ -d "${WORKDIR}" -a -w "${WORKDIR}" ]; then
1050                 echo -n "`basename $0`: "
1051                 echo -n "${_WORKDIR_bad}"
1052                 echo ${WORKDIR}
1053                 exit 1
1054         fi
1055         cd ${WORKDIR} || exit 1
1056
1057         # Generate release number.  The s/SECURITY/RELEASE/ bit exists
1058         # to provide an upgrade path for FreeBSD Update 1.x users, since
1059         # the kernels provided by FreeBSD Update 1.x are always labelled
1060         # as X.Y-SECURITY.
1061         RELNUM=`uname -r |
1062             sed -E 's,-p[0-9]+,,' |
1063             sed -E 's,-SECURITY,-RELEASE,'`
1064         ARCH=`uname -m`
1065         FETCHDIR=${RELNUM}/${ARCH}
1066         PATCHDIR=${RELNUM}/${ARCH}/bp
1067
1068         # Figure out what directory contains the running kernel
1069         BOOTFILE=`sysctl -n kern.bootfile`
1070         KERNELDIR=${BOOTFILE%/kernel}
1071         if ! [ -d ${KERNELDIR} ]; then
1072                 echo "Cannot identify running kernel"
1073                 exit 1
1074         fi
1075
1076         # Figure out what kernel configuration is running.  We start with
1077         # the output of `uname -i`, and then make the following adjustments:
1078         # 1. Replace "SMP-GENERIC" with "SMP".  Why the SMP kernel config
1079         # file says "ident SMP-GENERIC", I don't know...
1080         # 2. If the kernel claims to be GENERIC _and_ ${ARCH} is "amd64"
1081         # _and_ `sysctl kern.version` contains a line which ends "/SMP", then
1082         # we're running an SMP kernel.  This mis-identification is a bug
1083         # which was fixed in 6.2-STABLE.
1084         KERNCONF=`uname -i`
1085         if [ ${KERNCONF} = "SMP-GENERIC" ]; then
1086                 KERNCONF=SMP
1087         fi
1088         if [ ${KERNCONF} = "GENERIC" ] && [ ${ARCH} = "amd64" ]; then
1089                 if sysctl kern.version | grep -qE '/SMP$'; then
1090                         KERNCONF=SMP
1091                 fi
1092         fi
1093
1094         # Define some paths
1095         SHA256=/sbin/sha256
1096         PHTTPGET=/usr/libexec/phttpget
1097
1098         # Set up variables relating to VERBOSELEVEL
1099         fetch_setup_verboselevel
1100 }
1101
1102 # Packaged base and freebsd-update are incompatible.  Exit with an error if
1103 # packaged base is in use.
1104 check_pkgbase()
1105 {
1106         # Packaged base requires that pkg is bootstrapped.
1107         if ! pkg -c ${BASEDIR} -N >/dev/null 2>/dev/null; then
1108                 return
1109         fi
1110         # uname(1) is used by pkg to determine ABI, so it should exist.
1111         # If it comes from a package then this system uses packaged base.
1112         if ! pkg -c ${BASEDIR} which /usr/bin/uname >/dev/null; then
1113                 return
1114         fi
1115         cat <<EOF
1116 freebsd-update is incompatible with the use of packaged base.  Please see
1117 https://wiki.freebsd.org/PkgBase for more information.
1118 EOF
1119         exit 1
1120 }
1121
1122 #### Core functionality -- the actual work gets done here
1123
1124 # Use an SRV query to pick a server.  If the SRV query doesn't provide
1125 # a useful answer, use the server name specified by the user.
1126 # Put another way... look up _http._tcp.${SERVERNAME} and pick a server
1127 # from that; or if no servers are returned, use ${SERVERNAME}.
1128 # This allows a user to specify "update.FreeBSD.org" (in which case
1129 # freebsd-update will select one of the mirrors) or "update1.freebsd.org"
1130 # (in which case freebsd-update will use that particular server, since
1131 # there won't be an SRV entry for that name).
1132 #
1133 # We ignore the Port field, since we are always going to use port 80.
1134
1135 # Fetch the mirror list, but do not pick a mirror yet.  Returns 1 if
1136 # no mirrors are available for any reason.
1137 fetch_pick_server_init () {
1138         : > serverlist_tried
1139
1140 # Check that host(1) exists (i.e., that the system wasn't built with the
1141 # WITHOUT_BIND set) and don't try to find a mirror if it doesn't exist.
1142         if ! which -s host; then
1143                 : > serverlist_full
1144                 return 1
1145         fi
1146
1147         echo -n "Looking up ${SERVERNAME} mirrors... "
1148
1149 # Issue the SRV query and pull out the Priority, Weight, and Target fields.
1150 # BIND 9 prints "$name has SRV record ..." while BIND 8 prints
1151 # "$name server selection ..."; we allow either format.
1152         MLIST="_http._tcp.${SERVERNAME}"
1153         host -t srv "${MLIST}" |
1154             sed -nE "s/${MLIST} (has SRV record|server selection) //Ip" |
1155             cut -f 1,2,4 -d ' ' |
1156             sed -e 's/\.$//' |
1157             sort > serverlist_full
1158
1159 # If no records, give up -- we'll just use the server name we were given.
1160         if [ `wc -l < serverlist_full` -eq 0 ]; then
1161                 echo "none found."
1162                 return 1
1163         fi
1164
1165 # Report how many mirrors we found.
1166         echo `wc -l < serverlist_full` "mirrors found."
1167
1168 # Generate a random seed for use in picking mirrors.  If HTTP_PROXY
1169 # is set, this will be used to generate the seed; otherwise, the seed
1170 # will be random.
1171         if [ -n "${HTTP_PROXY}${http_proxy}" ]; then
1172                 RANDVALUE=`sha256 -qs "${HTTP_PROXY}${http_proxy}" |
1173                     tr -d 'a-f' |
1174                     cut -c 1-9`
1175         else
1176                 RANDVALUE=`jot -r 1 0 999999999`
1177         fi
1178 }
1179
1180 # Pick a mirror.  Returns 1 if we have run out of mirrors to try.
1181 fetch_pick_server () {
1182 # Generate a list of not-yet-tried mirrors
1183         sort serverlist_tried |
1184             comm -23 serverlist_full - > serverlist
1185
1186 # Have we run out of mirrors?
1187         if [ `wc -l < serverlist` -eq 0 ]; then
1188                 cat <<- EOF
1189                         No mirrors remaining, giving up.
1190
1191                         This may be because upgrading from this platform (${ARCH})
1192                         or release (${RELNUM}) is unsupported by `basename $0`. Only
1193                         platforms with Tier 1 support can be upgraded by `basename $0`.
1194                         See https://www.freebsd.org/platforms/ for more info.
1195
1196                         If unsupported, FreeBSD must be upgraded by source.
1197                 EOF
1198                 return 1
1199         fi
1200
1201 # Find the highest priority level (lowest numeric value).
1202         SRV_PRIORITY=`cut -f 1 -d ' ' serverlist | sort -n | head -1`
1203
1204 # Add up the weights of the response lines at that priority level.
1205         SRV_WSUM=0;
1206         while read X; do
1207                 case "$X" in
1208                 ${SRV_PRIORITY}\ *)
1209                         SRV_W=`echo $X | cut -f 2 -d ' '`
1210                         SRV_WSUM=$(($SRV_WSUM + $SRV_W))
1211                         ;;
1212                 esac
1213         done < serverlist
1214
1215 # If all the weights are 0, pretend that they are all 1 instead.
1216         if [ ${SRV_WSUM} -eq 0 ]; then
1217                 SRV_WSUM=`grep -E "^${SRV_PRIORITY} " serverlist | wc -l`
1218                 SRV_W_ADD=1
1219         else
1220                 SRV_W_ADD=0
1221         fi
1222
1223 # Pick a value between 0 and the sum of the weights - 1
1224         SRV_RND=`expr ${RANDVALUE} % ${SRV_WSUM}`
1225
1226 # Read through the list of mirrors and set SERVERNAME.  Write the line
1227 # corresponding to the mirror we selected into serverlist_tried so that
1228 # we won't try it again.
1229         while read X; do
1230                 case "$X" in
1231                 ${SRV_PRIORITY}\ *)
1232                         SRV_W=`echo $X | cut -f 2 -d ' '`
1233                         SRV_W=$(($SRV_W + $SRV_W_ADD))
1234                         if [ $SRV_RND -lt $SRV_W ]; then
1235                                 SERVERNAME=`echo $X | cut -f 3 -d ' '`
1236                                 echo "$X" >> serverlist_tried
1237                                 break
1238                         else
1239                                 SRV_RND=$(($SRV_RND - $SRV_W))
1240                         fi
1241                         ;;
1242                 esac
1243         done < serverlist
1244 }
1245
1246 # Take a list of ${oldhash}|${newhash} and output a list of needed patches,
1247 # i.e., those for which we have ${oldhash} and don't have ${newhash}.
1248 fetch_make_patchlist () {
1249         grep -vE "^([0-9a-f]{64})\|\1$" |
1250             tr '|' ' ' |
1251                 while read X Y; do
1252                         if [ -f "files/${Y}.gz" ] ||
1253                             [ ! -f "files/${X}.gz" ]; then
1254                                 continue
1255                         fi
1256                         echo "${X}|${Y}"
1257                 done | sort -u
1258 }
1259
1260 # Print user-friendly progress statistics
1261 fetch_progress () {
1262         LNC=0
1263         while read x; do
1264                 LNC=$(($LNC + 1))
1265                 if [ $(($LNC % 10)) = 0 ]; then
1266                         echo -n $LNC
1267                 elif [ $(($LNC % 2)) = 0 ]; then
1268                         echo -n .
1269                 fi
1270         done
1271         echo -n " "
1272 }
1273
1274 # Function for asking the user if everything is ok
1275 continuep () {
1276         while read -p "Does this look reasonable (y/n)? " CONTINUE; do
1277                 case "${CONTINUE}" in
1278                 [yY]*)
1279                         return 0
1280                         ;;
1281                 [nN]*)
1282                         return 1
1283                         ;;
1284                 esac
1285         done
1286 }
1287
1288 # Initialize the working directory
1289 workdir_init () {
1290         mkdir -p files
1291         touch tINDEX.present
1292 }
1293
1294 # Check that we have a public key with an appropriate hash, or
1295 # fetch the key if it doesn't exist.  Returns 1 if the key has
1296 # not yet been fetched.
1297 fetch_key () {
1298         if [ -r pub.ssl ] && [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1299                 return 0
1300         fi
1301
1302         echo -n "Fetching public key from ${SERVERNAME}... "
1303         rm -f pub.ssl
1304         fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/pub.ssl \
1305             2>${QUIETREDIR} || true
1306         if ! [ -r pub.ssl ]; then
1307                 echo "failed."
1308                 return 1
1309         fi
1310         if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
1311                 echo "key has incorrect hash."
1312                 rm -f pub.ssl
1313                 return 1
1314         fi
1315         echo "done."
1316 }
1317
1318 # Fetch metadata signature, aka "tag".
1319 fetch_tag () {
1320         echo -n "Fetching metadata signature "
1321         echo ${NDEBUG} "for ${RELNUM} from ${SERVERNAME}... "
1322         rm -f latest.ssl
1323         fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/latest.ssl  \
1324             2>${QUIETREDIR} || true
1325         if ! [ -r latest.ssl ]; then
1326                 echo "failed."
1327                 return 1
1328         fi
1329
1330         openssl pkeyutl -pubin -inkey pub.ssl -verifyrecover    \
1331             < latest.ssl > tag.new 2>${QUIETREDIR} || true
1332         rm latest.ssl
1333
1334         if ! [ `wc -l < tag.new` = 1 ] ||
1335             ! grep -qE  \
1336     "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1337                 tag.new; then
1338                 echo "invalid signature."
1339                 return 1
1340         fi
1341
1342         echo "done."
1343
1344         RELPATCHNUM=`cut -f 4 -d '|' < tag.new`
1345         TINDEXHASH=`cut -f 5 -d '|' < tag.new`
1346         EOLTIME=`cut -f 6 -d '|' < tag.new`
1347 }
1348
1349 # Sanity-check the patch number in a tag, to make sure that we're not
1350 # going to "update" backwards and to prevent replay attacks.
1351 fetch_tagsanity () {
1352         # Check that we're not going to move from -pX to -pY with Y < X.
1353         RELPX=`uname -r | sed -E 's,.*-,,'`
1354         if echo ${RELPX} | grep -qE '^p[0-9]+$'; then
1355                 RELPX=`echo ${RELPX} | cut -c 2-`
1356         else
1357                 RELPX=0
1358         fi
1359         if [ "${RELPATCHNUM}" -lt "${RELPX}" ]; then
1360                 echo
1361                 echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1362                 echo " appear older than what"
1363                 echo "we are currently running (`uname -r`)!"
1364                 echo "Cowardly refusing to proceed any further."
1365                 return 1
1366         fi
1367
1368         # If "tag" exists and corresponds to ${RELNUM}, make sure that
1369         # it contains a patch number <= RELPATCHNUM, in order to protect
1370         # against rollback (replay) attacks.
1371         if [ -f tag ] &&
1372             grep -qE    \
1373     "^freebsd-update\|${ARCH}\|${RELNUM}\|[0-9]+\|[0-9a-f]{64}\|[0-9]{10}" \
1374                 tag; then
1375                 LASTRELPATCHNUM=`cut -f 4 -d '|' < tag`
1376
1377                 if [ "${RELPATCHNUM}" -lt "${LASTRELPATCHNUM}" ]; then
1378                         echo
1379                         echo -n "Files on mirror (${RELNUM}-p${RELPATCHNUM})"
1380                         echo " are older than the"
1381                         echo -n "most recently seen updates"
1382                         echo " (${RELNUM}-p${LASTRELPATCHNUM})."
1383                         echo "Cowardly refusing to proceed any further."
1384                         return 1
1385                 fi
1386         fi
1387 }
1388
1389 # Fetch metadata index file
1390 fetch_metadata_index () {
1391         echo ${NDEBUG} "Fetching metadata index... "
1392         rm -f ${TINDEXHASH}
1393         fetch ${QUIETFLAG} http://${SERVERNAME}/${FETCHDIR}/t/${TINDEXHASH}
1394             2>${QUIETREDIR}
1395         if ! [ -f ${TINDEXHASH} ]; then
1396                 echo "failed."
1397                 return 1
1398         fi
1399         if [ `${SHA256} -q ${TINDEXHASH}` != ${TINDEXHASH} ]; then
1400                 echo "update metadata index corrupt."
1401                 return 1
1402         fi
1403         echo "done."
1404 }
1405
1406 # Print an error message about signed metadata being bogus.
1407 fetch_metadata_bogus () {
1408         echo
1409         echo "The update metadata$1 is correctly signed, but"
1410         echo "failed an integrity check."
1411         echo "Cowardly refusing to proceed any further."
1412         return 1
1413 }
1414
1415 # Construct tINDEX.new by merging the lines named in $1 from ${TINDEXHASH}
1416 # with the lines not named in $@ from tINDEX.present (if that file exists).
1417 fetch_metadata_index_merge () {
1418         for METAFILE in $@; do
1419                 if [ `grep -E "^${METAFILE}\|" ${TINDEXHASH} | wc -l`   \
1420                     -ne 1 ]; then
1421                         fetch_metadata_bogus " index"
1422                         return 1
1423                 fi
1424
1425                 grep -E "${METAFILE}\|" ${TINDEXHASH}
1426         done |
1427             sort > tINDEX.wanted
1428
1429         if [ -f tINDEX.present ]; then
1430                 join -t '|' -v 2 tINDEX.wanted tINDEX.present |
1431                     sort -m - tINDEX.wanted > tINDEX.new
1432                 rm tINDEX.wanted
1433         else
1434                 mv tINDEX.wanted tINDEX.new
1435         fi
1436 }
1437
1438 # Sanity check all the lines of tINDEX.new.  Even if more metadata lines
1439 # are added by future versions of the server, this won't cause problems,
1440 # since the only lines which appear in tINDEX.new are the ones which we
1441 # specifically grepped out of ${TINDEXHASH}.
1442 fetch_metadata_index_sanity () {
1443         if grep -qvE '^[0-9A-Z.-]+\|[0-9a-f]{64}$' tINDEX.new; then
1444                 fetch_metadata_bogus " index"
1445                 return 1
1446         fi
1447 }
1448
1449 # Sanity check the metadata file $1.
1450 fetch_metadata_sanity () {
1451         # Some aliases to save space later: ${P} is a character which can
1452         # appear in a path; ${M} is the four numeric metadata fields; and
1453         # ${H} is a sha256 hash.
1454         P="[-+./:=,%@_[~[:alnum:]]"
1455         M="[0-9]+\|[0-9]+\|[0-9]+\|[0-9]+"
1456         H="[0-9a-f]{64}"
1457
1458         # Check that the first four fields make sense.
1459         if gunzip -c < files/$1.gz |
1460             grep -qvE "^[a-z]+\|[0-9a-z-]+\|${P}+\|[fdL-]\|"; then
1461                 fetch_metadata_bogus ""
1462                 return 1
1463         fi
1464
1465         # Remove the first three fields.
1466         gunzip -c < files/$1.gz |
1467             cut -f 4- -d '|' > sanitycheck.tmp
1468
1469         # Sanity check entries with type 'f'
1470         if grep -E '^f' sanitycheck.tmp |
1471             grep -qvE "^f\|${M}\|${H}\|${P}*\$"; then
1472                 fetch_metadata_bogus ""
1473                 return 1
1474         fi
1475
1476         # Sanity check entries with type 'd'
1477         if grep -E '^d' sanitycheck.tmp |
1478             grep -qvE "^d\|${M}\|\|\$"; then
1479                 fetch_metadata_bogus ""
1480                 return 1
1481         fi
1482
1483         # Sanity check entries with type 'L'
1484         if grep -E '^L' sanitycheck.tmp |
1485             grep -qvE "^L\|${M}\|${P}*\|\$"; then
1486                 fetch_metadata_bogus ""
1487                 return 1
1488         fi
1489
1490         # Sanity check entries with type '-'
1491         if grep -E '^-' sanitycheck.tmp |
1492             grep -qvE "^-\|\|\|\|\|\|"; then
1493                 fetch_metadata_bogus ""
1494                 return 1
1495         fi
1496
1497         # Clean up
1498         rm sanitycheck.tmp
1499 }
1500
1501 # Fetch the metadata index and metadata files listed in $@,
1502 # taking advantage of metadata patches where possible.
1503 fetch_metadata () {
1504         fetch_metadata_index || return 1
1505         fetch_metadata_index_merge $@ || return 1
1506         fetch_metadata_index_sanity || return 1
1507
1508         # Generate a list of wanted metadata patches
1509         join -t '|' -o 1.2,2.2 tINDEX.present tINDEX.new |
1510             fetch_make_patchlist > patchlist
1511
1512         if [ -s patchlist ]; then
1513                 # Attempt to fetch metadata patches
1514                 echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
1515                 echo ${NDEBUG} "metadata patches.${DDSTATS}"
1516                 tr '|' '-' < patchlist |
1517                     lam -s "${FETCHDIR}/tp/" - -s ".gz" |
1518                     xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}   \
1519                         2>${STATSREDIR} | fetch_progress
1520                 echo "done."
1521
1522                 # Attempt to apply metadata patches
1523                 echo -n "Applying metadata patches... "
1524                 tr '|' ' ' < patchlist |
1525                     while read X Y; do
1526                         if [ ! -f "${X}-${Y}.gz" ]; then continue; fi
1527                         gunzip -c < ${X}-${Y}.gz > diff
1528                         gunzip -c < files/${X}.gz > diff-OLD
1529
1530                         # Figure out which lines are being added and removed
1531                         grep -E '^-' diff |
1532                             cut -c 2- |
1533                             while read PREFIX; do
1534                                 look "${PREFIX}" diff-OLD
1535                             done |
1536                             sort > diff-rm
1537                         grep -E '^\+' diff |
1538                             cut -c 2- > diff-add
1539
1540                         # Generate the new file
1541                         comm -23 diff-OLD diff-rm |
1542                             sort - diff-add > diff-NEW
1543
1544                         if [ `${SHA256} -q diff-NEW` = ${Y} ]; then
1545                                 mv diff-NEW files/${Y}
1546                                 gzip -n files/${Y}
1547                         else
1548                                 mv diff-NEW ${Y}.bad
1549                         fi
1550                         rm -f ${X}-${Y}.gz diff
1551                         rm -f diff-OLD diff-NEW diff-add diff-rm
1552                 done 2>${QUIETREDIR}
1553                 echo "done."
1554         fi
1555
1556         # Update metadata without patches
1557         cut -f 2 -d '|' < tINDEX.new |
1558             while read Y; do
1559                 if [ ! -f "files/${Y}.gz" ]; then
1560                         echo ${Y};
1561                 fi
1562             done |
1563             sort -u > filelist
1564
1565         if [ -s filelist ]; then
1566                 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
1567                 echo ${NDEBUG} "metadata files... "
1568                 lam -s "${FETCHDIR}/m/" - -s ".gz" < filelist |
1569                     xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}   \
1570                     2>${QUIETREDIR}
1571
1572                 while read Y; do
1573                         if ! [ -f ${Y}.gz ]; then
1574                                 echo "failed."
1575                                 return 1
1576                         fi
1577                         if [ `gunzip -c < ${Y}.gz |
1578                             ${SHA256} -q` = ${Y} ]; then
1579                                 mv ${Y}.gz files/${Y}.gz
1580                         else
1581                                 echo "metadata is corrupt."
1582                                 return 1
1583                         fi
1584                 done < filelist
1585                 echo "done."
1586         fi
1587
1588 # Sanity-check the metadata files.
1589         cut -f 2 -d '|' tINDEX.new > filelist
1590         while read X; do
1591                 fetch_metadata_sanity ${X} || return 1
1592         done < filelist
1593
1594 # Remove files which are no longer needed
1595         cut -f 2 -d '|' tINDEX.present |
1596             sort > oldfiles
1597         cut -f 2 -d '|' tINDEX.new |
1598             sort |
1599             comm -13 - oldfiles |
1600             lam -s "files/" - -s ".gz" |
1601             xargs rm -f
1602         rm patchlist filelist oldfiles
1603         rm ${TINDEXHASH}
1604
1605 # We're done!
1606         mv tINDEX.new tINDEX.present
1607         mv tag.new tag
1608
1609         return 0
1610 }
1611
1612 # Extract a subset of a downloaded metadata file containing only the parts
1613 # which are listed in COMPONENTS.
1614 fetch_filter_metadata_components () {
1615         METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
1616         gunzip -c < files/${METAHASH}.gz > $1.all
1617
1618         # Fish out the lines belonging to components we care about.
1619         for C in ${COMPONENTS}; do
1620                 look "`echo ${C} | tr '/' '|'`|" $1.all
1621         done > $1
1622
1623         # Remove temporary file.
1624         rm $1.all
1625 }
1626
1627 # Generate a filtered version of the metadata file $1 from the downloaded
1628 # file, by fishing out the lines corresponding to components we're trying
1629 # to keep updated, and then removing lines corresponding to paths we want
1630 # to ignore.
1631 fetch_filter_metadata () {
1632         # Fish out the lines belonging to components we care about.
1633         fetch_filter_metadata_components $1
1634
1635         # Canonicalize directory names by removing any trailing / in
1636         # order to avoid listing directories multiple times if they
1637         # belong to multiple components.  Turning "/" into "" doesn't
1638         # matter, since we add a leading "/" when we use paths later.
1639         cut -f 3- -d '|' $1 |
1640             sed -e 's,/|d|,|d|,' |
1641             sed -e 's,/|-|,|-|,' |
1642             sort -u > $1.tmp
1643
1644         # Figure out which lines to ignore and remove them.
1645         for X in ${IGNOREPATHS}; do
1646                 grep -E "^${X}" $1.tmp
1647         done |
1648             sort -u |
1649             comm -13 - $1.tmp > $1
1650
1651         # Remove temporary files.
1652         rm $1.tmp
1653 }
1654
1655 # Filter the metadata file $1 by adding lines with "/boot/$2"
1656 # replaced by ${KERNELDIR} (which is `sysctl -n kern.bootfile` minus the
1657 # trailing "/kernel"); and if "/boot/$2" does not exist, remove
1658 # the original lines which start with that.
1659 # Put another way: Deal with the fact that the FOO kernel is sometimes
1660 # installed in /boot/FOO/ and is sometimes installed elsewhere.
1661 fetch_filter_kernel_names () {
1662         grep ^/boot/$2 $1 |
1663             sed -e "s,/boot/$2,${KERNELDIR},g" |
1664             sort - $1 > $1.tmp
1665         mv $1.tmp $1
1666
1667         if ! [ -d /boot/$2 ]; then
1668                 grep -v ^/boot/$2 $1 > $1.tmp
1669                 mv $1.tmp $1
1670         fi
1671 }
1672
1673 # For all paths appearing in $1 or $3, inspect the system
1674 # and generate $2 describing what is currently installed.
1675 fetch_inspect_system () {
1676         # No errors yet...
1677         rm -f .err
1678
1679         # Tell the user why his disk is suddenly making lots of noise
1680         echo -n "Inspecting system... "
1681
1682         # Generate list of files to inspect
1683         cat $1 $3 |
1684             cut -f 1 -d '|' |
1685             sort -u > filelist
1686
1687         # Examine each file and output lines of the form
1688         # /path/to/file|type|device-inum|user|group|perm|flags|value
1689         # sorted by device and inode number.
1690         while read F; do
1691                 # If the symlink/file/directory does not exist, record this.
1692                 if ! [ -e ${BASEDIR}/${F} ]; then
1693                         echo "${F}|-||||||"
1694                         continue
1695                 fi
1696                 if ! [ -r ${BASEDIR}/${F} ]; then
1697                         echo "Cannot read file: ${BASEDIR}/${F}"        \
1698                             >/dev/stderr
1699                         touch .err
1700                         return 1
1701                 fi
1702
1703                 # Otherwise, output an index line.
1704                 if [ -L ${BASEDIR}/${F} ]; then
1705                         echo -n "${F}|L|"
1706                         stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1707                         readlink ${BASEDIR}/${F};
1708                 elif [ -f ${BASEDIR}/${F} ]; then
1709                         echo -n "${F}|f|"
1710                         stat -n -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1711                         sha256 -q ${BASEDIR}/${F};
1712                 elif [ -d ${BASEDIR}/${F} ]; then
1713                         echo -n "${F}|d|"
1714                         stat -f '%d-%i|%u|%g|%Mp%Lp|%Of|' ${BASEDIR}/${F};
1715                 else
1716                         echo "Unknown file type: ${BASEDIR}/${F}"       \
1717                             >/dev/stderr
1718                         touch .err
1719                         return 1
1720                 fi
1721         done < filelist |
1722             sort -k 3,3 -t '|' > $2.tmp
1723         rm filelist
1724
1725         # Check if an error occurred during system inspection
1726         if [ -f .err ]; then
1727                 return 1
1728         fi
1729
1730         # Convert to the form
1731         # /path/to/file|type|user|group|perm|flags|value|hlink
1732         # by resolving identical device and inode numbers into hard links.
1733         cut -f 1,3 -d '|' $2.tmp |
1734             sort -k 1,1 -t '|' |
1735             sort -s -u -k 2,2 -t '|' |
1736             join -1 2 -2 3 -t '|' - $2.tmp |
1737             awk -F \| -v OFS=\|         \
1738                 '{
1739                     if (($2 == $3) || ($4 == "-"))
1740                         print $3,$4,$5,$6,$7,$8,$9,""
1741                     else
1742                         print $3,$4,$5,$6,$7,$8,$9,$2
1743                 }' |
1744             sort > $2
1745         rm $2.tmp
1746
1747         # We're finished looking around
1748         echo "done."
1749 }
1750
1751 # For any paths matching ${MERGECHANGES}, compare $2 against $1 and $3 and
1752 # find any files with values unique to $2; generate $4 containing these paths
1753 # and their corresponding hashes from $1.
1754 fetch_filter_mergechanges () {
1755         # Pull out the paths and hashes of the files matching ${MERGECHANGES}.
1756         for F in $1 $2 $3; do
1757                 for X in ${MERGECHANGES}; do
1758                         grep -E "^${X}" ${F}
1759                 done |
1760                     cut -f 1,2,7 -d '|' |
1761                     sort > ${F}-values
1762         done
1763
1764         # Any line in $2-values which doesn't appear in $1-values or $3-values
1765         # and is a file means that we should list the path in $3.
1766         sort $1-values $3-values |
1767             comm -13 - $2-values |
1768             fgrep '|f|' |
1769             cut -f 1 -d '|' > $2-paths
1770
1771         # For each path, pull out one (and only one!) entry from $1-values.
1772         # Note that we cannot distinguish which "old" version the user made
1773         # changes to; but hopefully any changes which occur due to security
1774         # updates will exist in both the "new" version and the version which
1775         # the user has installed, so the merging will still work.
1776         while read X; do
1777                 look "${X}|" $1-values |
1778                     head -1
1779         done < $2-paths > $4
1780
1781         # Clean up
1782         rm $1-values $2-values $3-values $2-paths
1783 }
1784
1785 # For any paths matching ${UPDATEIFUNMODIFIED}, remove lines from $[123]
1786 # which correspond to lines in $2 with hashes not matching $1 or $3, unless
1787 # the paths are listed in $4.  For entries in $2 marked "not present"
1788 # (aka. type -), remove lines from $[123] unless there is a corresponding
1789 # entry in $1.
1790 fetch_filter_unmodified_notpresent () {
1791         # Figure out which lines of $1 and $3 correspond to bits which
1792         # should only be updated if they haven't changed, and fish out
1793         # the (path, type, value) tuples.
1794         # NOTE: We don't consider a file to be "modified" if it matches
1795         # the hash from $3.
1796         for X in ${UPDATEIFUNMODIFIED}; do
1797                 grep -E "^${X}" $1
1798                 grep -E "^${X}" $3
1799         done |
1800             cut -f 1,2,7 -d '|' |
1801             sort > $1-values
1802
1803         # Do the same for $2.
1804         for X in ${UPDATEIFUNMODIFIED}; do
1805                 grep -E "^${X}" $2
1806         done |
1807             cut -f 1,2,7 -d '|' |
1808             sort > $2-values
1809
1810         # Any entry in $2-values which is not in $1-values corresponds to
1811         # a path which we need to remove from $1, $2, and $3, unless it
1812         # that path appears in $4.
1813         comm -13 $1-values $2-values |
1814             sort -t '|' -k 1,1 > mlines.tmp
1815         cut -f 1 -d '|' $4 |
1816             sort |
1817             join -v 2 -t '|' - mlines.tmp |
1818             sort > mlines
1819         rm $1-values $2-values mlines.tmp
1820
1821         # Any lines in $2 which are not in $1 AND are "not present" lines
1822         # also belong in mlines.
1823         comm -13 $1 $2 |
1824             cut -f 1,2,7 -d '|' |
1825             fgrep '|-|' >> mlines
1826
1827         # Remove lines from $1, $2, and $3
1828         for X in $1 $2 $3; do
1829                 sort -t '|' -k 1,1 ${X} > ${X}.tmp
1830                 cut -f 1 -d '|' < mlines |
1831                     sort |
1832                     join -v 2 -t '|' - ${X}.tmp |
1833                     sort > ${X}
1834                 rm ${X}.tmp
1835         done
1836
1837         # Store a list of the modified files, for future reference
1838         fgrep -v '|-|' mlines |
1839             cut -f 1 -d '|' > modifiedfiles
1840         rm mlines
1841 }
1842
1843 # For each entry in $1 of type -, remove any corresponding
1844 # entry from $2 if ${ALLOWADD} != "yes".  Remove all entries
1845 # of type - from $1.
1846 fetch_filter_allowadd () {
1847         cut -f 1,2 -d '|' < $1 |
1848             fgrep '|-' |
1849             cut -f 1 -d '|' > filesnotpresent
1850
1851         if [ ${ALLOWADD} != "yes" ]; then
1852                 sort < $2 |
1853                     join -v 1 -t '|' - filesnotpresent |
1854                     sort > $2.tmp
1855                 mv $2.tmp $2
1856         fi
1857
1858         sort < $1 |
1859             join -v 1 -t '|' - filesnotpresent |
1860             sort > $1.tmp
1861         mv $1.tmp $1
1862         rm filesnotpresent
1863 }
1864
1865 # If ${ALLOWDELETE} != "yes", then remove any entries from $1
1866 # which don't correspond to entries in $2.
1867 fetch_filter_allowdelete () {
1868         # Produce a lists ${PATH}|${TYPE}
1869         for X in $1 $2; do
1870                 cut -f 1-2 -d '|' < ${X} |
1871                     sort -u > ${X}.nodes
1872         done
1873
1874         # Figure out which lines need to be removed from $1.
1875         if [ ${ALLOWDELETE} != "yes" ]; then
1876                 comm -23 $1.nodes $2.nodes > $1.badnodes
1877         else
1878                 : > $1.badnodes
1879         fi
1880
1881         # Remove the relevant lines from $1
1882         while read X; do
1883                 look "${X}|" $1
1884         done < $1.badnodes |
1885             comm -13 - $1 > $1.tmp
1886         mv $1.tmp $1
1887
1888         rm $1.badnodes $1.nodes $2.nodes
1889 }
1890
1891 # If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in $2
1892 # with metadata not matching any entry in $1, replace the corresponding
1893 # line of $3 with one having the same metadata as the entry in $2.
1894 fetch_filter_modified_metadata () {
1895         # Fish out the metadata from $1 and $2
1896         for X in $1 $2; do
1897                 cut -f 1-6 -d '|' < ${X} > ${X}.metadata
1898         done
1899
1900         # Find the metadata we need to keep
1901         if [ ${KEEPMODIFIEDMETADATA} = "yes" ]; then
1902                 comm -13 $1.metadata $2.metadata > keepmeta
1903         else
1904                 : > keepmeta
1905         fi
1906
1907         # Extract the lines which we need to remove from $3, and
1908         # construct the lines which we need to add to $3.
1909         : > $3.remove
1910         : > $3.add
1911         while read LINE; do
1912                 NODE=`echo "${LINE}" | cut -f 1-2 -d '|'`
1913                 look "${NODE}|" $3 >> $3.remove
1914                 look "${NODE}|" $3 |
1915                     cut -f 7- -d '|' |
1916                     lam -s "${LINE}|" - >> $3.add
1917         done < keepmeta
1918
1919         # Remove the specified lines and add the new lines.
1920         sort $3.remove |
1921             comm -13 - $3 |
1922             sort -u - $3.add > $3.tmp
1923         mv $3.tmp $3
1924
1925         rm keepmeta $1.metadata $2.metadata $3.add $3.remove
1926 }
1927
1928 # Remove lines from $1 and $2 which are identical;
1929 # no need to update a file if it isn't changing.
1930 fetch_filter_uptodate () {
1931         comm -23 $1 $2 > $1.tmp
1932         comm -13 $1 $2 > $2.tmp
1933
1934         mv $1.tmp $1
1935         mv $2.tmp $2
1936 }
1937
1938 # Fetch any "clean" old versions of files we need for merging changes.
1939 fetch_files_premerge () {
1940         # We only need to do anything if $1 is non-empty.
1941         if [ -s $1 ]; then
1942                 # Tell the user what we're doing
1943                 echo -n "Fetching files from ${OLDRELNUM} for merging... "
1944
1945                 # List of files wanted
1946                 fgrep '|f|' < $1 |
1947                     cut -f 3 -d '|' |
1948                     sort -u > files.wanted
1949
1950                 # Only fetch the files we don't already have
1951                 while read Y; do
1952                         if [ ! -f "files/${Y}.gz" ]; then
1953                                 echo ${Y};
1954                         fi
1955                 done < files.wanted > filelist
1956
1957                 # Actually fetch them
1958                 lam -s "${OLDFETCHDIR}/f/" - -s ".gz" < filelist |
1959                     xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}   \
1960                     2>${QUIETREDIR}
1961
1962                 # Make sure we got them all, and move them into /files/
1963                 while read Y; do
1964                         if ! [ -f ${Y}.gz ]; then
1965                                 echo "failed."
1966                                 return 1
1967                         fi
1968                         if [ `gunzip -c < ${Y}.gz |
1969                             ${SHA256} -q` = ${Y} ]; then
1970                                 mv ${Y}.gz files/${Y}.gz
1971                         else
1972                                 echo "${Y} has incorrect hash."
1973                                 return 1
1974                         fi
1975                 done < filelist
1976                 echo "done."
1977
1978                 # Clean up
1979                 rm filelist files.wanted
1980         fi
1981 }
1982
1983 # Prepare to fetch files: Generate a list of the files we need,
1984 # copy the unmodified files we have into /files/, and generate
1985 # a list of patches to download.
1986 fetch_files_prepare () {
1987         # Tell the user why his disk is suddenly making lots of noise
1988         echo -n "Preparing to download files... "
1989
1990         # Reduce indices to ${PATH}|${HASH} pairs
1991         for X in $1 $2 $3; do
1992                 cut -f 1,2,7 -d '|' < ${X} |
1993                     fgrep '|f|' |
1994                     cut -f 1,3 -d '|' |
1995                     sort > ${X}.hashes
1996         done
1997
1998         # List of files wanted
1999         cut -f 2 -d '|' < $3.hashes |
2000             sort -u |
2001             while read HASH; do
2002                 if ! [ -f files/${HASH}.gz ]; then
2003                         echo ${HASH}
2004                 fi
2005         done > files.wanted
2006
2007         # Generate a list of unmodified files
2008         comm -12 $1.hashes $2.hashes |
2009             sort -k 1,1 -t '|' > unmodified.files
2010
2011         # Copy all files into /files/.  We only need the unmodified files
2012         # for use in patching; but we'll want all of them if the user asks
2013         # to rollback the updates later.
2014         while read LINE; do
2015                 F=`echo "${LINE}" | cut -f 1 -d '|'`
2016                 HASH=`echo "${LINE}" | cut -f 2 -d '|'`
2017
2018                 # Skip files we already have.
2019                 if [ -f files/${HASH}.gz ]; then
2020                         continue
2021                 fi
2022
2023                 # Make sure the file hasn't changed.
2024                 cp "${BASEDIR}/${F}" tmpfile
2025                 if [ `sha256 -q tmpfile` != ${HASH} ]; then
2026                         echo
2027                         echo "File changed while FreeBSD Update running: ${F}"
2028                         return 1
2029                 fi
2030
2031                 # Place the file into storage.
2032                 gzip -c < tmpfile > files/${HASH}.gz
2033                 rm tmpfile
2034         done < $2.hashes
2035
2036         # Produce a list of patches to download
2037         sort -k 1,1 -t '|' $3.hashes |
2038             join -t '|' -o 2.2,1.2 - unmodified.files |
2039             fetch_make_patchlist > patchlist
2040
2041         # Garbage collect
2042         rm unmodified.files $1.hashes $2.hashes $3.hashes
2043
2044         # We don't need the list of possible old files any more.
2045         rm $1
2046
2047         # We're finished making noise
2048         echo "done."
2049 }
2050
2051 # Fetch files.
2052 fetch_files () {
2053         # Attempt to fetch patches
2054         if [ -s patchlist ]; then
2055                 echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
2056                 echo ${NDEBUG} "patches.${DDSTATS}"
2057                 tr '|' '-' < patchlist |
2058                     lam -s "${PATCHDIR}/" - |
2059                     xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}   \
2060                         2>${STATSREDIR} | fetch_progress
2061                 echo "done."
2062
2063                 # Attempt to apply patches
2064                 echo -n "Applying patches... "
2065                 tr '|' ' ' < patchlist |
2066                     while read X Y; do
2067                         if [ ! -f "${X}-${Y}" ]; then continue; fi
2068                         gunzip -c < files/${X}.gz > OLD
2069
2070                         bspatch OLD NEW ${X}-${Y}
2071
2072                         if [ `${SHA256} -q NEW` = ${Y} ]; then
2073                                 mv NEW files/${Y}
2074                                 gzip -n files/${Y}
2075                         fi
2076                         rm -f diff OLD NEW ${X}-${Y}
2077                 done 2>${QUIETREDIR}
2078                 echo "done."
2079         fi
2080
2081         # Download files which couldn't be generate via patching
2082         while read Y; do
2083                 if [ ! -f "files/${Y}.gz" ]; then
2084                         echo ${Y};
2085                 fi
2086         done < files.wanted > filelist
2087
2088         if [ -s filelist ]; then
2089                 echo -n "Fetching `wc -l < filelist | tr -d ' '` "
2090                 echo ${NDEBUG} "files... "
2091                 lam -s "${FETCHDIR}/f/" - -s ".gz" < filelist |
2092                     xargs ${XARGST} ${PHTTPGET} ${SERVERNAME}   \
2093                         2>${STATSREDIR} | fetch_progress
2094
2095                 while read Y; do
2096                         if ! [ -f ${Y}.gz ]; then
2097                                 echo "failed."
2098                                 return 1
2099                         fi
2100                         if [ `gunzip -c < ${Y}.gz |
2101                             ${SHA256} -q` = ${Y} ]; then
2102                                 mv ${Y}.gz files/${Y}.gz
2103                         else
2104                                 echo "${Y} has incorrect hash."
2105                                 return 1
2106                         fi
2107                 done < filelist
2108                 echo "done."
2109         fi
2110
2111         # Clean up
2112         rm files.wanted filelist patchlist
2113 }
2114
2115 # Create and populate install manifest directory; and report what updates
2116 # are available.
2117 fetch_create_manifest () {
2118         # If we have an existing install manifest, nuke it.
2119         if [ -L "${BDHASH}-install" ]; then
2120                 rm -r ${BDHASH}-install/
2121                 rm ${BDHASH}-install
2122         fi
2123
2124         # Report to the user if any updates were avoided due to local changes
2125         if [ -s modifiedfiles ]; then
2126                 cat - modifiedfiles <<- EOF | ${PAGER}
2127                         The following files are affected by updates. No changes have
2128                         been downloaded, however, because the files have been modified
2129                         locally:
2130                 EOF
2131         fi
2132         rm modifiedfiles
2133
2134         # If no files will be updated, tell the user and exit
2135         if ! [ -s INDEX-PRESENT ] &&
2136             ! [ -s INDEX-NEW ]; then
2137                 rm INDEX-PRESENT INDEX-NEW
2138                 echo
2139                 echo -n "No updates needed to update system to "
2140                 echo "${RELNUM}-p${RELPATCHNUM}."
2141                 return
2142         fi
2143
2144         # Divide files into (a) removed files, (b) added files, and
2145         # (c) updated files.
2146         cut -f 1 -d '|' < INDEX-PRESENT |
2147             sort > INDEX-PRESENT.flist
2148         cut -f 1 -d '|' < INDEX-NEW |
2149             sort > INDEX-NEW.flist
2150         comm -23 INDEX-PRESENT.flist INDEX-NEW.flist > files.removed
2151         comm -13 INDEX-PRESENT.flist INDEX-NEW.flist > files.added
2152         comm -12 INDEX-PRESENT.flist INDEX-NEW.flist > files.updated
2153         rm INDEX-PRESENT.flist INDEX-NEW.flist
2154
2155         # Report removed files, if any
2156         if [ -s files.removed ]; then
2157                 cat - files.removed <<- EOF | ${PAGER}
2158                         The following files will be removed as part of updating to
2159                         ${RELNUM}-p${RELPATCHNUM}:
2160                 EOF
2161         fi
2162         rm files.removed
2163
2164         # Report added files, if any
2165         if [ -s files.added ]; then
2166                 cat - files.added <<- EOF | ${PAGER}
2167                         The following files will be added as part of updating to
2168                         ${RELNUM}-p${RELPATCHNUM}:
2169                 EOF
2170         fi
2171         rm files.added
2172
2173         # Report updated files, if any
2174         if [ -s files.updated ]; then
2175                 cat - files.updated <<- EOF | ${PAGER}
2176                         The following files will be updated as part of updating to
2177                         ${RELNUM}-p${RELPATCHNUM}:
2178                 EOF
2179         fi
2180         rm files.updated
2181
2182         # Create a directory for the install manifest.
2183         MDIR=`mktemp -d install.XXXXXX` || return 1
2184
2185         # Populate it
2186         mv INDEX-PRESENT ${MDIR}/INDEX-OLD
2187         mv INDEX-NEW ${MDIR}/INDEX-NEW
2188
2189         # Link it into place
2190         ln -s ${MDIR} ${BDHASH}-install
2191 }
2192
2193 # Warn about any upcoming EoL
2194 fetch_warn_eol () {
2195         # What's the current time?
2196         NOWTIME=`date "+%s"`
2197
2198         # When did we last warn about the EoL date?
2199         if [ -f lasteolwarn ]; then
2200                 LASTWARN=`cat lasteolwarn`
2201         else
2202                 LASTWARN=`expr ${NOWTIME} - 63072000`
2203         fi
2204
2205         # If the EoL time is past, warn.
2206         if [ ${EOLTIME} -lt ${NOWTIME} ]; then
2207                 echo
2208                 cat <<-EOF
2209                 WARNING: `uname -sr` HAS PASSED ITS END-OF-LIFE DATE.
2210                 Any security issues discovered after `date -r ${EOLTIME}`
2211                 will not have been corrected.
2212                 EOF
2213                 return 1
2214         fi
2215
2216         # Figure out how long it has been since we last warned about the
2217         # upcoming EoL, and how much longer we have left.
2218         SINCEWARN=`expr ${NOWTIME} - ${LASTWARN}`
2219         TIMELEFT=`expr ${EOLTIME} - ${NOWTIME}`
2220
2221         # Don't warn if the EoL is more than 3 months away
2222         if [ ${TIMELEFT} -gt 7884000 ]; then
2223                 return 0
2224         fi
2225
2226         # Don't warn if the time remaining is more than 3 times the time
2227         # since the last warning.
2228         if [ ${TIMELEFT} -gt `expr ${SINCEWARN} \* 3` ]; then
2229                 return 0
2230         fi
2231
2232         # Figure out what time units to use.
2233         if [ ${TIMELEFT} -lt 604800 ]; then
2234                 UNIT="day"
2235                 SIZE=86400
2236         elif [ ${TIMELEFT} -lt 2678400 ]; then
2237                 UNIT="week"
2238                 SIZE=604800
2239         else
2240                 UNIT="month"
2241                 SIZE=2678400
2242         fi
2243
2244         # Compute the right number of units
2245         NUM=`expr ${TIMELEFT} / ${SIZE}`
2246         if [ ${NUM} != 1 ]; then
2247                 UNIT="${UNIT}s"
2248         fi
2249
2250         # Print the warning
2251         echo
2252         cat <<-EOF
2253                 WARNING: `uname -sr` is approaching its End-of-Life date.
2254                 It is strongly recommended that you upgrade to a newer
2255                 release within the next ${NUM} ${UNIT}.
2256         EOF
2257
2258         # Update the stored time of last warning
2259         echo ${NOWTIME} > lasteolwarn
2260 }
2261
2262 # Do the actual work involved in "fetch" / "cron".
2263 fetch_run () {
2264         workdir_init || return 1
2265
2266         # Prepare the mirror list.
2267         fetch_pick_server_init && fetch_pick_server
2268
2269         # Try to fetch the public key until we run out of servers.
2270         while ! fetch_key; do
2271                 fetch_pick_server || return 1
2272         done
2273
2274         # Try to fetch the metadata index signature ("tag") until we run
2275         # out of available servers; and sanity check the downloaded tag.
2276         while ! fetch_tag; do
2277                 fetch_pick_server || return 1
2278         done
2279         fetch_tagsanity || return 1
2280
2281         # Fetch the latest INDEX-NEW and INDEX-OLD files.
2282         fetch_metadata INDEX-NEW INDEX-OLD || return 1
2283
2284         # Generate filtered INDEX-NEW and INDEX-OLD files containing only
2285         # the lines which (a) belong to components we care about, and (b)
2286         # don't correspond to paths we're explicitly ignoring.
2287         fetch_filter_metadata INDEX-NEW || return 1
2288         fetch_filter_metadata INDEX-OLD || return 1
2289
2290         # Translate /boot/${KERNCONF} into ${KERNELDIR}
2291         fetch_filter_kernel_names INDEX-NEW ${KERNCONF}
2292         fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2293
2294         # For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2295         # system and generate an INDEX-PRESENT file.
2296         fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2297
2298         # Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2299         # correspond to lines in INDEX-PRESENT with hashes not appearing
2300         # in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2301         # INDEX-PRESENT has type - and there isn't a corresponding entry in
2302         # INDEX-OLD with type -.
2303         fetch_filter_unmodified_notpresent      \
2304             INDEX-OLD INDEX-PRESENT INDEX-NEW /dev/null
2305
2306         # For each entry in INDEX-PRESENT of type -, remove any corresponding
2307         # entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2308         # of type - from INDEX-PRESENT.
2309         fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2310
2311         # If ${ALLOWDELETE} != "yes", then remove any entries from
2312         # INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2313         fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2314
2315         # If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2316         # INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2317         # replace the corresponding line of INDEX-NEW with one having the
2318         # same metadata as the entry in INDEX-PRESENT.
2319         fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2320
2321         # Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2322         # no need to update a file if it isn't changing.
2323         fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2324
2325         # Prepare to fetch files: Generate a list of the files we need,
2326         # copy the unmodified files we have into /files/, and generate
2327         # a list of patches to download.
2328         fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2329
2330         # Fetch files.
2331         fetch_files || return 1
2332
2333         # Create and populate install manifest directory; and report what
2334         # updates are available.
2335         fetch_create_manifest || return 1
2336
2337         # Warn about any upcoming EoL
2338         fetch_warn_eol || return 1
2339 }
2340
2341 # If StrictComponents is not "yes", generate a new components list
2342 # with only the components which appear to be installed.
2343 upgrade_guess_components () {
2344         if [ "${STRICTCOMPONENTS}" = "no" ]; then
2345                 # Generate filtered INDEX-ALL with only the components listed
2346                 # in COMPONENTS.
2347                 fetch_filter_metadata_components $1 || return 1
2348
2349                 # Tell the user why his disk is suddenly making lots of noise
2350                 echo -n "Inspecting system... "
2351
2352                 # Look at the files on disk, and assume that a component is
2353                 # supposed to be present if it is more than half-present.
2354                 cut -f 1-3 -d '|' < INDEX-ALL |
2355                     tr '|' ' ' |
2356                     while read C S F; do
2357                         if [ -e ${BASEDIR}/${F} ]; then
2358                                 echo "+ ${C}|${S}"
2359                         fi
2360                         echo "= ${C}|${S}"
2361                     done |
2362                     sort |
2363                     uniq -c |
2364                     sed -E 's,^ +,,' > compfreq
2365                 grep ' = ' compfreq |
2366                     cut -f 1,3 -d ' ' |
2367                     sort -k 2,2 -t ' ' > compfreq.total
2368                 grep ' + ' compfreq |
2369                     cut -f 1,3 -d ' ' |
2370                     sort -k 2,2 -t ' ' > compfreq.present
2371                 join -t ' ' -1 2 -2 2 compfreq.present compfreq.total |
2372                     while read S P T; do
2373                         if [ ${T} -ne 0 -a ${P} -gt `expr ${T} / 2` ]; then
2374                                 echo ${S}
2375                         fi
2376                     done > comp.present
2377                 cut -f 2 -d ' ' < compfreq.total > comp.total
2378                 rm INDEX-ALL compfreq compfreq.total compfreq.present
2379
2380                 # We're done making noise.
2381                 echo "done."
2382
2383                 # Sometimes the kernel isn't installed where INDEX-ALL
2384                 # thinks that it should be: In particular, it is often in
2385                 # /boot/kernel instead of /boot/GENERIC or /boot/SMP.  To
2386                 # deal with this, if "kernel|X" is listed in comp.total
2387                 # (i.e., is a component which would be upgraded if it is
2388                 # found to be present) we will add it to comp.present.
2389                 # If "kernel|<anything>" is in comp.total but "kernel|X" is
2390                 # not, we print a warning -- the user is running a kernel
2391                 # which isn't part of the release.
2392                 KCOMP=`echo ${KERNCONF} | tr 'A-Z' 'a-z'`
2393                 grep -E "^kernel\|${KCOMP}\$" comp.total >> comp.present
2394
2395                 if grep -qE "^kernel\|" comp.total &&
2396                     ! grep -qE "^kernel\|${KCOMP}\$" comp.total; then
2397                         cat <<-EOF
2398
2399 WARNING: This system is running a "${KCOMP}" kernel, which is not a
2400 kernel configuration distributed as part of FreeBSD ${RELNUM}.
2401 This kernel will not be updated: you MUST update the kernel manually
2402 before running '`basename $0` [options] install'.
2403                         EOF
2404                 fi
2405
2406                 # Re-sort the list of installed components and generate
2407                 # the list of non-installed components.
2408                 sort -u < comp.present > comp.present.tmp
2409                 mv comp.present.tmp comp.present
2410                 comm -13 comp.present comp.total > comp.absent
2411
2412                 # Ask the user to confirm that what we have is correct.  To
2413                 # reduce user confusion, translate "X|Y" back to "X/Y" (as
2414                 # subcomponents must be listed in the configuration file).
2415                 echo
2416                 echo -n "The following components of FreeBSD "
2417                 echo "seem to be installed:"
2418                 tr '|' '/' < comp.present |
2419                     fmt -72
2420                 echo
2421                 echo -n "The following components of FreeBSD "
2422                 echo "do not seem to be installed:"
2423                 tr '|' '/' < comp.absent |
2424                     fmt -72
2425                 echo
2426                 continuep || return 1
2427                 echo
2428
2429                 # Suck the generated list of components into ${COMPONENTS}.
2430                 # Note that comp.present.tmp is used due to issues with
2431                 # pipelines and setting variables.
2432                 COMPONENTS=""
2433                 tr '|' '/' < comp.present > comp.present.tmp
2434                 while read C; do
2435                         COMPONENTS="${COMPONENTS} ${C}"
2436                 done < comp.present.tmp
2437
2438                 # Delete temporary files
2439                 rm comp.present comp.present.tmp comp.absent comp.total
2440         fi
2441 }
2442
2443 # If StrictComponents is not "yes", COMPONENTS contains an entry
2444 # corresponding to the currently running kernel, and said kernel
2445 # does not exist in the new release, add "kernel/generic" to the
2446 # list of components.
2447 upgrade_guess_new_kernel () {
2448         if [ "${STRICTCOMPONENTS}" = "no" ]; then
2449                 # Grab the unfiltered metadata file.
2450                 METAHASH=`look "$1|" tINDEX.present | cut -f 2 -d '|'`
2451                 gunzip -c < files/${METAHASH}.gz > $1.all
2452
2453                 # If "kernel/${KCOMP}" is in ${COMPONENTS} and that component
2454                 # isn't in $1.all, we need to add kernel/generic.
2455                 for C in ${COMPONENTS}; do
2456                         if [ ${C} = "kernel/${KCOMP}" ] &&
2457                             ! grep -qE "^kernel\|${KCOMP}\|" $1.all; then
2458                                 COMPONENTS="${COMPONENTS} kernel/generic"
2459                                 NKERNCONF="GENERIC"
2460                                 cat <<-EOF
2461
2462 WARNING: This system is running a "${KCOMP}" kernel, which is not a
2463 kernel configuration distributed as part of FreeBSD ${RELNUM}.
2464 As part of upgrading to FreeBSD ${RELNUM}, this kernel will be
2465 replaced with a "generic" kernel.
2466                                 EOF
2467                                 continuep || return 1
2468                         fi
2469                 done
2470
2471                 # Don't need this any more...
2472                 rm $1.all
2473         fi
2474 }
2475
2476 # Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2477 # INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2478 upgrade_oldall_to_oldnew () {
2479         # For each ${F}|... which appears in INDEX-ALL but does not appear
2480         # in INDEX-OLD, add ${F}|-|||||| to INDEX-OLD.
2481         cut -f 1 -d '|' < $1 |
2482             sort -u > $1.paths
2483         cut -f 1 -d '|' < $2 |
2484             sort -u |
2485             comm -13 $1.paths - |
2486             lam - -s "|-||||||" |
2487             sort - $1 > $1.tmp
2488         mv $1.tmp $1
2489
2490         # Remove lines from INDEX-OLD which also appear in INDEX-ALL
2491         comm -23 $1 $2 > $1.tmp
2492         mv $1.tmp $1
2493
2494         # Remove lines from INDEX-ALL which have a file name not appearing
2495         # anywhere in INDEX-OLD (since these must be files which haven't
2496         # changed -- if they were new, there would be an entry of type "-").
2497         cut -f 1 -d '|' < $1 |
2498             sort -u > $1.paths
2499         sort -k 1,1 -t '|' < $2 |
2500             join -t '|' - $1.paths |
2501             sort > $2.tmp
2502         rm $1.paths
2503         mv $2.tmp $2
2504
2505         # Rename INDEX-ALL to INDEX-NEW.
2506         mv $2 $3
2507 }
2508
2509 # Helper for upgrade_merge: Return zero true iff the two files differ only
2510 # in the contents of their RCS tags.
2511 samef () {
2512         X=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $1 | ${SHA256}`
2513         Y=`sed -E 's/\\$FreeBSD.*\\$/\$FreeBSD\$/' < $2 | ${SHA256}`
2514
2515         if [ $X = $Y ]; then
2516                 return 0;
2517         else
2518                 return 1;
2519         fi
2520 }
2521
2522 # From the list of "old" files in $1, merge changes in $2 with those in $3,
2523 # and update $3 to reflect the hashes of merged files.
2524 upgrade_merge () {
2525         # We only need to do anything if $1 is non-empty.
2526         if [ -s $1 ]; then
2527                 cut -f 1 -d '|' $1 |
2528                     sort > $1-paths
2529
2530                 # Create staging area for merging files
2531                 rm -rf merge/
2532                 while read F; do
2533                         D=`dirname ${F}`
2534                         mkdir -p merge/old/${D}
2535                         mkdir -p merge/${OLDRELNUM}/${D}
2536                         mkdir -p merge/${RELNUM}/${D}
2537                         mkdir -p merge/new/${D}
2538                 done < $1-paths
2539
2540                 # Copy in files
2541                 while read F; do
2542                         # Currently installed file
2543                         V=`look "${F}|" $2 | cut -f 7 -d '|'`
2544                         gunzip < files/${V}.gz > merge/old/${F}
2545
2546                         # Old release
2547                         if look "${F}|" $1 | fgrep -q "|f|"; then
2548                                 V=`look "${F}|" $1 | cut -f 3 -d '|'`
2549                                 gunzip < files/${V}.gz          \
2550                                     > merge/${OLDRELNUM}/${F}
2551                         fi
2552
2553                         # New release
2554                         if look "${F}|" $3 | cut -f 1,2,7 -d '|' |
2555                             fgrep -q "|f|"; then
2556                                 V=`look "${F}|" $3 | cut -f 7 -d '|'`
2557                                 gunzip < files/${V}.gz          \
2558                                     > merge/${RELNUM}/${F}
2559                         fi
2560                 done < $1-paths
2561
2562                 # Attempt to automatically merge changes
2563                 echo -n "Attempting to automatically merge "
2564                 echo -n "changes in files..."
2565                 : > failed.merges
2566                 while read F; do
2567                         # If the file doesn't exist in the new release,
2568                         # the result of "merging changes" is having the file
2569                         # not exist.
2570                         if ! [ -f merge/${RELNUM}/${F} ]; then
2571                                 continue
2572                         fi
2573
2574                         # If the file didn't exist in the old release, we're
2575                         # going to throw away the existing file and hope that
2576                         # the version from the new release is what we want.
2577                         if ! [ -f merge/${OLDRELNUM}/${F} ]; then
2578                                 cp merge/${RELNUM}/${F} merge/new/${F}
2579                                 continue
2580                         fi
2581
2582                         # Some files need special treatment.
2583                         case ${F} in
2584                         /etc/spwd.db | /etc/pwd.db | /etc/login.conf.db)
2585                                 # Don't merge these -- we're rebuild them
2586                                 # after updates are installed.
2587                                 cp merge/old/${F} merge/new/${F}
2588                                 ;;
2589                         *)
2590                                 if ! diff3 -E -m -L "current version"   \
2591                                     -L "${OLDRELNUM}" -L "${RELNUM}"    \
2592                                     merge/old/${F}                      \
2593                                     merge/${OLDRELNUM}/${F}             \
2594                                     merge/${RELNUM}/${F}                \
2595                                     > merge/new/${F} 2>/dev/null; then
2596                                         echo ${F} >> failed.merges
2597                                 fi
2598                                 ;;
2599                         esac
2600                 done < $1-paths
2601                 echo " done."
2602
2603                 # Ask the user to handle any files which didn't merge.
2604                 while read F; do
2605                         # If the installed file differs from the version in
2606                         # the old release only due to RCS tag expansion
2607                         # then just use the version in the new release.
2608                         if samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2609                                 cp merge/${RELNUM}/${F} merge/new/${F}
2610                                 continue
2611                         fi
2612
2613                         cat <<-EOF
2614
2615 The following file could not be merged automatically: ${F}
2616 Press Enter to edit this file in ${EDITOR} and resolve the conflicts
2617 manually...
2618                         EOF
2619                         while true; do
2620                                 read response </dev/tty
2621                                 if expr "${response}" : '[Aa][Cc][Cc][Ee][Pp][Tt]' > /dev/null; then
2622                                         echo
2623                                         break
2624                                 fi
2625                                 ${EDITOR} `pwd`/merge/new/${F} < /dev/tty
2626
2627                                 if ! grep -qE '^(<<<<<<<|=======|>>>>>>>)([[:space:]].*)?$' $(pwd)/merge/new/${F} ; then
2628                                         break
2629                                 fi
2630                                 cat <<-EOF
2631
2632 Merge conflict markers remain in: ${F}
2633 These must be resolved for the system to be functional.
2634
2635 Press Enter to return to editing this file, or type "ACCEPT" to carry on with
2636 these lines remaining in the file.
2637                                 EOF
2638                         done
2639                 done < failed.merges
2640                 rm failed.merges
2641
2642                 # Ask the user to confirm that he likes how the result
2643                 # of merging files.
2644                 while read F; do
2645                         # Skip files which haven't changed except possibly
2646                         # in their RCS tags.
2647                         if [ -f merge/old/${F} ] && [ -f merge/new/${F} ] &&
2648                             samef merge/old/${F} merge/new/${F}; then
2649                                 continue
2650                         fi
2651
2652                         # Skip files where the installed file differs from
2653                         # the old file only due to RCS tags.
2654                         if [ -f merge/old/${F} ] &&
2655                             [ -f merge/${OLDRELNUM}/${F} ] &&
2656                             samef merge/old/${F} merge/${OLDRELNUM}/${F}; then
2657                                 continue
2658                         fi
2659
2660                         # Warn about files which are ceasing to exist.
2661                         if ! [ -f merge/new/${F} ]; then
2662                                 cat <<-EOF
2663
2664 The following file will be removed, as it no longer exists in
2665 FreeBSD ${RELNUM}: ${F}
2666                                 EOF
2667                                 continuep < /dev/tty || return 1
2668                                 continue
2669                         fi
2670
2671                         # Print changes for the user's approval.
2672                         cat <<-EOF
2673
2674 The following changes, which occurred between FreeBSD ${OLDRELNUM} and
2675 FreeBSD ${RELNUM} have been merged into ${F}:
2676 EOF
2677                         diff -U 5 -L "current version" -L "new version" \
2678                             merge/old/${F} merge/new/${F} || true
2679                         continuep < /dev/tty || return 1
2680                 done < $1-paths
2681
2682                 # Store merged files.
2683                 while read F; do
2684                         if [ -f merge/new/${F} ]; then
2685                                 V=`${SHA256} -q merge/new/${F}`
2686
2687                                 gzip -c < merge/new/${F} > files/${V}.gz
2688                                 echo "${F}|${V}"
2689                         fi
2690                 done < $1-paths > newhashes
2691
2692                 # Pull lines out from $3 which need to be updated to
2693                 # reflect merged files.
2694                 while read F; do
2695                         look "${F}|" $3
2696                 done < $1-paths > $3-oldlines
2697
2698                 # Update lines to reflect merged files
2699                 join -t '|' -o 1.1,1.2,1.3,1.4,1.5,1.6,2.2,1.8          \
2700                     $3-oldlines newhashes > $3-newlines
2701
2702                 # Remove old lines from $3 and add new lines.
2703                 sort $3-oldlines |
2704                     comm -13 - $3 |
2705                     sort - $3-newlines > $3.tmp
2706                 mv $3.tmp $3
2707
2708                 # Clean up
2709                 rm $1-paths newhashes $3-oldlines $3-newlines
2710                 rm -rf merge/
2711         fi
2712
2713         # We're done with merging files.
2714         rm $1
2715 }
2716
2717 # Do the work involved in fetching upgrades to a new release
2718 upgrade_run () {
2719         workdir_init || return 1
2720
2721         # Prepare the mirror list.
2722         fetch_pick_server_init && fetch_pick_server
2723
2724         # Try to fetch the public key until we run out of servers.
2725         while ! fetch_key; do
2726                 fetch_pick_server || return 1
2727         done
2728
2729         # Try to fetch the metadata index signature ("tag") until we run
2730         # out of available servers; and sanity check the downloaded tag.
2731         while ! fetch_tag; do
2732                 fetch_pick_server || return 1
2733         done
2734         fetch_tagsanity || return 1
2735
2736         # Fetch the INDEX-OLD and INDEX-ALL.
2737         fetch_metadata INDEX-OLD INDEX-ALL || return 1
2738
2739         # If StrictComponents is not "yes", generate a new components list
2740         # with only the components which appear to be installed.
2741         upgrade_guess_components INDEX-ALL || return 1
2742
2743         # Generate filtered INDEX-OLD and INDEX-ALL files containing only
2744         # the components we want and without anything marked as "Ignore".
2745         fetch_filter_metadata INDEX-OLD || return 1
2746         fetch_filter_metadata INDEX-ALL || return 1
2747
2748         # Merge the INDEX-OLD and INDEX-ALL files into INDEX-OLD.
2749         sort INDEX-OLD INDEX-ALL > INDEX-OLD.tmp
2750         mv INDEX-OLD.tmp INDEX-OLD
2751         rm INDEX-ALL
2752
2753         # Adjust variables for fetching files from the new release.
2754         OLDRELNUM=${RELNUM}
2755         RELNUM=${TARGETRELEASE}
2756         OLDFETCHDIR=${FETCHDIR}
2757         FETCHDIR=${RELNUM}/${ARCH}
2758
2759         # Try to fetch the NEW metadata index signature ("tag") until we run
2760         # out of available servers; and sanity check the downloaded tag.
2761         while ! fetch_tag; do
2762                 fetch_pick_server || return 1
2763         done
2764
2765         # Fetch the new INDEX-ALL.
2766         fetch_metadata INDEX-ALL || return 1
2767
2768         # If StrictComponents is not "yes", COMPONENTS contains an entry
2769         # corresponding to the currently running kernel, and said kernel
2770         # does not exist in the new release, add "kernel/generic" to the
2771         # list of components.
2772         upgrade_guess_new_kernel INDEX-ALL || return 1
2773
2774         # Filter INDEX-ALL to contain only the components we want and without
2775         # anything marked as "Ignore".
2776         fetch_filter_metadata INDEX-ALL || return 1
2777
2778         # Convert INDEX-OLD (last release) and INDEX-ALL (new release) into
2779         # INDEX-OLD and INDEX-NEW files (in the sense of normal upgrades).
2780         upgrade_oldall_to_oldnew INDEX-OLD INDEX-ALL INDEX-NEW
2781
2782         # Translate /boot/${KERNCONF} or /boot/${NKERNCONF} into ${KERNELDIR}
2783         fetch_filter_kernel_names INDEX-NEW ${NKERNCONF}
2784         fetch_filter_kernel_names INDEX-OLD ${KERNCONF}
2785
2786         # For all paths appearing in INDEX-OLD or INDEX-NEW, inspect the
2787         # system and generate an INDEX-PRESENT file.
2788         fetch_inspect_system INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2789
2790         # Based on ${MERGECHANGES}, generate a file tomerge-old with the
2791         # paths and hashes of old versions of files to merge.
2792         fetch_filter_mergechanges INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2793
2794         # Based on ${UPDATEIFUNMODIFIED}, remove lines from INDEX-* which
2795         # correspond to lines in INDEX-PRESENT with hashes not appearing
2796         # in INDEX-OLD or INDEX-NEW.  Also remove lines where the entry in
2797         # INDEX-PRESENT has type - and there isn't a corresponding entry in
2798         # INDEX-OLD with type -.
2799         fetch_filter_unmodified_notpresent      \
2800             INDEX-OLD INDEX-PRESENT INDEX-NEW tomerge-old
2801
2802         # For each entry in INDEX-PRESENT of type -, remove any corresponding
2803         # entry from INDEX-NEW if ${ALLOWADD} != "yes".  Remove all entries
2804         # of type - from INDEX-PRESENT.
2805         fetch_filter_allowadd INDEX-PRESENT INDEX-NEW
2806
2807         # If ${ALLOWDELETE} != "yes", then remove any entries from
2808         # INDEX-PRESENT which don't correspond to entries in INDEX-NEW.
2809         fetch_filter_allowdelete INDEX-PRESENT INDEX-NEW
2810
2811         # If ${KEEPMODIFIEDMETADATA} == "yes", then for each entry in
2812         # INDEX-PRESENT with metadata not matching any entry in INDEX-OLD,
2813         # replace the corresponding line of INDEX-NEW with one having the
2814         # same metadata as the entry in INDEX-PRESENT.
2815         fetch_filter_modified_metadata INDEX-OLD INDEX-PRESENT INDEX-NEW
2816
2817         # Remove lines from INDEX-PRESENT and INDEX-NEW which are identical;
2818         # no need to update a file if it isn't changing.
2819         fetch_filter_uptodate INDEX-PRESENT INDEX-NEW
2820
2821         # Fetch "clean" files from the old release for merging changes.
2822         fetch_files_premerge tomerge-old
2823
2824         # Prepare to fetch files: Generate a list of the files we need,
2825         # copy the unmodified files we have into /files/, and generate
2826         # a list of patches to download.
2827         fetch_files_prepare INDEX-OLD INDEX-PRESENT INDEX-NEW || return 1
2828
2829         # Fetch patches from to-${RELNUM}/${ARCH}/bp/
2830         PATCHDIR=to-${RELNUM}/${ARCH}/bp
2831         fetch_files || return 1
2832
2833         # Merge configuration file changes.
2834         upgrade_merge tomerge-old INDEX-PRESENT INDEX-NEW || return 1
2835
2836         # Create and populate install manifest directory; and report what
2837         # updates are available.
2838         fetch_create_manifest || return 1
2839
2840         # Leave a note behind to tell the "install" command that the kernel
2841         # needs to be installed before the world.
2842         touch ${BDHASH}-install/kernelfirst
2843
2844         # Remind the user that they need to run "freebsd-update install"
2845         # to install the downloaded bits, in case they didn't RTFM.
2846         echo "To install the downloaded upgrades, run '`basename $0` [options] install'."
2847 }
2848
2849 # Make sure that all the file hashes mentioned in $@ have corresponding
2850 # gzipped files stored in /files/.
2851 install_verify () {
2852         # Generate a list of hashes
2853         cat $@ |
2854             cut -f 2,7 -d '|' |
2855             grep -E '^f' |
2856             cut -f 2 -d '|' |
2857             sort -u > filelist
2858
2859         # Make sure all the hashes exist
2860         while read HASH; do
2861                 if ! [ -f files/${HASH}.gz ]; then
2862                         echo -n "Update files missing -- "
2863                         echo "this should never happen."
2864                         echo "Re-run '`basename $0` [options] fetch'."
2865                         return 1
2866                 fi
2867         done < filelist
2868
2869         # Clean up
2870         rm filelist
2871 }
2872
2873 # Remove the system immutable flag from files
2874 install_unschg () {
2875         # Generate file list
2876         cat $@ |
2877             cut -f 1 -d '|' > filelist
2878
2879         # Remove flags
2880         while read F; do
2881                 if ! [ -e ${BASEDIR}/${F} ]; then
2882                         continue
2883                 else
2884                         echo ${BASEDIR}/${F}
2885                 fi
2886         done < filelist | xargs chflags noschg || return 1
2887
2888         # Clean up
2889         rm filelist
2890 }
2891
2892 # Decide which directory name to use for kernel backups.
2893 backup_kernel_finddir () {
2894         CNT=0
2895         while true ; do
2896                 # Pathname does not exist, so it is OK use that name
2897                 # for backup directory.
2898                 if [ ! -e $BASEDIR/$BACKUPKERNELDIR ]; then
2899                         return 0
2900                 fi
2901
2902                 # If directory do exist, we only use if it has our
2903                 # marker file.
2904                 if [ -d $BASEDIR/$BACKUPKERNELDIR -a \
2905                         -e $BASEDIR/$BACKUPKERNELDIR/.freebsd-update ]; then
2906                         return 0
2907                 fi
2908
2909                 # We could not use current directory name, so add counter to
2910                 # the end and try again.
2911                 CNT=$((CNT + 1))
2912                 if [ $CNT -gt 9 ]; then
2913                         echo "Could not find valid backup dir ($BASEDIR/$BACKUPKERNELDIR)"
2914                         exit 1
2915                 fi
2916                 BACKUPKERNELDIR="`echo $BACKUPKERNELDIR | sed -Ee 's/[0-9]\$//'`"
2917                 BACKUPKERNELDIR="${BACKUPKERNELDIR}${CNT}"
2918         done
2919 }
2920
2921 # Backup the current kernel using hardlinks, if not disabled by user.
2922 # Since we delete all files in the directory used for previous backups
2923 # we create a marker file called ".freebsd-update" in the directory so
2924 # we can determine on the next run that the directory was created by
2925 # freebsd-update and we then do not accidentally remove user files in
2926 # the unlikely case that the user has created a directory with a
2927 # conflicting name.
2928 backup_kernel () {
2929         # Only make kernel backup is so configured.
2930         if [ $BACKUPKERNEL != yes ]; then
2931                 return 0
2932         fi
2933
2934         # Decide which directory name to use for kernel backups.
2935         backup_kernel_finddir
2936
2937         # Remove old kernel backup files.  If $BACKUPKERNELDIR was
2938         # "not ours", backup_kernel_finddir would have exited, so
2939         # deleting the directory content is as safe as we can make it.
2940         if [ -d $BASEDIR/$BACKUPKERNELDIR ]; then
2941                 rm -fr $BASEDIR/$BACKUPKERNELDIR
2942         fi
2943
2944         # Create directories for backup.
2945         mkdir -p $BASEDIR/$BACKUPKERNELDIR
2946         mtree -cdn -p "${BASEDIR}/${KERNELDIR}" | \
2947             mtree -Ue -p "${BASEDIR}/${BACKUPKERNELDIR}" > /dev/null
2948
2949         # Mark the directory as having been created by freebsd-update.
2950         touch $BASEDIR/$BACKUPKERNELDIR/.freebsd-update
2951         if [ $? -ne 0 ]; then
2952                 echo "Could not create kernel backup directory"
2953                 exit 1
2954         fi
2955
2956         # Disable pathname expansion to be sure *.symbols is not
2957         # expanded.
2958         set -f
2959
2960         # Use find to ignore symbol files, unless disabled by user.
2961         if [ $BACKUPKERNELSYMBOLFILES = yes ]; then
2962                 FINDFILTER=""
2963         else
2964                 FINDFILTER="-a ! -name *.debug -a ! -name *.symbols"
2965         fi
2966
2967         # Backup all the kernel files using hardlinks.
2968         (cd ${BASEDIR}/${KERNELDIR} && find . -type f $FINDFILTER -exec \
2969             cp -pl '{}' ${BASEDIR}/${BACKUPKERNELDIR}/'{}' \;)
2970
2971         # Re-enable pathname expansion.
2972         set +f
2973 }
2974
2975 # Check for and remove an existing directory that conflicts with the file or
2976 # symlink that we are going to install.
2977 dir_conflict () {
2978         if [ -d "$1" ]; then
2979                 echo "Removing conflicting directory $1"
2980                 rm -rf -- "$1"
2981         fi
2982 }
2983
2984 # Install new files
2985 install_from_index () {
2986         # First pass: Do everything apart from setting file flags.  We
2987         # can't set flags yet, because schg inhibits hard linking.
2988         sort -k 1,1 -t '|' $1 |
2989             tr '|' ' ' |
2990             while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
2991                 case ${TYPE} in
2992                 d)
2993                         # Create a directory.  A file may change to a directory
2994                         # on upgrade (PR273661).  If that happens, remove the
2995                         # file first.
2996                         if [ -e "${BASEDIR}/${FPATH}" ] && \
2997                             ! [ -d "${BASEDIR}/${FPATH}" ]; then
2998                                 rm -f -- "${BASEDIR}/${FPATH}"
2999                         fi
3000                         install -d -o ${OWNER} -g ${GROUP}              \
3001                             -m ${PERM} ${BASEDIR}/${FPATH}
3002                         ;;
3003                 f)
3004                         dir_conflict "${BASEDIR}/${FPATH}"
3005                         if [ -z "${LINK}" ]; then
3006                                 # Create a file, without setting flags.
3007                                 gunzip < files/${HASH}.gz > ${HASH}
3008                                 install -S -o ${OWNER} -g ${GROUP}      \
3009                                     -m ${PERM} ${HASH} ${BASEDIR}/${FPATH}
3010                                 rm ${HASH}
3011                         else
3012                                 # Create a hard link.
3013                                 ln -f ${BASEDIR}/${LINK} ${BASEDIR}/${FPATH}
3014                         fi
3015                         ;;
3016                 L)
3017                         dir_conflict "${BASEDIR}/${FPATH}"
3018                         # Create a symlink
3019                         ln -sfh ${HASH} ${BASEDIR}/${FPATH}
3020                         ;;
3021                 esac
3022             done
3023
3024         # Perform a second pass, adding file flags.
3025         tr '|' ' ' < $1 |
3026             while read FPATH TYPE OWNER GROUP PERM FLAGS HASH LINK; do
3027                 if [ ${TYPE} = "f" ] &&
3028                     ! [ ${FLAGS} = "0" ]; then
3029                         chflags ${FLAGS} ${BASEDIR}/${FPATH}
3030                 fi
3031             done
3032 }
3033
3034 # Remove files which we want to delete
3035 install_delete () {
3036         # Generate list of new files
3037         cut -f 1 -d '|' < $2 |
3038             sort > newfiles
3039
3040         # Generate subindex of old files we want to nuke
3041         sort -k 1,1 -t '|' $1 |
3042             join -t '|' -v 1 - newfiles |
3043             sort -r -k 1,1 -t '|' |
3044             cut -f 1,2 -d '|' |
3045             tr '|' ' ' > killfiles
3046
3047         # Remove the offending bits
3048         while read FPATH TYPE; do
3049                 case ${TYPE} in
3050                 d)
3051                         rmdir ${BASEDIR}/${FPATH}
3052                         ;;
3053                 f)
3054                         if [ -f "${BASEDIR}/${FPATH}" ]; then
3055                                 rm "${BASEDIR}/${FPATH}"
3056                         fi
3057                         ;;
3058                 L)
3059                         if [ -L "${BASEDIR}/${FPATH}" ]; then
3060                                 rm "${BASEDIR}/${FPATH}"
3061                         fi
3062                         ;;
3063                 esac
3064         done < killfiles
3065
3066         # Clean up
3067         rm newfiles killfiles
3068 }
3069
3070 # Install new files, delete old files, and update generated files
3071 install_files () {
3072         # If we haven't already dealt with the kernel, deal with it.
3073         if ! [ -f $1/kerneldone ]; then
3074                 grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
3075                 grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
3076
3077                 # Backup current kernel before installing a new one
3078                 backup_kernel || return 1
3079
3080                 # Install new files
3081                 install_from_index INDEX-NEW || return 1
3082
3083                 # Remove files which need to be deleted
3084                 install_delete INDEX-OLD INDEX-NEW || return 1
3085
3086                 # Update linker.hints if necessary
3087                 if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3088                         kldxref -R ${BASEDIR}/boot/ 2>/dev/null
3089                 fi
3090
3091                 # We've finished updating the kernel.
3092                 touch $1/kerneldone
3093
3094                 # Do we need to ask for a reboot now?
3095                 if [ -f $1/kernelfirst ] &&
3096                     [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3097                         cat <<-EOF
3098
3099 Kernel updates have been installed.  Please reboot and run
3100 '`basename $0` [options] install' again to finish installing updates.
3101                         EOF
3102                         exit 0
3103                 fi
3104         fi
3105
3106         # If we haven't already dealt with the world, deal with it.
3107         if ! [ -f $1/worlddone ]; then
3108                 # Create any necessary directories first
3109                 grep -vE '^/boot/' $1/INDEX-NEW |
3110                     grep -E '^[^|]+\|d\|' > INDEX-NEW
3111                 install_from_index INDEX-NEW || return 1
3112
3113                 # Install new runtime linker
3114                 grep -vE '^/boot/' $1/INDEX-NEW |
3115                     grep -vE '^[^|]+\|d\|' |
3116                     grep -E '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3117                 install_from_index INDEX-NEW || return 1
3118
3119                 # Install new shared libraries next
3120                 grep -vE '^/boot/' $1/INDEX-NEW |
3121                     grep -vE '^[^|]+\|d\|' |
3122                     grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3123                     grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3124                 install_from_index INDEX-NEW || return 1
3125
3126                 # Deal with everything else
3127                 grep -vE '^/boot/' $1/INDEX-OLD |
3128                     grep -vE '^[^|]+\|d\|' |
3129                     grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3130                     grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3131                 grep -vE '^/boot/' $1/INDEX-NEW |
3132                     grep -vE '^[^|]+\|d\|' |
3133                     grep -vE '^/libexec/ld-elf[^|]*\.so\.[0-9]+\|' |
3134                     grep -vE '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3135                 install_from_index INDEX-NEW || return 1
3136                 install_delete INDEX-OLD INDEX-NEW || return 1
3137
3138                 # Restart host sshd if running (PR263489).  Note that this does
3139                 # not affect child sshd processes handling existing sessions.
3140                 if [ "$BASEDIR" = / ] && \
3141                     service sshd status >/dev/null 2>/dev/null; then
3142                         echo
3143                         echo "Restarting sshd after upgrade"
3144                         service sshd restart
3145                 fi
3146
3147                 # Rehash certs if we actually have certctl installed.
3148                 if which certctl>/dev/null; then
3149                         env DESTDIR=${BASEDIR} certctl rehash
3150                 fi
3151
3152                 # Rebuild generated pwd files and /etc/login.conf.db.
3153                 pwd_mkdb -d ${BASEDIR}/etc -p ${BASEDIR}/etc/master.passwd
3154                 cap_mkdb ${BASEDIR}/etc/login.conf
3155
3156                 # Rebuild man page databases, if necessary.
3157                 for D in /usr/share/man /usr/share/openssl/man; do
3158                         if [ ! -d ${BASEDIR}/$D ]; then
3159                                 continue
3160                         fi
3161                         if [ -f ${BASEDIR}/$D/mandoc.db ] && \
3162                             [ -z "$(find ${BASEDIR}/$D -type f -newer ${BASEDIR}/$D/mandoc.db)" ]; then
3163                                 continue;
3164                         fi
3165                         makewhatis ${BASEDIR}/$D
3166                 done
3167
3168                 # We've finished installing the world and deleting old files
3169                 # which are not shared libraries.
3170                 touch $1/worlddone
3171
3172                 # Do we need to ask the user to portupgrade now?
3173                 grep -vE '^/boot/' $1/INDEX-NEW |
3174                     grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3175                     cut -f 1 -d '|' |
3176                     sort > newfiles
3177                 if grep -vE '^/boot/' $1/INDEX-OLD |
3178                     grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' |
3179                     cut -f 1 -d '|' |
3180                     sort |
3181                     join -v 1 - newfiles |
3182                     grep -q .; then
3183                         cat <<-EOF
3184
3185 Completing this upgrade requires removing old shared object files.
3186 Please rebuild all installed 3rd party software (e.g., programs
3187 installed from the ports tree) and then run
3188 '`basename $0` [options] install' again to finish installing updates.
3189                         EOF
3190                         rm newfiles
3191                         exit 0
3192                 fi
3193                 rm newfiles
3194         fi
3195
3196         # Remove old shared libraries
3197         grep -vE '^/boot/' $1/INDEX-NEW |
3198             grep -vE '^[^|]+\|d\|' |
3199             grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-NEW
3200         grep -vE '^/boot/' $1/INDEX-OLD |
3201             grep -vE '^[^|]+\|d\|' |
3202             grep -E '^[^|]*/lib/[^|]*\.so\.[0-9]+\|' > INDEX-OLD
3203         install_delete INDEX-OLD INDEX-NEW || return 1
3204
3205         # Remove old directories
3206         grep -vE '^/boot/' $1/INDEX-NEW |
3207             grep -E '^[^|]+\|d\|' > INDEX-NEW
3208         grep -vE '^/boot/' $1/INDEX-OLD |
3209             grep -E '^[^|]+\|d\|' > INDEX-OLD
3210         install_delete INDEX-OLD INDEX-NEW || return 1
3211
3212         # Remove temporary files
3213         rm INDEX-OLD INDEX-NEW
3214 }
3215
3216 # Rearrange bits to allow the installed updates to be rolled back
3217 install_setup_rollback () {
3218         # Remove the "reboot after installing kernel", "kernel updated", and
3219         # "finished installing the world" flags if present -- they are
3220         # irrelevant when rolling back updates.
3221         if [ -f ${BDHASH}-install/kernelfirst ]; then
3222                 rm ${BDHASH}-install/kernelfirst
3223                 rm ${BDHASH}-install/kerneldone
3224         fi
3225         if [ -f ${BDHASH}-install/worlddone ]; then
3226                 rm ${BDHASH}-install/worlddone
3227         fi
3228
3229         if [ -L ${BDHASH}-rollback ]; then
3230                 mv ${BDHASH}-rollback ${BDHASH}-install/rollback
3231         fi
3232
3233         mv ${BDHASH}-install ${BDHASH}-rollback
3234 }
3235
3236 # Actually install updates
3237 install_run () {
3238         echo -n "Installing updates..."
3239
3240         # Make sure we have all the files we should have
3241         install_verify ${BDHASH}-install/INDEX-OLD      \
3242             ${BDHASH}-install/INDEX-NEW || return 1
3243
3244         # Remove system immutable flag from files
3245         install_unschg ${BDHASH}-install/INDEX-OLD      \
3246             ${BDHASH}-install/INDEX-NEW || return 1
3247
3248         # Install new files, delete old files, and update linker.hints
3249         install_files ${BDHASH}-install || return 1
3250
3251         # Rearrange bits to allow the installed updates to be rolled back
3252         install_setup_rollback
3253
3254         echo " done."
3255 }
3256
3257 # Rearrange bits to allow the previous set of updates to be rolled back next.
3258 rollback_setup_rollback () {
3259         if [ -L ${BDHASH}-rollback/rollback ]; then
3260                 mv ${BDHASH}-rollback/rollback rollback-tmp
3261                 rm -r ${BDHASH}-rollback/
3262                 rm ${BDHASH}-rollback
3263                 mv rollback-tmp ${BDHASH}-rollback
3264         else
3265                 rm -r ${BDHASH}-rollback/
3266                 rm ${BDHASH}-rollback
3267         fi
3268 }
3269
3270 # Install old files, delete new files, and update linker.hints
3271 rollback_files () {
3272         # Create directories first.  They may be needed by files we will
3273         # install in subsequent steps (PR273950).
3274         awk -F \| '{if ($2 == "d") print }' $1/INDEX-OLD > INDEX-OLD
3275         install_from_index INDEX-OLD || return 1
3276
3277         # Install old shared library files which don't have the same path as
3278         # a new shared library file.
3279         grep -vE '^/boot/' $1/INDEX-NEW |
3280             grep -E '/lib/.*\.so\.[0-9]+\|' |
3281             cut -f 1 -d '|' |
3282             sort > INDEX-NEW.libs.flist
3283         grep -vE '^/boot/' $1/INDEX-OLD |
3284             grep -E '/lib/.*\.so\.[0-9]+\|' |
3285             sort -k 1,1 -t '|' - |
3286             join -t '|' -v 1 - INDEX-NEW.libs.flist > INDEX-OLD
3287         install_from_index INDEX-OLD || return 1
3288
3289         # Deal with files which are neither kernel nor shared library
3290         grep -vE '^/boot/' $1/INDEX-OLD |
3291             grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3292         grep -vE '^/boot/' $1/INDEX-NEW |
3293             grep -vE '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3294         install_from_index INDEX-OLD || return 1
3295         install_delete INDEX-NEW INDEX-OLD || return 1
3296
3297         # Install any old shared library files which we didn't install above.
3298         grep -vE '^/boot/' $1/INDEX-OLD |
3299             grep -E '/lib/.*\.so\.[0-9]+\|' |
3300             sort -k 1,1 -t '|' - |
3301             join -t '|' - INDEX-NEW.libs.flist > INDEX-OLD
3302         install_from_index INDEX-OLD || return 1
3303
3304         # Delete unneeded shared library files
3305         grep -vE '^/boot/' $1/INDEX-OLD |
3306             grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-OLD
3307         grep -vE '^/boot/' $1/INDEX-NEW |
3308             grep -E '/lib/.*\.so\.[0-9]+\|' > INDEX-NEW
3309         install_delete INDEX-NEW INDEX-OLD || return 1
3310
3311         # Deal with kernel files
3312         grep -E '^/boot/' $1/INDEX-OLD > INDEX-OLD
3313         grep -E '^/boot/' $1/INDEX-NEW > INDEX-NEW
3314         install_from_index INDEX-OLD || return 1
3315         install_delete INDEX-NEW INDEX-OLD || return 1
3316         if [ -s INDEX-OLD -o -s INDEX-NEW ]; then
3317                 kldxref -R /boot/ 2>/dev/null
3318         fi
3319
3320         # Remove temporary files
3321         rm INDEX-OLD INDEX-NEW INDEX-NEW.libs.flist
3322 }
3323
3324 # Actually rollback updates
3325 rollback_run () {
3326         echo -n "Uninstalling updates..."
3327
3328         # If there are updates waiting to be installed, remove them; we
3329         # want the user to re-run 'fetch' after rolling back updates.
3330         if [ -L ${BDHASH}-install ]; then
3331                 rm -r ${BDHASH}-install/
3332                 rm ${BDHASH}-install
3333         fi
3334
3335         # Make sure we have all the files we should have
3336         install_verify ${BDHASH}-rollback/INDEX-NEW     \
3337             ${BDHASH}-rollback/INDEX-OLD || return 1
3338
3339         # Remove system immutable flag from files
3340         install_unschg ${BDHASH}-rollback/INDEX-NEW     \
3341             ${BDHASH}-rollback/INDEX-OLD || return 1
3342
3343         # Install old files, delete new files, and update linker.hints
3344         rollback_files ${BDHASH}-rollback || return 1
3345
3346         # Remove the rollback directory and the symlink pointing to it; and
3347         # rearrange bits to allow the previous set of updates to be rolled
3348         # back next.
3349         rollback_setup_rollback
3350
3351         echo " done."
3352 }
3353
3354 # Compare INDEX-ALL and INDEX-PRESENT and print warnings about differences.
3355 IDS_compare () {
3356         # Get all the lines which mismatch in something other than file
3357         # flags.  We ignore file flags because sysinstall doesn't seem to
3358         # set them when it installs FreeBSD; warning about these adds a
3359         # very large amount of noise.
3360         cut -f 1-5,7-8 -d '|' $1 > $1.noflags
3361         sort -k 1,1 -t '|' $1.noflags > $1.sorted
3362         cut -f 1-5,7-8 -d '|' $2 |
3363             comm -13 $1.noflags - |
3364             fgrep -v '|-|||||' |
3365             sort -k 1,1 -t '|' |
3366             join -t '|' $1.sorted - > INDEX-NOTMATCHING
3367
3368         # Ignore files which match IDSIGNOREPATHS.
3369         for X in ${IDSIGNOREPATHS}; do
3370                 grep -E "^${X}" INDEX-NOTMATCHING
3371         done |
3372             sort -u |
3373             comm -13 - INDEX-NOTMATCHING > INDEX-NOTMATCHING.tmp
3374         mv INDEX-NOTMATCHING.tmp INDEX-NOTMATCHING
3375
3376         # Go through the lines and print warnings.
3377         local IFS='|'
3378         while read FPATH TYPE OWNER GROUP PERM HASH LINK P_TYPE P_OWNER P_GROUP P_PERM P_HASH P_LINK; do
3379                 # Warn about different object types.
3380                 if ! [ "${TYPE}" = "${P_TYPE}" ]; then
3381                         echo -n "${FPATH} is a "
3382                         case "${P_TYPE}" in
3383                         f)      echo -n "regular file, "
3384                                 ;;
3385                         d)      echo -n "directory, "
3386                                 ;;
3387                         L)      echo -n "symlink, "
3388                                 ;;
3389                         esac
3390                         echo -n "but should be a "
3391                         case "${TYPE}" in
3392                         f)      echo -n "regular file."
3393                                 ;;
3394                         d)      echo -n "directory."
3395                                 ;;
3396                         L)      echo -n "symlink."
3397                                 ;;
3398                         esac
3399                         echo
3400
3401                         # Skip other tests, since they don't make sense if
3402                         # we're comparing different object types.
3403                         continue
3404                 fi
3405
3406                 # Warn about different owners.
3407                 if ! [ "${OWNER}" = "${P_OWNER}" ]; then
3408                         echo -n "${FPATH} is owned by user id ${P_OWNER}, "
3409                         echo "but should be owned by user id ${OWNER}."
3410                 fi
3411
3412                 # Warn about different groups.
3413                 if ! [ "${GROUP}" = "${P_GROUP}" ]; then
3414                         echo -n "${FPATH} is owned by group id ${P_GROUP}, "
3415                         echo "but should be owned by group id ${GROUP}."
3416                 fi
3417
3418                 # Warn about different permissions.  We do not warn about
3419                 # different permissions on symlinks, since some archivers
3420                 # don't extract symlink permissions correctly and they are
3421                 # ignored anyway.
3422                 if ! [ "${PERM}" = "${P_PERM}" ] &&
3423                     ! [ "${TYPE}" = "L" ]; then
3424                         echo -n "${FPATH} has ${P_PERM} permissions, "
3425                         echo "but should have ${PERM} permissions."
3426                 fi
3427
3428                 # Warn about different file hashes / symlink destinations.
3429                 if ! [ "${HASH}" = "${P_HASH}" ]; then
3430                         if [ "${TYPE}" = "L" ]; then
3431                                 echo -n "${FPATH} is a symlink to ${P_HASH}, "
3432                                 echo "but should be a symlink to ${HASH}."
3433                         fi
3434                         if [ "${TYPE}" = "f" ]; then
3435                                 echo -n "${FPATH} has SHA256 hash ${P_HASH}, "
3436                                 echo "but should have SHA256 hash ${HASH}."
3437                         fi
3438                 fi
3439
3440                 # We don't warn about different hard links, since some
3441                 # some archivers break hard links, and as long as the
3442                 # underlying data is correct they really don't matter.
3443         done < INDEX-NOTMATCHING
3444
3445         # Clean up
3446         rm $1 $1.noflags $1.sorted $2 INDEX-NOTMATCHING
3447 }
3448
3449 # Do the work involved in comparing the system to a "known good" index
3450 IDS_run () {
3451         workdir_init || return 1
3452
3453         # Prepare the mirror list.
3454         fetch_pick_server_init && fetch_pick_server
3455
3456         # Try to fetch the public key until we run out of servers.
3457         while ! fetch_key; do
3458                 fetch_pick_server || return 1
3459         done
3460
3461         # Try to fetch the metadata index signature ("tag") until we run
3462         # out of available servers; and sanity check the downloaded tag.
3463         while ! fetch_tag; do
3464                 fetch_pick_server || return 1
3465         done
3466         fetch_tagsanity || return 1
3467
3468         # Fetch INDEX-OLD and INDEX-ALL.
3469         fetch_metadata INDEX-OLD INDEX-ALL || return 1
3470
3471         # Generate filtered INDEX-OLD and INDEX-ALL files containing only
3472         # the components we want and without anything marked as "Ignore".
3473         fetch_filter_metadata INDEX-OLD || return 1
3474         fetch_filter_metadata INDEX-ALL || return 1
3475
3476         # Merge the INDEX-OLD and INDEX-ALL files into INDEX-ALL.
3477         sort INDEX-OLD INDEX-ALL > INDEX-ALL.tmp
3478         mv INDEX-ALL.tmp INDEX-ALL
3479         rm INDEX-OLD
3480
3481         # Translate /boot/${KERNCONF} to ${KERNELDIR}
3482         fetch_filter_kernel_names INDEX-ALL ${KERNCONF}
3483
3484         # Inspect the system and generate an INDEX-PRESENT file.
3485         fetch_inspect_system INDEX-ALL INDEX-PRESENT /dev/null || return 1
3486
3487         # Compare INDEX-ALL and INDEX-PRESENT and print warnings about any
3488         # differences.
3489         IDS_compare INDEX-ALL INDEX-PRESENT
3490 }
3491
3492 #### Main functions -- call parameter-handling and core functions
3493
3494 # Using the command line, configuration file, and defaults,
3495 # set all the parameters which are needed later.
3496 get_params () {
3497         init_params
3498         parse_cmdline $@
3499         parse_conffile
3500         default_params
3501 }
3502
3503 # Fetch command.  Make sure that we're being called
3504 # interactively, then run fetch_check_params and fetch_run
3505 cmd_fetch () {
3506         finalize_components_config ${COMPONENTS}
3507         if [ ! -t 0 -a $NOTTYOK -eq 0 ]; then
3508                 echo -n "`basename $0` fetch should not "
3509                 echo "be run non-interactively."
3510                 echo "Run `basename $0` cron instead."
3511                 exit 1
3512         fi
3513         fetch_check_params
3514         fetch_run || exit 1
3515         ISFETCHED=1
3516 }
3517
3518 # Cron command.  Make sure the parameters are sensible; wait
3519 # rand(3600) seconds; then fetch updates.  While fetching updates,
3520 # send output to a temporary file; only print that file if the
3521 # fetching failed.
3522 cmd_cron () {
3523         fetch_check_params
3524         sleep `jot -r 1 0 3600`
3525
3526         TMPFILE=`mktemp /tmp/freebsd-update.XXXXXX` || exit 1
3527         finalize_components_config ${COMPONENTS} >> ${TMPFILE}
3528         if ! fetch_run >> ${TMPFILE} ||
3529             ! grep -q "No updates needed" ${TMPFILE} ||
3530             [ ${VERBOSELEVEL} = "debug" ]; then
3531                 mail -s "`hostname` security updates" ${MAILTO} < ${TMPFILE}
3532         fi
3533         ISFETCHED=1
3534
3535         rm ${TMPFILE}
3536 }
3537
3538 # Fetch files for upgrading to a new release.
3539 cmd_upgrade () {
3540         finalize_components_config ${COMPONENTS}
3541         upgrade_check_params
3542         upgrade_check_kmod_ports
3543         upgrade_run || exit 1
3544 }
3545
3546 # Check if there are fetched updates ready to install.
3547 # Chdir into the working directory.
3548 cmd_updatesready () {
3549         finalize_components_config ${COMPONENTS}
3550         # Check if working directory exists (if not, no updates pending)
3551         if ! [ -e "${WORKDIR}" ]; then
3552                 echo "No updates are available to install."
3553                 exit 2
3554         fi
3555
3556         # Change into working directory (fail if no permission/directory etc.)
3557         cd ${WORKDIR} || exit 1
3558
3559         # Construct a unique name from ${BASEDIR}
3560         BDHASH=`echo ${BASEDIR} | sha256 -q`
3561
3562         # Check that we have updates ready to install
3563         if ! [ -L ${BDHASH}-install ]; then
3564                 echo "No updates are available to install."
3565                 exit 2
3566         fi
3567
3568         echo "There are updates available to install."
3569         echo "Run '`basename $0` [options] install' to proceed."
3570 }
3571
3572 # Install downloaded updates.
3573 cmd_install () {
3574         finalize_components_config ${COMPONENTS}
3575         install_check_params
3576         install_create_be
3577         install_run || exit 1
3578 }
3579
3580 # Rollback most recently installed updates.
3581 cmd_rollback () {
3582         finalize_components_config ${COMPONENTS}
3583         rollback_check_params
3584         rollback_run || exit 1
3585 }
3586
3587 # Compare system against a "known good" index.
3588 cmd_IDS () {
3589         finalize_components_config ${COMPONENTS}
3590         IDS_check_params
3591         IDS_run || exit 1
3592 }
3593
3594 # Output configuration.
3595 cmd_showconfig () {
3596         finalize_components_config ${COMPONENTS}
3597         for X in ${CONFIGOPTIONS}; do
3598                 echo $X=$(eval echo \$${X})
3599         done
3600 }
3601
3602 #### Entry point
3603
3604 # Make sure we find utilities from the base system
3605 export PATH=/sbin:/bin:/usr/sbin:/usr/bin:${PATH}
3606
3607 # Set a pager if the user doesn't
3608 if [ -z "$PAGER" ]; then
3609         PAGER=/usr/bin/less
3610 fi
3611
3612 # Set LC_ALL in order to avoid problems with character ranges like [A-Z].
3613 export LC_ALL=C
3614
3615 # Clear environment variables that may affect operation of tools that we use.
3616 unset GREP_OPTIONS
3617
3618 # Disallow use with packaged base.
3619 check_pkgbase
3620
3621 get_params $@
3622 for COMMAND in ${COMMANDS}; do
3623         cmd_${COMMAND}
3624 done