tools/update-verify/release/final-verification.sh

   1 #!/bin/bash
   2
   3 function usage {
   4     log "In the updates subdirectory of the directory this script is in,"
   5     log "there are a bunch of config files. You should call this script,"
   6     log "passing the names of one or more of those files as parameters"
   7     log "to this script."
   8     log ""
   9     log "This will validate that the update.xml files all exist for the"
  10     log "given config file, and that they report the correct file sizes"
  11     log "for the associated mar files, and that the associated mar files"
  12     log "are available on the update servers."
  13     log ""
  14     log "This script will spawn multiple curl processes to query the"
  15     log "snippets (update.xml file downloads) and the download urls in"
  16     log "parallel. The number of parallel curl processes can be managed"
  17     log "with the -p MAX_PROCS option."
  18     log ""
  19     log "Only the first three bytes of the mar files are downloaded"
  20     log "using curl -r 0-2 option to save time. GET requests are issued"
  21     log "rather than HEAD requests, since Akamai (one of our CDN"
  22     log "partners) caches GET and HEAD requests separately - therefore"
  23     log "they can be out-of-sync, and it is important that we validate"
  24     log "that the GET requests return the expected results."
  25     log ""
  26     log "Please note this script can run on linux and OS X. It has not"
  27     log "been tested on Windows, but may also work. It can be run"
  28     log "locally, and does not require access to the mozilla vpn or"
  29     log "any other special network, since the update servers are"
  30     log "available over the internet. However, it does require an"
  31     log "up-to-date checkout of the tools repository, as the updates/"
  32     log "subfolder changes over time, and reflects the currently"
  33     log "available updates. It makes no changes to the update servers"
  34     log "so there is no harm in running it. It simply generates a"
  35     log "report. However, please try to avoid hammering the update"
  36     log "servers aggressively, e.g. with thousands of parallel"
  37     log "processes. For example, feel free to run the examples below,"
  38     log "first making sure that your source code checkout is up-to-"
  39     log "date on your own machine, to get the latest configs in the"
  40     log "updates/ subdirectory."
  41     log ""
  42     log "Usage:"
  43     log "    $(basename "${0}") [-p MAX_PROCS] config1 [config2 config3 config4 ...]"
  44     log "    $(basename "${0}") -h"
  45     log ""
  46     log "Examples:"
  47     log "    1. $(basename "${0}") -p 128 mozBeta-thunderbird-linux.cfg mozBeta-thunderbird-linux64.cfg"
  48     log "    2. $(basename "${0}") mozBeta-thunderbird-linux64.cfg"
  49 }
  50
  51 function log {
  52     echo "$(date):  ${1}"
  53 }
  54
  55 # subprocesses don't log in real time, due to synchronisation
  56 # issues which can cause log entries to overwrite each other.
  57 # therefore this function outputs log entries written to
  58 # temporary files on disk, and then deletes them.
  59 function flush_logs {
  60     ls -1rt "${TMPDIR}" | grep '^log\.' | while read LOG
  61     do
  62         cat "${TMPDIR}/${LOG}"
  63         rm "${TMPDIR}/${LOG}"
  64     done
  65 }
  66
  67 # this function takes an update.xml url as an argument
  68 # and then logs a list of config files and their line
  69 # numbers, that led to this update.xml url being tested
  70 function show_cfg_file_entries {
  71     local update_xml_url="${1}"
  72     cat "${update_xml_urls}" | cut -f1 -d' ' | grep -Fn "${update_xml_url}" | sed 's/:.*//' | while read match_line_no
  73     do
  74         cfg_file="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f3 -d' ')"
  75         cfg_line_no="$(sed -n -e "${match_line_no}p" "${update_xml_urls}" | cut -f4 -d' ')"
  76         log "        ${cfg_file} line ${cfg_line_no}: $(sed -n -e "${cfg_line_no}p" "${cfg_file}")"
  77     done
  78 }
  79
  80 # this function takes a mar url as an argument and then
  81 # logs information about which update.xml urls referenced
  82 # this mar url, and which config files referenced those
  83 # mar urls - so you have a full understanding of why this
  84 # mar url was ever tested
  85 function show_update_xml_entries {
  86     local mar_url="${1}"
  87     grep -Frl "${mar_url}" "${TMPDIR}" | grep '/update_xml_to_mar\.' | while read update_xml_to_mar
  88     do
  89         mar_size="$(cat "${update_xml_to_mar}" | cut -f2 -d' ')"
  90         update_xml_url="$(cat "${update_xml_to_mar}" | cut -f3 -d' ')"
  91         patch_type="$(cat "${update_xml_to_mar}" | cut -f4 -d' ')"
  92         update_xml_actual_url="$(cat "${update_xml_to_mar}" | cut -f5 -d' ')"
  93         log "        ${update_xml_url}"
  94         [ -n "${update_xml_actual_url}" ] && log "            which redirected to: ${update_xml_actual_url}"
  95         log "            This contained an entry for:"
  96         log "                patch type: ${patch_type}"
  97         log "                mar size: ${mar_size}"
  98         log "                mar url: ${mar_url}"
  99         log "            The update.xml url above was retrieved because of the following cfg file entries:"
 100         show_cfg_file_entries "${update_xml_url}" | sed 's/        /                /'
 101     done
 102 }
 103
 104 echo -n "$(date):  Command called:"
 105 for ((INDEX=0; INDEX<=$#; INDEX+=1))
 106 do
 107     echo -n " '${!INDEX}'"
 108 done
 109 echo ''
 110 log "From directory: '$(pwd)'"
 111 log ''
 112 log "Parsing arguments..."
 113
 114 # Max procs lowered in bug 894368 to try to avoid spurious failures
 115 MAX_PROCS=48
 116 BAD_ARG=0
 117 BAD_FILE=0
 118 while getopts p:h OPT
 119 do
 120     case "${OPT}" in
 121         p) MAX_PROCS="${OPTARG}";;
 122         h) usage
 123            exit;;
 124         *) BAD_ARG=1;;
 125     esac
 126 done
 127 shift "$((OPTIND - 1))"
 128
 129 # invalid option specified
 130 [ "${BAD_ARG}" == 1 ] && exit 66
 131
 132 log "Checking one or more config files have been specified..."
 133 if [ $# -lt 1 ]
 134 then
 135     usage
 136     log "ERROR: You must specify one or more config files"
 137     exit 64
 138 fi
 139
 140 log "Checking whether MAX_PROCS is a number..."
 141 if ! let x=MAX_PROCS 2>/dev/null
 142 then
 143     usage
 144     log "ERROR: MAX_PROCS must be a number (-p option); you specified '${MAX_PROCS}' - this is not a number."
 145     exit 65
 146 fi
 147
 148 # config files are in updates subdirectory below this script
 149 if ! cd "$(dirname "${0}")/updates" 2>/dev/null
 150 then
 151     log "ERROR: Cannot cd into '$(dirname "${0}")/updates' from '$(pwd)'"
 152     exit 68
 153 fi
 154
 155 log "Checking specified config files (and downloading them if necessary):"
 156 log ''
 157 configs=()
 158 for file in "${@}"
 159 do
 160     if [[ ${file} == http* ]]
 161     then
 162         log "  Downloading config file '${file}'"
 163         cfg=$(mktemp)
 164         curl -fL --retry 5 --compressed "${file}" > "$cfg"
 165         if [ "$?" != 0 ]; then
 166             log "Error downloading config file '${file}'"
 167             BAD_FILE=1
 168         else
 169             log "  * '${file}' ok, downloaded to '${cfg}'"
 170             configs+=($cfg)
 171         fi
 172     elif [ -f "${file}" ]
 173     then
 174         log "  * '${file}' ok"
 175         configs+=(${file})
 176     else
 177         log "  * '${file}' missing"
 178         BAD_FILE=1
 179     fi
 180 done
 181 log ''
 182
 183 # invalid config specified
 184 if [ "${BAD_FILE}" == 1 ]
 185 then
 186     log "ERROR: Unable to download config file(s) or config files are missing from repo - see above"
 187     exit 67
 188 fi
 189
 190 log "All checks completed successfully."
 191 log ''
 192 log "Starting stopwatch..."
 193 log ''
 194 log "Please be aware output will now be buffered up, and only displayed after completion."
 195 log "Therefore do not be alarmed if you see no output for several minutes."
 196 log "See https://bugzilla.mozilla.org/show_bug.cgi?id=863602#c5 for details".
 197 log ''
 198
 199 START_TIME="$(date +%s)"
 200
 201 # Create a temporary directory for all temp files, that can easily be
 202 # deleted afterwards. See https://bugzilla.mozilla.org/show_bug.cgi?id=863602
 203 # to understand why we write everything in distinct temporary files rather
 204 # than writing to standard error/standard out or files shared across
 205 # processes.
 206 # Need to unset TMPDIR first since it affects mktemp behaviour on next line
 207 unset TMPDIR
 208 export TMPDIR="$(mktemp -d -t final_verification.XXXXXXXXXX)"
 209
 210 # this temporary file will list all update urls that need to be checked, in this format:
 211 # <update url> <comma separated list of patch types> <cfg file that requests it> <line number of config file>
 212 # e.g.
 213 # https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zh-TW/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64-major.cfg 3
 214 # https://aus4.mozilla.org/update/3/Firefox/18.0/20130104154748/Linux_x86_64-gcc3/zu/releasetest/default/default/default/update.xml?force=1 complete moz20-firefox-linux64.cfg 7
 215 # https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/ach/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64-major.cfg 11
 216 # https://aus4.mozilla.org/update/3/Firefox/19.0/20130215130331/Linux_x86_64-gcc3/af/releasetest/default/default/default/update.xml?force=1 complete,partial moz20-firefox-linux64.cfg 17
 217 update_xml_urls="$(mktemp -t update_xml_urls.XXXXXXXXXX)"
 218
 219 ####################################################################################
 220 # And now a summary of all temp files that will get generated during this process...
 221 #
 222 # 1) mktemp -t failure.XXXXXXXXXX
 223 #
 224 # Each failure will generate a one line temp file, which is a space separated
 225 # output of the error code, and the instance data for the failure.
 226 # e.g.
 227 #
 228 # PATCH_TYPE_MISSING https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/update.xml?force=1 complete https://aus4.mozilla.org/update/3/Firefox/4.0b12/20110222205441/Linux_x86-gcc3/dummy-locale/releasetest/default/default/default/update.xml?force=1
 229 #
 230 # 2) mktemp -t update_xml_to_mar.XXXXXXXXXX
 231 #
 232 # For each mar url referenced in an update.xml file, a temp file will be created to store the
 233 # association between update.xml url and mar url. This is later used (e.g. in function
 234 # show_update_xml_entries) to trace back the update.xml url(s) that led to a mar url being
 235 # tested. It is also used to keep a full list of mar urls to test.
 236 # e.g.
 237 #
 238 # <mar url> <mar size> <update.xml url> <patch type> <update.xml redirection url, if HTTP 302 returned>
 239 #
 240 # 3) mktemp -t log.XXXXXXXXXX
 241 #
 242 # For each log message logged by a subprocesses, we will create a temp log file with the
 243 # contents of the log message, since we cannot safely output the log message from the subprocess
 244 # and guarantee that it will be correctly output. By buffering log output in individual log files
 245 # we guarantee that log messages will not interfere with each other. We then flush them when all
 246 # forked subprocesses have completed.
 247 #
 248 # 4) mktemp -t mar_headers.XXXXXXXXXX
 249 #
 250 # We keep a copy of the mar url http headers retrieved in one file per mar url.
 251 #
 252 # 5) mktemp -t update.xml.headers.XXXXXXXXXX
 253 #
 254 # We keep a copy of the update.xml http headers retrieved in one file per update.xml url.
 255 #
 256 # 6) mktemp -t update.xml.XXXXXXXXXX
 257 #
 258 # We keep a copy of each update.xml file retrieved in individual files.
 259 ####################################################################################
 260
 261
 262 # generate full list of update.xml urls, followed by patch types,
 263 # as defined in the specified config files - and write into "${update_xml_urls}" file
 264 aus_server="https://aus5.mozilla.org"
 265 for cfg_file in "${configs[@]}"
 266 do
 267     line_no=0
 268     sed -e 's/localtest/cdntest/' "${cfg_file}" | while read config_line
 269     do
 270         let line_no++
 271         # to avoid contamination between iterations, reset variables
 272         # each loop in case they are not declared
 273         # aus_server is not "cleared" each iteration - to be consistent with previous behaviour of old
 274         # final-verification.sh script - might be worth reviewing if we really want this behaviour
 275         release="" product="" platform="" build_id="" locales="" channel="" from="" patch_types="complete"
 276         eval "${config_line}"
 277         for locale in ${locales}
 278         do
 279             echo "${aus_server}/update/3/$product/$release/$build_id/$platform/$locale/$channel/default/default/default/update.xml?force=1" "${patch_types// /,}" "${cfg_file}" "${line_no}"
 280         done
 281     done
 282 done > "${update_xml_urls}"
 283
 284 # download update.xml files and grab the mar urls from downloaded file for each patch type required
 285 cat "${update_xml_urls}" | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../get-update-xml.sh
 286 if [ "$?" != 0 ]; then
 287     flush_logs
 288     log "Error generating update requests"
 289     exit 70
 290 fi
 291
 292 flush_logs
 293
 294 # download http header for each mar url
 295 find "${TMPDIR}" -name 'update_xml_to_mar.*' -type f | xargs cat | cut -f1-2 -d' ' | sort -u | xargs -n2 "-P${MAX_PROCS}" ../test-mar-url.sh
 296 if [ "$?" != 0 ]; then
 297     flush_logs
 298     log "Error HEADing mar urls"
 299     exit 71
 300 fi
 301
 302 flush_logs
 303
 304 log ''
 305 log 'Stopping stopwatch...'
 306 STOP_TIME="$(date +%s)"
 307
 308 number_of_failures="$(find "${TMPDIR}" -name 'failure.*' -type f | wc -l | sed 's/ //g')"
 309 number_of_update_xml_urls="$(cat "${update_xml_urls}" | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')"
 310 number_of_mar_urls="$(find "${TMPDIR}" -name "update_xml_to_mar.*" | xargs cat | cut -f1 -d' ' | sort -u | wc -l | sed 's/ //g')"
 311
 312 if [ "${number_of_failures}" -eq 0 ]
 313 then
 314     log
 315     log "All tests passed successfully."
 316     log
 317     exit_code=0
 318 else
 319     log ''
 320     log '===================================='
 321     [ "${number_of_failures}" -gt 1 ] && log "${number_of_failures} FAILURES" || log '1 FAILURE'
 322     failure=0
 323     ls -1tr "${TMPDIR}" | grep '^failure\.' | while read failure_file
 324     do
 325         while read failure_code entry1 entry2 entry3 entry4 entry5 entry6 entry7
 326         do
 327             log '===================================='
 328             log ''
 329             case "${failure_code}" in
 330
 331                 UPDATE_XML_UNAVAILABLE)
 332                     update_xml_url="${entry1}"
 333                     update_xml="${entry2}"
 334                     update_xml_headers="${entry3}"
 335                     update_xml_debug="${entry4}"
 336                     update_xml_curl_exit_code="${entry5}"
 337                     log "FAILURE $((++failure)): Update xml file not available"
 338                     log ""
 339                     log "    Download url: ${update_xml_url}"
 340                     log "    Curl returned exit code: ${update_xml_curl_exit_code}"
 341                     log ""
 342                     log "    The HTTP headers were:"
 343                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
 344                     log ""
 345                     log "    The full curl debug output was:"
 346                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
 347                     log ""
 348                     log "    The returned update.xml file was:"
 349                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
 350                     log ""
 351                     log "    This url was tested because of the following cfg file entries:"
 352                     show_cfg_file_entries "${update_xml_url}"
 353                     log ""
 354
 355                     ;;
 356
 357                 UPDATE_XML_REDIRECT_FAILED)
 358                     update_xml_url="${entry1}"
 359                     update_xml_actual_url="${entry2}"
 360                     update_xml="${entry3}"
 361                     update_xml_headers="${entry4}"
 362                     update_xml_debug="${entry5}"
 363                     update_xml_curl_exit_code="${entry6}"
 364                     log "FAILURE $((++failure)): Update xml file not available at *redirected* location"
 365                     log ""
 366                     log "    Download url: ${update_xml_url}"
 367                     log "    Redirected to: ${update_xml_actual_url}"
 368                     log "    It could not be downloaded from this url."
 369                     log "    Curl returned exit code: ${update_xml_curl_exit_code}"
 370                     log ""
 371                     log "    The HTTP headers were:"
 372                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
 373                     log ""
 374                     log "    The full curl debug output was:"
 375                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
 376                     log ""
 377                     log "    The returned update.xml file was:"
 378                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
 379                     log ""
 380                     log "    This url was tested because of the following cfg file entries:"
 381                     show_cfg_file_entries "${update_xml_url}"
 382                     log ""
 383                     ;;
 384
 385                 PATCH_TYPE_MISSING)
 386                     update_xml_url="${entry1}"
 387                     patch_type="${entry2}"
 388                     update_xml="${entry3}"
 389                     update_xml_headers="${entry4}"
 390                     update_xml_debug="${entry5}"
 391                     update_xml_actual_url="${entry6}"
 392                     log "FAILURE $((++failure)): Patch type '${patch_type}' not present in the downloaded update.xml file."
 393                     log ""
 394                     log "    Update xml file downloaded from: ${update_xml_url}"
 395                     [ -n "${update_xml_actual_url}" ] && log "    This redirected to the download url: ${update_xml_actual_url}"
 396                     log "    Curl returned exit code: 0 (success)"
 397                     log ""
 398                     log "    The HTTP headers were:"
 399                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_headers}"
 400                     log ""
 401                     log "    The full curl debug output was:"
 402                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml_debug}"
 403                     log ""
 404                     log "    The returned update.xml file was:"
 405                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${update_xml}"
 406                     log ""
 407                     log "    This url and patch type combination was tested due to the following cfg file entries:"
 408                     show_cfg_file_entries "${update_xml_url}"
 409                     log ""
 410                     ;;
 411
 412                 NO_MAR_FILE)
 413                     mar_url="${entry1}"
 414                     mar_headers_file="${entry2}"
 415                     mar_headers_debug_file="${entry3}"
 416                     mar_file_curl_exit_code="${entry4}"
 417                     mar_actual_url="${entry5}"
 418                     log "FAILURE $((++failure)): Could not retrieve mar file"
 419                     log ""
 420                     log "    Mar file url: ${mar_url}"
 421                     [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
 422                     log "    The mar file could not be downloaded from this location."
 423                     log "    Curl returned exit code: ${mar_file_curl_exit_code}"
 424                     log ""
 425                     log "    The HTTP headers were:"
 426                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
 427                     log ""
 428                     log "    The full curl debug output was:"
 429                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
 430                     log ""
 431                     log "    The mar download was tested because it was referenced in the following update xml file(s):"
 432                     show_update_xml_entries "${mar_url}"
 433                     log ""
 434                     ;;
 435
 436                 MAR_FILE_WRONG_SIZE)
 437                     mar_url="${entry1}"
 438                     mar_required_size="${entry2}"
 439                     mar_actual_size="${entry3}"
 440                     mar_headers_file="${entry4}"
 441                     mar_headers_debug_file="${entry5}"
 442                     mar_file_curl_exit_code="${entry6}"
 443                     mar_actual_url="${entry7}"
 444                     log "FAILURE $((++failure)): Mar file is wrong size"
 445                     log ""
 446                     log "    Mar file url: ${mar_url}"
 447                     [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
 448                     log "    The http header of the mar file url says that the mar file is ${mar_actual_size} bytes."
 449                     log "    One or more of the following update.xml file(s) says that the file should be ${mar_required_size} bytes."
 450                     log ""
 451                     log "    These are the update xml file(s) that referenced this mar:"
 452                     show_update_xml_entries "${mar_url}"
 453                     log ""
 454                     log "    Curl returned exit code: ${mar_file_curl_exit_code}"
 455                     log ""
 456                     log "    The HTTP headers were:"
 457                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
 458                     log ""
 459                     log "    The full curl debug output was:"
 460                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
 461                     log ""
 462                     ;;
 463
 464                 BAD_HTTP_RESPONSE_CODE_FOR_MAR)
 465                     mar_url="${entry1}"
 466                     mar_headers_file="${entry2}"
 467                     mar_headers_debug_file="${entry3}"
 468                     mar_file_curl_exit_code="${entry4}"
 469                     mar_actual_url="${entry5}"
 470                     http_response_code="$(sed -e "s/$(printf '\r')//" -n -e '/^HTTP\//p' "${mar_headers_file}" | tail -1)"
 471                     log "FAILURE $((++failure)): '${http_response_code}' for mar file"
 472                     log ""
 473                     log "    Mar file url: ${mar_url}"
 474                     [ -n "${mar_actual_url}" ] && log "    This redirected to: ${mar_actual_url}"
 475                     log ""
 476                     log "    These are the update xml file(s) that referenced this mar:"
 477                     show_update_xml_entries "${mar_url}"
 478                     log ""
 479                     log "    Curl returned exit code: ${mar_file_curl_exit_code}"
 480                     log ""
 481                     log "    The HTTP headers were:"
 482                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_file}"
 483                     log ""
 484                     log "    The full curl debug output was:"
 485                     sed -e "s/$(printf '\r')//" -e "s/^/$(date):          /" -e '$a\' "${mar_headers_debug_file}"
 486                     log ""
 487                     ;;
 488
 489                 *)
 490                     log "ERROR: Unknown failure code - '${failure_code}'"
 491                     log "ERROR: This is a serious bug in this script."
 492                     log "ERROR: Only known failure codes are: UPDATE_XML_UNAVAILABLE, UPDATE_XML_REDIRECT_FAILED, PATCH_TYPE_MISSING, NO_MAR_FILE, MAR_FILE_WRONG_SIZE, BAD_HTTP_RESPONSE_CODE_FOR_MAR"
 493                     log ""
 494                     log "FAILURE $((++failure)): Data from failure is: ${entry1} ${entry2} ${entry3} ${entry4} ${entry5} ${entry6}"
 495                     log ""
 496                     ;;
 497
 498             esac
 499         done < "${TMPDIR}/${failure_file}"
 500     done
 501     exit_code=1
 502 fi
 503
 504
 505 log ''
 506 log '===================================='
 507 log 'KEY STATS'
 508 log '===================================='
 509 log ''
 510 log "Config files scanned:                       ${#@}"
 511 log "Update xml files downloaded and parsed:     ${number_of_update_xml_urls}"
 512 log "Unique mar urls found:                      ${number_of_mar_urls}"
 513 log "Failures:                                   ${number_of_failures}"
 514 log "Parallel processes used (maximum limit):    ${MAX_PROCS}"
 515 log "Execution time:                             $((STOP_TIME-START_TIME)) seconds"
 516 log ''
 517
 518 rm -rf "${TMPDIR}"
 519 exit ${exit_code}