src/depot.py

   1 #!/usr/bin/python2.7
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
  23 #
  24
  25 from __future__ import print_function
  26
  27 # pkg.depotd - package repository daemon
  28
  29 # XXX The prototype pkg.depotd combines both the version management server that
  30 # answers to pkgsend(1) sessions and the HTTP file server that answers to the
  31 # various GET operations that a pkg(1) client makes.  This split is expected to
  32 # be made more explicit, by constraining the pkg(1) operations such that they
  33 # can be served as a typical HTTP/HTTPS session.  Thus, pkg.depotd will reduce
  34 # to a special purpose HTTP/HTTPS server explicitly for the version management
  35 # operations, and must manipulate the various state files--catalogs, in
  36 # particular--such that the pkg(1) pull client can operately accurately with
  37 # only a basic HTTP/HTTPS server in place.
  38
  39 # XXX Although we pushed the evaluation of next-version, etc. to the pull
  40 # client, we should probably provide a query API to do same on the server, for
  41 # dumb clients (like a notification service).
  42
  43 # The default path for static and other web content.
  44 CONTENT_PATH_DEFAULT = "/usr/share/lib/pkg"
  45 # cherrypy has a max_request_body_size parameter that determines whether the
  46 # server should abort requests with REQUEST_ENTITY_TOO_LARGE when the request
  47 # body is larger than the specified size (in bytes).  The maximum size supported
  48 # by cherrypy is 2048 * 1024 * 1024 - 1 (just short of 2048MB), but the default
  49 # here is purposefully conservative.
  50 MAX_REQUEST_BODY_SIZE = 512 * 1024 * 1024
  51 # The default host/port(s) to serve data from.
  52 HOST_DEFAULT = "0.0.0.0"
  53 PORT_DEFAULT = 80
  54 SSL_PORT_DEFAULT = 443
  55 # The minimum number of threads allowed.
  56 THREADS_MIN = 1
  57 # The default number of threads to start.
  58 THREADS_DEFAULT = 60
  59 # The maximum number of threads that can be started.
  60 THREADS_MAX = 5000
  61 # The default server socket timeout in seconds. We want this to be longer than
  62 # the normal default of 10 seconds to accommodate clients with poor quality
  63 # connections.
  64 SOCKET_TIMEOUT_DEFAULT = 60
  65
  66 import getopt
  67 import gettext
  68 import locale
  69 import logging
  70 import os
  71 import os.path
  72 import OpenSSL.crypto as crypto
  73 import subprocess
  74 import sys
  75 import tempfile
  76 import urlparse
  77
  78 from imp import reload
  79
  80 try:
  81         import cherrypy
  82         version = cherrypy.__version__.split('.')
  83         if map(int, version) < [3, 1, 0]:
  84                 raise ImportError
  85         elif map(int, version) >= [3, 2, 0]:
  86                 raise ImportError
  87 except ImportError:
  88         print("""cherrypy 3.1.0 or greater (but less than """
  89             """3.2.0) is required to use this program.""", file=sys.stderr)
  90         sys.exit(2)
  91
  92 import cherrypy.process.servers
  93 from cherrypy.process.plugins import Daemonizer
  94
  95 from pkg.misc import msg, emsg, setlocale
  96 from pkg.client.debugvalues import DebugValues
  97
  98 import pkg
  99 import pkg.client.api_errors as api_errors
 100 import pkg.config as cfg
 101 import pkg.portable.util as os_util
 102 import pkg.search_errors as search_errors
 103 import pkg.server.depot as ds
 104 import pkg.server.depotresponse as dr
 105 import pkg.server.repository as sr
 106
 107
 108 class LogSink(object):
 109         """This is a dummy object that we can use to discard log entries
 110         without relying on non-portable interfaces such as /dev/null."""
 111
 112         def write(self, *args, **kwargs):
 113                 """Discard the bits."""
 114                 pass
 115
 116         def flush(self, *args, **kwargs):
 117                 """Discard the bits."""
 118                 pass
 119
 120
 121 def usage(text=None, retcode=2, full=False):
 122         """Optionally emit a usage message and then exit using the specified
 123         exit code."""
 124
 125         if text:
 126                 emsg(text)
 127
 128         if not full:
 129                 # The full usage message isn't desired.
 130                 emsg(_("Try `pkg.depotd --help or -?' for more "
 131                     "information."))
 132                 sys.exit(retcode)
 133
 134         print("""\
 135 Usage: /usr/lib/pkg.depotd [-a address] [-d inst_root] [-p port] [-s threads]
 136            [-t socket_timeout] [--cfg] [--content-root]
 137            [--disable-ops op[/1][,...]] [--debug feature_list]
 138            [--image-root dir] [--log-access dest] [--log-errors dest]
 139            [--mirror] [--nasty] [--nasty-sleep] [--proxy-base url]
 140            [--readonly] [--ssl-cert-file] [--ssl-dialog] [--ssl-key-file]
 141            [--sort-file-max-size size] [--writable-root dir]
 142
 143         -a address      The IP address on which to listen for connections.  The
 144                         default value is 0.0.0.0 (INADDR_ANY) which will listen
 145                         on all active interfaces.  To listen on all active IPv6
 146                         interfaces, use '::'.
 147         -d inst_root    The file system path at which the server should find its
 148                         repository data.  Required unless PKG_REPO has been set
 149                         in the environment.
 150         -p port         The port number on which the instance should listen for
 151                         incoming package requests.  The default value is 80 if
 152                         ssl certificate and key information has not been
 153                         provided; otherwise, the default value is 443.
 154         -s threads      The number of threads that will be started to serve
 155                         requests.  The default value is 10.
 156         -t timeout      The maximum number of seconds the server should wait for
 157                         a response from a client before closing a connection.
 158                         The default value is 60.
 159         --cfg           The pathname of the file to use when reading and writing
 160                         depot configuration data, or a fully qualified service
 161                         fault management resource identifier (FMRI) of the SMF
 162                         service or instance to read configuration data from.
 163         --content-root  The file system path to the directory containing the
 164                         the static and other web content used by the depot's
 165                         browser user interface.  The default value is
 166                         '/usr/share/lib/pkg'.
 167         --disable-ops   A comma separated list of operations that the depot
 168                         should not configure.  If, for example, you wanted
 169                         to omit loading search v1, 'search/1' should be
 170                         provided as an argument, or to disable all search
 171                         operations, simply 'search'.
 172         --debug         The name of a debug feature to enable; or a whitespace
 173                         or comma separated list of features to enable.
 174                         Possible values are: headers, hash=sha1+sha256,
 175                         hash=sha256, hash=sha1+sha512_256, hash=sha512_256
 176         --image-root    The path to the image whose file information will be
 177                         used as a cache for file data.
 178         --log-access    The destination for any access related information
 179                         logged by the depot process.  Possible values are:
 180                         stderr, stdout, none, or an absolute pathname.  The
 181                         default value is stdout if stdout is a tty; otherwise
 182                         the default value is none.
 183         --log-errors    The destination for any errors or other information
 184                         logged by the depot process.  Possible values are:
 185                         stderr, stdout, none, or an absolute pathname.  The
 186                         default value is stderr.
 187         --mirror        Package mirror mode; publishing and metadata operations
 188                         disallowed.  Cannot be used with --readonly or
 189                         --rebuild.
 190         --nasty         Instruct the server to misbehave.  At random intervals
 191                         it will time-out, send bad responses, hang up on
 192                         clients, and generally be hostile.  The option
 193                         takes a value (1 to 100) for how nasty the server
 194                         should be.
 195         --nasty-sleep   In nasty mode (see --nasty), how many seconds to
 196                         randomly sleep when a random sleep occurs.
 197         --proxy-base    The url to use as the base for generating internal
 198                         redirects and content.
 199         --readonly      Read-only operation; modifying operations disallowed.
 200                         Cannot be used with --mirror or --rebuild.
 201         --ssl-cert-file The absolute pathname to a PEM-encoded Certificate file.
 202                         This option must be used with --ssl-key-file.  Usage of
 203                         this option will cause the depot to only respond to SSL
 204                         requests on the provided port.
 205         --ssl-dialog    Specifies what method should be used to obtain the
 206                         passphrase needed to decrypt the file specified by
 207                         --ssl-key-file.  Supported values are: builtin,
 208                         exec:/path/to/program, smf, or an SMF FMRI.  The
 209                         default value is builtin.  If smf is specified, an
 210                         SMF FMRI must be provided using the --cfg option.
 211         --ssl-key-file  The absolute pathname to a PEM-encoded Private Key file.
 212                         This option must be used with --ssl-cert-file.  Usage of
 213                         this option will cause the depot to only respond to SSL
 214                         requests on the provided port.
 215         --sort-file-max-size
 216                         The maximum size of the indexer sort file. Used to
 217                         limit the amount of RAM the depot uses for indexing,
 218                         or increase it for speed.
 219         --writable-root The path to a directory to which the program has write
 220                         access.  Used with --readonly to allow server to
 221                         create needed files, such as search indices, without
 222                         needing write access to the package information.
 223 Options:
 224         --help or -?
 225
 226 Environment:
 227         PKG_REPO                Used as default inst_root if -d not provided.
 228         PKG_DEPOT_CONTENT       Used as default content_root if --content-root
 229                                 not provided.""")
 230         sys.exit(retcode)
 231
 232 class OptionError(Exception):
 233         """Option exception. """
 234
 235         def __init__(self, *args):
 236                 Exception.__init__(self, *args)
 237
 238 if __name__ == "__main__":
 239
 240         setlocale(locale.LC_ALL, "")
 241         gettext.install("pkg", "/usr/share/locale",
 242             codeset=locale.getpreferredencoding())
 243
 244         add_content = False
 245         exit_ready = False
 246         rebuild = False
 247         reindex = False
 248         nasty = False
 249
 250         # Track initial configuration values.
 251         ivalues = { "pkg": {}, "nasty": {} }
 252         if "PKG_REPO" in os.environ:
 253                 ivalues["pkg"]["inst_root"] = os.environ["PKG_REPO"]
 254
 255         try:
 256                 content_root = os.environ["PKG_DEPOT_CONTENT"]
 257                 ivalues["pkg"]["content_root"] = content_root
 258         except KeyError:
 259                 try:
 260                         content_root = os.path.join(os.environ['PKG_HOME'],
 261                             'share/lib/pkg')
 262                         ivalues["pkg"]["content_root"] = content_root
 263                 except KeyError:
 264                         pass
 265
 266         opt = None
 267         addresses = set()
 268         debug_features = []
 269         disable_ops = []
 270         repo_props = {}
 271         socket_path = ""
 272         user_cfg = None
 273         try:
 274                 long_opts = ["add-content", "cfg=", "cfg-file=",
 275                     "content-root=", "debug=", "disable-ops=", "exit-ready",
 276                     "help", "image-root=", "log-access=", "log-errors=",
 277                     "llmirror", "mirror", "nasty=", "nasty-sleep=",
 278                     "proxy-base=", "readonly", "rebuild", "refresh-index",
 279                     "set-property=", "ssl-cert-file=", "ssl-dialog=",
 280                     "ssl-key-file=", "sort-file-max-size=", "writable-root="]
 281
 282                 opts, pargs = getopt.getopt(sys.argv[1:], "a:d:np:s:t:?",
 283                     long_opts)
 284
 285                 show_usage = False
 286                 for opt, arg in opts:
 287                         if opt == "-a":
 288                                 addresses.add(arg)
 289                         elif opt == "-n":
 290                                 sys.exit(0)
 291                         elif opt == "-d":
 292                                 ivalues["pkg"]["inst_root"] = arg
 293                         elif opt == "-p":
 294                                 ivalues["pkg"]["port"] = arg
 295                         elif opt == "-s":
 296                                 threads = int(arg)
 297                                 if threads < THREADS_MIN:
 298                                         raise OptionError(
 299                                             "minimum value is {0:d}".format(
 300                                             THREADS_MIN))
 301                                 if threads > THREADS_MAX:
 302                                         raise OptionError(
 303                                             "maximum value is {0:d}".format(
 304                                             THREADS_MAX))
 305                                 ivalues["pkg"]["threads"] = threads
 306                         elif opt == "-t":
 307                                 ivalues["pkg"]["socket_timeout"] = arg
 308                         elif opt == "--add-content":
 309                                 add_content = True
 310                         elif opt == "--cfg":
 311                                 user_cfg  = arg
 312                         elif opt == "--cfg-file":
 313                                 ivalues["pkg"]["cfg_file"] = arg
 314                         elif opt == "--content-root":
 315                                 ivalues["pkg"]["content_root"] = arg
 316                         elif opt == "--debug":
 317                                 if arg is None or arg == "":
 318                                         continue
 319
 320                                 # A list of features can be specified using a
 321                                 # "," or any whitespace character as separators.
 322                                 if "," in arg:
 323                                         features = arg.split(",")
 324                                 else:
 325                                         features = arg.split()
 326                                 debug_features.extend(features)
 327
 328                                 # We also allow key=value debug flags, which
 329                                 # get set in pkg.client.debugvalues
 330                                 for feature in features:
 331                                         try:
 332                                                 key, val = feature.split("=", 1)
 333                                                 DebugValues.set_value(key, val)
 334                                         except (AttributeError, ValueError):
 335                                                 pass
 336
 337                         elif opt == "--disable-ops":
 338                                 if arg is None or arg == "":
 339                                         raise OptionError(
 340                                             "An argument must be specified.")
 341
 342                                 disableops = arg.split(",")
 343                                 for s in disableops:
 344                                         if "/" in s:
 345                                                 op, ver = s.rsplit("/", 1)
 346                                         else:
 347                                                 op = s
 348                                                 ver = "*"
 349
 350                                         if op not in \
 351                                             ds.DepotHTTP.REPO_OPS_DEFAULT:
 352                                                 raise OptionError(
 353                                                     "Invalid operation "
 354                                                     "'{0}'.".format(s))
 355                                         disable_ops.append(s)
 356                         elif opt == "--exit-ready":
 357                                 exit_ready = True
 358                         elif opt == "--image-root":
 359                                 ivalues["pkg"]["image_root"] = arg
 360                         elif opt.startswith("--log-"):
 361                                 prop = "log_{0}".format(opt.lstrip("--log-"))
 362                                 ivalues["pkg"][prop] = arg
 363                         elif opt in ("--help", "-?"):
 364                                 show_usage = True
 365                         elif opt == "--mirror":
 366                                 ivalues["pkg"]["mirror"] = True
 367                         elif opt == "--llmirror":
 368                                 ivalues["pkg"]["mirror"] = True
 369                                 ivalues["pkg"]["ll_mirror"] = True
 370                                 ivalues["pkg"]["readonly"] = True
 371                         elif opt == "--nasty":
 372                                 # ValueError is caught by caller.
 373                                 nasty_value = int(arg)
 374                                 if (nasty_value > 100 or nasty_value < 1):
 375                                         raise OptionError("Invalid value "
 376                                             "for nasty option.\n Please "
 377                                             "choose a value between 1 and 100.")
 378                                 nasty = True
 379                                 ivalues["nasty"]["nasty_level"] = nasty_value
 380                         elif opt == "--nasty-sleep":
 381                                 # ValueError is caught by caller.
 382                                 sleep_value = int(arg)
 383                                 ivalues["nasty"]["nasty_sleep"] = sleep_value
 384                         elif opt == "--proxy-base":
 385                                 # Attempt to decompose the url provided into
 386                                 # its base parts.  This is done so we can
 387                                 # remove any scheme information since we
 388                                 # don't need it.
 389                                 scheme, netloc, path, params, query, \
 390                                     fragment = urlparse.urlparse(arg,
 391                                     "http", allow_fragments=0)
 392
 393                                 if not netloc:
 394                                         raise OptionError("Unable to "
 395                                             "determine the hostname from "
 396                                             "the provided URL; please use a "
 397                                             "fully qualified URL.")
 398
 399                                 scheme = scheme.lower()
 400                                 if scheme not in ("http", "https"):
 401                                         raise OptionError("Invalid URL; http "
 402                                             "and https are the only supported "
 403                                             "schemes.")
 404
 405                                 # Rebuild the url with the sanitized components.
 406                                 ivalues["pkg"]["proxy_base"] = \
 407                                     urlparse.urlunparse((scheme, netloc, path,
 408                                     params, query, fragment))
 409                         elif opt == "--readonly":
 410                                 ivalues["pkg"]["readonly"] = True
 411                         elif opt == "--rebuild":
 412                                 rebuild = True
 413                         elif opt == "--refresh-index":
 414                                 # Note: This argument is for internal use
 415                                 # only.
 416                                 #
 417                                 # This flag is purposefully omitted in usage.
 418                                 # The supported way to forcefully reindex is to
 419                                 # kill any pkg.depot using that directory,
 420                                 # remove the index directory, and restart the
 421                                 # pkg.depot process. The index will be rebuilt
 422                                 # automatically on startup.
 423                                 reindex = True
 424                                 exit_ready = True
 425                         elif opt == "--set-property":
 426                                 try:
 427                                         prop, p_value = arg.split("=", 1)
 428                                         p_sec, p_name = prop.split(".", 1)
 429                                 except ValueError:
 430                                         usage(_("property arguments must be of "
 431                                             "the form '<section.property>="
 432                                             "<value>'."))
 433                                 repo_props.setdefault(p_sec, {})
 434                                 repo_props[p_sec][p_name] = p_value
 435                         elif opt == "--ssl-cert-file":
 436                                 if arg == "none" or arg == "":
 437                                         # Assume this is an override to clear
 438                                         # the value.
 439                                         arg = ""
 440                                 elif not os.path.isabs(arg):
 441                                         raise OptionError("The path to "
 442                                            "the Certificate file must be "
 443                                            "absolute.")
 444                                 elif not os.path.exists(arg):
 445                                         raise OptionError("The specified "
 446                                             "file does not exist.")
 447                                 elif not os.path.isfile(arg):
 448                                         raise OptionError("The specified "
 449                                             "pathname is not a file.")
 450                                 ivalues["pkg"]["ssl_cert_file"] = arg
 451                         elif opt == "--ssl-key-file":
 452                                 if arg == "none" or arg == "":
 453                                         # Assume this is an override to clear
 454                                         # the value.
 455                                         arg = ""
 456                                 elif not os.path.isabs(arg):
 457                                         raise OptionError("The path to "
 458                                            "the Private Key file must be "
 459                                            "absolute.")
 460                                 elif not os.path.exists(arg):
 461                                         raise OptionError("The specified "
 462                                             "file does not exist.")
 463                                 elif not os.path.isfile(arg):
 464                                         raise OptionError("The specified "
 465                                             "pathname is not a file.")
 466                                 ivalues["pkg"]["ssl_key_file"] = arg
 467                         elif opt == "--ssl-dialog":
 468                                 if arg != "builtin" and \
 469                                     arg != "smf" and not \
 470                                     arg.startswith("exec:/") and not \
 471                                     arg.startswith("svc:"):
 472                                         raise OptionError("Invalid value "
 473                                             "specified.  Expected: builtin, "
 474                                             "exec:/path/to/program, smf, or "
 475                                             "an SMF FMRI.")
 476
 477                                 if arg.startswith("exec:"):
 478                                         if os_util.get_canonical_os_type() != \
 479                                           "unix":
 480                                                 # Don't allow a somewhat
 481                                                 # insecure authentication method
 482                                                 # on some platforms.
 483                                                 raise OptionError("exec is "
 484                                                     "not a supported dialog "
 485                                                     "type for this operating "
 486                                                     "system.")
 487
 488                                         f = os.path.abspath(arg.split(
 489                                             "exec:")[1])
 490                                         if not os.path.isfile(f):
 491                                                 raise OptionError("Invalid "
 492                                                     "file path specified for "
 493                                                     "exec.")
 494                                 ivalues["pkg"]["ssl_dialog"] = arg
 495                         elif opt == "--sort-file-max-size":
 496                                 ivalues["pkg"]["sort_file_max_size"] = arg
 497                         elif opt == "--writable-root":
 498                                 ivalues["pkg"]["writable_root"] = arg
 499
 500                 # Set accumulated values.
 501                 if debug_features:
 502                         ivalues["pkg"]["debug"] = debug_features
 503                 if disable_ops:
 504                         ivalues["pkg"]["disable_ops"] = disable_ops
 505                 if addresses:
 506                         ivalues["pkg"]["address"] = list(addresses)
 507
 508                 if DebugValues:
 509                         reload(pkg.digest)
 510
 511                 # Build configuration object.
 512                 dconf = ds.DepotConfig(target=user_cfg, overrides=ivalues)
 513         except getopt.GetoptError as _e:
 514                 usage("pkg.depotd: {0}".format(_e.msg))
 515         except api_errors.ApiException as _e:
 516                 usage("pkg.depotd: {0}".format(str(_e)))
 517         except OptionError as _e:
 518                 usage("pkg.depotd: option: {0} -- {1}".format(opt, _e))
 519         except (ArithmeticError, ValueError):
 520                 usage("pkg.depotd: illegal option value: {0} specified " \
 521                     "for option: {1}".format(arg, opt))
 522
 523         if show_usage:
 524                 usage(retcode=0, full=True)
 525
 526         if not dconf.get_property("pkg", "log_errors"):
 527                 dconf.set_property("pkg", "log_errors", "stderr")
 528
 529         # If stdout is a tty, then send access output there by default instead
 530         # of discarding it.
 531         if not dconf.get_property("pkg", "log_access"):
 532                 if os.isatty(sys.stdout.fileno()):
 533                         dconf.set_property("pkg", "log_access", "stdout")
 534                 else:
 535                         dconf.set_property("pkg", "log_access", "none")
 536
 537         # Check for invalid option combinations.
 538         image_root = dconf.get_property("pkg", "image_root")
 539         inst_root = dconf.get_property("pkg", "inst_root")
 540         mirror = dconf.get_property("pkg", "mirror")
 541         ll_mirror = dconf.get_property("pkg", "ll_mirror")
 542         readonly = dconf.get_property("pkg", "readonly")
 543         writable_root = dconf.get_property("pkg", "writable_root")
 544         if rebuild and add_content:
 545                 usage("--add-content cannot be used with --rebuild")
 546         if rebuild and reindex:
 547                 usage("--refresh-index cannot be used with --rebuild")
 548         if (rebuild or add_content) and (readonly or mirror):
 549                 usage("--readonly and --mirror cannot be used with --rebuild "
 550                     "or --add-content")
 551         if reindex and mirror:
 552                 usage("--mirror cannot be used with --refresh-index")
 553         if reindex and readonly and not writable_root:
 554                 usage("--readonly can only be used with --refresh-index if "
 555                     "--writable-root is used")
 556         if image_root and not ll_mirror:
 557                 usage("--image-root can only be used with --llmirror.")
 558         if image_root and writable_root:
 559                 usage("--image_root and --writable-root cannot be used "
 560                     "together.")
 561         if image_root and inst_root:
 562                 usage("--image-root and -d cannot be used together.")
 563
 564         # If the image format changes this may need to be reexamined.
 565         if image_root:
 566                 inst_root = os.path.join(image_root, "var", "pkg")
 567
 568         # Set any values using defaults if they weren't provided.
 569
 570         # Only use the first value for now; multiple bind addresses may be
 571         # supported later.
 572         address = dconf.get_property("pkg", "address")
 573         if address:
 574                 address = address[0]
 575         elif not address:
 576                 dconf.set_property("pkg", "address", [HOST_DEFAULT])
 577                 address = dconf.get_property("pkg", "address")[0]
 578
 579         if not inst_root:
 580                 usage("Either PKG_REPO or -d must be provided")
 581
 582         content_root = dconf.get_property("pkg", "content_root")
 583         if not content_root:
 584                 dconf.set_property("pkg", "content_root", CONTENT_PATH_DEFAULT)
 585                 content_root = dconf.get_property("pkg", "content_root")
 586
 587         port = dconf.get_property("pkg", "port")
 588         ssl_cert_file = dconf.get_property("pkg", "ssl_cert_file")
 589         ssl_key_file = dconf.get_property("pkg", "ssl_key_file")
 590         if (ssl_cert_file and not ssl_key_file) or (ssl_key_file and not
 591             ssl_cert_file):
 592                 usage("The --ssl-cert-file and --ssl-key-file options must "
 593                     "must both be provided when using either option.")
 594         elif not port:
 595                 if ssl_cert_file and ssl_key_file:
 596                         dconf.set_property("pkg", "port", SSL_PORT_DEFAULT)
 597                 else:
 598                         dconf.set_property("pkg", "port", PORT_DEFAULT)
 599                 port = dconf.get_property("pkg", "port")
 600
 601         socket_timeout = dconf.get_property("pkg", "socket_timeout")
 602         if not socket_timeout:
 603                 dconf.set_property("pkg", "socket_timeout",
 604                     SOCKET_TIMEOUT_DEFAULT)
 605                 socket_timeout = dconf.get_property("pkg", "socket_timeout")
 606
 607         threads = dconf.get_property("pkg", "threads")
 608         if not threads:
 609                 dconf.set_property("pkg", "threads", THREADS_DEFAULT)
 610                 threads = dconf.get_property("pkg", "threads")
 611
 612         # If the program is going to reindex, the port is irrelevant since
 613         # the program will not bind to a port.
 614         if not exit_ready:
 615                 try:
 616                         cherrypy.process.servers.check_port(address, port)
 617                 except Exception as e:
 618                         emsg("pkg.depotd: unable to bind to the specified "
 619                             "port: {0:d}. Reason: {1}".format(port, e))
 620                         sys.exit(1)
 621         else:
 622                 # Not applicable if we're not going to serve content
 623                 dconf.set_property("pkg", "content_root", "")
 624
 625         # Any relative paths should be made absolute using pkg_root.  'pkg_root'
 626         # is a special property that was added to enable internal deployment of
 627         # multiple disparate versions of the pkg.depotd software.
 628         pkg_root = dconf.get_property("pkg", "pkg_root")
 629
 630         repo_config_file = dconf.get_property("pkg", "cfg_file")
 631         if repo_config_file and not os.path.isabs(repo_config_file):
 632                 repo_config_file = os.path.join(pkg_root, repo_config_file)
 633
 634         if content_root and not os.path.isabs(content_root):
 635                 content_root = os.path.join(pkg_root, content_root)
 636
 637         if inst_root and not os.path.isabs(inst_root):
 638                 inst_root = os.path.join(pkg_root, inst_root)
 639
 640         if ssl_cert_file:
 641                 if ssl_cert_file == "none":
 642                         ssl_cert_file = None
 643                 elif not os.path.isabs(ssl_cert_file):
 644                         ssl_cert_file = os.path.join(pkg_root, ssl_cert_file)
 645
 646         if ssl_key_file:
 647                 if ssl_key_file == "none":
 648                         ssl_key_file = None
 649                 elif not os.path.isabs(ssl_key_file):
 650                         ssl_key_file = os.path.join(pkg_root, ssl_key_file)
 651
 652         if writable_root and not os.path.isabs(writable_root):
 653                 writable_root = os.path.join(pkg_root, writable_root)
 654
 655         # Setup SSL if requested.
 656         key_data = None
 657         ssl_dialog = dconf.get_property("pkg", "ssl_dialog")
 658         if not exit_ready and ssl_cert_file and ssl_key_file and \
 659             ssl_dialog != "builtin":
 660                 cmdline = None
 661                 def get_ssl_passphrase(*ignored):
 662                         p = None
 663                         try:
 664                                 p = subprocess.Popen(cmdline, shell=True,
 665                                         stdout=subprocess.PIPE,
 666                                         stderr=None)
 667                                 p.wait()
 668                         except Exception as __e:
 669                                 emsg("pkg.depotd: an error occurred while "
 670                                     "executing [{0}]; unable to obtain the "
 671                                     "passphrase needed to decrypt the SSL "
 672                                     "private key file: {1}".format(cmdline,
 673                                     __e))
 674                                 sys.exit(1)
 675                         return p.stdout.read().strip("\n")
 676
 677                 if ssl_dialog.startswith("exec:"):
 678                         exec_path = ssl_dialog.split("exec:")[1]
 679                         if not os.path.isabs(exec_path):
 680                                 exec_path = os.path.join(pkg_root, exec_path)
 681                         cmdline = "{0} {1} {2:d}".format(exec_path, "''", port)
 682                 elif ssl_dialog == "smf" or ssl_dialog.startswith("svc:"):
 683                         if ssl_dialog == "smf":
 684                                 # Assume the configuration target was an SMF
 685                                 # FMRI and let svcprop fail with an error if
 686                                 # it wasn't.
 687                                 svc_fmri = dconf.target
 688                         else:
 689                                 svc_fmri = ssl_dialog
 690                         cmdline = "/usr/bin/svcprop -p " \
 691                             "pkg_secure/ssl_key_passphrase {0}".format(svc_fmri)
 692
 693                 # The key file requires decryption, but the user has requested
 694                 # exec-based authentication, so it will have to be decoded first
 695                 # to an un-named temporary file.
 696                 try:
 697                         with file(ssl_key_file, "rb") as key_file:
 698                                 pkey = crypto.load_privatekey(
 699                                     crypto.FILETYPE_PEM, key_file.read(),
 700                                     get_ssl_passphrase)
 701
 702                         key_data = tempfile.TemporaryFile()
 703                         key_data.write(crypto.dump_privatekey(
 704                             crypto.FILETYPE_PEM, pkey))
 705                         key_data.seek(0)
 706                 except EnvironmentError as _e:
 707                         emsg("pkg.depotd: unable to read the SSL private key "
 708                             "file: {0}".format(_e))
 709                         sys.exit(1)
 710                 except crypto.Error as _e:
 711                         emsg("pkg.depotd: authentication or cryptography "
 712                             "failure while attempting to decode\nthe SSL "
 713                             "private key file: {0}".format(_e))
 714                         sys.exit(1)
 715                 else:
 716                         # Redirect the server to the decrypted key file.
 717                         ssl_key_file = "/dev/fd/{0:d}".format(key_data.fileno())
 718
 719         # Setup our global configuration.
 720         gconf = {
 721             "checker.on": True,
 722             "environment": "production",
 723             "log.screen": False,
 724             "server.max_request_body_size": MAX_REQUEST_BODY_SIZE,
 725             "server.shutdown_timeout": 0,
 726             "server.socket_host": address,
 727             "server.socket_port": port,
 728             "server.socket_timeout": socket_timeout,
 729             "server.ssl_certificate": ssl_cert_file,
 730             "server.ssl_private_key": ssl_key_file,
 731             "server.thread_pool": threads,
 732             "tools.log_headers.on": True,
 733             "tools.encode.on": True
 734         }
 735
 736         if "headers" in dconf.get_property("pkg", "debug"):
 737                 # Despite its name, this only logs headers when there is an
 738                 # error; it's redundant with the debug feature enabled.
 739                 gconf["tools.log_headers.on"] = False
 740
 741                 # Causes the headers of every request to be logged to the error
 742                 # log; even if an exception occurs.
 743                 gconf["tools.log_headers_always.on"] = True
 744                 cherrypy.tools.log_headers_always = cherrypy.Tool(
 745                     "on_start_resource",
 746                     cherrypy.lib.cptools.log_request_headers)
 747
 748         log_cfg = {
 749             "access": dconf.get_property("pkg", "log_access"),
 750             "errors": dconf.get_property("pkg", "log_errors")
 751         }
 752
 753         # If stdin is not a tty and the pkgdepot controller isn't being used,
 754         # then assume process will be daemonized and redirect output.
 755         if not os.environ.get("PKGDEPOT_CONTROLLER") and \
 756             not os.isatty(sys.stdin.fileno()):
 757                 # Ensure log handlers are setup to use the file descriptors for
 758                 # stdout and stderr as the Daemonizer (used for test suite and
 759                 # SMF service) requires this.
 760                 if log_cfg["access"] == "stdout":
 761                         log_cfg["access"] = "/dev/fd/{0:d}".format(
 762                             sys.stdout.fileno())
 763                 elif log_cfg["access"] == "stderr":
 764                         log_cfg["access"] = "/dev/fd/{0:d}".format(
 765                             sys.stderr.fileno())
 766                 elif log_cfg["access"] == "none":
 767                         log_cfg["access"] = "/dev/null"
 768
 769                 if log_cfg["errors"] == "stderr":
 770                         log_cfg["errors"] = "/dev/fd/{0:d}".format(
 771                             sys.stderr.fileno())
 772                 elif log_cfg["errors"] == "stdout":
 773                         log_cfg["errors"] = "/dev/fd/{0:d}".format(
 774                             sys.stdout.fileno())
 775                 elif log_cfg["errors"] == "none":
 776                         log_cfg["errors"] = "/dev/null"
 777
 778         log_type_map = {
 779             "errors": {
 780                 "param": "log.error_file",
 781                 "attr": "error_log"
 782             },
 783             "access": {
 784                 "param": "log.access_file",
 785                 "attr": "access_log"
 786             }
 787         }
 788
 789         for log_type in log_type_map:
 790                 dest = log_cfg[log_type]
 791                 if dest in ("stdout", "stderr", "none"):
 792                         if dest == "none":
 793                                 h = logging.StreamHandler(LogSink())
 794                         else:
 795                                 h = logging.StreamHandler(eval("sys.{0}".format(
 796                                     dest)))
 797
 798                         h.setLevel(logging.DEBUG)
 799                         h.setFormatter(cherrypy._cplogging.logfmt)
 800                         log_obj = eval("cherrypy.log.{0}".format(
 801                             log_type_map[log_type]["attr"]))
 802                         log_obj.addHandler(h)
 803                         # Since we've replaced cherrypy's log handler with our
 804                         # own, we don't want the output directed to a file.
 805                         dest = ""
 806                 elif dest:
 807                         if not os.path.isabs(dest):
 808                                 dest = os.path.join(pkg_root, dest)
 809                 gconf[log_type_map[log_type]["param"]] = dest
 810
 811         cherrypy.config.update(gconf)
 812
 813         # Now that our logging, etc. has been setup, it's safe to perform any
 814         # remaining preparation.
 815
 816         # Initialize repository state.
 817         if not readonly:
 818                 # Not readonly, so assume a new repository should be created.
 819                 try:
 820                         sr.repository_create(inst_root, properties=repo_props)
 821                 except sr.RepositoryExistsError:
 822                         # Already exists, nothing to do.
 823                         pass
 824                 except (api_errors.ApiException, sr.RepositoryError) as _e:
 825                         emsg("pkg.depotd: {0}".format(_e))
 826                         sys.exit(1)
 827
 828         try:
 829                 sort_file_max_size = dconf.get_property("pkg",
 830                     "sort_file_max_size")
 831
 832                 repo = sr.Repository(cfgpathname=repo_config_file,
 833                     log_obj=cherrypy, mirror=mirror, properties=repo_props,
 834                     read_only=readonly, root=inst_root,
 835                     sort_file_max_size=sort_file_max_size,
 836                     writable_root=writable_root)
 837         except (RuntimeError, sr.RepositoryError) as _e:
 838                 emsg("pkg.depotd: {0}".format(_e))
 839                 sys.exit(1)
 840         except search_errors.IndexingException as _e:
 841                 emsg("pkg.depotd: {0}".format(str(_e)), "INDEX")
 842                 sys.exit(1)
 843         except api_errors.ApiException as _e:
 844                 emsg("pkg.depotd: {0}".format(str(_e)))
 845                 sys.exit(1)
 846
 847         if not rebuild and not add_content and not repo.mirror and \
 848             not (repo.read_only and not repo.writable_root):
 849                 # Automatically update search indexes on startup if not already
 850                 # told to, and not in readonly/mirror mode.
 851                 reindex = True
 852
 853         if reindex:
 854                 try:
 855                         # Only execute a index refresh here if --exit-ready was
 856                         # requested; it will be handled later in the setup
 857                         # process for other cases.
 858                         if repo.root and exit_ready:
 859                                 repo.refresh_index()
 860                 except (sr.RepositoryError, search_errors.IndexingException,
 861                     api_errors.ApiException) as e:
 862                         emsg(str(e), "INDEX")
 863                         sys.exit(1)
 864         elif rebuild:
 865                 try:
 866                         repo.rebuild(build_index=True)
 867                 except sr.RepositoryError as e:
 868                         emsg(str(e), "REBUILD")
 869                         sys.exit(1)
 870                 except (search_errors.IndexingException,
 871                     api_errors.UnknownErrors,
 872                     api_errors.PermissionsException) as e:
 873                         emsg(str(e), "INDEX")
 874                         sys.exit(1)
 875         elif add_content:
 876                 try:
 877                         repo.add_content()
 878                         repo.refresh_index()
 879                 except sr.RepositoryError as e:
 880                         emsg(str(e), "ADD_CONTENT")
 881                         sys.exit(1)
 882                 except (search_errors.IndexingException,
 883                     api_errors.UnknownErrors,
 884                     api_errors.PermissionsException) as e:
 885                         emsg(str(e), "INDEX")
 886                         sys.exit(1)
 887
 888         # Ready to start depot; exit now if requested.
 889         if exit_ready:
 890                 sys.exit(0)
 891
 892         # Next, initialize depot.
 893         if nasty:
 894                 depot = ds.NastyDepotHTTP(repo, dconf)
 895         else:
 896                 depot = ds.DepotHTTP(repo, dconf)
 897
 898         # Now build our site configuration.
 899         conf = {
 900             "/": {
 901                 # We have to override cherrypy's default response_class so that
 902                 # we have access to the write() callable to stream data
 903                 # directly to the client.
 904                 "wsgi.response_class": dr.DepotResponse,
 905             },
 906             "/robots.txt": {
 907                 "tools.staticfile.on": True,
 908                 "tools.staticfile.filename": os.path.join(depot.web_root,
 909                     "robots.txt")
 910             },
 911         }
 912
 913         proxy_base = dconf.get_property("pkg", "proxy_base")
 914         if proxy_base:
 915                 # This changes the base URL for our server, and is primarily
 916                 # intended to allow our depot process to operate behind Apache
 917                 # or some other webserver process.
 918                 #
 919                 # Visit the following URL for more information:
 920                 #    http://cherrypy.org/wiki/BuiltinTools#tools.proxy
 921                 proxy_conf = {
 922                         "tools.proxy.on": True,
 923                         "tools.proxy.local": "",
 924                         "tools.proxy.base": proxy_base
 925                 }
 926
 927                 # Now merge or add our proxy configuration information into the
 928                 # existing configuration.
 929                 for entry in proxy_conf:
 930                         conf["/"][entry] = proxy_conf[entry]
 931
 932         if reindex:
 933                 # Tell depot to update search indexes when possible;
 934                 # this is done as a background task so that packages
 935                 # can be served immediately while search indexes are
 936                 # still being updated.
 937                 depot._queue_refresh_index()
 938
 939         # If stdin is not a tty and the pkgdepot controller isn't being used,
 940         # then assume process should be daemonized.
 941         if not os.environ.get("PKGDEPOT_CONTROLLER") and \
 942             not os.isatty(sys.stdin.fileno()):
 943                 # Translate the values in log_cfg into paths.
 944                 Daemonizer(cherrypy.engine, stderr=log_cfg["errors"],
 945                     stdout=log_cfg["access"]).subscribe()
 946
 947         try:
 948                 root = cherrypy.Application(depot)
 949                 cherrypy.quickstart(root, config=conf)
 950         except Exception as _e:
 951                 emsg("pkg.depotd: unknown error starting depot server, " \
 952                     "illegal option value specified?")
 953                 emsg(_e)
 954                 sys.exit(1)