src/sysrepo.py

   1 #!/usr/bin/python2.7
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22
  23 #
  24 # Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
  25 #
  26
  27 import atexit
  28 import errno
  29 import getopt
  30 import gettext
  31 import locale
  32 import logging
  33 import os
  34 import shutil
  35 import simplejson
  36 import socket
  37 import stat
  38 import sys
  39 import traceback
  40 import urllib2
  41 import warnings
  42
  43 from mako.template import Template
  44
  45 from pkg.client import global_settings
  46 from pkg.misc import msg, PipeError
  47
  48 import pkg
  49 import pkg.catalog
  50 import pkg.client.api
  51 import pkg.client.progress as progress
  52 import pkg.client.api_errors as apx
  53 import pkg.digest as digest
  54 import pkg.misc as misc
  55 import pkg.portable as portable
  56 import pkg.p5p as p5p
  57
  58 logger = global_settings.logger
  59 orig_cwd = None
  60
  61 PKG_CLIENT_NAME = "pkg.sysrepo"
  62 CLIENT_API_VERSION = 82
  63 pkg.client.global_settings.client_name = PKG_CLIENT_NAME
  64
  65 # exit codes
  66 EXIT_OK      = 0
  67 EXIT_OOPS    = 1
  68 EXIT_BADOPT  = 2
  69
  70 #
  71 # This is a simple python script, run from the method script that starts
  72 # svc:/application/pkg/system-repository:default.
  73 #
  74 # It writes an Apache configuration that is used to serve responses to pkg
  75 # clients querying the system repository, as well as providing http/https proxy
  76 # services to those clients, accessing external repositories.
  77 # file:// repositories on the system running the system repository are also
  78 # exposed to pkg clients, via Alias directives.
  79 #
  80 # See src/util/apache2/sysrepo/*.mako for the templates used to create the
  81 # Apache configuration.
  82 #
  83 # The following filesystem locations are used:
  84 #
  85 # variable      default install path          description
  86 # ---------     ---------------------         ------------
  87 # runtime_dir   system/volatile/pkg/sysrepo   runtime .conf, htdocs, pid files
  88 # template_dir  etc/pkg/sysrepo               mako templates
  89 # log_dir       var/log/pkg/sysrepo           log files
  90 # cache_dir     var/cache/pkg/sysrepo         apache proxy cache
  91 #
  92 # all of the above can be modified with command line arguments.
  93 #
  94
  95 SYSREPO_CRYPTO_FILENAME = "crypto.txt"
  96 SYSREPO_HTTP_TEMPLATE = "sysrepo_httpd.conf.mako"
  97 SYSREPO_HTTP_FILENAME = "sysrepo_httpd.conf"
  98
  99 SYSREPO_PUB_TEMPLATE = "sysrepo_publisher_response.mako"
 100 SYSREPO_PUB_FILENAME = "index.html"
 101
 102 SYSREPO_HTDOCS_DIRNAME = "htdocs"
 103
 104 SYSREPO_VERSIONS_DIRNAME = ["versions", "0"]
 105 SYSREPO_SYSPUB_DIRNAME = ["syspub", "0"]
 106 SYSREPO_PUB_DIRNAME = ["publisher", "0"]
 107
 108 # static string with our versions response
 109 SYSREPO_VERSIONS_STR = """\
 110 pkg-server {0}
 111 publisher 0
 112 versions 0
 113 catalog 1
 114 file 1
 115 syspub 0
 116 manifest 0
 117 """.format(pkg.VERSION)
 118
 119 SYSREPO_USER = "pkg5srv"
 120 SYSREPO_GROUP = "pkg5srv"
 121
 122 class SysrepoException(Exception):
 123         def __unicode__(self):
 124         # To workaround python issues 6108 and 2517, this provides a
 125         # a standard wrapper for this class' exceptions so that they
 126         # have a chance of being stringified correctly.
 127                 return str(self)
 128
 129 @atexit.register
 130 def cleanup():
 131         """To be called at program finish."""
 132         pass
 133
 134 def error(text, cmd=None):
 135         """Emit an error message prefixed by the command name """
 136
 137         if cmd:
 138                 text = "{0}: {1}".format(cmd, text)
 139                 pkg_cmd = "pkg.sysrepo "
 140         else:
 141                 pkg_cmd = "pkg.sysrepo: "
 142
 143                 # If we get passed something like an Exception, we can convert
 144                 # it down to a string.
 145                 text = str(text)
 146
 147         # If the message starts with whitespace, assume that it should come
 148         # *before* the command-name prefix.
 149         text_nows = text.lstrip()
 150         ws = text[:len(text) - len(text_nows)]
 151
 152         # This has to be a constant value as we can't reliably get our actual
 153         # program name on all platforms.
 154         logger.error(ws + pkg_cmd + text_nows)
 155
 156 def usage(usage_error=None, cmd=None, retcode=EXIT_BADOPT):
 157         """Emit a usage message and optionally prefix it with a more
 158         specific error message.  Causes program to exit.
 159         """
 160
 161         if usage_error:
 162                 error(usage_error, cmd=cmd)
 163
 164         msg(_("""\
 165 Usage:
 166         pkg.sysrepo -p <port> [-R image_root] [ -c cache_dir] [-h hostname]
 167                 [-l logs_dir] [-r runtime_dir] [-s cache_size] [-t template_dir]
 168                 [-T http_timeout] [-w http_proxy] [-W https_proxy]
 169      """))
 170         sys.exit(retcode)
 171
 172 def _get_image(image_dir):
 173         """Return a pkg.client.api.ImageInterface for the provided
 174         image directory."""
 175
 176         cdir = os.getcwd()
 177         if not image_dir:
 178                 image_dir = "/"
 179         api_inst = None
 180         tracker = progress.QuietProgressTracker()
 181         try:
 182                 api_inst = pkg.client.api.ImageInterface(
 183                     image_dir, CLIENT_API_VERSION,
 184                     tracker, None, PKG_CLIENT_NAME)
 185
 186                 if api_inst.root != image_dir:
 187                         msg(_("Problem getting image at {0}").format(
 188                             image_dir))
 189         except Exception as err:
 190                 raise SysrepoException(
 191                     _("Unable to get image at {dir}: {reason}").format(
 192                     dir=image_dir,
 193                     reason=str(err)))
 194
 195         # restore the current directory, which ImageInterace had changed
 196         os.chdir(cdir)
 197         return api_inst
 198
 199 def _follow_redirects(uri_list, http_timeout):
 200         """ Follow HTTP redirects from servers.  Needed so that we can create
 201         RewriteRules for all repository URLs that pkg clients may encounter.
 202
 203         We return a sorted list of URIs that were found having followed all
 204         redirects in 'uri_list'.  We also return a boolean, True if we timed out
 205         when following any of the URIs.
 206         """
 207
 208         ret_uris = set(uri_list)
 209         timed_out = False
 210
 211         class SysrepoRedirectHandler(urllib2.HTTPRedirectHandler):
 212                 """ A HTTPRedirectHandler that saves URIs we've been
 213                 redirected to along the path to our eventual destination."""
 214                 def __init__(self):
 215                         self.redirects = set()
 216
 217                 def redirect_request(self, req, fp, code, msg, hdrs, newurl):
 218                         self.redirects.add(newurl)
 219                         return urllib2.HTTPRedirectHandler.redirect_request(
 220                             self, req, fp, code, msg, hdrs, newurl)
 221
 222         for uri in uri_list:
 223                 handler = SysrepoRedirectHandler()
 224                 opener = urllib2.build_opener(handler)
 225                 if not uri.startswith("http:"):
 226                         ret_uris.update([uri])
 227                         continue
 228
 229                 # otherwise, open a known url to check for redirects
 230                 try:
 231                         opener.open("{0}/versions/0".format(uri), None,
 232                             http_timeout)
 233                         ret_uris.update(set(
 234                             [item.replace("/versions/0", "").rstrip("/")
 235                             for item in handler.redirects]))
 236                 except urllib2.URLError as err:
 237                         # We need to log this, and carry on - the url
 238                         # could become available at a later date.
 239                         msg(_("WARNING: unable to access {uri} when checking "
 240                             "for redirects: {err}").format(**locals()))
 241                         timed_out = True
 242
 243         return sorted(list(ret_uris)), timed_out
 244
 245 def __validate_pub_info(pub_info, no_uri_pubs, api_inst):
 246         """Determine if pub_info and no_uri_pubs objects, which may have been
 247         decoded from a json representation are valid, raising a SysrepoException
 248         if they are not.
 249
 250         We use the api_inst to sanity-check that all publishers configured in
 251         the image are represented in pub_info or no_uri_pubs, and that their
 252         URIs are present.
 253
 254         SysrepoExceptions are raised with developer-oriented debug messages
 255         which are not to be translated or shown to users.
 256         """
 257
 258         # validate the structure of the pub_info object
 259         if not isinstance(pub_info, dict):
 260                 raise SysrepoException("{0} is not a dict".format(pub_info))
 261         for uri in pub_info:
 262                 if not isinstance(uri, basestring):
 263                         raise SysrepoException("{0} is not a basestring".format(
 264                             uri))
 265                 uri_info = pub_info[uri]
 266                 if not isinstance(uri_info, list):
 267                         raise SysrepoException("{0} is not a list".format(
 268                             uri_info))
 269                 for props in uri_info:
 270                         if len(props) != 6:
 271                                 raise SysrepoException("{0} does not have 6 "
 272                                     "items".format(props))
 273                         # props [0] and [3] must be strings
 274                         if not isinstance(props[0], basestring) or \
 275                             not isinstance(props[3], basestring):
 276                                 raise SysrepoException("indices 0 and 3 of {0} "
 277                                     "are not basestrings".format(props))
 278                         # prop[5] must be a string, either "file" or "dir"
 279                         # and prop[0] must start with file://
 280                         if not isinstance(props[5], basestring) or \
 281                             (props[5] not in ["file", "dir"] and
 282                             props[0].startswith("file://")):
 283                                 raise SysrepoException("index 5 of {0} is not a "
 284                                     "basestring or is not 'file' or 'dir'".format(
 285                                     props))
 286         # validate the structure of the no_uri_pubs object
 287         if not isinstance(no_uri_pubs, list):
 288                 raise SysrepoException("{0} is not a list".format(no_uri_pubs))
 289         for item in no_uri_pubs:
 290                 if not isinstance(item, basestring):
 291                         raise SysrepoException(
 292                             "{0} is not a basestring".format(item))
 293
 294         # check that we have entries for each URI for each publisher.
 295         # (we may have more URIs than these, due to server-side http redirects
 296         # that are not reflected as origins or mirrors in the image itself)
 297         for pub in api_inst.get_publishers():
 298                 if pub.disabled:
 299                         continue
 300                 repo = pub.repository
 301                 for uri in repo.mirrors + repo.origins:
 302                         uri_key = uri.uri.rstrip("/")
 303                         if uri_key not in pub_info:
 304                                 raise SysrepoException("{0} is not in {1}".format(
 305                                     uri_key, pub_info))
 306                 if repo.mirrors + repo.origins == []:
 307                         if pub.prefix not in no_uri_pubs:
 308                                 raise SysrepoException("{0} is not in {1}".format(
 309                                     pub.prefix, no_uri_pubs))
 310         return
 311
 312 def _load_publisher_info(api_inst, image_dir):
 313         """Loads information about the publishers configured for the
 314         given ImageInterface from image_dir in a format identical to that
 315         returned by _get_publisher_info(..)  that is, a dictionary mapping
 316         URIs to a list of lists. An example entry might be:
 317             pub_info[uri] = [[prefix, cert, key, hash of the uri, proxy], ... ]
 318
 319         and a list of publishers which have no origin or mirror URIs.
 320
 321         If the cache doesn't exist, or is in a format we don't recognise, or
 322         we've managed to determine that it's stale, we return None, None
 323         indicating that the publisher_info must be rebuilt.
 324         """
 325         pub_info = None
 326         no_uri_pubs = None
 327         cache_path = os.path.join(image_dir,
 328             pkg.client.global_settings.sysrepo_pub_cache_path)
 329         try:
 330                 try:
 331                         st_cache = os.lstat(cache_path)
 332                 except OSError as e:
 333                         if e.errno == errno.ENOENT:
 334                                 return None, None
 335                         else:
 336                                 raise
 337
 338                 # the cache must be a regular file
 339                 if not stat.S_ISREG(st_cache.st_mode):
 340                         raise IOError("not a regular file")
 341
 342                 with open(cache_path, "r") as cache_file:
 343                         try:
 344                                 pub_info_tuple = simplejson.load(cache_file)
 345                         except simplejson.JSONDecodeError:
 346                                 error(_("Invalid config cache file at {0} "
 347                                     "generating fresh configuration.").format(
 348                                     cache_path))
 349                                 return None, None
 350
 351                         if len(pub_info_tuple) != 2:
 352                                 error(_("Invalid config cache at {0} "
 353                                     "generating fresh configuration.").format(
 354                                     cache_path))
 355                                 return None, None
 356
 357                         pub_info, no_uri_pubs = pub_info_tuple
 358                         # sanity-check the cached configuration
 359                         try:
 360                                 __validate_pub_info(pub_info, no_uri_pubs,
 361                                     api_inst)
 362                         except SysrepoException as e:
 363                                 error(_("Invalid config cache at {0} "
 364                                     "generating fresh configuration.").format(
 365                                     cache_path))
 366                                 return None, None
 367
 368         # If we have any problems loading the publisher info, we explain why.
 369         except IOError as e:
 370                 error(_("Unable to load config from {cache_path}: {e}").format(
 371                     **locals()))
 372                 return None, None
 373
 374         return pub_info, no_uri_pubs
 375
 376 def _store_publisher_info(uri_pub_map, no_uri_pubs, image_dir):
 377         """Stores a given pair of (uri_pub_map, no_uri_pubs) objects to a
 378         configuration cache file beneath image_dir."""
 379         cache_path = os.path.join(image_dir,
 380             pkg.client.global_settings.sysrepo_pub_cache_path)
 381         cache_dir = os.path.dirname(cache_path)
 382         try:
 383                 if not os.path.exists(cache_dir):
 384                         os.makedirs(cache_dir, 0o700)
 385                 try:
 386                         # if the cache exists, it must be a file
 387                         st_cache = os.lstat(cache_path)
 388                         if not stat.S_ISREG(st_cache.st_mode):
 389                                 raise IOError("not a regular file")
 390                 except OSError:
 391                         pass
 392
 393                 with open(cache_path, "wb") as cache_file:
 394                         simplejson.dump((uri_pub_map, no_uri_pubs), cache_file,
 395                             indent=True)
 396                         os.chmod(cache_path, 0o600)
 397         except IOError as e:
 398                 error(_("Unable to store config to {cache_path}: {e}").format(
 399                     **locals()))
 400
 401 def _valid_proxy(proxy):
 402         """Checks the given proxy string to make sure that it does not contain
 403         any authentication details since these are not supported by ProxyRemote.
 404         """
 405
 406         u = urllib2.urlparse.urlparse(proxy)
 407         netloc_parts = u.netloc.split("@")
 408         # If we don't have any authentication details, return.
 409         if len(netloc_parts) == 1:
 410                 return True
 411         return False
 412
 413 def _get_publisher_info(api_inst, http_timeout, image_dir):
 414         """Returns information about the publishers configured for the given
 415         ImageInterface.
 416
 417         The first item returned is a map of uris to a list of lists of the form
 418         [[prefix, cert, key, hash of the uri, proxy, uri type], ... ]
 419
 420         The second item returned is a list of publisher prefixes which specify
 421         no uris.
 422
 423         Where possible, we attempt to load cached publisher information, but if
 424         that cached information is stale or unavailable, we fall back to
 425         querying the image for the publisher information, verifying repository
 426         URIs and checking for redirects and write that information to the
 427         cache."""
 428
 429         # the cache gets deleted by pkg.client.image.Image.save_config()
 430         # any time publisher configuration changes are made.
 431         uri_pub_map, no_uri_pubs = _load_publisher_info(api_inst, image_dir)
 432         if uri_pub_map:
 433                 return uri_pub_map, no_uri_pubs
 434
 435         # map URIs to (pub.prefix, cert, key, hash, proxy, utype) tuples
 436         uri_pub_map = {}
 437         no_uri_pubs = []
 438         timed_out = False
 439
 440         for pub in api_inst.get_publishers():
 441                 if pub.disabled:
 442                         continue
 443
 444                 prefix = pub.prefix
 445                 repo = pub.repository
 446
 447                 # Determine the proxies to use per URI
 448                 proxy_map = {}
 449                 for uri in repo.mirrors + repo.origins:
 450                         key = uri.uri.rstrip("/")
 451                         if uri.proxies:
 452                                 # Apache can only use a single proxy, even
 453                                 # if many are configured. Use the first we find.
 454                                 proxy_map[key] = uri.proxies[0].uri
 455
 456                 # Apache's ProxyRemote directive does not allow proxies that
 457                 # require authentication.
 458                 for uri in proxy_map:
 459                         if not _valid_proxy(proxy_map[uri]):
 460                                 raise SysrepoException("proxy value {val} "
 461                                     "for {uri} is not supported.".format(
 462                                     uri=uri, val=proxy_map[uri]))
 463
 464                 uri_list, timed_out = _follow_redirects(
 465                     [repo_uri.uri.rstrip("/")
 466                     for repo_uri in repo.mirrors + repo.origins],
 467                     http_timeout)
 468
 469                 for uri in uri_list:
 470
 471                         # We keep a field to store information about the type
 472                         # of URI we're looking at, which saves us
 473                         # from needing to make os.path.isdir(..) or
 474                         # os.path.isfile(..) calls when processing the template.
 475                         # This is important when we're rebuilding the
 476                         # configuration from cached publisher info and an
 477                         # file:// repository is temporarily unreachable.
 478                         utype = ""
 479                         if uri.startswith("file:"):
 480                                 # we only support p5p files and directory-based
 481                                 # repositories of >= version 4.
 482                                 urlresult = urllib2.urlparse.urlparse(uri)
 483                                 utype = "dir"
 484                                 if not os.path.exists(urlresult.path):
 485                                         raise SysrepoException(
 486                                             _("file repository {0} does not "
 487                                             "exist or is not accessible").format(uri))
 488                                 if os.path.isdir(urlresult.path) and \
 489                                     not os.path.exists(os.path.join(
 490                                     urlresult.path, "pkg5.repository")):
 491                                         raise SysrepoException(
 492                                             _("file repository {0} cannot be "
 493                                             "proxied. Only file "
 494                                             "repositories of version 4 or "
 495                                             "later are supported.").format(uri))
 496                                 if not os.path.isdir(urlresult.path):
 497                                         utype = "file"
 498                                         try:
 499                                                 p5p.Archive(urlresult.path)
 500                                         except p5p.InvalidArchive:
 501                                                 raise SysrepoException(
 502                                                     _("unable to read p5p "
 503                                                     "archive file at {0}").format(
 504                                                     urlresult.path))
 505
 506                         hash = _uri_hash(uri)
 507                         # we don't have per-uri ssl key/cert information yet,
 508                         # so we just pull it from one of the RepositoryURIs.
 509                         cert = repo_uri.ssl_cert
 510                         key = repo_uri.ssl_key
 511                         uri_pub_map.setdefault(uri, []).append(
 512                             (prefix, cert, key, hash, proxy_map.get(uri), utype)
 513                             )
 514
 515                 if not repo.mirrors + repo.origins:
 516                         no_uri_pubs.append(prefix)
 517
 518         # if we weren't able to follow all redirects, then we don't write a new
 519         # cache, because it could be incomplete.
 520         if not timed_out:
 521                 _store_publisher_info(uri_pub_map, no_uri_pubs, image_dir)
 522         return uri_pub_map, no_uri_pubs
 523
 524 def _chown_cache_dir(dir):
 525         """Sets ownership for cache directory as pkg5srv:bin"""
 526
 527         uid = portable.get_user_by_name(SYSREPO_USER, None, False)
 528         gid = portable.get_group_by_name("bin", None, False)
 529         try:
 530                 os.chown(dir, uid, gid)
 531         except OSError as err:
 532                 if not os.environ.get("PKG5_TEST_ENV", None):
 533                         raise SysrepoException(
 534                             _("Unable to chown to {user}:{group}: "
 535                             "{err}").format(
 536                             user=SYSREPO_USER, group="bin",
 537                             err=err))
 538
 539 def _write_httpd_conf(runtime_dir, log_dir, template_dir, host, port, cache_dir,
 540     cache_size, uri_pub_map, http_proxy, https_proxy):
 541         """Writes the apache configuration for the system repository.
 542
 543         If http_proxy or http_proxy is supplied, it will override any proxy
 544         values set in the image we're reading configuration from.
 545         """
 546
 547         try:
 548                 # check our hostname
 549                 socket.gethostbyname(host)
 550
 551                 # check our directories
 552                 dirs = [runtime_dir, log_dir]
 553                 if cache_dir not in ["None", "memory"]:
 554                         dirs.append(cache_dir)
 555                 for dir in dirs + [template_dir]:
 556                         if os.path.exists(dir) and not os.path.isdir(dir):
 557                                 raise SysrepoException(
 558                                     _("{0} is not a directory").format(dir))
 559
 560                 for dir in dirs:
 561                         try:
 562                                 os.makedirs(dir, 0o755)
 563                                 # set pkg5srv:bin as ownership for cache
 564                                 # directory.
 565                                 if dir == cache_dir:
 566                                         _chown_cache_dir(dir)
 567                         except OSError as err:
 568                                 if err.errno != errno.EEXIST:
 569                                         raise
 570
 571                 # check our port
 572                 try:
 573                         num = int(port)
 574                         if num <= 0 or num >= 65535:
 575                                 raise SysrepoException(
 576                                     _("invalid port: {0}").format(port))
 577                 except ValueError:
 578                         raise SysrepoException(_("invalid port: {0}").format(
 579                             port))
 580
 581                 # check our cache size
 582                 try:
 583                         num = int(cache_size)
 584                         if num <= 0:
 585                                 raise SysrepoException(_("invalid cache size: "
 586                                    "{0}").format(num))
 587                 except ValueError:
 588                         raise SysrepoException(
 589                             _("invalid cache size: {0}").format(cache_size))
 590
 591                 # check our proxy arguments - we can use a proxy to handle
 592                 # incoming http or https requests, but that proxy must use http.
 593                 for key, val in [("http_proxy", http_proxy),
 594                     ("https_proxy", https_proxy)]:
 595                         if not val:
 596                                 continue
 597                         try:
 598                                 result = urllib2.urlparse.urlparse(val)
 599                                 if result.scheme != "http":
 600                                         raise Exception(
 601                                             _("scheme must be http"))
 602                                 if not result.netloc:
 603                                         raise Exception("missing netloc")
 604                                 if not _valid_proxy(val):
 605                                         raise Exception("unsupported proxy")
 606                         except Exception as e:
 607                                 raise SysrepoException(
 608                                     _("invalid {key}: {val}: {err}").format(
 609                                     key=key, val=val, err=str(e)))
 610
 611                 httpd_conf_template_path = os.path.join(template_dir,
 612                     SYSREPO_HTTP_TEMPLATE)
 613
 614                 # we're disabling unicode here because we want Mako to
 615                 # passthrough any filesystem path names, whatever the
 616                 # original encoding.
 617                 httpd_conf_template = Template(
 618                     filename=httpd_conf_template_path,
 619                     disable_unicode=True)
 620
 621                 # our template expects cache size expressed in Kb
 622                 httpd_conf_text = httpd_conf_template.render(
 623                     sysrepo_log_dir=log_dir,
 624                     sysrepo_runtime_dir=runtime_dir,
 625                     sysrepo_template_dir=template_dir,
 626                     uri_pub_map=uri_pub_map,
 627                     ipv6_addr="::1",
 628                     host=host,
 629                     port=port,
 630                     cache_dir=cache_dir,
 631                     cache_size=int(cache_size) * 1024,
 632                     http_proxy=http_proxy,
 633                     https_proxy=https_proxy)
 634                 httpd_conf_path = os.path.join(runtime_dir,
 635                     SYSREPO_HTTP_FILENAME)
 636                 httpd_conf_file = file(httpd_conf_path, "wb")
 637                 httpd_conf_file.write(httpd_conf_text)
 638                 httpd_conf_file.close()
 639         except socket.gaierror as err:
 640                 raise SysrepoException(
 641                     _("Unable to write sysrepo_httpd.conf: {host}: "
 642                     "{err}").format(**locals()))
 643         except (OSError, IOError) as err:
 644                 raise SysrepoException(
 645                     _("Unable to write sysrepo_httpd.conf: {0}").format(err))
 646
 647 def _write_crypto_conf(runtime_dir, uri_pub_map):
 648         """Writes the crypto.txt file, containing keys and certificates
 649         in order for the system repository to proxy to https repositories."""
 650
 651         try:
 652                 crypto_path = os.path.join(runtime_dir, SYSREPO_CRYPTO_FILENAME)
 653                 file(crypto_path, "w").close()
 654                 os.chmod(crypto_path, 0o600)
 655                 written_crypto_content = False
 656
 657                 for repo_list in uri_pub_map.values():
 658                         for (pub, cert_path, key_path, hash, proxy, utype) in \
 659                             repo_list:
 660                                 if cert_path and key_path:
 661                                        crypto_file = file(crypto_path, "a")
 662                                        crypto_file.writelines(file(cert_path))
 663                                        crypto_file.writelines(file(key_path))
 664                                        crypto_file.close()
 665                                        written_crypto_content = True
 666
 667                 # Apache needs us to have some content in this file
 668                 if not written_crypto_content:
 669                         crypto_file = file(crypto_path, "w")
 670                         crypto_file.write(
 671                             "# this space intentionally left blank\n")
 672                         crypto_file.close()
 673                 os.chmod(crypto_path, 0o400)
 674         except OSError as err:
 675                 raise SysrepoException(
 676                     _("unable to write crypto.txt file: {0}").format(err))
 677
 678 def _write_publisher_response(uri_pub_map, htdocs_path, template_dir):
 679         """Writes static html for all file-repository-based publishers that
 680         is served as their publisher/0 responses.  Responses for
 681         non-file-based publishers are handled by rewrite rules in our
 682         Apache configuration."""
 683
 684         try:
 685                 # build a version of our uri_pub_map, keyed by publisher
 686                 pub_uri_map = {}
 687                 for uri in uri_pub_map:
 688                         for (pub, cert, key, hash, proxy, utype) in \
 689                             uri_pub_map[uri]:
 690                                 if pub not in pub_uri_map:
 691                                         pub_uri_map[pub] = []
 692                                 pub_uri_map[pub].append(
 693                                     (uri, cert, key, hash, proxy, utype))
 694
 695                 publisher_template_path = os.path.join(template_dir,
 696                     SYSREPO_PUB_TEMPLATE)
 697                 publisher_template = Template(filename=publisher_template_path)
 698
 699                 for pub in pub_uri_map:
 700                         for (uri, cert_path, key_path, hash, proxy, utype) in \
 701                             pub_uri_map[pub]:
 702                                 if uri.startswith("file:"):
 703                                         publisher_text = \
 704                                             publisher_template.render(
 705                                             uri=uri, pub=pub)
 706                                         publisher_path = os.path.sep.join(
 707                                             [htdocs_path, pub, hash] +
 708                                             SYSREPO_PUB_DIRNAME)
 709                                         os.makedirs(publisher_path)
 710                                         publisher_file = file(
 711                                             os.path.sep.join([publisher_path,
 712                                             SYSREPO_PUB_FILENAME]), "w")
 713                                         publisher_file.write(publisher_text)
 714                                         publisher_file.close()
 715         except OSError as err:
 716                 raise SysrepoException(
 717                     _("unable to write publisher response: {0}").format(err))
 718
 719 def _write_versions_response(htdocs_path):
 720         """Writes a static versions/0 response for the system repository."""
 721
 722         try:
 723                 versions_path = os.path.join(htdocs_path,
 724                     os.path.sep.join(SYSREPO_VERSIONS_DIRNAME))
 725                 os.makedirs(versions_path)
 726
 727                 versions_file = file(os.path.join(versions_path, "index.html"),
 728                     "w")
 729                 versions_file.write(SYSREPO_VERSIONS_STR)
 730                 versions_file.close()
 731         except OSError as err:
 732                 raise SysrepoException(
 733                     _("Unable to write versions response: {0}").format(err))
 734
 735 def _write_sysrepo_response(api_inst, htdocs_path, uri_pub_map, no_uri_pubs):
 736         """Writes a static syspub/0 response for the system repository."""
 737
 738         try:
 739                 sysrepo_path = os.path.join(htdocs_path,
 740                     os.path.sep.join(SYSREPO_SYSPUB_DIRNAME))
 741                 os.makedirs(sysrepo_path)
 742                 pub_prefixes = [
 743                     info[0]
 744                     for uri in uri_pub_map.keys()
 745                     for info in uri_pub_map[uri]
 746                 ]
 747                 pub_prefixes.extend(no_uri_pubs)
 748                 api_inst.write_syspub(os.path.join(sysrepo_path, "index.html"),
 749                     pub_prefixes, 0)
 750         except (OSError, apx.ApiException) as err:
 751                 raise SysrepoException(
 752                     _("Unable to write syspub response: {0}").format(err))
 753
 754 def _uri_hash(uri):
 755         """Returns a string hash of the given URI"""
 756         return digest.DEFAULT_HASH_FUNC(uri).hexdigest()
 757
 758 def _chown_runtime_dir(runtime_dir):
 759         """Change the ownership of all files under runtime_dir to our sysrepo
 760         user/group"""
 761
 762         uid = portable.get_user_by_name(SYSREPO_USER, None, False)
 763         gid = portable.get_group_by_name(SYSREPO_GROUP, None, False)
 764         try:
 765                 misc.recursive_chown_dir(runtime_dir, uid, gid)
 766         except OSError as err:
 767                 if not os.environ.get("PKG5_TEST_ENV", None):
 768                         raise SysrepoException(
 769                             _("Unable to chown to {user}:{group}: "
 770                             "{err}").format(
 771                             user=SYSREPO_USER, group=SYSREPO_GROUP,
 772                             err=err))
 773
 774 def cleanup_conf(runtime_dir=None):
 775         """Destroys an old configuration."""
 776         try:
 777                 shutil.rmtree(runtime_dir, ignore_errors=True)
 778         except OSError as err:
 779                 raise SysrepoException(
 780                     _("Unable to cleanup old configuration: {0}").format(err))
 781
 782 def refresh_conf(image_root="/", port=None, runtime_dir=None,
 783     log_dir=None, template_dir=None, host="127.0.0.1", cache_dir=None,
 784     cache_size=1024, http_timeout=3, http_proxy=None, https_proxy=None):
 785         """Creates a new configuration for the system repository.
 786
 787         TODO: a way to map only given zones to given publishers
 788         """
 789         try:
 790                 ret = EXIT_OK
 791                 cleanup_conf(runtime_dir=runtime_dir)
 792                 try:
 793                         http_timeout = int(http_timeout)
 794                 except ValueError as err:
 795                         raise SysrepoException(
 796                             _("invalid value for http_timeout: {0}").format(err))
 797                 if http_timeout < 1:
 798                         raise SysrepoException(
 799                             _("http_timeout must a positive integer"))
 800                 try:
 801                         api_inst = _get_image(image_root)
 802                         uri_pub_map, no_uri_pubs = _get_publisher_info(api_inst,
 803                             http_timeout, api_inst.root)
 804                 except SysrepoException as err:
 805                         raise SysrepoException(
 806                             _("unable to get publisher information: {0}").format(
 807                             err))
 808                 try:
 809                         htdocs_path = os.path.join(runtime_dir,
 810                             SYSREPO_HTDOCS_DIRNAME)
 811                         os.makedirs(htdocs_path)
 812                 except OSError as err:
 813                         raise SysrepoException(
 814                             _("unable to create htdocs dir: {0}").format(err))
 815
 816                 _write_httpd_conf(runtime_dir, log_dir, template_dir, host,
 817                     port, cache_dir, cache_size, uri_pub_map, http_proxy,
 818                     https_proxy)
 819                 _write_crypto_conf(runtime_dir, uri_pub_map)
 820                 _write_publisher_response(uri_pub_map, htdocs_path,
 821                     template_dir)
 822                 _write_versions_response(htdocs_path)
 823                 _write_sysrepo_response(api_inst, htdocs_path, uri_pub_map,
 824                     no_uri_pubs)
 825                 _chown_runtime_dir(runtime_dir)
 826         except SysrepoException as err:
 827                 error(err)
 828                 ret = EXIT_OOPS
 829         return ret
 830
 831 def main_func():
 832         global_settings.client_name = PKG_CLIENT_NAME
 833
 834         global orig_cwd
 835
 836         try:
 837                 orig_cwd = os.getcwd()
 838         except OSError as e:
 839                 try:
 840                         orig_cwd = os.environ["PWD"]
 841                         if not orig_cwd or orig_cwd[0] != "/":
 842                                 orig_cwd = None
 843                 except KeyError:
 844                         orig_cwd = None
 845
 846         # some sensible defaults
 847         host = "127.0.0.1"
 848         port = None
 849         # an empty image_root means we don't get '//' in the below
 850         # _get_image() deals with "" in a sane manner.
 851         image_root = ""
 852         cache_dir = "{0}/var/cache/pkg/sysrepo".format(image_root)
 853         cache_size = "1024"
 854         template_dir = "{0}/etc/pkg/sysrepo".format(image_root)
 855         runtime_dir = "{0}/var/run/pkg/sysrepo".format(image_root)
 856         log_dir = "{0}/var/log/pkg/sysrepo".format(image_root)
 857         http_timeout = 4
 858         http_proxy = None
 859         https_proxy = None
 860
 861         try:
 862                 opts, pargs = getopt.getopt(sys.argv[1:],
 863                     "c:h:l:p:r:R:s:t:T:w:W:?", ["help"])
 864                 for opt, arg in opts:
 865                         if opt == "-c":
 866                                 cache_dir = arg
 867                         elif opt == "-h":
 868                                 host = arg
 869                         elif opt == "-l":
 870                                 log_dir = arg
 871                         elif opt == "-p":
 872                                 port = arg
 873                         elif opt == "-r":
 874                                 runtime_dir = arg
 875                         elif opt == "-R":
 876                                 image_root = arg
 877                         elif opt == "-s":
 878                                 cache_size = arg
 879                         elif opt == "-t":
 880                                 template_dir = arg
 881                         elif opt == "-T":
 882                                 http_timeout = arg
 883                         elif opt == "-w":
 884                                 http_proxy = arg
 885                         elif opt == "-W":
 886                                 https_proxy = arg
 887                         else:
 888                                 usage()
 889
 890         except getopt.GetoptError as e:
 891                 usage(_("illegal global option -- {0}").format(e.opt))
 892
 893         if not port:
 894                 usage(_("required port option missing."))
 895
 896         ret = refresh_conf(image_root=image_root, log_dir=log_dir,
 897             host=host, port=port, runtime_dir=runtime_dir,
 898             template_dir=template_dir, cache_dir=cache_dir,
 899             cache_size=cache_size, http_timeout=http_timeout,
 900             http_proxy=http_proxy, https_proxy=https_proxy)
 901         return ret
 902
 903 #
 904 # Establish a specific exit status which means: "python barfed an exception"
 905 # so that we can more easily detect these in testing of the CLI commands.
 906 #
 907 def handle_errors(func, *args, **kwargs):
 908         """Catch exceptions raised by the main program function and then print
 909         a message and/or exit with an appropriate return code.
 910         """
 911
 912         traceback_str = misc.get_traceback_message()
 913
 914         try:
 915                 # Out of memory errors can be raised as EnvironmentErrors with
 916                 # an errno of ENOMEM, so in order to handle those exceptions
 917                 # with other errnos, we nest this try block and have the outer
 918                 # one handle the other instances.
 919                 try:
 920                         __ret = func(*args, **kwargs)
 921                 except (MemoryError, EnvironmentError) as __e:
 922                         if isinstance(__e, EnvironmentError) and \
 923                             __e.errno != errno.ENOMEM:
 924                                 raise
 925                         error("\n" + misc.out_of_memory())
 926                         __ret = EXIT_OOPS
 927         except SystemExit as __e:
 928                 raise __e
 929         except (PipeError, KeyboardInterrupt):
 930                 # Don't display any messages here to prevent possible further
 931                 # broken pipe (EPIPE) errors.
 932                 __ret = EXIT_OOPS
 933         except apx.VersionException as __e:
 934                 error(_("The sysrepo command appears out of sync with the "
 935                     "libraries provided\nby pkg:/package/pkg. The client "
 936                     "version is {client} while the library\nAPI version is "
 937                     "{api}.").format(client=__e.received_version,
 938                      api=__e.expected_version))
 939                 __ret = EXIT_OOPS
 940         except:
 941                 traceback.print_exc()
 942                 error(traceback_str)
 943                 __ret = 99
 944         return __ret
 945
 946
 947 if __name__ == "__main__":
 948         misc.setlocale(locale.LC_ALL, "", error)
 949         gettext.install("pkg", "/usr/share/locale",
 950             codeset=locale.getpreferredencoding())
 951
 952         # Make all warnings be errors.
 953         warnings.simplefilter('error')
 954
 955         __retval = handle_errors(main_func)
 956         try:
 957                 logging.shutdown()
 958         except IOError:
 959                 # Ignore python's spurious pipe problems.
 960                 pass
 961         sys.exit(__retval)