src/pkgrepo.py

   1 #!/usr/bin/python2.7
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22
  23 #
  24 # Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
  25 #
  26
  27 PKG_CLIENT_NAME = "pkgrepo"
  28
  29 # pkgrepo exit codes
  30 EXIT_OK      = 0
  31 EXIT_OOPS    = 1
  32 EXIT_BADOPT  = 2
  33 EXIT_PARTIAL = 3
  34
  35 # listing constants
  36 LISTING_FORMATS = ("default", "json", "json-formatted", "tsv")
  37
  38 # globals
  39 tmpdirs = []
  40
  41 import atexit
  42 import collections
  43 import copy
  44 import errno
  45 import getopt
  46 import gettext
  47 import locale
  48 import logging
  49 import os
  50 import shlex
  51 import shutil
  52 import sys
  53 import tempfile
  54 import textwrap
  55 import traceback
  56 import warnings
  57 import itertools
  58
  59 from pkg.client import global_settings
  60 from pkg.client.debugvalues import DebugValues
  61 from pkg.misc import msg, PipeError
  62 import pkg
  63 import pkg.catalog
  64 import pkg.client.api_errors as apx
  65 import pkg.client.pkgdefs as pkgdefs
  66 import pkg.client.progress
  67 import pkg.client.publisher as publisher
  68 import pkg.client.transport.transport as transport
  69 import pkg.misc as misc
  70 import pkg.server.repository as sr
  71
  72 logger = global_settings.logger
  73
  74 @atexit.register
  75 def cleanup():
  76         """To be called at program finish."""
  77         for d in tmpdirs:
  78                 shutil.rmtree(d, True)
  79
  80
  81 def error(text, cmd=None):
  82         """Emit an error message prefixed by the command name """
  83
  84         if cmd:
  85                 text = "{0}: {1}".format(cmd, text)
  86                 pkg_cmd = "pkgrepo "
  87         else:
  88                 pkg_cmd = "pkgrepo: "
  89
  90                 # If we get passed something like an Exception, we can convert
  91                 # it down to a string.
  92                 text = str(text)
  93
  94         # If the message starts with whitespace, assume that it should come
  95         # *before* the command-name prefix.
  96         text_nows = text.lstrip()
  97         ws = text[:len(text) - len(text_nows)]
  98
  99         # This has to be a constant value as we can't reliably get our actual
 100         # program name on all platforms.
 101         logger.error(ws + pkg_cmd + text_nows)
 102
 103
 104 def get_tracker(quiet=False):
 105         if quiet:
 106                 progtrack = pkg.client.progress.QuietProgressTracker()
 107         else:
 108                 try:
 109                         progtrack = \
 110                             pkg.client.progress.FancyUNIXProgressTracker()
 111                 except pkg.client.progress.ProgressTrackerException:
 112                         progtrack = \
 113                             pkg.client.progress.CommandLineProgressTracker()
 114         return progtrack
 115
 116
 117 def usage(usage_error=None, cmd=None, retcode=2, full=False):
 118         """Emit a usage message and optionally prefix it with a more
 119         specific error message.  Causes program to exit.
 120         """
 121
 122         if usage_error:
 123                 error(usage_error, cmd=cmd)
 124
 125         if not full:
 126                 # The full usage message isn't desired.
 127                 logger.error(_("Try `pkgrepo --help or -?' for more "
 128                     "information."))
 129                 sys.exit(retcode)
 130
 131         msg(_("""\
 132 Usage:
 133         pkgrepo [options] command [cmd_options] [operands]
 134
 135 Subcommands:
 136      pkgrepo create [--version ver] uri_or_path
 137
 138      pkgrepo add-publisher -s repo_uri_or_path publisher ...
 139
 140      pkgrepo remove-publisher [-n] [--synchronous] -s repo_uri_or_path
 141          publisher ...
 142
 143      pkgrepo get [-F format] [-p publisher ...] -s repo_uri_or_path
 144          [--key ssl_key ... --cert ssl_cert ...] [section/property ...]
 145
 146      pkgrepo info [-F format] [-H] [-p publisher ...] -s repo_uri_or_path
 147          [--key ssl_key ... --cert ssl_cert ...]
 148
 149      pkgrepo list [-F format] [-H] [-p publisher ...] -s repo_uri_or_path
 150          [--key ssl_key ... --cert ssl_cert ...] [pkg_fmri_pattern ...]
 151
 152      pkgrepo contents [-m] [-t action_type ...] -s repo_uri_or_path
 153          [--key ssl_key ... --cert ssl_cert ...] [pkg_fmri_pattern ...]
 154
 155      pkgrepo rebuild [-p publisher ...] -s repo_uri_or_path [--key ssl_key ...
 156          --cert ssl_cert ...] [--no-catalog] [--no-index]
 157
 158      pkgrepo refresh [-p publisher ...] -s repo_uri_or_path [--key ssl_key ...
 159          --cert ssl_cert ...] [--no-catalog] [--no-index]
 160
 161      pkgrepo remove [-n] [-p publisher ...] -s repo_uri_or_path
 162          pkg_fmri_pattern ...
 163
 164      pkgrepo set [-p publisher ...] -s repo_uri_or_path
 165          section/property[+|-]=[value] ... or
 166          section/property[+|-]=([value]) ...
 167
 168      pkgrepo verify [-d] [-p publisher ...] [-i ignored_dep_file ...]
 169          [--disable verification ...] -s repo_uri_or_path
 170
 171      pkgrepo fix [-v] [-p publisher ...] -s repo_uri_or_path
 172
 173      pkgrepo help
 174      pkgrepo version
 175
 176 Options:
 177         --help or -?
 178             Displays a usage message."""))
 179
 180         sys.exit(retcode)
 181
 182
 183 class OptionError(Exception):
 184         """Option exception. """
 185
 186         def __init__(self, *args):
 187                 Exception.__init__(self, *args)
 188
 189
 190 def parse_uri(uri):
 191         """Parse the repository location provided and attempt to transform it
 192         into a valid repository URI.
 193         """
 194
 195         return publisher.RepositoryURI(misc.parse_uri(uri))
 196
 197
 198 def subcmd_remove(conf, args):
 199         subcommand = "remove"
 200
 201         opts, pargs = getopt.getopt(args, "np:s:")
 202
 203         dry_run = False
 204         pubs = set()
 205         for opt, arg in opts:
 206                 if opt == "-n":
 207                         dry_run = True
 208                 elif opt == "-p":
 209                         pubs.add(arg)
 210                 elif opt == "-s":
 211                         conf["repo_uri"] = parse_uri(arg)
 212
 213         if not pargs:
 214                 usage(_("At least one package pattern must be provided."),
 215                     cmd=subcommand)
 216
 217         # Get repository object.
 218         if not conf.get("repo_uri", None):
 219                 usage(_("A package repository location must be provided "
 220                     "using -s."), cmd=subcommand)
 221         repo = get_repo(conf, read_only=False, subcommand=subcommand)
 222
 223         if "all" in pubs:
 224                 pubs = set()
 225
 226         # Find matching packages.
 227         try:
 228                 matching, refs = repo.get_matching_fmris(pargs, pubs=pubs)
 229         except apx.PackageMatchErrors as e:
 230                 error(str(e), cmd=subcommand)
 231                 return EXIT_OOPS
 232
 233         if dry_run:
 234                 # Don't make any changes; display list of packages to be
 235                 # removed and exit.
 236                 packages = set(f for m in matching.values() for f in m)
 237                 count = len(packages)
 238                 plist = "\n".join("\t{0}".format(
 239                     p.get_fmri(include_build=False))
 240                     for p in sorted(packages))
 241                 logger.info(_("{count:d} package(s) will be removed:\n"
 242                     "{plist}").format(**locals()))
 243                 return EXIT_OK
 244
 245         progtrack = get_tracker()
 246         packages = collections.defaultdict(list)
 247         for m in matching.values():
 248                 for f in m:
 249                         packages[f.publisher].append(f)
 250
 251         for pub in packages:
 252                 logger.info(
 253                     _("Removing packages for publisher {0} ...").format(pub))
 254                 repo.remove_packages(packages[pub], progtrack=progtrack,
 255                     pub=pub)
 256                 if len(packages) > 1:
 257                         # Add a newline between each publisher.
 258                         logger.info("")
 259
 260         return EXIT_OK
 261
 262
 263 def get_repo(conf, allow_invalid=False, read_only=True, subcommand=None):
 264         """Return the repository object for current program configuration.
 265
 266         'allow_invalid' specifies whether potentially corrupt repositories are
 267         allowed; should only be True if performing a rebuild operation."""
 268
 269         repo_uri = conf["repo_uri"]
 270         if repo_uri.scheme != "file":
 271                 usage(_("Network repositories are not currently supported "
 272                     "for this operation."), cmd=subcommand)
 273
 274         path = repo_uri.get_pathname()
 275         if not path:
 276                 # Bad URI?
 277                 raise sr.RepositoryInvalidError(str(repo_uri))
 278         return sr.Repository(allow_invalid=allow_invalid, read_only=read_only,
 279             root=path)
 280
 281
 282 def setup_transport(conf, subcommand=None, prefix=None, verbose=False,
 283     remote_prefix=True, ssl_key=None, ssl_cert=None):
 284         repo_uri = conf.get("repo_uri", None)
 285         if not repo_uri:
 286                 usage(_("No repository location specified."), cmd=subcommand)
 287
 288         global tmpdirs
 289         temp_root = misc.config_temp_root()
 290
 291         tmp_dir = tempfile.mkdtemp(dir=temp_root)
 292         tmpdirs.append(tmp_dir)
 293
 294         incoming_dir = tempfile.mkdtemp(dir=temp_root)
 295         tmpdirs.append(incoming_dir)
 296
 297         cache_dir = tempfile.mkdtemp(dir=temp_root)
 298         tmpdirs.append(cache_dir)
 299
 300         # Create transport and transport config.
 301         xport, xport_cfg = transport.setup_transport()
 302         xport_cfg.add_cache(cache_dir, readonly=False)
 303         xport_cfg.incoming_root = incoming_dir
 304         xport_cfg.pkg_root = tmp_dir
 305
 306         if not prefix:
 307                 pub = "target"
 308         else:
 309                 pub = prefix
 310
 311         # Configure target publisher.
 312         src_pub = transport.setup_publisher(str(repo_uri), pub, xport,
 313             xport_cfg, remote_prefix=remote_prefix, ssl_key=ssl_key,
 314             ssl_cert=ssl_cert)
 315
 316         return xport, src_pub, tmp_dir
 317
 318
 319 def subcmd_add_publisher(conf, args):
 320         """Add publisher(s) to the specified repository."""
 321
 322         subcommand = "add-publisher"
 323
 324         opts, pargs = getopt.getopt(args, "s:")
 325         for opt, arg in opts:
 326                 if opt == "-s":
 327                         conf["repo_uri"] = parse_uri(arg)
 328
 329         repo_uri = conf.get("repo_uri", None)
 330         if not repo_uri:
 331                 usage(_("No repository location specified."), cmd=subcommand)
 332         if repo_uri.scheme != "file":
 333                 usage(_("Network repositories are not currently supported "
 334                     "for this operation."), cmd=subcommand)
 335
 336         if not pargs:
 337                 usage(_("At least one publisher must be specified"),
 338                     cmd=subcommand)
 339
 340         abort = False
 341         for pfx in pargs:
 342                 if not misc.valid_pub_prefix(pfx):
 343                         error(_("Invalid publisher prefix '{0}'").format(pfx),
 344                             cmd=subcommand)
 345                         abort = True
 346         if abort:
 347                 return EXIT_OOPS
 348
 349         repo = get_repo(conf, read_only=False, subcommand=subcommand)
 350         make_default = not repo.publishers
 351         existing = repo.publishers & set(pargs)
 352
 353         # Elide the publishers that already exist, but retain the order
 354         # publishers were specified in.
 355         new_pubs = [
 356             pfx for pfx in pargs
 357             if pfx not in repo.publishers
 358         ]
 359
 360         # Tricky logic; _set_pub will happily add new publishers if necessary
 361         # and not set any properties if you didn't specify any.
 362         rval = _set_pub(conf, subcommand, {}, new_pubs, repo)
 363
 364         if make_default:
 365                 # No publisher existed previously, so set the default publisher
 366                 # to be the first new one that was added.
 367                 _set_repo(conf, subcommand, { "publisher": {
 368                     "prefix": new_pubs[0] } }, repo)
 369
 370         if rval == EXIT_OK and existing:
 371                 # Some of the publishers that were requested for addition
 372                 # were already known.
 373                 error(_("specified publisher(s) already exist: {0}").format(
 374                     ", ".join(existing)), cmd=subcommand)
 375                 if new_pubs:
 376                         return EXIT_PARTIAL
 377                 return EXIT_OOPS
 378         return rval
 379
 380 def subcmd_remove_publisher(conf, args):
 381         """Remove publisher(s) from a repository"""
 382
 383         subcommand = "remove-publisher"
 384
 385         dry_run = False
 386         synch = False
 387         opts, pargs = getopt.getopt(args, "ns:", ["synchronous"])
 388         for opt, arg in opts:
 389                 if opt == "-s":
 390                         conf["repo_uri"] = parse_uri(arg)
 391                 elif opt == "-n":
 392                         dry_run = True
 393                 elif opt == "--synchronous":
 394                         synch = True
 395         repo_uri = conf.get("repo_uri", None)
 396         if not repo_uri:
 397                 usage(_("No repository location specified."), cmd=subcommand)
 398         if repo_uri.scheme != "file":
 399                 usage(_("Network repositories are not currently supported "
 400                     "for this operation."), cmd=subcommand)
 401
 402         if not pargs:
 403                 usage(_("At least one publisher must be specified"),
 404                     cmd=subcommand)
 405
 406         inv_pfxs = []
 407         for pfx in pargs:
 408                 if not misc.valid_pub_prefix(pfx):
 409                         inv_pfxs.append(pfx)
 410
 411         if inv_pfxs:
 412                 error(_("Invalid publisher prefix(es):\n {0}").format(
 413                     "\n ".join(inv_pfxs)), cmd=subcommand)
 414                 return EXIT_OOPS
 415
 416         repo = get_repo(conf, read_only=False, subcommand=subcommand)
 417         existing = repo.publishers & set(pargs)
 418         noexisting = [pfx for pfx in pargs
 419             if pfx not in repo.publishers]
 420         # Publishers left if remove succeeds.
 421         left = [pfx for pfx in repo.publishers if pfx not in pargs]
 422
 423         if noexisting:
 424                 error(_("The following publisher(s) could not be found:\n "
 425                     "{0}").format("\n ".join(noexisting)), cmd=subcommand)
 426                 return EXIT_OOPS
 427
 428         logger.info(_("Removing publisher(s)"))
 429         for pfx in existing:
 430                 rstore = repo.get_pub_rstore(pfx)
 431                 numpkg = rstore.catalog.package_count
 432                 logger.info(_("\'{pfx}\'\t({num} package(s))").format(
 433                     pfx=pfx, num=str(numpkg)))
 434
 435         if dry_run:
 436                 return EXIT_OK
 437
 438         defaultpfx = repo.cfg.get_property("publisher", "prefix")
 439         repo_path = repo_uri.get_pathname()
 440
 441         repo.remove_publisher(existing, repo_path, synch)
 442         # Change the repository publisher/prefix property, if necessary.
 443         if defaultpfx in existing:
 444                 if len(left) == 1:
 445                         _set_repo(conf, subcommand, { "publisher" :  {
 446                             "prefix" :  left[0]} }, repo)
 447                         msg(_("The default publisher was removed."
 448                             " Setting 'publisher/prefix' to '{0}',"
 449                             " the only publisher left").format(left[0]))
 450                 else:
 451                         _set_repo(conf, subcommand, { "publisher": {
 452                             "prefix" :  ""} }, repo)
 453                         msg(_("The default publisher was removed."
 454                             " The 'publisher/prefix' property has been"
 455                             " unset"))
 456
 457         return EXIT_OK
 458
 459 def subcmd_create(conf, args):
 460         """Create a package repository at the given location."""
 461
 462         subcommand = "create"
 463
 464         opts, pargs = getopt.getopt(args, "s:", ["version="])
 465
 466         version = None
 467         for opt, arg in opts:
 468                 if opt == "-s":
 469                         conf["repo_uri"] = parse_uri(arg)
 470                 elif opt == "--version":
 471                         # This option is currently private and allows creating a
 472                         # repository with a specific format based on version.
 473                         try:
 474                                 version = int(arg)
 475                         except ValueError:
 476                                 usage(_("Version must be an integer value."),
 477                                     cmd=subcommand)
 478
 479         if len(pargs) > 1:
 480                 usage(_("Only one repository location may be specified."),
 481                     cmd=subcommand)
 482         elif pargs:
 483                 conf["repo_uri"] = parse_uri(pargs[0])
 484
 485         repo_uri = conf.get("repo_uri", None)
 486         if not repo_uri:
 487                 usage(_("No repository location specified."), cmd=subcommand)
 488         if repo_uri.scheme != "file":
 489                 usage(_("Network repositories are not currently supported "
 490                     "for this operation."), cmd=subcommand)
 491
 492         # Attempt to create a repository at the specified location.  Allow
 493         # whatever exceptions are raised to bubble up.
 494         sr.repository_create(repo_uri, version=version)
 495
 496         return EXIT_OK
 497
 498
 499 def subcmd_get(conf, args):
 500         """Display repository properties."""
 501
 502         subcommand = "get"
 503         omit_headers = False
 504         out_format = "default"
 505         pubs = set()
 506         key = None
 507         cert = None
 508
 509         opts, pargs = getopt.getopt(args, "F:Hp:s:", ["key=", "cert="])
 510         for opt, arg in opts:
 511                 if opt == "-F":
 512                         if arg not in LISTING_FORMATS:
 513                                 raise apx.InvalidOptionError(
 514                                     apx.InvalidOptionError.ARG_INVALID,
 515                                     [arg, opt])
 516                         out_format = arg
 517                 elif opt == "-H":
 518                         omit_headers = True
 519                 elif opt == "-p":
 520                         pubs.add(arg)
 521                 elif opt == "-s":
 522                         conf["repo_uri"] = parse_uri(arg)
 523                 elif opt == "--key":
 524                         key = arg
 525                 elif opt == "--cert":
 526                         cert = arg
 527
 528         # Setup transport so configuration can be retrieved.
 529         if not conf.get("repo_uri", None):
 530                 usage(_("A package repository location must be provided "
 531                     "using -s."), cmd=subcommand)
 532         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
 533             ssl_key=key, ssl_cert=cert)
 534
 535         # Get properties.
 536         if pubs:
 537                 return _get_pub(conf, subcommand, xport, xpub, omit_headers,
 538                     out_format, pubs, pargs)
 539         return _get_repo(conf, subcommand, xport, xpub, omit_headers,
 540             out_format, pargs)
 541
 542
 543 def _get_repo(conf, subcommand, xport, xpub, omit_headers, out_format, pargs):
 544         """Display repository properties."""
 545
 546         # Configuration index is indexed by section name and property name.
 547         # Retrieve and flatten it to simplify listing process.
 548         stat_idx = xport.get_status(xpub)
 549         cfg_idx = stat_idx.get("repository", {}).get("configuration", {})
 550         props = set()
 551
 552         # Set minimum widths for section and property name columns by using the
 553         # length of the column headers.
 554         max_sname_len = len(_("SECTION"))
 555         max_pname_len = len(_("PROPERTY"))
 556
 557         for sname in cfg_idx:
 558                 max_sname_len = max(max_sname_len, len(sname))
 559                 for pname in cfg_idx[sname]:
 560                         max_pname_len = max(max_pname_len, len(pname))
 561                         props.add("/".join((sname, pname)))
 562
 563         req_props = set(pargs)
 564         if len(req_props) >= 1:
 565                 found = props & req_props
 566                 notfound = req_props - found
 567                 del props
 568         else:
 569                 found = props
 570                 notfound = set()
 571
 572         def gen_listing():
 573                 for prop in sorted(found):
 574                         sname, pname = prop.rsplit("/", 1)
 575                         sval = cfg_idx[sname][pname]
 576                         yield {
 577                             "section": sname,
 578                             "property": pname,
 579                             "value": sval,
 580                         }
 581
 582         #    SECTION PROPERTY VALUE
 583         #    <sec_1> <prop_1> <prop_1_value>
 584         #    <sec_2> <prop_2> <prop_2_value>
 585         #    ...
 586         field_data = {
 587             "section" : [("default", "json", "tsv"), _("SECTION"), ""],
 588             "property" : [("default", "json", "tsv"), _("PROPERTY"), ""],
 589             "value" : [("default", "json", "tsv"), _("VALUE"), ""],
 590         }
 591         desired_field_order = (_("SECTION"), _("PROPERTY"), _("VALUE"))
 592
 593         # Default output formatting.
 594         def_fmt = "{0:" + str(max_sname_len) + "} {1:" + str(max_pname_len) + \
 595             "} {2}"
 596
 597         if found or (not req_props and out_format == "default"):
 598                 # print without trailing newline.
 599                 sys.stdout.write(misc.get_listing(desired_field_order,
 600                     field_data, gen_listing(), out_format, def_fmt,
 601                     omit_headers))
 602
 603         if found and notfound:
 604                 return EXIT_PARTIAL
 605         if req_props and not found:
 606                 if out_format == "default":
 607                         # Don't pollute other output formats.
 608                         error(_("no matching properties found"),
 609                             cmd=subcommand)
 610                 return EXIT_OOPS
 611         return EXIT_OK
 612
 613
 614 def _get_matching_pubs(subcommand, pubs, xport, xpub, out_format="default",
 615     use_transport=False):
 616
 617         # Retrieve publisher information.
 618         pub_data = xport.get_publisherdata(xpub)
 619         known_pubs = set(p.prefix for p in pub_data)
 620         if len(pubs) > 0 and "all" not in pubs:
 621                 found = known_pubs & pubs
 622                 notfound = pubs - found
 623                 pub_data = [p for p in pub_data if p.prefix in found]
 624         else:
 625                 found = known_pubs
 626                 notfound = set()
 627
 628         if use_transport:
 629                 # Assign transport information.
 630                 for p in pub_data:
 631                         p.repository = xpub.repository
 632
 633         # Establish initial return value and perform early exit if appropriate.
 634         rval = EXIT_OK
 635         if found and notfound:
 636                 rval = EXIT_PARTIAL
 637         elif pubs and not found:
 638                 if out_format == "default":
 639                         # Don't pollute other output formats.
 640                         error(_("no matching publishers found"),
 641                             cmd=subcommand)
 642                 return EXIT_OOPS, None, None
 643         return rval, found, pub_data
 644
 645
 646 def _get_pub(conf, subcommand, xport, xpub, omit_headers, out_format, pubs,
 647     pargs):
 648         """Display publisher properties."""
 649
 650         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
 651             xpub, out_format=out_format)
 652         if rval == EXIT_OOPS:
 653                 return rval
 654
 655         # Set minimum widths for section and property name columns by using the
 656         # length of the column headers and data.
 657         max_pubname_len = str(max(
 658             [len(_("PUBLISHER"))] + [len(p) for p in found]
 659         ))
 660         max_sname_len = len(_("SECTION"))
 661         max_pname_len = len(_("PROPERTY"))
 662
 663         # For each requested publisher, retrieve the requested property data.
 664         pub_idx = {}
 665         for pub in pub_data:
 666                 pub_idx[pub.prefix] = {
 667                     "publisher": {
 668                         "alias": pub.alias,
 669                         "prefix": pub.prefix,
 670                     },
 671                 }
 672
 673                 pub_repo = pub.repository
 674                 if pub_repo:
 675                         pub_idx[pub.prefix]["repository"] = {
 676                             "collection-type": pub_repo.collection_type,
 677                             "description": pub_repo.description,
 678                             "legal-uris": pub_repo.legal_uris,
 679                             "mirrors": pub_repo.mirrors,
 680                             "name": pub_repo.name,
 681                             "origins": pub_repo.origins,
 682                             "refresh-seconds": pub_repo.refresh_seconds,
 683                             "registration-uri": pub_repo.registration_uri,
 684                             "related-uris": pub_repo.related_uris,
 685                         }
 686                 else:
 687                         pub_idx[pub.prefix]["repository"] = {
 688                             "collection-type": "core",
 689                             "description": "",
 690                             "legal-uris": [],
 691                             "mirrors": [],
 692                             "name": "",
 693                             "origins": [],
 694                             "refresh-seconds": "",
 695                             "registration-uri": "",
 696                             "related-uris": [],
 697                         }
 698
 699         # Determine possible set of properties and lengths.
 700         props = set()
 701         for pub in pub_idx:
 702                 for sname in pub_idx[pub]:
 703                         max_sname_len = max(max_sname_len, len(sname))
 704                         for pname in pub_idx[pub][sname]:
 705                                 max_pname_len = max(max_pname_len, len(pname))
 706                                 props.add("/".join((sname, pname)))
 707
 708         # Determine properties to display.
 709         req_props = set(pargs)
 710         if len(req_props) >= 1:
 711                 found = props & req_props
 712                 notfound = req_props - found
 713                 del props
 714         else:
 715                 found = props
 716                 notfound = set()
 717
 718         def gen_listing():
 719                 for pub in sorted(pub_idx.keys()):
 720                         for prop in sorted(found):
 721                                 sname, pname = prop.rsplit("/", 1)
 722                                 sval = pub_idx[pub][sname][pname]
 723                                 yield {
 724                                     "publisher": pub,
 725                                     "section": sname,
 726                                     "property": pname,
 727                                     "value": sval,
 728                                 }
 729
 730         #    PUBLISHER SECTION PROPERTY VALUE
 731         #    <pub_1>   <sec_1> <prop_1> <prop_1_value>
 732         #    <pub_1>   <sec_2> <prop_2> <prop_2_value>
 733         #    ...
 734         field_data = {
 735             "publisher" : [("default", "json", "tsv"), _("PUBLISHER"), ""],
 736             "section" : [("default", "json", "tsv"), _("SECTION"), ""],
 737             "property" : [("default", "json", "tsv"), _("PROPERTY"), ""],
 738             "value" : [("default", "json", "tsv"), _("VALUE"), ""],
 739         }
 740         desired_field_order = (_("PUBLISHER"), _("SECTION"), _("PROPERTY"),
 741             _("VALUE"))
 742
 743         # Default output formatting.
 744         def_fmt = "{0:" + str(max_pubname_len) + "} {1:" + str(max_sname_len) + \
 745             "} {2:" + str(max_pname_len) + "} {3}"
 746
 747         if found or (not req_props and out_format == "default"):
 748                 # print without trailing newline.
 749                 sys.stdout.write(misc.get_listing(desired_field_order,
 750                     field_data, gen_listing(), out_format, def_fmt,
 751                     omit_headers))
 752
 753         if found and notfound:
 754                 rval = EXIT_PARTIAL
 755         if req_props and not found:
 756                 if out_format == "default":
 757                         # Don't pollute other output formats.
 758                         error(_("no matching properties found"),
 759                             cmd=subcommand)
 760                 rval = EXIT_OOPS
 761         return rval
 762
 763
 764 def subcmd_info(conf, args):
 765         """Display a list of known publishers and a summary of known packages
 766         and when the package data for the given publisher was last updated.
 767         """
 768
 769         subcommand = "info"
 770         omit_headers = False
 771         out_format = "default"
 772         pubs = set()
 773         key = None
 774         cert = None
 775
 776         opts, pargs = getopt.getopt(args, "F:Hp:s:", ["key=", "cert="])
 777         for opt, arg in opts:
 778                 if opt == "-F":
 779                         if arg not in LISTING_FORMATS:
 780                                 raise apx.InvalidOptionError(
 781                                     apx.InvalidOptionError.ARG_INVALID,
 782                                     [arg, opt])
 783                         out_format = arg
 784                 elif opt == "-H":
 785                         omit_headers = True
 786                 elif opt == "-p":
 787                         pubs.add(arg)
 788                 elif opt == "-s":
 789                         conf["repo_uri"] = parse_uri(arg)
 790                 elif opt == "--key":
 791                         key = arg
 792                 elif opt == "--cert":
 793                         cert = arg
 794
 795         if pargs:
 796                 usage(_("command does not take operands"), cmd=subcommand)
 797
 798         # Setup transport so status can be retrieved.
 799         if not conf.get("repo_uri", None):
 800                 usage(_("A package repository location must be provided "
 801                     "using -s."), cmd=subcommand)
 802         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
 803             ssl_key=key, ssl_cert=cert)
 804
 805         # Retrieve repository status information.
 806         stat_idx = xport.get_status(xpub)
 807         pub_idx = stat_idx.get("repository", {}).get("publishers", {})
 808         if len(pubs) > 0 and "all" not in pubs:
 809                 found = set(pub_idx.keys()) & pubs
 810                 notfound = pubs - found
 811         else:
 812                 found = set(pub_idx.keys())
 813                 notfound = set()
 814
 815         def gen_listing():
 816                 for pfx in found:
 817                         pdata = pub_idx[pfx]
 818                         pkg_count = pdata.get("package-count", 0)
 819                         last_update = pdata.get("last-catalog-update", "")
 820                         if last_update:
 821                                 # Reformat the date into something more user
 822                                 # friendly (and locale specific).
 823                                 last_update = pkg.catalog.basic_ts_to_datetime(
 824                                     last_update)
 825                                 last_update = "{0}Z".format(
 826                                     pkg.catalog.datetime_to_ts(last_update))
 827                         rstatus = _(pub_idx[pfx].get("status", "online"))
 828                         yield {
 829                             "publisher": pfx,
 830                             "packages": pkg_count,
 831                             "status": rstatus,
 832                             "updated": last_update,
 833                         }
 834
 835         #    PUBLISHER PACKAGES        STATUS   UPDATED
 836         #    <pub_1>   <num_uniq_pkgs> <status> <cat_last_modified>
 837         #    <pub_2>   <num_uniq_pkgs> <status> <cat_last_modified>
 838         #    ...
 839         field_data = {
 840             "publisher" : [("default", "json", "tsv"), _("PUBLISHER"), ""],
 841             "packages" : [("default", "json", "tsv"), _("PACKAGES"), ""],
 842             "status" : [("default", "json", "tsv"), _("STATUS"), ""],
 843             "updated" : [("default", "json", "tsv"), _("UPDATED"), ""],
 844         }
 845
 846         desired_field_order = (_("PUBLISHER"), "", _("PACKAGES"), _("STATUS"),
 847             _("UPDATED"))
 848
 849         # Default output formatting.
 850         pub_len = str(max(
 851             [len(desired_field_order[0])] + [len(p) for p in found]
 852         ))
 853         def_fmt = "{0:" + pub_len + "} {1:8} {2:16} {3}"
 854
 855         if found or (not pubs and out_format == "default"):
 856                 # print without trailing newline.
 857                 sys.stdout.write(misc.get_listing(desired_field_order,
 858                     field_data, gen_listing(), out_format, def_fmt,
 859                     omit_headers))
 860
 861         if found and notfound:
 862                 return EXIT_PARTIAL
 863         if pubs and not found:
 864                 if out_format == "default":
 865                         # Don't pollute other output formats.
 866                         error(_("no matching publishers found"),
 867                             cmd=subcommand)
 868                 return EXIT_OOPS
 869         return EXIT_OK
 870
 871 def subcmd_list(conf, args):
 872         """List all packages matching the specified patterns."""
 873
 874         subcommand = "list"
 875         omit_headers = False
 876         out_format = "default"
 877         pubs = set()
 878         key = None
 879         cert = None
 880
 881         opts, pargs = getopt.getopt(args, "F:Hp:s:", ["key=", "cert="])
 882         for opt, arg in opts:
 883                 if opt == "-F":
 884                         if arg not in LISTING_FORMATS:
 885                                 raise apx.InvalidOptionError(
 886                                     apx.InvalidOptionError.ARG_INVALID,
 887                                     [arg, opt])
 888                         out_format = arg
 889                 elif opt == "-H":
 890                         omit_headers = True
 891                 elif opt == "-p":
 892                         pubs.add(arg)
 893                 elif opt == "-s":
 894                         conf["repo_uri"] = parse_uri(arg)
 895                 elif opt == "--key":
 896                         key = arg
 897                 elif opt == "--cert":
 898                         cert = arg
 899
 900
 901         # Setup transport so configuration can be retrieved.
 902         if not conf.get("repo_uri", None):
 903                 usage(_("A package repository location must be provided "
 904                     "using -s."), cmd=subcommand)
 905         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
 906             ssl_key=key, ssl_cert=cert)
 907
 908         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
 909             xpub, out_format=out_format, use_transport=True)
 910         if rval == EXIT_OOPS:
 911                 return rval
 912
 913         refresh_pub(pub_data, xport)
 914         listed = {}
 915         matched = set()
 916         unmatched = set()
 917
 918         def gen_listing():
 919                 collect_attrs = out_format.startswith("json")
 920                 for pub in pub_data:
 921                         cat = pub.catalog
 922                         for f, states, attrs in cat.gen_packages(
 923                             collect_attrs=collect_attrs, matched=matched,
 924                             patterns=pargs, pubs=[pub.prefix],
 925                             unmatched=unmatched, return_fmris=True):
 926                                 if not listed:
 927                                         listed["packages"] = True
 928
 929                                 state = None
 930                                 if out_format == "default" or \
 931                                     out_format == "tsv":
 932                                         if pkgdefs.PKG_STATE_OBSOLETE in \
 933                                             states:
 934                                                 state = "o"
 935                                         elif pkgdefs.PKG_STATE_RENAMED in \
 936                                             states:
 937                                                 state = "r"
 938
 939                                 if out_format == "default":
 940                                     fver = str(f.version.get_version(
 941                                         include_build=False))
 942                                     ffmri = str(f.get_fmri(include_build=False))
 943                                 else:
 944                                     fver = str(f.version)
 945                                     ffmri = str(f)
 946
 947                                 ret = {
 948                                     "publisher": f.publisher,
 949                                     "name": f.pkg_name,
 950                                     "version": fver,
 951                                     "release": str(f.version.release),
 952                                     "build-release":
 953                                         str(f.version.build_release),
 954                                     "branch": str(f.version.branch),
 955                                     "timestamp":
 956                                         str(f.version.timestr),
 957                                     "pkg.fmri": ffmri,
 958                                     "short_state": state,
 959                                 }
 960
 961                                 for attr in attrs:
 962                                         ret[attr] = []
 963                                         for mods in attrs[attr]:
 964                                                 d = dict(mods)
 965                                                 d["value"] = \
 966                                                     attrs[attr][mods]
 967                                                 ret[attr].append(d)
 968                                 yield ret
 969
 970                         unmatched.difference_update(matched)
 971
 972         field_data = {
 973             "publisher": [("default", "json", "tsv"), _("PUBLISHER"), ""],
 974             "name": [("default", "json", "tsv"), _("NAME"), ""],
 975             "version": [("default", "json"), _("VERSION"), ""],
 976             "release": [("json", "tsv",), _("RELEASE"), ""],
 977             "build-release": [("json", "tsv",), _("BUILD RELEASE"), ""],
 978             "branch": [("json", "tsv",), _("BRANCH"), ""],
 979             "timestamp": [("json", "tsv",), _("PACKAGING DATE"), ""],
 980             "pkg.fmri": [("json", "tsv",), _("FMRI"), ""],
 981             "short_state": [("default", "tsv"), "O", ""],
 982          }
 983
 984         desired_field_order = (_("PUBLISHER"), _("NAME"), "O", _("VERSION"),
 985             _("SUMMARY"), _("DESCRIPTION"), _("CATEGORIES"), _("RELEASE"),
 986             _("BUILD RELEASE"), _("BRANCH"), _("PACKAGING DATE"), _("FMRI"),
 987             _("STATE"))
 988
 989         # Default output formatting.
 990         max_pub_name_len = str(
 991             max(list(len(p) for p in found) + [len(_("PUBLISHER"))]))
 992         def_fmt = "{0:" + max_pub_name_len + "} {1:45} {2:1} {3}"
 993
 994         # print without trailing newline.
 995         sys.stdout.write(misc.get_listing(
 996             desired_field_order, field_data, gen_listing(),
 997             out_format, def_fmt, omit_headers))
 998
 999         if not listed and pargs:
1000                 # No matching packages.
1001                 logger.error("")
1002                 if not unmatched:
1003                         unmatched = pargs
1004                 error(apx.PackageMatchErrors(unmatched_fmris=unmatched),
1005                     cmd=subcommand)
1006                 return EXIT_OOPS
1007         elif unmatched:
1008                 # One or more patterns didn't match a package from any
1009                 # publisher; only display the error.
1010                 logger.error("")
1011                 error(apx.PackageMatchErrors(unmatched_fmris=unmatched),
1012                     cmd=subcommand)
1013                 return EXIT_PARTIAL
1014
1015         return EXIT_OK
1016
1017
1018 def refresh_pub(pub_data, xport):
1019         """A helper function to refresh all specified publishers."""
1020
1021         global tmpdirs
1022         temp_root = misc.config_temp_root()
1023         progtrack = get_tracker()
1024         progtrack.set_purpose(progtrack.PURPOSE_LISTING)
1025
1026         progtrack.refresh_start(pub_cnt=len(pub_data), full_refresh=True,
1027             target_catalog=False)
1028
1029         for pub in pub_data:
1030                 progtrack.refresh_start_pub(pub)
1031                 meta_root = tempfile.mkdtemp(dir=temp_root)
1032                 tmpdirs.append(meta_root)
1033                 pub.meta_root = meta_root
1034                 pub.transport = xport
1035
1036                 try:
1037                         pub.refresh(True, True, progtrack=progtrack)
1038                 except apx.TransportError:
1039                         # Assume that a catalog doesn't exist for the target
1040                         # publisher and drive on.
1041                         pass
1042                 progtrack.refresh_end_pub(pub)
1043
1044         progtrack.refresh_done()
1045
1046
1047 def subcmd_contents(conf, args):
1048         """List package contents."""
1049
1050         subcommand = "contents"
1051         display_raw = False
1052         pubs = set()
1053         key = None
1054         cert = None
1055         attrs = []
1056         action_types = []
1057
1058         opts, pargs = getopt.getopt(args, "ms:t:", ["key=", "cert="])
1059         for opt, arg in opts:
1060                 if opt == "-s":
1061                         conf["repo_uri"] = parse_uri(arg)
1062                 elif opt == "-m":
1063                         display_raw = True
1064                 elif opt == "-t":
1065                         action_types.extend(arg.split(","))
1066                 elif opt == "--key":
1067                         key = arg
1068                 elif opt == "--cert":
1069                         cert = arg
1070
1071         # Setup transport so configuration can be retrieved.
1072         if not conf.get("repo_uri", None):
1073                 usage(_("A package repository location must be provided "
1074                     "using -s."), cmd=subcommand)
1075
1076         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
1077             ssl_key=key, ssl_cert=cert)
1078
1079         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
1080             xpub, use_transport=True)
1081         if rval == EXIT_OOPS:
1082                 return rval
1083
1084         # Default output prints out the raw manifest. The -m option is implicit
1085         # for now and supported to make the interface equivalent to pkg
1086         # contents.
1087         if not attrs or display_raw:
1088                 attrs = ["action.raw"]
1089
1090         refresh_pub(pub_data, xport)
1091         listed = False
1092         matched = set()
1093         unmatched = set()
1094         manifests = []
1095
1096         for pub in pub_data:
1097                 cat = pub.catalog
1098                 for f, states, attr in cat.gen_packages(matched=matched,
1099                         patterns=pargs, pubs=[pub.prefix],
1100                         unmatched=unmatched, return_fmris=True):
1101                         if not listed:
1102                                 listed = True
1103                         manifests.append(xport.get_manifest(f))
1104                 unmatched.difference_update(matched)
1105
1106         # Build a generator expression based on whether specific action types
1107         # were provided.
1108         if action_types:
1109                 # If query is limited to specific action types, use the more
1110                 # efficient type-based generation mechanism.
1111                 gen_expr = (
1112                     (m.fmri, a, None, None, None)
1113                     for m in manifests
1114                     for a in m.gen_actions_by_types(action_types)
1115                 )
1116         else:
1117                 gen_expr = (
1118                     (m.fmri, a, None, None, None)
1119                     for m in manifests
1120                     for a in m.gen_actions()
1121                 )
1122
1123         # Determine if the query returned any results by "peeking" at the first
1124         # value returned from the generator expression.
1125         try:
1126                 got = gen_expr.next()
1127         except StopIteration:
1128                 got = None
1129                 actionlist = []
1130
1131         if got:
1132                 actionlist = itertools.chain([got], gen_expr)
1133
1134         rval = EXIT_OK
1135         if action_types and manifests and not got:
1136                 logger.error(_(gettext.ngettext("""\
1137 pkgrepo: contents: This package contains no actions with the types specified
1138 using the -t option""", """\
1139 pkgrepo: contents: These packages contain no actions with the types specified
1140 using the -t option.""", len(pargs))))
1141                 rval = EXIT_OOPS
1142
1143         if manifests and rval == EXIT_OK:
1144                 lines = misc.list_actions_by_attrs(actionlist, attrs)
1145                 for line in lines:
1146                         text = ("{0}".format(*line)).rstrip()
1147                         if not text:
1148                                continue
1149                         msg(text)
1150
1151         if unmatched:
1152                 if manifests:
1153                         logger.error("")
1154                 logger.error(_("""\
1155 pkgrepo: contents: no packages matching the following patterns you specified
1156 were found in the repository."""))
1157                 logger.error("")
1158                 for p in unmatched:
1159                         logger.error("        {0}".format(p))
1160                 rval = EXIT_OOPS
1161
1162         return rval
1163
1164
1165 def __rebuild_local(subcommand, conf, pubs, build_catalog, build_index):
1166         """In an attempt to allow operations on potentially corrupt
1167         repositories, 'local' repositories (filesystem-basd ones) are handled
1168         separately."""
1169
1170         repo = get_repo(conf, allow_invalid=build_catalog, read_only=False,
1171             subcommand=subcommand)
1172
1173         rpubs = set(repo.publishers)
1174         if not pubs:
1175                 found = rpubs
1176         else:
1177                 found = rpubs & pubs
1178         notfound = pubs - found
1179
1180         rval = EXIT_OK
1181         if found and notfound:
1182                 rval = EXIT_PARTIAL
1183         elif pubs and not found:
1184                 error(_("no matching publishers found"), cmd=subcommand)
1185                 return EXIT_OOPS
1186
1187         logger.info("Initiating repository rebuild.")
1188         for pfx in found:
1189                 repo.rebuild(build_catalog=build_catalog,
1190                     build_index=build_index, pub=pfx)
1191
1192         return rval
1193
1194
1195 def __rebuild_remote(subcommand, conf, pubs, key, cert, build_catalog,
1196     build_index):
1197         def do_rebuild(xport, xpub):
1198                 if build_catalog and build_index:
1199                         xport.publish_rebuild(xpub)
1200                 elif build_catalog:
1201                         xport.publish_rebuild_packages(xpub)
1202                 elif build_index:
1203                         xport.publish_rebuild_indexes(xpub)
1204
1205         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
1206             ssl_key=key, ssl_cert=cert)
1207         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
1208             xpub)
1209         if rval == EXIT_OOPS:
1210                 return rval
1211
1212         logger.info("Initiating repository rebuild.")
1213         for pfx in found:
1214                 xpub.prefix = pfx
1215                 do_rebuild(xport, xpub)
1216
1217         return rval
1218
1219
1220 def subcmd_rebuild(conf, args):
1221         """Rebuild the repository's catalog and index data (as permitted)."""
1222
1223         subcommand = "rebuild"
1224         build_catalog = True
1225         build_index = True
1226         key = None
1227         cert = None
1228
1229         opts, pargs = getopt.getopt(args, "p:s:", ["no-catalog", "no-index",
1230             "key=", "cert="])
1231         pubs = set()
1232         for opt, arg in opts:
1233                 if opt == "-p":
1234                         if not misc.valid_pub_prefix(arg):
1235                                 error(_("Invalid publisher prefix '{0}'").format(
1236                                     arg), cmd=subcommand)
1237                         pubs.add(arg)
1238                 elif opt == "-s":
1239                         conf["repo_uri"] = parse_uri(arg)
1240                 elif opt == "--no-catalog":
1241                         build_catalog = False
1242                 elif opt == "--no-index":
1243                         build_index = False
1244                 elif opt == "--key":
1245                         key = arg
1246                 elif opt == "--cert":
1247                         cert = arg
1248
1249         if pargs:
1250                 usage(_("command does not take operands"), cmd=subcommand)
1251
1252         if not build_catalog and not build_index:
1253                 # Why?  Who knows; but do what was requested--nothing!
1254                 return EXIT_OK
1255
1256         # Setup transport so operation can be performed.
1257         if not conf.get("repo_uri", None):
1258                 usage(_("A package repository location must be provided "
1259                     "using -s."), cmd=subcommand)
1260
1261         if conf["repo_uri"].scheme == "file":
1262                 return __rebuild_local(subcommand, conf, pubs, build_catalog,
1263                     build_index)
1264
1265         return __rebuild_remote(subcommand, conf, pubs, key, cert,
1266             build_catalog, build_index)
1267
1268
1269 def subcmd_refresh(conf, args):
1270         """Refresh the repository's catalog and index data (as permitted)."""
1271
1272         subcommand = "refresh"
1273         add_content = True
1274         refresh_index = True
1275         key = None
1276         cert = None
1277
1278         opts, pargs = getopt.getopt(args, "p:s:", ["no-catalog", "no-index",
1279             "key=", "cert="])
1280         pubs = set()
1281         for opt, arg in opts:
1282                 if opt == "-p":
1283                         if not misc.valid_pub_prefix(arg):
1284                                 error(_("Invalid publisher prefix '{0}'").format(
1285                                     arg), cmd=subcommand)
1286                         pubs.add(arg)
1287                 elif opt == "-s":
1288                         conf["repo_uri"] = parse_uri(arg)
1289                 elif opt == "--no-catalog":
1290                         add_content = False
1291                 elif opt == "--no-index":
1292                         refresh_index = False
1293                 elif opt == "--key":
1294                         key = arg
1295                 elif opt == "--cert":
1296                         cert = arg
1297
1298         if pargs:
1299                 usage(_("command does not take operands"), cmd=subcommand)
1300
1301         if not add_content and not refresh_index:
1302                 # Why?  Who knows; but do what was requested--nothing!
1303                 return EXIT_OK
1304
1305         # Setup transport so operation can be performed.
1306         if not conf.get("repo_uri", None):
1307                 usage(_("A package repository location must be provided "
1308                     "using -s."), cmd=subcommand)
1309
1310         def do_refresh(xport, xpub):
1311                 if add_content and refresh_index:
1312                         xport.publish_refresh(xpub)
1313                 elif add_content:
1314                         xport.publish_refresh_packages(xpub)
1315                 elif refresh_index:
1316                         xport.publish_refresh_indexes(xpub)
1317
1318         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand,
1319             ssl_key=key, ssl_cert=cert)
1320         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
1321             xpub)
1322         if rval == EXIT_OOPS:
1323                 return rval
1324
1325         logger.info("Initiating repository refresh.")
1326         for pfx in found:
1327                 xpub.prefix = pfx
1328                 do_refresh(xport, xpub)
1329
1330         return rval
1331
1332
1333 def subcmd_set(conf, args):
1334         """Set repository properties."""
1335
1336         subcommand = "set"
1337         pubs = set()
1338
1339         opts, pargs = getopt.getopt(args, "p:s:")
1340         for opt, arg in opts:
1341                 if opt == "-p":
1342                         pubs.add(arg)
1343                 elif opt == "-s":
1344                         conf["repo_uri"] = parse_uri(arg)
1345
1346         bad_args = False
1347         props = {}
1348         if not pargs:
1349                 bad_args = True
1350         else:
1351                 for arg in pargs:
1352                         try:
1353                                 # Attempt to parse property into components.
1354                                 prop, val = arg.split("=", 1)
1355                                 sname, pname = prop.rsplit("/", 1)
1356
1357                                 # Store property values by section.
1358                                 props.setdefault(sname, {})
1359
1360                                 # Parse the property value into a list if
1361                                 # necessary, otherwise append it to the list
1362                                 # of values for the property.
1363                                 if len(val) > 0  and val[0] == "(" and \
1364                                     val[-1] == ")":
1365                                         val = shlex.split(val.strip("()"))
1366
1367                                 if sname in props and pname in props[sname]:
1368                                         # Determine if previous value is already
1369                                         # a list, and if not, convert and append
1370                                         # the value.
1371                                         pval = props[sname][pname]
1372                                         if not isinstance(pval, list):
1373                                                 pval = [pval]
1374                                         if isinstance(val, list):
1375                                                 pval.extend(val)
1376                                         else:
1377                                                 pval.append(val)
1378                                         props[sname][pname] = pval
1379                                 else:
1380                                         # Otherwise, just store the value.
1381                                         props[sname][pname] = val
1382                         except ValueError:
1383                                 bad_args = True
1384                                 break
1385
1386         if bad_args:
1387                 usage(_("a property name and value must be provided in the "
1388                     "form <section/property>=<value> or "
1389                     "<section/property>=([\"<value>\" ...])"))
1390
1391         # Get repository object.
1392         if not conf.get("repo_uri", None):
1393                 usage(_("A package repository location must be provided "
1394                     "using -s."), cmd=subcommand)
1395         repo = get_repo(conf, read_only=False, subcommand=subcommand)
1396
1397         # Set properties.
1398         if pubs:
1399                 return _set_pub(conf, subcommand, props, pubs, repo)
1400
1401         return _set_repo(conf, subcommand, props, repo)
1402
1403
1404 def _set_pub(conf, subcommand, props, pubs, repo):
1405         """Set publisher properties."""
1406
1407         for sname, sprops in props.iteritems():
1408                 if sname not in ("publisher", "repository"):
1409                         usage(_("unknown property section "
1410                             "'{0}'").format(sname), cmd=subcommand)
1411                 for pname in sprops:
1412                         if sname == "publisher" and pname == "prefix":
1413                                 usage(_("'{0}' may not be set using "
1414                                     "this command".format(pname)))
1415                         attrname = pname.replace("-", "_")
1416                         if not hasattr(publisher.Publisher, attrname) and \
1417                             not hasattr(publisher.Repository, attrname):
1418                                 usage(_("unknown property '{0}'").format(
1419                                     pname), cmd=subcommand)
1420
1421         if "all" in pubs:
1422                 # Default to list of all publishers.
1423                 pubs = repo.publishers
1424                 if not pubs:
1425                         # If there are still no known publishers, this
1426                         # operation cannot succeed, so fail now.
1427                         usage(_("One or more publishers must be specified to "
1428                             "create and set properties for as none exist yet."),
1429                             cmd=subcommand)
1430
1431         # Get publishers and update properties.
1432         failed = []
1433         new_pub = False
1434         for pfx in pubs:
1435                 try:
1436                         # Get a copy of the existing publisher.
1437                         pub = copy.copy(repo.get_publisher(pfx))
1438                 except sr.RepositoryUnknownPublisher as e:
1439                         pub = publisher.Publisher(pfx)
1440                         new_pub = True
1441                 except sr.RepositoryError as e:
1442                         failed.append((pfx, e))
1443                         continue
1444
1445                 try:
1446                         # Set/update the publisher's properties.
1447                         for sname, sprops in props.iteritems():
1448                                 if sname == "publisher":
1449                                         target = pub
1450                                 elif sname == "repository":
1451                                         target = pub.repository
1452                                         if not target:
1453                                                 target = publisher.Repository()
1454                                                 pub.repository = target
1455
1456                                 for pname, val in sprops.iteritems():
1457                                         attrname = pname.replace("-", "_")
1458                                         pval = getattr(target, attrname)
1459                                         if isinstance(pval, list) and \
1460                                             not isinstance(val, list):
1461                                                 # If the target property expects
1462                                                 # a list, transform the provided
1463                                                 # value into one if it isn't
1464                                                 # already.
1465                                                 if val == "":
1466                                                         val = []
1467                                                 else:
1468                                                         val = [val]
1469                                         setattr(target, attrname, val)
1470                 except apx.ApiException as e:
1471                         failed.append((pfx, e))
1472                         continue
1473
1474                 if new_pub:
1475                         repo.add_publisher(pub)
1476                 else:
1477                         repo.update_publisher(pub)
1478
1479         if failed:
1480                 for pfx, details in failed:
1481                         error(_("Unable to set properties for publisher "
1482                             "'{pfx}':\n{details}").format(**locals()))
1483                 if len(failed) < len(pubs):
1484                         return EXIT_PARTIAL
1485                 return EXIT_OOPS
1486         return EXIT_OK
1487
1488
1489 def _set_repo(conf, subcommand, props, repo):
1490         """Set repository properties."""
1491
1492         # Set properties.
1493         for sname, props in props.iteritems():
1494                 for pname, val in props.iteritems():
1495                         repo.cfg.set_property(sname, pname, val)
1496         repo.write_config()
1497
1498         return EXIT_OK
1499
1500
1501 def subcmd_version(conf, args):
1502         """Display the version of the pkg(5) API."""
1503
1504         subcommand = "version"
1505         if args:
1506                 usage(_("command does not take operands"), cmd=subcommand)
1507         msg(pkg.VERSION)
1508         return EXIT_OK
1509
1510
1511 verify_error_header = None
1512 verify_warning_header = None
1513 verify_reason_headers = None
1514
1515 def __load_verify_msgs():
1516         """Since our gettext isn't loaded we need to ensure our globals have
1517         correct content by calling this method.  These values are used by both
1518         fix when in verbose mode, and verify"""
1519
1520         global verify_error_header
1521         global verify_warning_header
1522         global verify_reason_headers
1523
1524         # A map of error detail types to the human-readable description of each
1525         # type.  These correspond to keys in the dictionary returned by
1526         # sr.Repository.verify(..)
1527         verify_reason_headers = {
1528             "path": _("Repository path"),
1529             "actual": _("Computed hash"),
1530             "fpath": _("Path"),
1531             "permissionspath": _("Path"),
1532             "pkg": _("Package"),
1533             "depend": _("Dependency"),
1534             "type":_("Dependency type"),
1535             "err": _("Detail")
1536         }
1537
1538         verify_error_header = _("ERROR")
1539         verify_warning_header = _("WARNING")
1540
1541
1542 def __fmt_verify(verify_tuple):
1543         """Format a verify_tuple, of the form (error, path, message, reason)
1544         returning a formatted error message, and an FMRI indicating what
1545         packages within the repository are affected. Note that the returned FMRI
1546         may not be valid, in which case a path to the broken manifest in the
1547         repository is returned instead."""
1548
1549         error, path, message, reason = verify_tuple
1550
1551         formatted_message = "{error_type:>16}: {message}\n".format(
1552             error_type=verify_error_header, message=message)
1553         reason["path"] = path
1554
1555         if error == sr.REPO_VERIFY_BADMANIFEST:
1556                 reason_keys = ["path", "err"]
1557         elif error in [sr.REPO_VERIFY_PERM, sr.REPO_VERIFY_MFPERM]:
1558                 reason_keys = ["pkg", "path"]
1559         elif error == sr.REPO_VERIFY_BADHASH:
1560                 reason_keys = ["pkg", "path", "actual", "fpath"]
1561         elif error == sr.REPO_VERIFY_UNKNOWN:
1562                 reason_keys = ["path", "err"]
1563         elif error == sr.REPO_VERIFY_BADSIG:
1564                 reason_keys = ["pkg", "path", "err"]
1565         elif error == sr.REPO_VERIFY_DEPENDERROR:
1566                 reason_keys = ["pkg", "depend", "type"]
1567         elif error == sr.REPO_VERIFY_WARN_OPENPERMS:
1568                 formatted_message = \
1569                     "{error_type:>16}: {message}\n".format(
1570                     error_type=verify_warning_header, message=message)
1571                 reason_keys = ["permissionspath", "err"]
1572         else:
1573                 # A list of the details we provide.  Some error codes
1574                 # have different details associated with them.
1575                 reason_keys = ["pkg", "path", "fpath"]
1576
1577
1578         # the detailed error message can be long, so we'll wrap it.  If what we
1579         # have fits on a single line, use it, otherwise begin displaying the
1580         # message on the next line.
1581         if "err" in reason_keys:
1582                 err_str = ""
1583                 lines = textwrap.wrap(reason["err"])
1584                 if len(lines) != 1:
1585                         for line in lines:
1586                                 err_str += "{0:>18}\n".format(line)
1587                         reason["err"] = "\n" + err_str.rstrip()
1588                 else:
1589                         reason["err"] = lines[0]
1590
1591         for key in reason_keys:
1592                 # sometimes we don't have the key we want, for example we may
1593                 # not have a file path from the package if the error is a
1594                 # missing repository file for a 'license' action (which don't
1595                 # have 'path' attributes, hence no 'fpath' dictionary entry)
1596                 if key not in reason:
1597                         continue
1598                 formatted_message += "{key:>16}: {value}\n".format(
1599                     key=verify_reason_headers[key], value=reason[key])
1600
1601         formatted_message += "\n"
1602
1603         if error == sr.REPO_VERIFY_WARN_OPENPERMS:
1604                 return formatted_message, None
1605         elif "depend" in reason:
1606                 return formatted_message, reason["depend"]
1607         elif "pkg" in reason:
1608                 return formatted_message, reason["pkg"]
1609         return formatted_message, reason["path"]
1610
1611
1612 def __collect_default_ignore_dep_files(ignored_dep_files):
1613         """Helpler function to collect default ignored-dependency files."""
1614
1615         root_ignored = "/usr/share/pkg/ignored_deps"
1616         altroot = DebugValues.get_value("ignored_deps")
1617         if altroot:
1618                 root_ignored = altroot
1619         if os.path.exists(root_ignored):
1620                 igfiles = os.listdir(root_ignored)
1621                 for igf in igfiles:
1622                         ignored_dep_files.append(os.path.join(root_ignored,
1623                             igf))
1624
1625
1626 def subcmd_verify(conf, args):
1627         """Verify the repository content (file, manifest content and
1628         dependencies only)."""
1629
1630         subcommand = "verify"
1631         __load_verify_msgs()
1632
1633         opts, pargs = getopt.getopt(args, "dp:s:i:", ["disable="])
1634         allowed_checks = set(sr.verify_default_checks)
1635         force_dep_check = False
1636         ignored_dep_files = []
1637         pubs = set()
1638         for opt, arg in opts:
1639                 if opt == "-s":
1640                         conf["repo_uri"] = parse_uri(arg)
1641                 elif opt == "-p":
1642                         if not misc.valid_pub_prefix(arg):
1643                                 error(_("Invalid publisher prefix '{0}'").format(
1644                                     arg), cmd=subcommand)
1645                         pubs.add(arg)
1646                 elif opt == "-d":
1647                         force_dep_check = True
1648                 elif opt == "--disable":
1649                         arg = arg.lower()
1650                         if arg in sr.verify_default_checks:
1651                                 if arg in allowed_checks:
1652                                         allowed_checks.remove(arg)
1653                         else:
1654                                 usage(_("Invalid verification to be disabled, "
1655                                     "please consider: {0}").format(", ".join(
1656                                     sr.verify_default_checks)), cmd=subcommand)
1657                 elif opt == "-i":
1658                         ignored_dep_files.append(arg)
1659
1660         if pargs:
1661                 usage(_("command does not take operands"), cmd=subcommand)
1662
1663         repo_uri = conf.get("repo_uri", None)
1664         if not repo_uri:
1665                 usage(_("A package repository location must be provided "
1666                     "using -s."), cmd=subcommand)
1667
1668         if repo_uri.scheme != "file":
1669                 usage(_("Network repositories are not currently supported "
1670                     "for this operation."), cmd=subcommand)
1671
1672         if sr.VERIFY_DEPENDENCY not in allowed_checks and \
1673             (force_dep_check or len(ignored_dep_files) > 0):
1674                 usage(_("-d or -i option cannot be used when dependency "
1675                     "verification is disabled."), cmd=subcommand)
1676
1677         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand)
1678         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
1679             xpub)
1680
1681         if rval == EXIT_OOPS:
1682                 return rval
1683
1684         logger.info("Initiating repository verification.")
1685         bad_fmris = set()
1686         progtrack = get_tracker()
1687
1688         def report_error(verify_tuple):
1689                 message, bad_fmri = __fmt_verify(verify_tuple)
1690                 if bad_fmri:
1691                         bad_fmris.add(bad_fmri)
1692                 progtrack.repo_verify_yield_error(bad_fmri, message)
1693
1694         if sr.VERIFY_DEPENDENCY in allowed_checks or not force_dep_check:
1695                 __collect_default_ignore_dep_files(ignored_dep_files)
1696
1697         repo = sr.Repository(root=repo_uri.get_pathname())
1698
1699         found_pubs = []
1700         for pfx in found:
1701                 xport, xpub, tmp_dir = setup_transport(conf, prefix=pfx,
1702                     remote_prefix=False,
1703                     subcommand=subcommand)
1704                 xpub.transport = xport
1705                 found_pubs.append(xpub)
1706
1707         for verify_tuple in repo.verify(pubs=found_pubs,
1708             allowed_checks=allowed_checks, force_dep_check=force_dep_check,
1709             ignored_dep_files=ignored_dep_files, progtrack=progtrack):
1710                 report_error(verify_tuple)
1711
1712         if bad_fmris:
1713                 return EXIT_OOPS
1714         return EXIT_OK
1715
1716
1717 def subcmd_fix(conf, args):
1718         """Fix the repository content (file and manifest content only)
1719         For index and catalog content corruption, a rebuild should be
1720         performed."""
1721
1722         subcommand = "fix"
1723         __load_verify_msgs()
1724         verbose = False
1725
1726         # Dependency verification. Note fix will not force dependency check.
1727         force_dep_check = False
1728         ignored_dep_files = []
1729
1730         opts, pargs = getopt.getopt(args, "vp:s:")
1731         pubs = set()
1732         for opt, arg in opts:
1733                 if opt == "-s":
1734                         conf["repo_uri"] = parse_uri(arg)
1735                 if opt == "-v":
1736                         verbose = True
1737                 if opt == "-p":
1738                         if not misc.valid_pub_prefix(arg):
1739                                 error(_("Invalid publisher prefix '{0}'").format(
1740                                     arg), cmd=subcommand)
1741                         pubs.add(arg)
1742
1743         if pargs:
1744                 usage(_("command does not take operands"), cmd=subcommand)
1745
1746         repo_uri = conf.get("repo_uri", None)
1747         if not repo_uri:
1748                 usage(_("A package repository location must be provided "
1749                     "using -s."), cmd=subcommand)
1750
1751         if repo_uri.scheme != "file":
1752                 usage(_("Network repositories are not currently supported "
1753                     "for this operation."), cmd=subcommand)
1754
1755         xport, xpub, tmp_dir = setup_transport(conf, subcommand=subcommand)
1756         rval, found, pub_data = _get_matching_pubs(subcommand, pubs, xport,
1757             xpub)
1758         if rval == EXIT_OOPS:
1759                 return rval
1760
1761         logger.info("Initiating repository fix.")
1762         progtrack = get_tracker()
1763
1764         def verify_cb(tracker, verify_tuple):
1765                 """A method passed to sr.Repository.fix(..) to emit verify
1766                 messages if verbose mode is enabled."""
1767                 if not verbose:
1768                         return
1769                 formatted_message, bad_fmri = __fmt_verify(verify_tuple)
1770                 tracker.repo_verify_yield_error(bad_fmri, formatted_message)
1771
1772         repo = sr.Repository(root=repo_uri.get_pathname())
1773         bad_deps = set()
1774         broken_fmris = set()
1775         failed_fix_paths = set()
1776         progtrack = get_tracker()
1777         __collect_default_ignore_dep_files(ignored_dep_files)
1778
1779         found_pubs = []
1780         for pfx in found:
1781                 xport, xpub, tmp_dir = setup_transport(conf, prefix=pfx,
1782                     remote_prefix=False,
1783                     subcommand=subcommand)
1784                 xpub.transport = xport
1785                 found_pubs.append(xpub)
1786
1787         for status_code, path, message, reason in \
1788             repo.fix(pubs=found_pubs, force_dep_check=force_dep_check,
1789                 ignored_dep_files=ignored_dep_files,
1790                 progtrack=progtrack,
1791                 verify_callback=verify_cb):
1792                 if status_code == sr.REPO_FIX_ITEM:
1793                         # When we can't get the FMRI, eg. in the case
1794                         # of a corrupt manifest, use the path instead.
1795                         fmri = reason["pkg"]
1796                         if not fmri:
1797                                 fmri = path
1798                         broken_fmris.add(fmri)
1799                         if verbose:
1800                                 progtrack.repo_fix_yield_info(fmri,
1801                                     message)
1802                 elif status_code == sr.REPO_VERIFY_DEPENDERROR:
1803                         bad_deps.add(reason["depend"])
1804                 else:
1805                         failed_fix_paths.add(path)
1806
1807         progtrack.flush()
1808         logger.info("")
1809
1810         if broken_fmris:
1811                 logger.info(_("Use pkgsend(1) or pkgrecv(1) to republish the\n"
1812                     "following packages or paths which were quarantined:\n\n\t"
1813                     "{0}").format(
1814                     "\n\t".join([str(f) for f in broken_fmris])))
1815         if failed_fix_paths:
1816                 logger.info(_("\npkgrepo could not repair the following paths "
1817                     "in the repository:\n\n\t{0}").format(
1818                     "\n\t".join([p for p in failed_fix_paths])))
1819         if bad_deps:
1820                 logger.info(_("\npkgrepo could not repair the following "
1821                     "dependency issues in the repository:\n\n\t{0}").format(
1822                     "\n\t".join([p for p in bad_deps])))
1823         if not (broken_fmris or failed_fix_paths or bad_deps):
1824                 logger.info(_("No repository fixes required."))
1825         else:
1826                 logger.info(_("Repository repairs completed."))
1827
1828         if failed_fix_paths or bad_deps:
1829                 return EXIT_OOPS
1830         return EXIT_OK
1831
1832
1833 def main_func():
1834         global_settings.client_name = PKG_CLIENT_NAME
1835
1836         try:
1837                 opts, pargs = getopt.getopt(sys.argv[1:], "s:D:?",
1838                     ["help", "debug="])
1839         except getopt.GetoptError as e:
1840                 usage(_("illegal global option -- {0}").format(e.opt))
1841
1842         conf = {}
1843         show_usage = False
1844         for opt, arg in opts:
1845                 if opt == "-s":
1846                         conf["repo_uri"] = parse_uri(arg)
1847                 elif opt in ("--help", "-?"):
1848                         show_usage = True
1849                 elif opt == "-D" or opt == "--debug":
1850                         try:
1851                                 key, value = arg.split("=", 1)
1852                         except (AttributeError, ValueError):
1853                                 usage(_("{opt} takes argument of form "
1854                                    "name=value, not {arg}").format(
1855                                    opt=opt, arg=arg))
1856                         DebugValues.set_value(key, value)
1857
1858         if DebugValues:
1859                 reload(pkg.digest)
1860
1861         subcommand = None
1862         if pargs:
1863                 subcommand = pargs.pop(0)
1864                 if subcommand == "help":
1865                         show_usage = True
1866
1867         if show_usage:
1868                 usage(retcode=0, full=True)
1869         elif not subcommand:
1870                 usage(_("no subcommand specified"))
1871
1872         subcommand = subcommand.replace("-", "_")
1873         func = globals().get("subcmd_{0}".format(subcommand), None)
1874         if not func:
1875                 subcommand = subcommand.replace("_", "-")
1876                 usage(_("unknown subcommand '{0}'").format(subcommand))
1877
1878         try:
1879                 return func(conf, pargs)
1880         except getopt.GetoptError as e:
1881                 if e.opt in ("help", "?"):
1882                         usage(full=True)
1883                 usage(_("illegal option -- {0}").format(e.opt), cmd=subcommand)
1884
1885
1886 #
1887 # Establish a specific exit status which means: "python barfed an exception"
1888 # so that we can more easily detect these in testing of the CLI commands.
1889 #
1890 def handle_errors(func, *args, **kwargs):
1891         """Catch exceptions raised by the main program function and then print
1892         a message and/or exit with an appropriate return code.
1893         """
1894
1895         traceback_str = misc.get_traceback_message()
1896
1897         try:
1898                 # Out of memory errors can be raised as EnvironmentErrors with
1899                 # an errno of ENOMEM, so in order to handle those exceptions
1900                 # with other errnos, we nest this try block and have the outer
1901                 # one handle the other instances.
1902                 try:
1903                         __ret = func(*args, **kwargs)
1904                 except (MemoryError, EnvironmentError) as __e:
1905                         if isinstance(__e, EnvironmentError) and \
1906                             __e.errno != errno.ENOMEM:
1907                                 raise
1908                         error("\n" + misc.out_of_memory())
1909                         __ret = EXIT_OOPS
1910         except SystemExit as __e:
1911                 raise __e
1912         except (IOError, PipeError, KeyboardInterrupt) as __e:
1913                 # Don't display any messages here to prevent possible further
1914                 # broken pipe (EPIPE) errors.
1915                 if isinstance(__e, IOError) and __e.errno != errno.EPIPE:
1916                         error(str(__e))
1917                 __ret = EXIT_OOPS
1918         except apx.VersionException as __e:
1919                 error(_("The pkgrepo command appears out of sync with the "
1920                     "libraries provided\nby pkg:/package/pkg. The client "
1921                     "version is {client} while the library\nAPI version is "
1922                     "{api}.").format(client=__e.received_version,
1923                      api=__e.expected_version))
1924                 __ret = EXIT_OOPS
1925         except apx.BadRepositoryURI as __e:
1926                 error(str(__e))
1927                 __ret = EXIT_BADOPT
1928         except apx.InvalidOptionError as __e:
1929                 error("{0} Supported formats: {1}".format(
1930                     str(__e), LISTING_FORMATS))
1931                 __ret = EXIT_BADOPT
1932         except (apx.ApiException, sr.RepositoryError) as __e:
1933                 error(str(__e))
1934                 __ret = EXIT_OOPS
1935         except:
1936                 traceback.print_exc()
1937                 error(traceback_str)
1938                 __ret = 99
1939         return __ret
1940
1941
1942 if __name__ == "__main__":
1943         misc.setlocale(locale.LC_ALL, "", error)
1944         gettext.install("pkg", "/usr/share/locale",
1945             codeset=locale.getpreferredencoding())
1946
1947         # Make all warnings be errors.
1948         warnings.simplefilter('error')
1949
1950         __retval = handle_errors(main_func)
1951         try:
1952                 logging.shutdown()
1953         except IOError:
1954                 # Ignore python's spurious pipe problems.
1955                 pass
1956         sys.exit(__retval)