tools/python/pkglint/userland.py

   1 #!/usr/bin/python3.5
   2 #
   3 # CDDL HEADER START
   4 #
   5 # The contents of this file are subject to the terms of the
   6 # Common Development and Distribution License (the "License").
   7 # You may not use this file except in compliance with the License.
   8 #
   9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10 # or http://www.opensolaris.org/os/licensing.
  11 # See the License for the specific language governing permissions
  12 # and limitations under the License.
  13 #
  14 # When distributing Covered Code, include this CDDL HEADER in each
  15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16 # If applicable, add the following below this CDDL HEADER, with the
  17 # fields enclosed by brackets "[]" replaced with your own identifying
  18 # information: Portions Copyright [yyyy] [name of copyright owner]
  19 #
  20 # CDDL HEADER END
  21 #
  22
  23 #
  24 # Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
  25 #
  26
  27 # Some userland consolidation specific lint checks
  28
  29 import pkg.lint.base as base
  30 from pkg.lint.engine import lint_fmri_successor
  31 import pkg.fmri
  32 import pkg.elf as elf
  33 import re
  34 import os.path
  35 import subprocess
  36 import pkg.client.api
  37 import pkg.client.api_errors
  38 import pkg.client.progress
  39
  40 class UserlandActionChecker(base.ActionChecker):
  41         """An opensolaris.org-specific class to check actions."""
  42
  43         name = "userland.action"
  44
  45         def __init__(self, config):
  46                 self.description = _(
  47                     "checks Userland packages for common content errors")
  48                 path = os.getenv('PROTO_PATH')
  49                 if path != None:
  50                         self.proto_path = path.split()
  51                 else:
  52                         self.proto_path = None
  53                 #
  54                 # These lists are used to check if a 32/64-bit binary
  55                 # is in a proper 32/64-bit directory.
  56                 #
  57                 self.pathlist32 = [
  58                         "i86",
  59                         "sparcv7",
  60                         "32",
  61                         "i86pc-solaris-64int",  # perl path
  62                         "sun4-solaris-64int"    # perl path
  63                 ]
  64                 self.pathlist64 = [
  65                         "amd64",
  66                         "sparcv9",
  67                         "64",
  68                         "i86pc-solaris-thread-multi-64",     # perl path
  69                         "sun4-solaris-thread-multi-64"       # perl path
  70                 ]
  71                 self.runpath_re = [
  72                         re.compile('^/lib(/.*)?$'),
  73                         re.compile('^/usr/'),
  74                         re.compile('^\$ORIGIN/')
  75                 ]
  76                 self.runpath_64_re = [
  77                         re.compile('^.*/64(/.*)?$'),
  78                         re.compile('^.*/amd64(/.*)?$'),
  79                         re.compile('^.*/sparcv9(/.*)?$'),
  80                         re.compile('^.*/i86pc-solaris-thread-multi-64(/.*)?$'), # perl path
  81                         re.compile('^.*/sun4-solaris-thread-multi-64(/.*)?$')   # perl path
  82                 ]
  83                 self.initscript_re = re.compile("^etc/(rc.|init)\.d")
  84
  85                 self.lint_paths = {}
  86                 self.ref_paths = {}
  87
  88                 super(UserlandActionChecker, self).__init__(config)
  89
  90         def startup(self, engine):
  91                 """Initialize the checker with a dictionary of paths, so that we
  92                 can do link resolution.
  93
  94                 This is copied from the core pkglint code, but should eventually
  95                 be made common.
  96                 """
  97
  98                 def seed_dict(mf, attr, dic, atype=None, verbose=False):
  99                         """Updates a dictionary of { attr: [(fmri, action), ..]}
 100                         where attr is the value of that attribute from
 101                         actions of a given type atype, in the given
 102                         manifest."""
 103
 104                         pkg_vars = mf.get_all_variants()
 105
 106                         if atype:
 107                                 mfg = (a for a in mf.gen_actions_by_type(atype))
 108                         else:
 109                                 mfg = (a for a in mf.gen_actions())
 110
 111                         for action in mfg:
 112                                 if atype and action.name != atype:
 113                                         continue
 114                                 if attr not in action.attrs:
 115                                         continue
 116
 117                                 variants = action.get_variant_template()
 118                                 variants.merge_unknown(pkg_vars)
 119                                 action.attrs.update(variants)
 120
 121                                 p = action.attrs[attr]
 122                                 dic.setdefault(p, []).append((mf.fmri, action))
 123
 124                 # construct a set of FMRIs being presented for linting, and
 125                 # avoid seeding the reference dictionary with any for which
 126                 # we're delivering new packages.
 127                 lint_fmris = {}
 128                 for m in engine.gen_manifests(engine.lint_api_inst,
 129                     release=engine.release, pattern=engine.pattern):
 130                         lint_fmris.setdefault(m.fmri.get_name(), []).append(m.fmri)
 131                 for m in engine.lint_manifests:
 132                         lint_fmris.setdefault(m.fmri.get_name(), []).append(m.fmri)
 133
 134                 engine.logger.debug(
 135                     _("Seeding reference action path dictionaries."))
 136
 137                 for manifest in engine.gen_manifests(engine.ref_api_inst,
 138                     release=engine.release):
 139                         # Only put this manifest into the reference dictionary
 140                         # if it's not an older version of the same package.
 141                         if not any(
 142                             lint_fmri_successor(fmri, manifest.fmri)
 143                             for fmri
 144                             in lint_fmris.get(manifest.fmri.get_name(), [])
 145                         ):
 146                                 seed_dict(manifest, "path", self.ref_paths)
 147
 148                 engine.logger.debug(
 149                     _("Seeding lint action path dictionaries."))
 150
 151                 # we provide a search pattern, to allow users to lint a
 152                 # subset of the packages in the lint_repository
 153                 for manifest in engine.gen_manifests(engine.lint_api_inst,
 154                     release=engine.release, pattern=engine.pattern):
 155                         seed_dict(manifest, "path", self.lint_paths)
 156
 157                 engine.logger.debug(
 158                     _("Seeding local action path dictionaries."))
 159
 160                 for manifest in engine.lint_manifests:
 161                         seed_dict(manifest, "path", self.lint_paths)
 162
 163                 self.__merge_dict(self.lint_paths, self.ref_paths,
 164                     ignore_pubs=engine.ignore_pubs)
 165
 166         def __merge_dict(self, src, target, ignore_pubs=True):
 167                 """Merges the given src dictionary into the target
 168                 dictionary, giving us the target content as it would appear,
 169                 were the packages in src to get published to the
 170                 repositories that made up target.
 171
 172                 We need to only merge packages at the same or successive
 173                 version from the src dictionary into the target dictionary.
 174                 If the src dictionary contains a package with no version
 175                 information, it is assumed to be more recent than the same
 176                 package with no version in the target."""
 177
 178                 for p in src:
 179                         if p not in target:
 180                                 target[p] = src[p]
 181                                 continue
 182
 183                         def build_dic(arr):
 184                                 """Builds a dictionary of fmri:action entries"""
 185                                 dic = {}
 186                                 for (pfmri, action) in arr:
 187                                         if pfmri in dic:
 188                                                 dic[pfmri].append(action)
 189                                         else:
 190                                                 dic[pfmri] = [action]
 191                                 return dic
 192
 193                         src_dic = build_dic(src[p])
 194                         targ_dic = build_dic(target[p])
 195
 196                         for src_pfmri in src_dic:
 197                                 # we want to remove entries deemed older than
 198                                 # src_pfmri from targ_dic.
 199                                 for targ_pfmri in targ_dic.copy():
 200                                         sname = src_pfmri.get_name()
 201                                         tname = targ_pfmri.get_name()
 202                                         if lint_fmri_successor(src_pfmri,
 203                                             targ_pfmri,
 204                                             ignore_pubs=ignore_pubs):
 205                                                 targ_dic.pop(targ_pfmri)
 206                         targ_dic.update(src_dic)
 207                         l = []
 208                         for pfmri in targ_dic:
 209                                 for action in targ_dic[pfmri]:
 210                                         l.append((pfmri, action))
 211                         target[p] = l
 212
 213         def __realpath(self, path, target):
 214                 """Combine path and target to get the real path."""
 215
 216                 result = os.path.dirname(path)
 217
 218                 for frag in target.split(os.sep):
 219                         if frag == '..':
 220                                 result = os.path.dirname(result)
 221                         elif frag == '.':
 222                                 pass
 223                         else:
 224                                 result = os.path.join(result, frag)
 225
 226                 return result
 227
 228         def __elf_aslr_check(self, path, engine):
 229                 result = None
 230
 231                 ei = elf.get_info(path)
 232                 type = ei.get("type");
 233                 if type != "exe":
 234                         return result
 235
 236                 # get the ASLR tag string for this binary
 237                 aslr_tag_process = subprocess.Popen(
 238                         "/usr/bin/elfedit -r -e 'dyn:sunw_aslr' "
 239                         + path, shell=True,
 240                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 241
 242                 # aslr_tag_string will get stdout; err will get stderr
 243                 aslr_tag_string, err = aslr_tag_process.communicate()
 244
 245                 # No ASLR tag was found; everthing must be tagged
 246                 if aslr_tag_process.returncode != 0:
 247                         engine.error(
 248                                 _("'%s' is not tagged for aslr") % (path),
 249                                 msgid="%s%s.5" % (self.name, "001"))
 250                         return result
 251
 252                 # look for "ENABLE" anywhere in the string;
 253                 # warn about binaries which are not ASLR enabled
 254                 if re.search("ENABLE", aslr_tag_string) is not None:
 255                         return result
 256                 engine.warning(
 257                         _("'%s' does not have aslr enabled") % (path),
 258                         msgid="%s%s.6" % (self.name, "001"))
 259                 return result
 260
 261         def __elf_runpath_check(self, path, engine):
 262                 result = None
 263                 list = []
 264
 265                 ed = elf.get_dynamic(path)
 266                 ei = elf.get_info(path)
 267                 bits = ei.get("bits")
 268                 for dir in ed.get("runpath", "").split(":"):
 269                         if dir == None or dir == '':
 270                                 continue
 271
 272                         match = False
 273                         for expr in self.runpath_re:
 274                                 if expr.match(dir):
 275                                         match = True
 276                                         break
 277
 278                         # The RUNPATH shouldn't contain any runtime linker
 279                         # default paths (or the /64 equivalent link)
 280                         if dir in ['/lib', '/lib/64',
 281                                    '/lib/amd64', '/lib/sparcv9',
 282                                    '/usr/lib', '/usr/lib/64',
 283                                    '/usr/lib/amd64', '/usr/lib/sparcv9' ]:
 284                                 list.append(dir)
 285
 286                         if match == False:
 287                                 list.append(dir)
 288
 289                         if bits == 32:
 290                                 for expr in self.runpath_64_re:
 291                                         if expr.search(dir):
 292                                                 engine.warning(
 293                                                         _("64-bit runpath in 32-bit binary, '%s' includes '%s'") % (path, dir),
 294                                                         msgid="%s%s.3" % (self.name, "001"))
 295                         else:
 296                                 match = False
 297                                 for expr in self.runpath_64_re:
 298                                         if expr.search(dir):
 299                                                 match = True
 300                                                 break
 301                                 if match == False:
 302                                         engine.warning(
 303                                                 _("32-bit runpath in 64-bit binary, '%s' includes '%s'") % (path, dir),
 304                                                 msgid="%s%s.3" % (self.name, "001"))
 305                 if len(list) > 0:
 306                         result = _("bad RUNPATH, '%%s' includes '%s'" %
 307                                    ":".join(list))
 308
 309                 return result
 310
 311         def __elf_wrong_location_check(self, path):
 312                 result = None
 313
 314                 ei = elf.get_info(path)
 315                 bits = ei.get("bits")
 316                 type = ei.get("type");
 317                 elems = os.path.dirname(path).split("/")
 318
 319                 path64 = False
 320                 for p in self.pathlist64:
 321                     if (p in elems):
 322                             path64 = True
 323
 324                 path32 = False
 325                 for p in self.pathlist32:
 326                     if (p in elems):
 327                             path32 = True
 328
 329                 # ignore 64-bit executables in normal (non-32-bit-specific)
 330                 # locations, that's ok now.
 331                 if (type == "exe" and bits == 64 and path32 == False and path64 == False):
 332                         return result
 333
 334                 if bits == 32 and path64:
 335                         result = _("32-bit object '%s' in 64-bit path")
 336                 elif bits == 64 and not path64:
 337                         result = _("64-bit object '%s' in 32-bit path")
 338                 return result
 339
 340         def file_action(self, action, manifest, engine, pkglint_id="001"):
 341                 """Checks for existence in the proto area."""
 342
 343                 if action.name not in ["file"]:
 344                         return
 345
 346                 path = action.hash
 347                 if path == None or path == 'NOHASH':
 348                         path = action.attrs["path"]
 349
 350                 # check for writable files without a preserve attribute
 351                 if "mode" in action.attrs:
 352                         mode = action.attrs["mode"]
 353
 354                         if (int(mode, 8) & 0o222) != 0 and "preserve" not in action.attrs:
 355                                 engine.error(
 356                                 _("%(path)s is writable (%(mode)s), but missing a preserve"
 357                                   " attribute") %  {"path": path, "mode": mode},
 358                                 msgid="%s%s.0" % (self.name, pkglint_id))
 359                 elif "preserve" in action.attrs:
 360                         if "mode" in action.attrs:
 361                                 mode = action.attrs["mode"]
 362                                 if (int(mode, 8) & 0o222) == 0:
 363                                         engine.error(
 364                                         _("%(path)s has a preserve action, but is not writable (%(mode)s)") %  {"path": path, "mode": mode},
 365                                 msgid="%s%s.4" % (self.name, pkglint_id))
 366                         else:
 367                                 engine.error(
 368                                 _("%(path)s has a preserve action, but no mode") %  {"path": path, "mode": mode},
 369                                 msgid="%s%s.3" % (self.name, pkglint_id))
 370
 371                 # checks that require a physical file to look at
 372                 if self.proto_path is not None:
 373                         for directory in self.proto_path:
 374                                 fullpath = directory + "/" + path
 375
 376                                 if os.path.exists(fullpath):
 377                                         break
 378
 379                         if not os.path.exists(fullpath):
 380                                 engine.info(
 381                                         _("%s missing from proto area, skipping"
 382                                           " content checks") % path,
 383                                         msgid="%s%s.1" % (self.name, pkglint_id))
 384                         elif elf.is_elf_object(fullpath):
 385                                 # 32/64 bit in wrong place
 386                                 result = self.__elf_wrong_location_check(fullpath)
 387                                 if result != None:
 388                                         engine.error(result % path,
 389                                                 msgid="%s%s.2" % (self.name, pkglint_id))
 390                                 result = self.__elf_runpath_check(fullpath, engine)
 391                                 if result != None:
 392                                         engine.error(result % path,
 393                                                 msgid="%s%s.3" % (self.name, pkglint_id))
 394                                 # illumos does not support ASLR
 395                                 #result = self.__elf_aslr_check(fullpath, engine)
 396
 397         file_action.pkglint_desc = _("Paths should exist in the proto area.")
 398
 399         def link_resolves(self, action, manifest, engine, pkglint_id="002"):
 400                 """Checks for link resolution."""
 401
 402                 if action.name not in ["link", "hardlink"]:
 403                         return
 404
 405                 path = action.attrs["path"]
 406                 target = action.attrs["target"]
 407                 realtarget = self.__realpath(path, target)
 408
 409                 # Check against the target image (ref_paths), since links might
 410                 # resolve outside the packages delivering a particular
 411                 # component.
 412
 413                 # links to files should directly match a patch in the reference
 414                 # repo.
 415                 if self.ref_paths.get(realtarget, None):
 416                         return
 417
 418                 # If it didn't match a path in the reference repo, it may still
 419                 # be a link to a directory that has no action because it uses
 420                 # the default attributes.  Look for a path that starts with
 421                 # this value plus a trailing slash to be sure this it will be
 422                 # resolvable on a fully installed system.
 423                 realtarget += '/'
 424                 for key in self.ref_paths:
 425                         if key.startswith(realtarget):
 426                                 return
 427
 428                 engine.error(_("%s %s has unresolvable target '%s'") %
 429                                 (action.name, path, target),
 430                         msgid="%s%s.0" % (self.name, pkglint_id))
 431
 432         link_resolves.pkglint_desc = _("links should resolve.")
 433
 434         def init_script(self, action, manifest, engine, pkglint_id="003"):
 435                 """Checks for SVR4 startup scripts."""
 436
 437                 if action.name not in ["file", "dir", "link", "hardlink"]:
 438                         return
 439
 440                 path = action.attrs["path"]
 441                 if self.initscript_re.match(path):
 442                         engine.warning(
 443                                 _("SVR4 startup '%s', deliver SMF"
 444                                   " service instead") % path,
 445                                 msgid="%s%s.0" % (self.name, pkglint_id))
 446
 447         init_script.pkglint_desc = _(
 448                 "SVR4 startup scripts should not be delivered.")
 449
 450 class UserlandManifestChecker(base.ManifestChecker):
 451         """An opensolaris.org-specific class to check manifests."""
 452
 453         name = "userland.manifest"
 454
 455         def __init__(self, config):
 456                 super(UserlandManifestChecker, self).__init__(config)
 457
 458         def forbidden_publisher(self, manifest, engine, pkglint_id="1001"):
 459                 if not os.environ.get("ENCUMBERED"):
 460                         for action in manifest.gen_actions_by_type("depend"):
 461                                 for f in action.attrlist("fmri"):
 462                                         pkg_name=pkg.fmri.PkgFmri(f).pkg_name
 463                                         info_needed = pkg.client.api.PackageInfo.ALL_OPTIONS - \
 464                                             (pkg.client.api.PackageInfo.ACTION_OPTIONS |
 465                                              frozenset([pkg.client.api.PackageInfo.LICENSES]))
 466                                         progtracker = pkg.client.progress.NullProgressTracker()
 467                                         interface=pkg.client.api.ImageInterface("/", pkg.client.api.CURRENT_API_VERSION, progtracker, lambda x: False, None,None)
 468                                         ret = interface.info([pkg_name],True,info_needed)
 469                                         if ret[pkg.client.api.ImageInterface.INFO_FOUND]:
 470                                                 allowed_pubs = engine.get_param("%s.allowed_pubs" % self.name).split(" ") + ["openindiana.org","on-nightly"]
 471                                                 for i in ret[pkg.client.api.ImageInterface.INFO_FOUND]:
 472                                                         if i.publisher not in allowed_pubs:
 473                                                                 engine.error(_("package %(pkg)s depends on %(name)s, which comes from forbidden publisher %(publisher)s") %
 474                                                                         {"pkg":manifest.fmri,"name":pkg_name,"publisher":i.publisher}, msgid="%s%s.1" % (self.name, pkglint_id))
 475
 476         forbidden_publisher.pkglint_desc = _(
 477                 "Dependencies should come from standard publishers" )
 478
 479         def component_check(self, manifest, engine, pkglint_id="001"):
 480                 manifest_paths = []
 481                 files = False
 482                 license = False
 483
 484                 for action in manifest.gen_actions_by_type("file"):
 485                         files = True
 486                         break
 487
 488                 if files == False:
 489                         return
 490
 491                 for action in manifest.gen_actions_by_type("license"):
 492                         if not action.attrs['license']:
 493                                 engine.error( _("missing vaue for action license attribute 'license' like 'CDDL','MIT','GPL'..."),
 494                                     msgid="%s%s.0" % (self.name, pkglint_id))
 495                         else:
 496                                 license = True
 497                                 break
 498
 499                 if license == False:
 500                         engine.error( _("missing license action"),
 501                                 msgid="%s%s.0" % (self.name, pkglint_id))
 502
 503 #                if 'org.opensolaris.arc-caseid' not in manifest:
 504 #                        engine.error( _("missing ARC data (org.opensolaris.arc-caseid)"),
 505 #                                msgid="%s%s.0" % (self.name, pkglint_id))
 506
 507         component_check.pkglint_desc = _(
 508                 "license actions and ARC information are required if you deliver files.")
 509
 510         def publisher_in_fmri(self, manifest, engine, pkglint_id="002"):
 511                 allowed_pubs = engine.get_param(
 512                     "%s.allowed_pubs" % self.name).split(" ")
 513
 514                 fmri = manifest.fmri
 515                 if fmri.publisher and fmri.publisher not in allowed_pubs:
 516                         engine.error(_("package %s has a publisher set!") %
 517                             manifest.fmri,
 518                             msgid="%s%s.2" % (self.name, pkglint_id))
 519
 520         publisher_in_fmri.pkglint_desc = _(
 521                 "extra publisher set" )