pym/portage/sets/revdep.py

   1 # Copyright 2008 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 import os
   5 import re
   6 import time
   7 from portage.sets.libs import LibraryConsumerSet
   8 from portage.sets import get_boolean
   9 from portage.dbapi.vartree import dblink
  10 from portage.versions import catsplit
  11
  12 __all__ = ["MissingLibraryConsumerSet"]
  13
  14 class MissingLibraryConsumerSet(LibraryConsumerSet):
  15
  16         """
  17         This class is the set of packages to emerge due to missing libraries.
  18
  19         This class scans binaries for missing and broken shared library dependencies
  20         and fixes them by emerging the packages containing the broken binaries.
  21
  22         The user may also emerge packages containing consumers of specified
  23         libraries by passing the name or a python regular expression through the
  24         environment variable, LIBRARY.  Due to a limitation in passing flags to
  25         package sets through the portage cli, the user must set environment
  26         variables to modify the behaviour of this package set.  So if the
  27         environment variable LIBRARY is set, the behaviour of this set changes.
  28
  29         """
  30
  31         description = "The set of packages to emerge due to missing libraries."
  32         _operations = ["merge"]
  33
  34         def __init__(self, vardbapi, debug=False):
  35                 super(MissingLibraryConsumerSet, self).__init__(vardbapi, debug)
  36                 # FIXME Since we can't get command line arguments from the user, the
  37                 # soname can be passed through an environment variable for now.
  38                 self.libraryRegexp = os.getenv("LIBRARY")
  39                 self.root = self.dbapi.root
  40                 self.linkmap = self.dbapi.linkmap
  41
  42         def load(self):
  43                 # brokenDependencies: object -> set-of-unsatisfied-dependencies, where
  44                 # object is an installed binary/library and
  45                 # set-of-unsatisfied-dependencies are sonames or libraries required by
  46                 # the object but have no corresponding libraries to fulfill the
  47                 # dependency.
  48                 brokenDependencies = {}
  49                 atoms = set()
  50
  51                 # If the LIBRARY environment variable is set, the resulting package set
  52                 # will be packages containing consumers of the libraries matched by the
  53                 # variable.
  54                 if self.libraryRegexp:
  55                         atoms = self.findAtomsOfLibraryConsumers(self.libraryRegexp)
  56                         self._setAtoms(atoms)
  57                         if self.debug:
  58                                 print
  59                                 print "atoms to be emerged:"
  60                                 for x in sorted(atoms):
  61                                         print x
  62                         return
  63
  64                 # Get the list of broken dependencies from LinkageMap.
  65                 if self.debug:
  66                         timeStart = time.time()
  67                 brokenDependencies = self.linkmap.listBrokenDependencies()
  68                 if self.debug:
  69                         timeListBrokenBinaries = time.time() - timeStart
  70
  71                 # Add broken libtool libraries into the brokenDependencies dict.
  72                 if self.debug:
  73                         timeStart = time.time()
  74                 brokenDependencies.update(self.listBrokenLibtoolLibraries())
  75                 if self.debug:
  76                         timeLibtool = time.time() - timeStart
  77
  78                 # FIXME Too many atoms may be emerged because libraries in binary
  79                 # packages are not being handled properly eg openoffice, nvidia-drivers,
  80                 # sun-jdk.  Certain binaries are run in an environment where additional
  81                 # library paths are added via LD_LIBRARY_PATH.  Since these paths aren't
  82                 # registered in _obj_properties, they appear broken (and are if not run
  83                 # in the correct environment).  I have to determine if libraries and lib
  84                 # paths should be masked using /etc/revdep-rebuild/* as done in
  85                 # revdep-rebuild or if there is a better way to identify and deal with
  86                 # these problematic packages (or if something entirely different should
  87                 # be done).  For now directory and library masks are used.
  88
  89                 # Remove masked directories and libraries.
  90                 if self.debug:
  91                         timeStart = time.time()
  92                 if brokenDependencies:
  93                         brokenDependencies = self.removeMaskedDependencies(brokenDependencies)
  94                 if self.debug:
  95                         timeMask = time.time() - timeStart
  96
  97                 # Determine atoms to emerge based on broken objects in
  98                 # brokenDependencies.
  99                 if self.debug:
 100                         timeStart = time.time()
 101                 if brokenDependencies:
 102                         atoms = self.mapPathsToAtoms(set(brokenDependencies.keys()))
 103                 if self.debug:
 104                         timeAtoms = time.time() - timeStart
 105
 106                 # Debug output
 107                 if self.debug:
 108                         print
 109                         print len(brokenDependencies), "brokenDependencies:"
 110                         for x in sorted(brokenDependencies.keys()):
 111                                 print
 112                                 print x, "->"
 113                                 print '\t', brokenDependencies[x]
 114                         print
 115                         print "atoms to be emerged:"
 116                         for x in sorted(atoms):
 117                                 print x
 118                         print
 119                         print "listBrokenBinaries time:", timeListBrokenBinaries
 120                         print "Libtool time:", timeLibtool
 121                         print "Mask time:", timeMask
 122                         print "mapPathsToAtoms time:", timeAtoms
 123                         print
 124
 125                 self._setAtoms(atoms)
 126
 127         def removeMaskedDependencies(self, dependencies):
 128                 """
 129                 Remove all masked dependencies and return the updated mapping.
 130
 131                 @param dependencies: dependencies from which to removed masked
 132                         dependencies
 133                 @type dependencies: dict (example: {'/usr/bin/foo': set(['libfoo.so'])})
 134                 @rtype: dict
 135                 @return: shallow copy of dependencies with masked items removed
 136
 137                 """
 138                 rValue = dependencies.copy()
 139                 dirMask, libMask = self.getDependencyMasks()
 140
 141                 # Remove entries that are masked.
 142                 if dirMask or libMask:
 143                         if self.debug:
 144                                 print "The following are masked:"
 145                         for binary, libSet in rValue.items():
 146                                 for dir in dirMask:
 147                                         # Check if the broken binary lies within the masked directory or
 148                                         # its subdirectories.
 149                                         # TODO Perhaps we should allow regexps as masks.
 150                                         if os.path.commonprefix([dir, binary]) == dir:
 151                                                 del rValue[binary]
 152                                                 if self.debug:
 153                                                         print "dirMask:",binary
 154                                                 break
 155                                 # Check if all the required libraries are masked.
 156                                 if binary in rValue and libSet.issubset(set(libMask)):
 157                                         del rValue[binary]
 158                                         if self.debug:
 159                                                 print "libMask:", binary, libSet & set(libMask)
 160
 161                 if self.debug:
 162                         print
 163                         print "Directory mask:", dirMask
 164                         print
 165                         print "Library mask:", libMask
 166
 167                 return rValue
 168
 169         def getDependencyMasks(self):
 170                 """
 171                 Return all dependency masks as a tuple.
 172
 173                 @rtype: 2-tuple of sets of strings
 174                 @return: 2-tuple in which the first component is a set of directory
 175                         masks and the second component is a set of library masks
 176
 177                 """
 178                 dirMask = set()
 179                 libMask = set()
 180                 _dirMask_re = re.compile(r'SEARCH_DIRS_MASK\s*=\s*"([^"]*)"')
 181                 _libMask_re = re.compile(r'LD_LIBRARY_MASK\s*=\s*"([^"]*)"')
 182                 lines = []
 183
 184                 # Reads the contents of /etc/revdep-rebuild/*
 185                 libMaskDir = os.path.join(self.root, "etc", "revdep-rebuild")
 186                 if os.path.exists(libMaskDir):
 187                         for file in os.listdir(libMaskDir):
 188                                 try:
 189                                         f = open(os.path.join(libMaskDir, file), "r")
 190                                         try:
 191                                                 lines.extend(f.readlines())
 192                                         finally:
 193                                                 f.close()
 194                                 except IOError: # OSError?
 195                                         continue
 196                         # The following parses SEARCH_DIRS_MASK and LD_LIBRARY_MASK variables
 197                         # from /etc/revdep-rebuild/*
 198                         for line in lines:
 199                                 matchDir = _dirMask_re.match(line)
 200                                 matchLib = _libMask_re.match(line)
 201                                 if matchDir:
 202                                         dirMask.update(set(matchDir.group(1).split()))
 203                                 if matchLib:
 204                                         libMask.update(set(matchLib.group(1).split()))
 205
 206                 # These directories contain specially evaluated libraries.
 207                 # app-emulation/vmware-workstation-6.0.1.55017
 208                 dirMask.add('/opt/vmware/workstation/lib')
 209                 # app-emulation/vmware-server-console-1.0.6.91891
 210                 dirMask.add('/opt/vmware/server/console/lib')
 211                 # www-client/mozilla-firefox-2.0.0.15
 212                 dirMask.add('/usr/lib/mozilla-firefox/plugins')
 213                 dirMask.add('/usr/lib64/mozilla-firefox/plugins')
 214
 215                 return (dirMask, libMask)
 216
 217         def findAtomsOfLibraryConsumers(self, searchString):
 218                 """
 219                 Return atoms containing consumers of libraries matching the argument.
 220
 221                 @param searchString: a string used to search for libraries
 222                 @type searchString: string to be compiled as a regular expression
 223                         (example: 'libfoo.*')
 224                 @rtype: set of strings
 225                 @return: the returned set of atoms are valid to be used by package sets
 226
 227                 """
 228                 atoms = set()
 229                 consumers = set()
 230                 matchedLibraries = set()
 231                 libraryObjects = []
 232                 _librarySearch_re = re.compile(searchString)
 233
 234                 # Find libraries matching searchString.
 235                 libraryObjects = self.linkmap.listLibraryObjects()
 236                 for library in libraryObjects:
 237                         m = _librarySearch_re.search(library)
 238                         if m:
 239                                 matchedLibraries.add(library)
 240                                 consumers.update(self.linkmap.findConsumers(library))
 241
 242                 if self.debug:
 243                         print
 244                         print "Consumers of the following libraries will be emerged:"
 245                         for x in matchedLibraries:
 246                                 print x
 247
 248                 if consumers:
 249                         # The following prevents emerging the packages that own the matched
 250                         # libraries.  Note that this will prevent updating the packages owning
 251                         # the libraries if there are newer versions available in the installed
 252                         # slot.  See bug #30095
 253                         atoms = self.mapPathsToAtoms(consumers)
 254                         libraryOwners = self.mapPathsToAtoms(matchedLibraries)
 255                         atoms.difference_update(libraryOwners)
 256
 257                 return atoms
 258
 259         def listBrokenLibtoolLibraries(self):
 260                 """
 261                 Find broken libtool libraries and their missing dependencies.
 262
 263                 @rtype: dict (example: {'/lib/libfoo.la': set(['/lib/libbar.la'])})
 264                 @return: The return value is a library -> set-of-libraries mapping, where
 265                         library is a broken library and the set consists of dependencies
 266                         needed by library that do not exist on the filesystem.
 267
 268                 """
 269                 rValue = {}
 270                 lines = []
 271                 dependencies = []
 272                 _la_re = re.compile(r".*\.la$")
 273                 _dependency_libs_re = re.compile(r"^dependency_libs\s*=\s*'(.*)'")
 274
 275                 # Loop over the contents of all packages.
 276                 for cpv in self.dbapi.cpv_all():
 277                         mysplit = catsplit(cpv)
 278                         link = dblink(mysplit[0], mysplit[1], myroot=self.dbapi.root, \
 279                                         mysettings=self.dbapi.settings, treetype='vartree', \
 280                                         vartree=self.dbapi.vartree)
 281                         for file in link.getcontents():
 282                                 # Check if the file end with '.la'.
 283                                 matchLib = _la_re.match(file)
 284                                 if matchLib:
 285                                         # Read the lines from the library.
 286                                         lines = []
 287                                         try:
 288                                                 f = open(file, "r")
 289                                                 try:
 290                                                         lines.extend(f.readlines())
 291                                                 finally:
 292                                                         f.close()
 293                                         except IOError:
 294                                                 continue
 295                                         # Find the line listing the dependencies.
 296                                         for line in lines:
 297                                                 matchLine = _dependency_libs_re.match(line)
 298                                                 if matchLine:
 299                                                         dependencies = matchLine.group(1).split()
 300                                                         # For each dependency that is a pathname (begins with
 301                                                         # os.sep), check that it exists on the filesystem.  If it
 302                                                         # does not exist, then add the library and the missing
 303                                                         # dependency to rValue.
 304                                                         for dependency in dependencies:
 305                                                                 if dependency[0:1] == os.sep and \
 306                                                                                 not os.path.isfile(dependency):
 307                                                                         rValue.setdefault(file, set()).add(dependency)
 308
 309                 return rValue
 310
 311         def singleBuilder(self, options, settings, trees):
 312                 debug = get_boolean(options, "debug", False)
 313                 return MissingLibraryConsumerSet(trees["vartree"].dbapi, debug)
 314         singleBuilder = classmethod(singleBuilder)