| blob | e9c4351235c745fe38e7eb5cdaae741e1b7630cd |
1 --- libs.py.2.2_rc8 2008-08-20 20:49:17.000000000 -0500
2 +++ pym/portage/sets/libs.py 2008-08-20 22:53:19.000000000 -0500
3 @@ -2,10 +2,18 @@
4 # Distributed under the terms of the GNU General Public License v2
5 # $Id: libs.py 10759 2008-06-22 04:04:50Z zmedico $
7 +import os
8 +import re
9 +import time
10 +from portage.dbapi.vartree import dblink
11 +from portage.versions import catsplit
12 from portage.sets.base import PackageSet
13 from portage.sets import get_boolean
14 from portage.versions import catpkgsplit
16 +__all__ = ["LibraryConsumerSet", "PreservedLibraryConsumerSet",
17 + "MissingLibraryConsumerSet"]
18 +
19 class LibraryConsumerSet(PackageSet):
20 _operations = ["merge", "unmerge"]
22 @@ -45,3 +53,338 @@
23 debug = get_boolean(options, "debug", False)
24 return PreservedLibraryConsumerSet(trees["vartree"].dbapi, debug)
25 singleBuilder = classmethod(singleBuilder)
26 +
27 +
28 +class MissingLibraryConsumerSet(LibraryConsumerSet):
29 +
30 + """
31 + This class is the set of packages to emerge due to missing libraries.
32 +
33 + This class scans binaries for missing and broken shared library dependencies
34 + and fixes them by emerging the packages containing the broken binaries.
35 +
36 + The user may also emerge packages containing consumers of specified
37 + libraries by passing the name or a python regular expression through the
38 + environment variable, LIBRARY. Due to a limitation in passing flags to
39 + package sets through the portage cli, the user must set environment
40 + variables to modify the behaviour of this package set. So if the
41 + environment variable LIBRARY is set, the behaviour of this set changes.
42 +
43 + """
44 +
45 + description = "The set of packages to emerge due to missing libraries."
46 + _operations = ["merge"]
47 +
48 + def __init__(self, vardbapi, debug=False):
49 + super(MissingLibraryConsumerSet, self).__init__(vardbapi, debug)
50 + # FIXME Since we can't get command line arguments from the user, the
51 + # soname can be passed through an environment variable for now.
52 + self.libraryRegexp = os.getenv("LIBRARY")
53 + self.root = self.dbapi.root
54 + self.linkmap = self.dbapi.linkmap
55 +
56 + def load(self):
57 + # brokenDependencies: object -> set-of-unsatisfied-dependencies, where
58 + # object is an installed binary/library and
59 + # set-of-unsatisfied-dependencies are sonames or libraries required by
60 + # the object but have no corresponding libraries to fulfill the
61 + # dependency.
62 + brokenDependencies = {}
63 + atoms = set()
64 +
65 + # If the LIBRARY environment variable is set, the resulting package set
66 + # will be packages containing consumers of the libraries matched by the
67 + # variable.
68 + if self.libraryRegexp:
69 + atoms = self.findAtomsOfLibraryConsumers(self.libraryRegexp)
70 + self._setAtoms(atoms)
71 + if self.debug:
72 + print
73 + print "atoms to be emerged:"
74 + for x in sorted(atoms):
75 + print x
76 + return
77 +
78 + # Rebuild LinkageMap.
79 + if self.debug:
80 + timeStart = time.time()
81 + self.linkmap.rebuild()
82 + if self.debug:
83 + timeRebuild = time.time() - timeStart
84 +
85 + # Get the list of broken dependencies from LinkageMap.
86 + if self.debug:
87 + timeStart = time.time()
88 + brokenDependencies = self.linkmap.listBrokenBinaries(self.debug)
89 + if self.debug:
90 + timeListBrokenBinaries = time.time() - timeStart
91 +
92 + # Add broken libtool libraries into the brokenDependencies dict.
93 + if self.debug:
94 + timeStart = time.time()
95 + brokenDependencies.update(self.listBrokenLibtoolLibraries())
96 + if self.debug:
97 + timeLibtool = time.time() - timeStart
98 +
99 + # FIXME Too many atoms may be emerged because libraries in binary
100 + # packages are not being handled properly eg openoffice, nvidia-drivers,
101 + # sun-jdk. Certain binaries are run in an environment where additional
102 + # library paths are added via LD_LIBRARY_PATH. Since these paths aren't
103 + # registered in _obj_properties, they appear broken (and are if not run
104 + # in the correct environment). I have to determine if libraries and lib
105 + # paths should be masked using /etc/revdep-rebuild/* as done in
106 + # revdep-rebuild or if there is a better way to identify and deal with
107 + # these problematic packages (or if something entirely different should
108 + # be done). For now directory and library masks are used.
109 +
110 + # Remove masked directories and libraries.
111 + if self.debug:
112 + timeStart = time.time()
113 + if brokenDependencies:
114 + brokenDependencies = self.removeMaskedDependencies(brokenDependencies)
115 + if self.debug:
116 + timeMask = time.time() - timeStart
117 +
118 + # Determine atoms to emerge based on broken objects in
119 + # brokenDependencies.
120 + if self.debug:
121 + timeStart = time.time()
122 + if brokenDependencies:
123 + atoms = self.mapPathsToAtoms(set(brokenDependencies.keys()))
124 + if self.debug:
125 + timeAtoms = time.time() - timeStart
126 +
127 + # Debug output
128 + if self.debug:
129 + print
130 + print len(brokenDependencies), "brokenDependencies:"
131 + for x in sorted(brokenDependencies.keys()):
132 + print
133 + print x, "->"
134 + print '\t', brokenDependencies[x]
135 + print
136 + print "atoms to be emerged:"
137 + for x in sorted(atoms):
138 + print x
139 + print
140 + print "Rebuild time:", timeRebuild
141 + print "Broken binaries time:", timeListBrokenBinaries
142 + print "Broken libtool time:", timeLibtool
143 + print "Remove mask time:", timeMask
144 + print "mapPathsToAtoms time:", timeAtoms
145 + print
146 +
147 + self._setAtoms(atoms)
148 +
149 + def removeMaskedDependencies(self, dependencies):
150 + """
151 + Remove all masked dependencies and return the updated mapping.
152 +
153 + @param dependencies: dependencies from which to removed masked
154 + dependencies
155 + @type dependencies: dict (example: {'/usr/bin/foo': set(['libfoo.so'])})
156 + @rtype: dict
157 + @return: shallow copy of dependencies with masked items removed
158 +
159 + """
160 + rValue = dependencies.copy()
161 + dirMask, libMask = self.getDependencyMasks()
162 +
163 + # Remove entries that are masked.
164 + if dirMask or libMask:
165 + if self.debug:
166 + print "The following are masked:"
167 + for binary, libSet in rValue.items():
168 + for directory in dirMask:
169 + # Check if the broken binary lies within the masked directory or
170 + # its subdirectories.
171 + # XXX Perhaps we should allow regexps as masks.
172 + if binary.startswith(directory):
173 + del rValue[binary]
174 + if self.debug:
175 + print "dirMask:",binary
176 + break
177 + # Check if all the required libraries are masked.
178 + if binary in rValue and libSet.issubset(libMask):
179 + del rValue[binary]
180 + if self.debug:
181 + print "libMask:", binary, libSet & libMask
182 +
183 + if self.debug:
184 + print
185 + print "Directory mask:", dirMask
186 + print
187 + print "Library mask:", libMask
188 +
189 + return rValue
190 +
191 + def getDependencyMasks(self):
192 + """
193 + Return all dependency masks as a tuple.
194 +
195 + @rtype: 2-tuple of sets of strings
196 + @return: 2-tuple in which the first component is a set of directory
197 + masks and the second component is a set of library masks
198 +
199 + """
200 + dirMask = set()
201 + libMask = set()
202 + _dirMask_re = re.compile(r'SEARCH_DIRS_MASK\s*=\s*"([^"]*)"')
203 + _libMask_re = re.compile(r'LD_LIBRARY_MASK\s*=\s*"([^"]*)"')
204 + lines = []
205 +
206 + # Reads the contents of /etc/revdep-rebuild/*
207 + libMaskDir = os.path.join(self.root, "etc", "revdep-rebuild")
208 + if os.path.exists(libMaskDir):
209 + for file in os.listdir(libMaskDir):
210 + try:
211 + f = open(os.path.join(libMaskDir, file), "r")
212 + try:
213 + lines.extend(f.readlines())
214 + finally:
215 + f.close()
216 + except IOError: # OSError?
217 + continue
218 + # The following parses SEARCH_DIRS_MASK and LD_LIBRARY_MASK variables
219 + # from /etc/revdep-rebuild/*
220 + for line in lines:
221 + matchDir = _dirMask_re.match(line)
222 + matchLib = _libMask_re.match(line)
223 + if matchDir:
224 + dirMask.update(set(matchDir.group(1).split()))
225 + if matchLib:
226 + libMask.update(set(matchLib.group(1).split()))
227 +
228 + # These directories contain specially evaluated libraries.
229 + # app-emulation/vmware-workstation-6.0.1.55017
230 + dirMask.add('/opt/vmware/workstation/lib')
231 + # app-emulation/vmware-server-console-1.0.6.91891
232 + dirMask.add('/opt/vmware/server/console/lib')
233 + # www-client/mozilla-firefox-2.0.0.15
234 + dirMask.add('/usr/lib/mozilla-firefox/plugins')
235 + dirMask.add('/usr/lib64/mozilla-firefox/plugins')
236 + # app-office/openoffice-2.4.1
237 + dirMask.add('/opt/OpenOffice')
238 + dirMask.add('/usr/lib/openoffice')
239 + # dev-libs/libmix-2.05 libmix.so is missing soname entry
240 + libMask.add('libmix.so')
241 + # app-accessibility/speech-tools-1.2.96_beta missing sonames
242 + libMask.add('libestools.so')
243 + libMask.add('libestbase.so')
244 + libMask.add('libeststring.so')
245 + # app-emulation/emul-linux-x86-soundlibs-20080418
246 + dirMask.add('/usr/kde/3.5/lib32')
247 +
248 + return (dirMask, libMask)
249 +
250 + def listBrokenLibtoolLibraries(self):
251 + """
252 + Find broken libtool libraries and their missing dependencies.
253 +
254 + Consider adding new entry into /var/db/pkg to catalog libtool libraries
255 + (similar in nature to NEEDED.ELF.2). Currently, the contents of all
256 + packages are searched in order to find libtool libraries, so having them
257 + available in the vdb_path will speed things up.
258 +
259 + @rtype: dict (example: {'/lib/libfoo.la': set(['/lib/libbar.la'])})
260 + @return: The return value is a library -> set-of-libraries mapping, where
261 + library is a broken library and the set consists of dependencies
262 + needed by library that do not exist on the filesystem.
263 +
264 + """
265 + rValue = {}
266 + lines = []
267 + dependencies = []
268 + _la_re = re.compile(r".*\.la$")
269 + _dependency_libs_re = re.compile(r"^dependency_libs\s*=\s*'(.*)'")
270 +
271 + # Loop over the contents of all packages.
272 + for cpv in self.dbapi.cpv_all():
273 + mysplit = catsplit(cpv)
274 + link = dblink(mysplit[0], mysplit[1], myroot=self.dbapi.root, \
275 + mysettings=self.dbapi.settings, treetype='vartree', \
276 + vartree=self.dbapi.vartree)
277 + for file in link.getcontents():
278 + # Check if the file ends with '.la'.
279 + matchLib = _la_re.match(file)
280 + if matchLib:
281 + # Read the lines from the library.
282 + lines = []
283 + try:
284 + f = open(file, "r")
285 + try:
286 + lines.extend(f.readlines())
287 + finally:
288 + f.close()
289 + except IOError:
290 + continue
291 + # Find the line listing the dependencies.
292 + for line in lines:
293 + matchLine = _dependency_libs_re.match(line)
294 + if matchLine:
295 + dependencies = matchLine.group(1).split()
296 + # For each dependency that is a pathname (begins with
297 + # os.sep), check that it exists on the filesystem. If it
298 + # does not exist, then add the library and the missing
299 + # dependency to rValue.
300 + for dependency in dependencies:
301 + if dependency[0] == os.sep and \
302 + not os.path.isfile(dependency):
303 + rValue.setdefault(file, set()).add(dependency)
304 +
305 + return rValue
306 +
307 + def findAtomsOfLibraryConsumers(self, searchString):
308 + """
309 + Return atoms containing consumers of libraries matching the argument.
310 +
311 + The libraries returned by LinkageMap.listLibraryObjects() are matched
312 + against the searchString regular expression. Consequently, only files
313 + cataloged in LinkageMap will be considered. Since symlinks aren't
314 + entered into NEEDED.ELF.2 files, LinkageMap doesn't catalog them. So if
315 + /usr/lib were a symlink to /usr/lib64 and the regular expression were
316 + /usr/lib/libfoo*, nothing would be matched (libfoo would work however).
317 + This can be fixed by adding symlink entries into LinkageMap, searching
318 + CONTENTS files for symlinks, or utilizing the find utility.
319 +
320 + @param searchString: a string used to search for libraries
321 + @type searchString: string to be compiled as a regular expression
322 + (example: 'libfoo.*')
323 + @rtype: set of strings
324 + @return: the returned set of atoms are valid to be used by package sets
325 +
326 + """
327 + atoms = set()
328 + consumers = set()
329 + matchedLibraries = set()
330 + libraryObjects = self.linkmap.listLibraryObjects()
331 + _librarySearch_re = re.compile(searchString)
332 +
333 + # Find libraries matching searchString.
334 + for library in libraryObjects:
335 + m = _librarySearch_re.search(library)
336 + if m:
337 + matchedLibraries.add(library)
338 + consumers.update(self.linkmap.findConsumers(library))
339 +
340 + if self.debug:
341 + print
342 + print "Consumers of the following libraries will be emerged:"
343 + for x in matchedLibraries:
344 + print x
345 +
346 + if consumers:
347 + # The following prevents emerging the packages that own the matched
348 + # libraries. Note that this will prevent updating the packages owning
349 + # the libraries if there are newer versions available in the installed
350 + # slot. See bug #30095
351 + atoms = self.mapPathsToAtoms(consumers)
352 + libraryOwners = self.mapPathsToAtoms(matchedLibraries)
353 + atoms.difference_update(libraryOwners)
354 +
355 + return atoms
356 +
357 + def singleBuilder(self, options, settings, trees):
358 + debug = get_boolean(options, "debug", False)
359 + return MissingLibraryConsumerSet(trees["vartree"].dbapi, debug)
360 + singleBuilder = classmethod(singleBuilder)