Lib/plat-mac/pimp.py

   1 """Package Install Manager for Python.
   2
   3 This is currently a MacOSX-only strawman implementation.
   4 Despite other rumours the name stands for "Packman IMPlementation".
   5
   6 Tools to allow easy installation of packages. The idea is that there is
   7 an online XML database per (platform, python-version) containing packages
   8 known to work with that combination. This module contains tools for getting
   9 and parsing the database, testing whether packages are installed, computing
  10 dependencies and installing packages.
  11
  12 There is a minimal main program that works as a command line tool, but the
  13 intention is that the end user will use this through a GUI.
  14 """
  15 import sys
  16 import os
  17 import popen2
  18 import urllib
  19 import urllib2
  20 import urlparse
  21 import plistlib
  22 import distutils.util
  23 import distutils.sysconfig
  24 import hashlib
  25 import tarfile
  26 import tempfile
  27 import shutil
  28 import time
  29
  30 __all__ = ["PimpPreferences", "PimpDatabase", "PimpPackage", "main",
  31     "getDefaultDatabase", "PIMP_VERSION", "main"]
  32
  33 _scriptExc_NotInstalled = "pimp._scriptExc_NotInstalled"
  34 _scriptExc_OldInstalled = "pimp._scriptExc_OldInstalled"
  35 _scriptExc_BadInstalled = "pimp._scriptExc_BadInstalled"
  36
  37 NO_EXECUTE=0
  38
  39 PIMP_VERSION="0.5"
  40
  41 # Flavors:
  42 # source: setup-based package
  43 # binary: tar (or other) archive created with setup.py bdist.
  44 # installer: something that can be opened
  45 DEFAULT_FLAVORORDER=['source', 'binary', 'installer']
  46 DEFAULT_DOWNLOADDIR='/tmp'
  47 DEFAULT_BUILDDIR='/tmp'
  48 DEFAULT_INSTALLDIR=distutils.sysconfig.get_python_lib()
  49 DEFAULT_PIMPDATABASE_FMT="http://www.python.org/packman/version-%s/%s-%s-%s-%s-%s.plist"
  50
  51 def getDefaultDatabase(experimental=False):
  52     if experimental:
  53         status = "exp"
  54     else:
  55         status = "prod"
  56
  57     major, minor, micro, state, extra = sys.version_info
  58     pyvers = '%d.%d' % (major, minor)
  59     if micro == 0 and state != 'final':
  60         pyvers = pyvers + '%s%d' % (state, extra)
  61
  62     longplatform = distutils.util.get_platform()
  63     osname, release, machine = longplatform.split('-')
  64     # For some platforms we may want to differentiate between
  65     # installation types
  66     if osname == 'darwin':
  67         if sys.prefix.startswith('/System/Library/Frameworks/Python.framework'):
  68             osname = 'darwin_apple'
  69         elif sys.prefix.startswith('/Library/Frameworks/Python.framework'):
  70             osname = 'darwin_macpython'
  71         # Otherwise we don't know...
  72     # Now we try various URLs by playing with the release string.
  73     # We remove numbers off the end until we find a match.
  74     rel = release
  75     while True:
  76         url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, rel, machine)
  77         try:
  78             urllib2.urlopen(url)
  79         except urllib2.HTTPError, arg:
  80             pass
  81         else:
  82             break
  83         if not rel:
  84             # We're out of version numbers to try. Use the
  85             # full release number, this will give a reasonable
  86             # error message later
  87             url = DEFAULT_PIMPDATABASE_FMT % (PIMP_VERSION, status, pyvers, osname, release, machine)
  88             break
  89         idx = rel.rfind('.')
  90         if idx < 0:
  91             rel = ''
  92         else:
  93             rel = rel[:idx]
  94     return url
  95
  96 def _cmd(output, dir, *cmditems):
  97     """Internal routine to run a shell command in a given directory."""
  98
  99     cmd = ("cd \"%s\"; " % dir) + " ".join(cmditems)
 100     if output:
 101         output.write("+ %s\n" % cmd)
 102     if NO_EXECUTE:
 103         return 0
 104     child = popen2.Popen4(cmd)
 105     child.tochild.close()
 106     while 1:
 107         line = child.fromchild.readline()
 108         if not line:
 109             break
 110         if output:
 111             output.write(line)
 112     return child.wait()
 113
 114 class PimpDownloader:
 115     """Abstract base class - Downloader for archives"""
 116
 117     def __init__(self, argument,
 118             dir="",
 119             watcher=None):
 120         self.argument = argument
 121         self._dir = dir
 122         self._watcher = watcher
 123
 124     def download(self, url, filename, output=None):
 125         return None
 126
 127     def update(self, str):
 128         if self._watcher:
 129             return self._watcher.update(str)
 130         return True
 131
 132 class PimpCurlDownloader(PimpDownloader):
 133
 134     def download(self, url, filename, output=None):
 135         self.update("Downloading %s..." % url)
 136         exitstatus = _cmd(output, self._dir,
 137                     "curl",
 138                     "--output", filename,
 139                     url)
 140         self.update("Downloading %s: finished" % url)
 141         return (not exitstatus)
 142
 143 class PimpUrllibDownloader(PimpDownloader):
 144
 145     def download(self, url, filename, output=None):
 146         output = open(filename, 'wb')
 147         self.update("Downloading %s: opening connection" % url)
 148         keepgoing = True
 149         download = urllib2.urlopen(url)
 150         if download.headers.has_key("content-length"):
 151             length = long(download.headers['content-length'])
 152         else:
 153             length = -1
 154
 155         data = download.read(4096) #read 4K at a time
 156         dlsize = 0
 157         lasttime = 0
 158         while keepgoing:
 159             dlsize = dlsize + len(data)
 160             if len(data) == 0:
 161                 #this is our exit condition
 162                 break
 163             output.write(data)
 164             if int(time.time()) != lasttime:
 165                 # Update at most once per second
 166                 lasttime = int(time.time())
 167                 if length == -1:
 168                     keepgoing = self.update("Downloading %s: %d bytes..." % (url, dlsize))
 169                 else:
 170                     keepgoing = self.update("Downloading %s: %d%% (%d bytes)..." % (url, int(100.0*dlsize/length), dlsize))
 171             data = download.read(4096)
 172         if keepgoing:
 173             self.update("Downloading %s: finished" % url)
 174         return keepgoing
 175
 176 class PimpUnpacker:
 177     """Abstract base class - Unpacker for archives"""
 178
 179     _can_rename = False
 180
 181     def __init__(self, argument,
 182             dir="",
 183             renames=[],
 184             watcher=None):
 185         self.argument = argument
 186         if renames and not self._can_rename:
 187             raise RuntimeError, "This unpacker cannot rename files"
 188         self._dir = dir
 189         self._renames = renames
 190         self._watcher = watcher
 191
 192     def unpack(self, archive, output=None, package=None):
 193         return None
 194
 195     def update(self, str):
 196         if self._watcher:
 197             return self._watcher.update(str)
 198         return True
 199
 200 class PimpCommandUnpacker(PimpUnpacker):
 201     """Unpack archives by calling a Unix utility"""
 202
 203     _can_rename = False
 204
 205     def unpack(self, archive, output=None, package=None):
 206         cmd = self.argument % archive
 207         if _cmd(output, self._dir, cmd):
 208             return "unpack command failed"
 209
 210 class PimpTarUnpacker(PimpUnpacker):
 211     """Unpack tarfiles using the builtin tarfile module"""
 212
 213     _can_rename = True
 214
 215     def unpack(self, archive, output=None, package=None):
 216         tf = tarfile.open(archive, "r")
 217         members = tf.getmembers()
 218         skip = []
 219         if self._renames:
 220             for member in members:
 221                 for oldprefix, newprefix in self._renames:
 222                     if oldprefix[:len(self._dir)] == self._dir:
 223                         oldprefix2 = oldprefix[len(self._dir):]
 224                     else:
 225                         oldprefix2 = None
 226                     if member.name[:len(oldprefix)] == oldprefix:
 227                         if newprefix is None:
 228                             skip.append(member)
 229                             #print 'SKIP', member.name
 230                         else:
 231                             member.name = newprefix + member.name[len(oldprefix):]
 232                             print '    ', member.name
 233                         break
 234                     elif oldprefix2 and member.name[:len(oldprefix2)] == oldprefix2:
 235                         if newprefix is None:
 236                             skip.append(member)
 237                             #print 'SKIP', member.name
 238                         else:
 239                             member.name = newprefix + member.name[len(oldprefix2):]
 240                             #print '    ', member.name
 241                         break
 242                 else:
 243                     skip.append(member)
 244                     #print '????', member.name
 245         for member in members:
 246             if member in skip:
 247                 self.update("Skipping %s" % member.name)
 248                 continue
 249             self.update("Extracting %s" % member.name)
 250             tf.extract(member, self._dir)
 251         if skip:
 252             names = [member.name for member in skip if member.name[-1] != '/']
 253             if package:
 254                 names = package.filterExpectedSkips(names)
 255             if names:
 256                 return "Not all files were unpacked: %s" % " ".join(names)
 257
 258 ARCHIVE_FORMATS = [
 259     (".tar.Z", PimpTarUnpacker, None),
 260     (".taz", PimpTarUnpacker, None),
 261     (".tar.gz", PimpTarUnpacker, None),
 262     (".tgz", PimpTarUnpacker, None),
 263     (".tar.bz", PimpTarUnpacker, None),
 264     (".zip", PimpCommandUnpacker, "unzip \"%s\""),
 265 ]
 266
 267 class PimpPreferences:
 268     """Container for per-user preferences, such as the database to use
 269     and where to install packages."""
 270
 271     def __init__(self,
 272             flavorOrder=None,
 273             downloadDir=None,
 274             buildDir=None,
 275             installDir=None,
 276             pimpDatabase=None):
 277         if not flavorOrder:
 278             flavorOrder = DEFAULT_FLAVORORDER
 279         if not downloadDir:
 280             downloadDir = DEFAULT_DOWNLOADDIR
 281         if not buildDir:
 282             buildDir = DEFAULT_BUILDDIR
 283         if not pimpDatabase:
 284             pimpDatabase = getDefaultDatabase()
 285         self.setInstallDir(installDir)
 286         self.flavorOrder = flavorOrder
 287         self.downloadDir = downloadDir
 288         self.buildDir = buildDir
 289         self.pimpDatabase = pimpDatabase
 290         self.watcher = None
 291
 292     def setWatcher(self, watcher):
 293         self.watcher = watcher
 294
 295     def setInstallDir(self, installDir=None):
 296         if installDir:
 297             # Installing to non-standard location.
 298             self.installLocations = [
 299                 ('--install-lib', installDir),
 300                 ('--install-headers', None),
 301                 ('--install-scripts', None),
 302                 ('--install-data', None)]
 303         else:
 304             installDir = DEFAULT_INSTALLDIR
 305             self.installLocations = []
 306         self.installDir = installDir
 307
 308     def isUserInstall(self):
 309         return self.installDir != DEFAULT_INSTALLDIR
 310
 311     def check(self):
 312         """Check that the preferences make sense: directories exist and are
 313         writable, the install directory is on sys.path, etc."""
 314
 315         rv = ""
 316         RWX_OK = os.R_OK|os.W_OK|os.X_OK
 317         if not os.path.exists(self.downloadDir):
 318             rv += "Warning: Download directory \"%s\" does not exist\n" % self.downloadDir
 319         elif not os.access(self.downloadDir, RWX_OK):
 320             rv += "Warning: Download directory \"%s\" is not writable or not readable\n" % self.downloadDir
 321         if not os.path.exists(self.buildDir):
 322             rv += "Warning: Build directory \"%s\" does not exist\n" % self.buildDir
 323         elif not os.access(self.buildDir, RWX_OK):
 324             rv += "Warning: Build directory \"%s\" is not writable or not readable\n" % self.buildDir
 325         if not os.path.exists(self.installDir):
 326             rv += "Warning: Install directory \"%s\" does not exist\n" % self.installDir
 327         elif not os.access(self.installDir, RWX_OK):
 328             rv += "Warning: Install directory \"%s\" is not writable or not readable\n" % self.installDir
 329         else:
 330             installDir = os.path.realpath(self.installDir)
 331             for p in sys.path:
 332                 try:
 333                     realpath = os.path.realpath(p)
 334                 except:
 335                     pass
 336                 if installDir == realpath:
 337                     break
 338             else:
 339                 rv += "Warning: Install directory \"%s\" is not on sys.path\n" % self.installDir
 340         return rv
 341
 342     def compareFlavors(self, left, right):
 343         """Compare two flavor strings. This is part of your preferences
 344         because whether the user prefers installing from source or binary is."""
 345         if left in self.flavorOrder:
 346             if right in self.flavorOrder:
 347                 return cmp(self.flavorOrder.index(left), self.flavorOrder.index(right))
 348             return -1
 349         if right in self.flavorOrder:
 350             return 1
 351         return cmp(left, right)
 352
 353 class PimpDatabase:
 354     """Class representing a pimp database. It can actually contain
 355     information from multiple databases through inclusion, but the
 356     toplevel database is considered the master, as its maintainer is
 357     "responsible" for the contents."""
 358
 359     def __init__(self, prefs):
 360         self._packages = []
 361         self.preferences = prefs
 362         self._url = ""
 363         self._urllist = []
 364         self._version = ""
 365         self._maintainer = ""
 366         self._description = ""
 367
 368     # Accessor functions
 369     def url(self): return self._url
 370     def version(self): return self._version
 371     def maintainer(self): return self._maintainer
 372     def description(self): return self._description
 373
 374     def close(self):
 375         """Clean up"""
 376         self._packages = []
 377         self.preferences = None
 378
 379     def appendURL(self, url, included=0):
 380         """Append packages from the database with the given URL.
 381         Only the first database should specify included=0, so the
 382         global information (maintainer, description) get stored."""
 383
 384         if url in self._urllist:
 385             return
 386         self._urllist.append(url)
 387         fp = urllib2.urlopen(url).fp
 388         plistdata = plistlib.Plist.fromFile(fp)
 389         # Test here for Pimp version, etc
 390         if included:
 391             version = plistdata.get('Version')
 392             if version and version > self._version:
 393                 sys.stderr.write("Warning: included database %s is for pimp version %s\n" %
 394                     (url, version))
 395         else:
 396             self._version = plistdata.get('Version')
 397             if not self._version:
 398                 sys.stderr.write("Warning: database has no Version information\n")
 399             elif self._version > PIMP_VERSION:
 400                 sys.stderr.write("Warning: database version %s newer than pimp version %s\n"
 401                     % (self._version, PIMP_VERSION))
 402             self._maintainer = plistdata.get('Maintainer', '')
 403             self._description = plistdata.get('Description', '').strip()
 404             self._url = url
 405         self._appendPackages(plistdata['Packages'], url)
 406         others = plistdata.get('Include', [])
 407         for o in others:
 408             o = urllib.basejoin(url, o)
 409             self.appendURL(o, included=1)
 410
 411     def _appendPackages(self, packages, url):
 412         """Given a list of dictionaries containing package
 413         descriptions create the PimpPackage objects and append them
 414         to our internal storage."""
 415
 416         for p in packages:
 417             p = dict(p)
 418             if p.has_key('Download-URL'):
 419                 p['Download-URL'] = urllib.basejoin(url, p['Download-URL'])
 420             flavor = p.get('Flavor')
 421             if flavor == 'source':
 422                 pkg = PimpPackage_source(self, p)
 423             elif flavor == 'binary':
 424                 pkg = PimpPackage_binary(self, p)
 425             elif flavor == 'installer':
 426                 pkg = PimpPackage_installer(self, p)
 427             elif flavor == 'hidden':
 428                 pkg = PimpPackage_installer(self, p)
 429             else:
 430                 pkg = PimpPackage(self, dict(p))
 431             self._packages.append(pkg)
 432
 433     def list(self):
 434         """Return a list of all PimpPackage objects in the database."""
 435
 436         return self._packages
 437
 438     def listnames(self):
 439         """Return a list of names of all packages in the database."""
 440
 441         rv = []
 442         for pkg in self._packages:
 443             rv.append(pkg.fullname())
 444         rv.sort()
 445         return rv
 446
 447     def dump(self, pathOrFile):
 448         """Dump the contents of the database to an XML .plist file.
 449
 450         The file can be passed as either a file object or a pathname.
 451         All data, including included databases, is dumped."""
 452
 453         packages = []
 454         for pkg in self._packages:
 455             packages.append(pkg.dump())
 456         plistdata = {
 457             'Version': self._version,
 458             'Maintainer': self._maintainer,
 459             'Description': self._description,
 460             'Packages': packages
 461             }
 462         plist = plistlib.Plist(**plistdata)
 463         plist.write(pathOrFile)
 464
 465     def find(self, ident):
 466         """Find a package. The package can be specified by name
 467         or as a dictionary with name, version and flavor entries.
 468
 469         Only name is obligatory. If there are multiple matches the
 470         best one (higher version number, flavors ordered according to
 471         users' preference) is returned."""
 472
 473         if type(ident) == str:
 474             # Remove ( and ) for pseudo-packages
 475             if ident[0] == '(' and ident[-1] == ')':
 476                 ident = ident[1:-1]
 477             # Split into name-version-flavor
 478             fields = ident.split('-')
 479             if len(fields) < 1 or len(fields) > 3:
 480                 return None
 481             name = fields[0]
 482             if len(fields) > 1:
 483                 version = fields[1]
 484             else:
 485                 version = None
 486             if len(fields) > 2:
 487                 flavor = fields[2]
 488             else:
 489                 flavor = None
 490         else:
 491             name = ident['Name']
 492             version = ident.get('Version')
 493             flavor = ident.get('Flavor')
 494         found = None
 495         for p in self._packages:
 496             if name == p.name() and \
 497                     (not version or version == p.version()) and \
 498                     (not flavor or flavor == p.flavor()):
 499                 if not found or found < p:
 500                     found = p
 501         return found
 502
 503 ALLOWED_KEYS = [
 504     "Name",
 505     "Version",
 506     "Flavor",
 507     "Description",
 508     "Home-page",
 509     "Download-URL",
 510     "Install-test",
 511     "Install-command",
 512     "Pre-install-command",
 513     "Post-install-command",
 514     "Prerequisites",
 515     "MD5Sum",
 516     "User-install-skips",
 517     "Systemwide-only",
 518 ]
 519
 520 class PimpPackage:
 521     """Class representing a single package."""
 522
 523     def __init__(self, db, plistdata):
 524         self._db = db
 525         name = plistdata["Name"]
 526         for k in plistdata.keys():
 527             if not k in ALLOWED_KEYS:
 528                 sys.stderr.write("Warning: %s: unknown key %s\n" % (name, k))
 529         self._dict = plistdata
 530
 531     def __getitem__(self, key):
 532         return self._dict[key]
 533
 534     def name(self): return self._dict['Name']
 535     def version(self): return self._dict.get('Version')
 536     def flavor(self): return self._dict.get('Flavor')
 537     def description(self): return self._dict['Description'].strip()
 538     def shortdescription(self): return self.description().splitlines()[0]
 539     def homepage(self): return self._dict.get('Home-page')
 540     def downloadURL(self): return self._dict.get('Download-URL')
 541     def systemwideOnly(self): return self._dict.get('Systemwide-only')
 542
 543     def fullname(self):
 544         """Return the full name "name-version-flavor" of a package.
 545
 546         If the package is a pseudo-package, something that cannot be
 547         installed through pimp, return the name in (parentheses)."""
 548
 549         rv = self._dict['Name']
 550         if self._dict.has_key('Version'):
 551             rv = rv + '-%s' % self._dict['Version']
 552         if self._dict.has_key('Flavor'):
 553             rv = rv + '-%s' % self._dict['Flavor']
 554         if self._dict.get('Flavor') == 'hidden':
 555             # Pseudo-package, show in parentheses
 556             rv = '(%s)' % rv
 557         return rv
 558
 559     def dump(self):
 560         """Return a dict object containing the information on the package."""
 561         return self._dict
 562
 563     def __cmp__(self, other):
 564         """Compare two packages, where the "better" package sorts lower."""
 565
 566         if not isinstance(other, PimpPackage):
 567             return cmp(id(self), id(other))
 568         if self.name() != other.name():
 569             return cmp(self.name(), other.name())
 570         if self.version() != other.version():
 571             return -cmp(self.version(), other.version())
 572         return self._db.preferences.compareFlavors(self.flavor(), other.flavor())
 573
 574     def installed(self):
 575         """Test wheter the package is installed.
 576
 577         Returns two values: a status indicator which is one of
 578         "yes", "no", "old" (an older version is installed) or "bad"
 579         (something went wrong during the install test) and a human
 580         readable string which may contain more details."""
 581
 582         namespace = {
 583             "NotInstalled": _scriptExc_NotInstalled,
 584             "OldInstalled": _scriptExc_OldInstalled,
 585             "BadInstalled": _scriptExc_BadInstalled,
 586             "os": os,
 587             "sys": sys,
 588             }
 589         installTest = self._dict['Install-test'].strip() + '\n'
 590         try:
 591             exec installTest in namespace
 592         except ImportError, arg:
 593             return "no", str(arg)
 594         except _scriptExc_NotInstalled, arg:
 595             return "no", str(arg)
 596         except _scriptExc_OldInstalled, arg:
 597             return "old", str(arg)
 598         except _scriptExc_BadInstalled, arg:
 599             return "bad", str(arg)
 600         except:
 601             sys.stderr.write("-------------------------------------\n")
 602             sys.stderr.write("---- %s: install test got exception\n" % self.fullname())
 603             sys.stderr.write("---- source:\n")
 604             sys.stderr.write(installTest)
 605             sys.stderr.write("---- exception:\n")
 606             import traceback
 607             traceback.print_exc(file=sys.stderr)
 608             if self._db._maintainer:
 609                 sys.stderr.write("---- Please copy this and mail to %s\n" % self._db._maintainer)
 610             sys.stderr.write("-------------------------------------\n")
 611             return "bad", "Package install test got exception"
 612         return "yes", ""
 613
 614     def prerequisites(self):
 615         """Return a list of prerequisites for this package.
 616
 617         The list contains 2-tuples, of which the first item is either
 618         a PimpPackage object or None, and the second is a descriptive
 619         string. The first item can be None if this package depends on
 620         something that isn't pimp-installable, in which case the descriptive
 621         string should tell the user what to do."""
 622
 623         rv = []
 624         if not self._dict.get('Download-URL'):
 625             # For pseudo-packages that are already installed we don't
 626             # return an error message
 627             status, _  = self.installed()
 628             if status == "yes":
 629                 return []
 630             return [(None,
 631                 "Package %s cannot be installed automatically, see the description" %
 632                     self.fullname())]
 633         if self.systemwideOnly() and self._db.preferences.isUserInstall():
 634             return [(None,
 635                 "Package %s can only be installed system-wide" %
 636                     self.fullname())]
 637         if not self._dict.get('Prerequisites'):
 638             return []
 639         for item in self._dict['Prerequisites']:
 640             if type(item) == str:
 641                 pkg = None
 642                 descr = str(item)
 643             else:
 644                 name = item['Name']
 645                 if item.has_key('Version'):
 646                     name = name + '-' + item['Version']
 647                 if item.has_key('Flavor'):
 648                     name = name + '-' + item['Flavor']
 649                 pkg = self._db.find(name)
 650                 if not pkg:
 651                     descr = "Requires unknown %s"%name
 652                 else:
 653                     descr = pkg.shortdescription()
 654             rv.append((pkg, descr))
 655         return rv
 656
 657
 658     def downloadPackageOnly(self, output=None):
 659         """Download a single package, if needed.
 660
 661         An MD5 signature is used to determine whether download is needed,
 662         and to test that we actually downloaded what we expected.
 663         If output is given it is a file-like object that will receive a log
 664         of what happens.
 665
 666         If anything unforeseen happened the method returns an error message
 667         string.
 668         """
 669
 670         scheme, loc, path, query, frag = urlparse.urlsplit(self._dict['Download-URL'])
 671         path = urllib.url2pathname(path)
 672         filename = os.path.split(path)[1]
 673         self.archiveFilename = os.path.join(self._db.preferences.downloadDir, filename)
 674         if not self._archiveOK():
 675             if scheme == 'manual':
 676                 return "Please download package manually and save as %s" % self.archiveFilename
 677             downloader = PimpUrllibDownloader(None, self._db.preferences.downloadDir,
 678                 watcher=self._db.preferences.watcher)
 679             if not downloader.download(self._dict['Download-URL'],
 680                     self.archiveFilename, output):
 681                 return "download command failed"
 682         if not os.path.exists(self.archiveFilename) and not NO_EXECUTE:
 683             return "archive not found after download"
 684         if not self._archiveOK():
 685             return "archive does not have correct MD5 checksum"
 686
 687     def _archiveOK(self):
 688         """Test an archive. It should exist and the MD5 checksum should be correct."""
 689
 690         if not os.path.exists(self.archiveFilename):
 691             return 0
 692         if not self._dict.get('MD5Sum'):
 693             sys.stderr.write("Warning: no MD5Sum for %s\n" % self.fullname())
 694             return 1
 695         data = open(self.archiveFilename, 'rb').read()
 696         checksum = hashlib.md5(data).hexdigest()
 697         return checksum == self._dict['MD5Sum']
 698
 699     def unpackPackageOnly(self, output=None):
 700         """Unpack a downloaded package archive."""
 701
 702         filename = os.path.split(self.archiveFilename)[1]
 703         for ext, unpackerClass, arg in ARCHIVE_FORMATS:
 704             if filename[-len(ext):] == ext:
 705                 break
 706         else:
 707             return "unknown extension for archive file: %s" % filename
 708         self.basename = filename[:-len(ext)]
 709         unpacker = unpackerClass(arg, dir=self._db.preferences.buildDir,
 710                 watcher=self._db.preferences.watcher)
 711         rv = unpacker.unpack(self.archiveFilename, output=output)
 712         if rv:
 713             return rv
 714
 715     def installPackageOnly(self, output=None):
 716         """Default install method, to be overridden by subclasses"""
 717         return "%s: This package needs to be installed manually (no support for flavor=\"%s\")" \
 718             % (self.fullname(), self._dict.get(flavor, ""))
 719
 720     def installSinglePackage(self, output=None):
 721         """Download, unpack and install a single package.
 722
 723         If output is given it should be a file-like object and it
 724         will receive a log of what happened."""
 725
 726         if not self._dict.get('Download-URL'):
 727             return "%s: This package needs to be installed manually (no Download-URL field)" % self.fullname()
 728         msg = self.downloadPackageOnly(output)
 729         if msg:
 730             return "%s: download: %s" % (self.fullname(), msg)
 731
 732         msg = self.unpackPackageOnly(output)
 733         if msg:
 734             return "%s: unpack: %s" % (self.fullname(), msg)
 735
 736         return self.installPackageOnly(output)
 737
 738     def beforeInstall(self):
 739         """Bookkeeping before installation: remember what we have in site-packages"""
 740         self._old_contents = os.listdir(self._db.preferences.installDir)
 741
 742     def afterInstall(self):
 743         """Bookkeeping after installation: interpret any new .pth files that have
 744         appeared"""
 745
 746         new_contents = os.listdir(self._db.preferences.installDir)
 747         for fn in new_contents:
 748             if fn in self._old_contents:
 749                 continue
 750             if fn[-4:] != '.pth':
 751                 continue
 752             fullname = os.path.join(self._db.preferences.installDir, fn)
 753             f = open(fullname)
 754             for line in f.readlines():
 755                 if not line:
 756                     continue
 757                 if line[0] == '#':
 758                     continue
 759                 if line[:6] == 'import':
 760                     exec line
 761                     continue
 762                 if line[-1] == '\n':
 763                     line = line[:-1]
 764                 if not os.path.isabs(line):
 765                     line = os.path.join(self._db.preferences.installDir, line)
 766                 line = os.path.realpath(line)
 767                 if not line in sys.path:
 768                     sys.path.append(line)
 769
 770     def filterExpectedSkips(self, names):
 771         """Return a list that contains only unpexpected skips"""
 772         if not self._db.preferences.isUserInstall():
 773             return names
 774         expected_skips = self._dict.get('User-install-skips')
 775         if not expected_skips:
 776             return names
 777         newnames = []
 778         for name in names:
 779             for skip in expected_skips:
 780                 if name[:len(skip)] == skip:
 781                     break
 782             else:
 783                 newnames.append(name)
 784         return newnames
 785
 786 class PimpPackage_binary(PimpPackage):
 787
 788     def unpackPackageOnly(self, output=None):
 789         """We don't unpack binary packages until installing"""
 790         pass
 791
 792     def installPackageOnly(self, output=None):
 793         """Install a single source package.
 794
 795         If output is given it should be a file-like object and it
 796         will receive a log of what happened."""
 797
 798         if self._dict.has_key('Install-command'):
 799             return "%s: Binary package cannot have Install-command" % self.fullname()
 800
 801         if self._dict.has_key('Pre-install-command'):
 802             if _cmd(output, '/tmp', self._dict['Pre-install-command']):
 803                 return "pre-install %s: running \"%s\" failed" % \
 804                     (self.fullname(), self._dict['Pre-install-command'])
 805
 806         self.beforeInstall()
 807
 808         # Install by unpacking
 809         filename = os.path.split(self.archiveFilename)[1]
 810         for ext, unpackerClass, arg in ARCHIVE_FORMATS:
 811             if filename[-len(ext):] == ext:
 812                 break
 813         else:
 814             return "%s: unknown extension for archive file: %s" % (self.fullname(), filename)
 815         self.basename = filename[:-len(ext)]
 816
 817         install_renames = []
 818         for k, newloc in self._db.preferences.installLocations:
 819             if not newloc:
 820                 continue
 821             if k == "--install-lib":
 822                 oldloc = DEFAULT_INSTALLDIR
 823             else:
 824                 return "%s: Don't know installLocation %s" % (self.fullname(), k)
 825             install_renames.append((oldloc, newloc))
 826
 827         unpacker = unpackerClass(arg, dir="/", renames=install_renames)
 828         rv = unpacker.unpack(self.archiveFilename, output=output, package=self)
 829         if rv:
 830             return rv
 831
 832         self.afterInstall()
 833
 834         if self._dict.has_key('Post-install-command'):
 835             if _cmd(output, '/tmp', self._dict['Post-install-command']):
 836                 return "%s: post-install: running \"%s\" failed" % \
 837                     (self.fullname(), self._dict['Post-install-command'])
 838
 839         return None
 840
 841
 842 class PimpPackage_source(PimpPackage):
 843
 844     def unpackPackageOnly(self, output=None):
 845         """Unpack a source package and check that setup.py exists"""
 846         PimpPackage.unpackPackageOnly(self, output)
 847         # Test that a setup script has been create
 848         self._buildDirname = os.path.join(self._db.preferences.buildDir, self.basename)
 849         setupname = os.path.join(self._buildDirname, "setup.py")
 850         if not os.path.exists(setupname) and not NO_EXECUTE:
 851             return "no setup.py found after unpack of archive"
 852
 853     def installPackageOnly(self, output=None):
 854         """Install a single source package.
 855
 856         If output is given it should be a file-like object and it
 857         will receive a log of what happened."""
 858
 859         if self._dict.has_key('Pre-install-command'):
 860             if _cmd(output, self._buildDirname, self._dict['Pre-install-command']):
 861                 return "pre-install %s: running \"%s\" failed" % \
 862                     (self.fullname(), self._dict['Pre-install-command'])
 863
 864         self.beforeInstall()
 865         installcmd = self._dict.get('Install-command')
 866         if installcmd and self._install_renames:
 867             return "Package has install-command and can only be installed to standard location"
 868         # This is the "bit-bucket" for installations: everything we don't
 869         # want. After installation we check that it is actually empty
 870         unwanted_install_dir = None
 871         if not installcmd:
 872             extra_args = ""
 873             for k, v in self._db.preferences.installLocations:
 874                 if not v:
 875                     # We don't want these files installed. Send them
 876                     # to the bit-bucket.
 877                     if not unwanted_install_dir:
 878                         unwanted_install_dir = tempfile.mkdtemp()
 879                     v = unwanted_install_dir
 880                 extra_args = extra_args + " %s \"%s\"" % (k, v)
 881             installcmd = '"%s" setup.py install %s' % (sys.executable, extra_args)
 882         if _cmd(output, self._buildDirname, installcmd):
 883             return "install %s: running \"%s\" failed" % \
 884                 (self.fullname(), installcmd)
 885         if unwanted_install_dir and os.path.exists(unwanted_install_dir):
 886             unwanted_files = os.listdir(unwanted_install_dir)
 887             if unwanted_files:
 888                 rv = "Warning: some files were not installed: %s" % " ".join(unwanted_files)
 889             else:
 890                 rv = None
 891             shutil.rmtree(unwanted_install_dir)
 892             return rv
 893
 894         self.afterInstall()
 895
 896         if self._dict.has_key('Post-install-command'):
 897             if _cmd(output, self._buildDirname, self._dict['Post-install-command']):
 898                 return "post-install %s: running \"%s\" failed" % \
 899                     (self.fullname(), self._dict['Post-install-command'])
 900         return None
 901
 902 class PimpPackage_installer(PimpPackage):
 903
 904     def unpackPackageOnly(self, output=None):
 905         """We don't unpack dmg packages until installing"""
 906         pass
 907
 908     def installPackageOnly(self, output=None):
 909         """Install a single source package.
 910
 911         If output is given it should be a file-like object and it
 912         will receive a log of what happened."""
 913
 914         if self._dict.has_key('Post-install-command'):
 915             return "%s: Installer package cannot have Post-install-command" % self.fullname()
 916
 917         if self._dict.has_key('Pre-install-command'):
 918             if _cmd(output, '/tmp', self._dict['Pre-install-command']):
 919                 return "pre-install %s: running \"%s\" failed" % \
 920                     (self.fullname(), self._dict['Pre-install-command'])
 921
 922         self.beforeInstall()
 923
 924         installcmd = self._dict.get('Install-command')
 925         if installcmd:
 926             if '%' in installcmd:
 927                 installcmd = installcmd % self.archiveFilename
 928         else:
 929             installcmd = 'open \"%s\"' % self.archiveFilename
 930         if _cmd(output, "/tmp", installcmd):
 931             return '%s: install command failed (use verbose for details)' % self.fullname()
 932         return '%s: downloaded and opened. Install manually and restart Package Manager' % self.archiveFilename
 933
 934 class PimpInstaller:
 935     """Installer engine: computes dependencies and installs
 936     packages in the right order."""
 937
 938     def __init__(self, db):
 939         self._todo = []
 940         self._db = db
 941         self._curtodo = []
 942         self._curmessages = []
 943
 944     def __contains__(self, package):
 945         return package in self._todo
 946
 947     def _addPackages(self, packages):
 948         for package in packages:
 949             if not package in self._todo:
 950                 self._todo.append(package)
 951
 952     def _prepareInstall(self, package, force=0, recursive=1):
 953         """Internal routine, recursive engine for prepareInstall.
 954
 955         Test whether the package is installed and (if not installed
 956         or if force==1) prepend it to the temporary todo list and
 957         call ourselves recursively on all prerequisites."""
 958
 959         if not force:
 960             status, message = package.installed()
 961             if status == "yes":
 962                 return
 963         if package in self._todo or package in self._curtodo:
 964             return
 965         self._curtodo.insert(0, package)
 966         if not recursive:
 967             return
 968         prereqs = package.prerequisites()
 969         for pkg, descr in prereqs:
 970             if pkg:
 971                 self._prepareInstall(pkg, False, recursive)
 972             else:
 973                 self._curmessages.append("Problem with dependency: %s" % descr)
 974
 975     def prepareInstall(self, package, force=0, recursive=1):
 976         """Prepare installation of a package.
 977
 978         If the package is already installed and force is false nothing
 979         is done. If recursive is true prerequisites are installed first.
 980
 981         Returns a list of packages (to be passed to install) and a list
 982         of messages of any problems encountered.
 983         """
 984
 985         self._curtodo = []
 986         self._curmessages = []
 987         self._prepareInstall(package, force, recursive)
 988         rv = self._curtodo, self._curmessages
 989         self._curtodo = []
 990         self._curmessages = []
 991         return rv
 992
 993     def install(self, packages, output):
 994         """Install a list of packages."""
 995
 996         self._addPackages(packages)
 997         status = []
 998         for pkg in self._todo:
 999             msg = pkg.installSinglePackage(output)
1000             if msg:
1001                 status.append(msg)
1002         return status
1003
1004
1005
1006 def _run(mode, verbose, force, args, prefargs, watcher):
1007     """Engine for the main program"""
1008
1009     prefs = PimpPreferences(**prefargs)
1010     if watcher:
1011         prefs.setWatcher(watcher)
1012     rv = prefs.check()
1013     if rv:
1014         sys.stdout.write(rv)
1015     db = PimpDatabase(prefs)
1016     db.appendURL(prefs.pimpDatabase)
1017
1018     if mode == 'dump':
1019         db.dump(sys.stdout)
1020     elif mode =='list':
1021         if not args:
1022             args = db.listnames()
1023         print "%-20.20s\t%s" % ("Package", "Description")
1024         print
1025         for pkgname in args:
1026             pkg = db.find(pkgname)
1027             if pkg:
1028                 description = pkg.shortdescription()
1029                 pkgname = pkg.fullname()
1030             else:
1031                 description = 'Error: no such package'
1032             print "%-20.20s\t%s" % (pkgname, description)
1033             if verbose:
1034                 print "\tHome page:\t", pkg.homepage()
1035                 try:
1036                     print "\tDownload URL:\t", pkg.downloadURL()
1037                 except KeyError:
1038                     pass
1039                 description = pkg.description()
1040                 description = '\n\t\t\t\t\t'.join(description.splitlines())
1041                 print "\tDescription:\t%s" % description
1042     elif mode =='status':
1043         if not args:
1044             args = db.listnames()
1045             print "%-20.20s\t%s\t%s" % ("Package", "Installed", "Message")
1046             print
1047         for pkgname in args:
1048             pkg = db.find(pkgname)
1049             if pkg:
1050                 status, msg = pkg.installed()
1051                 pkgname = pkg.fullname()
1052             else:
1053                 status = 'error'
1054                 msg = 'No such package'
1055             print "%-20.20s\t%-9.9s\t%s" % (pkgname, status, msg)
1056             if verbose and status == "no":
1057                 prereq = pkg.prerequisites()
1058                 for pkg, msg in prereq:
1059                     if not pkg:
1060                         pkg = ''
1061                     else:
1062                         pkg = pkg.fullname()
1063                     print "%-20.20s\tRequirement: %s %s" % ("", pkg, msg)
1064     elif mode == 'install':
1065         if not args:
1066             print 'Please specify packages to install'
1067             sys.exit(1)
1068         inst = PimpInstaller(db)
1069         for pkgname in args:
1070             pkg = db.find(pkgname)
1071             if not pkg:
1072                 print '%s: No such package' % pkgname
1073                 continue
1074             list, messages = inst.prepareInstall(pkg, force)
1075             if messages and not force:
1076                 print "%s: Not installed:" % pkgname
1077                 for m in messages:
1078                     print "\t", m
1079             else:
1080                 if verbose:
1081                     output = sys.stdout
1082                 else:
1083                     output = None
1084                 messages = inst.install(list, output)
1085                 if messages:
1086                     print "%s: Not installed:" % pkgname
1087                     for m in messages:
1088                         print "\t", m
1089
1090 def main():
1091     """Minimal commandline tool to drive pimp."""
1092
1093     import getopt
1094     def _help():
1095         print "Usage: pimp [options] -s [package ...]  List installed status"
1096         print "       pimp [options] -l [package ...]  Show package information"
1097         print "       pimp [options] -i package ...    Install packages"
1098         print "       pimp -d                          Dump database to stdout"
1099         print "       pimp -V                          Print version number"
1100         print "Options:"
1101         print "       -v     Verbose"
1102         print "       -f     Force installation"
1103         print "       -D dir Set destination directory"
1104         print "              (default: %s)" % DEFAULT_INSTALLDIR
1105         print "       -u url URL for database"
1106         sys.exit(1)
1107
1108     class _Watcher:
1109         def update(self, msg):
1110             sys.stderr.write(msg + '\r')
1111             return 1
1112
1113     try:
1114         opts, args = getopt.getopt(sys.argv[1:], "slifvdD:Vu:")
1115     except getopt.GetoptError:
1116         _help()
1117     if not opts and not args:
1118         _help()
1119     mode = None
1120     force = 0
1121     verbose = 0
1122     prefargs = {}
1123     watcher = None
1124     for o, a in opts:
1125         if o == '-s':
1126             if mode:
1127                 _help()
1128             mode = 'status'
1129         if o == '-l':
1130             if mode:
1131                 _help()
1132             mode = 'list'
1133         if o == '-d':
1134             if mode:
1135                 _help()
1136             mode = 'dump'
1137         if o == '-V':
1138             if mode:
1139                 _help()
1140             mode = 'version'
1141         if o == '-i':
1142             mode = 'install'
1143         if o == '-f':
1144             force = 1
1145         if o == '-v':
1146             verbose = 1
1147             watcher = _Watcher()
1148         if o == '-D':
1149             prefargs['installDir'] = a
1150         if o == '-u':
1151             prefargs['pimpDatabase'] = a
1152     if not mode:
1153         _help()
1154     if mode == 'version':
1155         print 'Pimp version %s; module name is %s' % (PIMP_VERSION, __name__)
1156     else:
1157         _run(mode, verbose, force, args, prefargs, watcher)
1158
1159 # Finally, try to update ourselves to a newer version.
1160 # If the end-user updates pimp through pimp the new version
1161 # will be called pimp_update and live in site-packages
1162 # or somewhere similar
1163 if __name__ != 'pimp_update':
1164     try:
1165         import pimp_update
1166     except ImportError:
1167         pass
1168     else:
1169         if pimp_update.PIMP_VERSION <= PIMP_VERSION:
1170             import warnings
1171             warnings.warn("pimp_update is version %s, not newer than pimp version %s" %
1172                 (pimp_update.PIMP_VERSION, PIMP_VERSION))
1173         else:
1174             from pimp_update import *
1175
1176 if __name__ == '__main__':
1177     main()