feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.injector import model, gpg, namespaces, qdom
  13
  14 from display import checking, result, error, highlight, error_new_line
  15
  16 now = time.time()
  17
  18 version = '0.7'
  19
  20 WEEK = 60 * 60 * 24 * 7
  21
  22 def host(address):
  23         if hasattr(address, 'hostname'):
  24                 return address.hostname
  25         else:
  26                 return address[1].split(':', 1)[0]
  27
  28 def port(address):
  29         if hasattr(address, 'port'):
  30                 return address.port
  31         else:
  32                 port = address[1].split(':', 1)[1:]
  33                 if port:
  34                         return int(port[0])
  35                 else:
  36                         return None
  37
  38 assert port(('http', 'foo:81')) == 81
  39 assert port(urlparse.urlparse('http://foo:81')) == 81
  40
  41 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  42 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  43 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  44 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  45 parser.add_option("-V", "--version", help="display version information", action='store_true')
  46
  47 (options, args) = parser.parse_args()
  48
  49 if options.version:
  50         print "FeedLint (zero-install) " + version
  51         print "Copyright (C) 2007 Thomas Leonard"
  52         print "This program comes with ABSOLUTELY NO WARRANTY,"
  53         print "to the extent permitted by law."
  54         print "You may redistribute copies of this program"
  55         print "under the terms of the GNU General Public License."
  56         print "For more information about these matters, see the file named COPYING."
  57         sys.exit(0)
  58
  59 if options.verbose:
  60         logger = logging.getLogger()
  61         if options.verbose == 1:
  62                 logger.setLevel(logging.INFO)
  63         else:
  64                 logger.setLevel(logging.DEBUG)
  65
  66 if len(args) < 1:
  67         parser.print_help()
  68         sys.exit(1)
  69
  70 checked = set()
  71
  72 try:
  73         to_check = [model.canonical_iface_uri(a) for a in args]
  74 except SafeException, ex:
  75         if options.verbose: raise
  76         print >>sys.stderr, ex
  77         sys.exit(1)
  78
  79 def check_key(feed, fingerprint):
  80         for line in os.popen('gpg --with-colons --list-keys %s' % s.fingerprint):
  81                 if line.startswith('pub:'):
  82                         key_id = line.split(':')[4]
  83                         break
  84         else:
  85                 raise SafeException('Failed to find key with fingerprint %s on your keyring' % fingerprint)
  86
  87         key_url = urlparse.urljoin(feed, '%s.gpg' % key_id)
  88
  89         if key_url in checked:
  90                 info("(already checked key URL %s)", key_url)
  91         else:
  92                 checking("Checking key %s" % key_url)
  93                 urllib2.urlopen(key_url).read()
  94                 result('OK')
  95                 checked.add(key_url)
  96
  97 def get_http_size(url, ttl = 3):
  98         address = urlparse.urlparse(url)
  99
 100         if url.lower().startswith('http://'):
 101                 http = httplib.HTTPConnection(host(address), port(address) or 80)
 102         elif url.lower().startswith('https://'):
 103                 http = httplib.HTTPSConnection(host(address), port(address) or 443)
 104         else:
 105                 assert False, url
 106
 107         parts = url.split('/', 3)
 108         if len(parts) == 4:
 109                 path = parts[3]
 110         else:
 111                 path = ''
 112
 113         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 114         response = http.getresponse()
 115         try:
 116                 if response.status == 200:
 117                         return response.getheader('Content-Length')
 118                 elif response.status in (301, 302, 303):
 119                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 120                         new_url = urlparse.urljoin(url, new_url_rel)
 121                 else:
 122                         raise SafeException("HTTP error: got status code %s" % response.status)
 123         finally:
 124                 response.close()
 125
 126         if ttl:
 127                 result("Moved", 'YELLOW')
 128                 checking("Checking new URL %s" % new_url)
 129                 assert new_url
 130                 return get_http_size(new_url, ttl - 1)
 131         else:
 132                 raise SafeException('Too many redirections.')
 133
 134 def get_ftp_size(url):
 135         address = urlparse.urlparse(url)
 136         ftp = ftplib.FTP(host(address))
 137         try:
 138                 ftp.login()
 139                 ftp.voidcmd('TYPE I')
 140                 return ftp.size(url.split('/', 3)[3])
 141         finally:
 142                 ftp.close()
 143
 144 def get_size(url):
 145         scheme = urlparse.urlparse(url)[0].lower()
 146         if scheme.startswith('http') or scheme.startswith('https'):
 147                 return get_http_size(url)
 148         elif scheme.startswith('ftp'):
 149                 return get_ftp_size(url)
 150         else:
 151                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 152
 153 def check_source(source):
 154         if hasattr(source, 'url'):
 155                 checking("Checking archive %s" % source.url)
 156                 actual_size = get_size(source.url)
 157                 if actual_size is None:
 158                         result("No Content-Length for archive; can't check", 'YELLOW')
 159                 else:
 160                         actual_size = int(actual_size)
 161                         expected_size = source.size + (source.start_offset or 0)
 162                         if actual_size != expected_size:
 163                                 error('Bad length')
 164                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 165                                                 (expected_size, actual_size))
 166                         result('OK')
 167         elif hasattr(source, 'steps'):
 168                 for step in source.steps:
 169                         check_source(step)
 170
 171 existing_urls = set()
 172 def check_exists(url):
 173         if url in existing_urls: return         # Already checked
 174
 175         checking("Checking URL exists %s" % url)
 176         get_size(url)
 177         result('OK')
 178         existing_urls.add(url)
 179
 180 n_errors = 0
 181
 182 while to_check:
 183         feed = to_check.pop()
 184         if feed in checked:
 185                 info("Already checked feed %s", feed)
 186                 continue
 187
 188         checked.add(feed)
 189
 190         checking("Checking " + feed, indent = 0)
 191
 192         try:
 193                 if feed.startswith('/'):
 194                         with open(feed) as stream:
 195                                 feed_obj = model.ZeroInstallFeed(qdom.parse(stream), local_path = feed)
 196                         result('Local')
 197                 else:
 198                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 199                         try:
 200                                 stream = urllib2.urlopen(feed)
 201                                 shutil.copyfileobj(stream, tmp)
 202                                 tmp.seek(0)
 203                                 start = tmp.read(5)
 204                                 if start == '<?xml':
 205                                         result('Fetched')
 206                                 elif start == '-----':
 207                                         result('Old sig', colour = 'YELLOW')
 208                                         error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 209                                                         colour = 'YELLOW')
 210                                 else:
 211                                         result('Fetched')
 212                                         error_new_line('  Unknown format. File starts "%s"' % start)
 213
 214                                 tmp.seek(0)
 215                                 data, sigs = gpg.check_stream(tmp)
 216
 217                                 for s in sigs:
 218                                         if isinstance(s, gpg.ValidSig):
 219                                                 check_key(feed, s.fingerprint)
 220                                         else:
 221                                                 raise SafeException("Can't check sig: %s" % s)
 222
 223                                 feed_obj = model.ZeroInstallFeed(qdom.parse(data))
 224
 225                                 if feed_obj.url != feed:
 226                                         raise SafeException('Incorrect URL "%s"' % feed_obj.url)
 227
 228                         finally:
 229                                 tmp.close()
 230
 231                 for f in feed_obj.feeds:
 232                         info("Will check feed %s", f.uri)
 233                         to_check.append(f.uri)
 234
 235                 highest_version = None
 236                 for impl in feed_obj.implementations.values():
 237                         if hasattr(impl, 'dependencies'):
 238                                 for r in impl.dependencies.values():
 239                                         if r.interface not in checked:
 240                                                 info("Will check dependency %s", r)
 241                                                 if options.dependencies:
 242                                                         to_check.append(r.interface)
 243                                                 else:
 244                                                         check_exists(r.interface)
 245                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 246                                 for source in impl.download_sources:
 247                                         check_source(source)
 248                         stability = impl.upstream_stability or model.testing
 249                         if highest_version is None or impl.version > highest_version.version:
 250                                 highest_version = impl
 251                         if stability == model.testing:
 252                                 testing_error = None
 253                                 if not impl.released:
 254                                         testing_error = "No release data on testing version"
 255                                 else:
 256                                         try:
 257                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 258                                         except ValueError, ex:
 259                                                 testing_error = "Can't parse date"
 260                                         else:
 261                                                 ago = now - time.mktime(released)
 262                                                 if ago < 0:
 263                                                         testing_error = 'Release data is in the future!'
 264                                 if testing_error:
 265                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 266
 267                 if highest_version and (highest_version.upstream_stability or model.testing) is model.testing:
 268                         print highlight('  Highest version (%s) is still "testing"' % highest_version.get_version(), 'YELLOW')
 269
 270                 for homepage in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 271                         check_exists(homepage.content)
 272
 273                 for icon in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 274                         check_exists(icon.getAttribute('href'))
 275
 276         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 277                 err_msg = str(ex).strip() or str(type(ex))
 278                 error_new_line('  ' + err_msg)
 279                 n_errors += 1
 280                 if options.verbose: traceback.print_exc()
 281         except SafeException, ex:
 282                 if options.verbose: raise
 283                 error_new_line('  ' + str(ex))
 284                 n_errors += 1
 285
 286 if n_errors == 0:
 287         print "OK"
 288 else:
 289         print "\nERRORS FOUND:", n_errors
 290         sys.exit(1)