feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.injector import model, gpg, reader, namespaces
  13
  14 from display import checking, result, error, highlight, error_new_line
  15
  16 now = time.time()
  17
  18 version = '0.4'
  19
  20 WEEK = 60 * 60 * 24 * 7
  21
  22 def host(address):
  23         if hasattr(address, 'hostname'):
  24                 return address.hostname
  25         else:
  26                 return address[1].split(':', 1)[0]
  27
  28 def port(address):
  29         if hasattr(address, 'port'):
  30                 return address.port
  31         else:
  32                 port = address[1].split(':', 1)[1:]
  33                 if port:
  34                         return int(port[0])
  35                 else:
  36                         return None
  37
  38 assert port(('http', 'foo:81')) == 81
  39 assert port(urlparse.urlparse('http://foo:81')) == 81
  40
  41 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  42 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  43 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  44 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  45 parser.add_option("-V", "--version", help="display version information", action='store_true')
  46
  47 (options, args) = parser.parse_args()
  48
  49 if options.version:
  50         print "FeedLint (zero-install) " + version
  51         print "Copyright (C) 2007 Thomas Leonard"
  52         print "This program comes with ABSOLUTELY NO WARRANTY,"
  53         print "to the extent permitted by law."
  54         print "You may redistribute copies of this program"
  55         print "under the terms of the GNU General Public License."
  56         print "For more information about these matters, see the file named COPYING."
  57         sys.exit(0)
  58
  59 if options.verbose:
  60         logger = logging.getLogger()
  61         if options.verbose == 1:
  62                 logger.setLevel(logging.INFO)
  63         else:
  64                 logger.setLevel(logging.DEBUG)
  65
  66 if len(args) < 1:
  67         parser.print_help()
  68         sys.exit(1)
  69
  70 checked = set()
  71
  72 try:
  73         to_check = [model.canonical_iface_uri(a) for a in args]
  74 except SafeException, ex:
  75         if options.verbose: raise
  76         print >>sys.stderr, ex
  77         sys.exit(1)
  78
  79 def check_key(feed, fingerprint):
  80         for line in os.popen('gpg --with-colons --list-keys %s' % s.fingerprint):
  81                 if line.startswith('pub:'):
  82                         key_id = line.split(':')[4]
  83                         break
  84         else:
  85                 raise SafeException('Failed to find key with fingerprint %s on your keyring' % fingerprint)
  86
  87         key_url = urlparse.urljoin(feed, '%s.gpg' % key_id)
  88
  89         if key_url in checked:
  90                 info("(already checked key URL %s)", key_url)
  91         else:
  92                 checking("Checking key %s" % key_url)
  93                 urllib2.urlopen(key_url).read()
  94                 result('OK')
  95                 checked.add(key_url)
  96
  97 def get_http_size(url, ttl = 1):
  98         assert url.lower().startswith('http://')
  99
 100         address = urlparse.urlparse(url)
 101         http = httplib.HTTPConnection(host(address), port(address) or 80)
 102
 103         parts = url.split('/', 3)
 104         if len(parts) == 4:
 105                 path = parts[3]
 106         else:
 107                 path = ''
 108
 109         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 110         response = http.getresponse()
 111         try:
 112                 if response.status == 200:
 113                         return response.getheader('Content-Length')
 114                 elif response.status in (301, 302):
 115                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 116                         new_url = urlparse.urljoin(url, new_url_rel)
 117                 else:
 118                         raise SafeException("HTTP error: got status code %s" % response.status)
 119         finally:
 120                 response.close()
 121
 122         if ttl:
 123                 result("Moved", 'YELLOW')
 124                 checking("Checking new URL %s" % new_url)
 125                 assert new_url
 126                 return get_http_size(new_url, ttl - 1)
 127         else:
 128                 raise SafeException('Too many redirections.')
 129
 130 def get_ftp_size(url):
 131         address = urlparse.urlparse(url)
 132         ftp = ftplib.FTP(host(address))
 133         try:
 134                 ftp.login()
 135                 ftp.voidcmd('TYPE I')
 136                 return ftp.size(url.split('/', 3)[3])
 137         finally:
 138                 ftp.close()
 139
 140 def get_size(url):
 141         scheme = urlparse.urlparse(url)[0].lower()
 142         if scheme.startswith('http'):
 143                 return get_http_size(url)
 144         elif scheme.startswith('ftp'):
 145                 return get_ftp_size(url)
 146         else:
 147                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 148
 149 def check_source(source):
 150         if hasattr(source, 'url'):
 151                 checking("Checking archive %s" % source.url)
 152                 actual_size = get_size(source.url)
 153                 if actual_size is None:
 154                         result("No Content-Length for archive; can't check", 'YELLOW')
 155                 else:
 156                         actual_size = int(actual_size)
 157                         expected_size = source.size + (source.start_offset or 0)
 158                         if actual_size != expected_size:
 159                                 error('Bad length')
 160                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 161                                                 (expected_size, actual_size))
 162                         result('OK')
 163         elif hasattr(source, 'steps'):
 164                 for step in source.steps:
 165                         check_source(step)
 166
 167 existing_urls = set()
 168 def check_exists(url):
 169         if url in existing_urls: return         # Already checked
 170
 171         checking("Checking URL exists %s" % url)
 172         get_size(url)
 173         result('OK')
 174         existing_urls.add(url)
 175
 176 n_errors = 0
 177
 178 while to_check:
 179         feed = to_check.pop()
 180         if feed in checked:
 181                 info("Already checked feed %s", feed)
 182                 continue
 183
 184         checked.add(feed)
 185
 186         checking("Checking " + feed, indent = 0)
 187
 188         try:
 189                 iface = model.Interface(feed)
 190
 191                 if feed.startswith('/'):
 192                         reader.update(iface, feed, local = True)
 193                         result('Local')
 194                 else:
 195                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 196                         try:
 197                                 stream = urllib2.urlopen(feed)
 198                                 shutil.copyfileobj(stream, tmp)
 199                                 tmp.seek(0)
 200                                 start = tmp.read(5)
 201                                 if start == '<?xml':
 202                                         result('Fetched')
 203                                 elif start == '-----':
 204                                         result('Old sig', colour = 'YELLOW')
 205                                         error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 206                                                         colour = 'YELLOW')
 207                                 else:
 208                                         result('Fetched')
 209                                         error_new_line('  Unknown format. File starts "%s"' % start)
 210
 211                                 tmp.seek(0)
 212                                 data, sigs = gpg.check_stream(tmp)
 213
 214                                 for s in sigs:
 215                                         if isinstance(s, gpg.ValidSig):
 216                                                 check_key(feed, s.fingerprint)
 217                                         else:
 218                                                 raise SafeException("Can't check sig: %s" % s)
 219
 220                                 feed_tmp = tempfile.NamedTemporaryFile(prefix = 'feedlint-')
 221                                 try:
 222                                         shutil.copyfileobj(data, feed_tmp)
 223                                         feed_tmp.seek(0)
 224                                         reader.update(iface, feed_tmp.name)
 225
 226                                         for f in iface.feeds:
 227                                                 info("Will check feed %s", f.uri)
 228                                                 to_check.append(f.uri)
 229                                 finally:
 230                                         feed_tmp.close()
 231                         finally:
 232                                 tmp.close()
 233
 234                 for impl in iface.implementations.values():
 235                         if hasattr(impl, 'dependencies'):
 236                                 for r in impl.dependencies.values():
 237                                         if r.interface not in checked:
 238                                                 info("Will check dependency %s", r)
 239                                                 if options.dependencies:
 240                                                         to_check.append(r.interface)
 241                                                 else:
 242                                                         check_exists(r.interface)
 243                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 244                                 for source in impl.download_sources:
 245                                         check_source(source)
 246                         stability = impl.upstream_stability or model.testing
 247                         if stability == model.testing:
 248                                 testing_error = None
 249                                 if not impl.released:
 250                                         testing_error = "No release data on testing version"
 251                                 else:
 252                                         try:
 253                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 254                                         except ValueError, ex:
 255                                                 testing_error = "Can't parse date"
 256                                         else:
 257                                                 ago = now - time.mktime(released)
 258                                                 if ago < 0:
 259                                                         testing_error = 'Release data is in the future!'
 260                                                 elif ago > 1 * WEEK:
 261                                                         print highlight('  Version %s has been Testing for more than a week' % impl.get_version(),
 262                                                                         'YELLOW')
 263                                 if testing_error:
 264                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 265
 266                 for homepage in iface.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 267                         check_exists(homepage.content)
 268
 269                 for icon in iface.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 270                         check_exists(icon.getAttribute('href'))
 271
 272         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 273                 err_msg = str(ex).strip() or str(type(ex))
 274                 error_new_line('  ' + err_msg)
 275                 n_errors += 1
 276                 if options.verbose: traceback.print_exc()
 277         except SafeException, ex:
 278                 if options.verbose: raise
 279                 error_new_line('  ' + str(ex))
 280                 n_errors += 1
 281
 282 if n_errors == 0:
 283         print "OK"
 284 else:
 285         print "\nERRORS FOUND:", n_errors
 286         sys.exit(1)