feedlint

   1 #!/usr/bin/env python
   2
   3 from optparse import OptionParser
   4 import sys, shutil, tempfile, urlparse
   5 import socket
   6 import urllib2, os, httplib
   7 import ftplib
   8 import logging, time, traceback
   9 from logging import info
  10
  11 from zeroinstall import SafeException
  12 from zeroinstall.injector import model, gpg, namespaces, qdom
  13 from zeroinstall.injector.config import load_config
  14
  15 from display import checking, result, error, highlight, error_new_line
  16
  17 config = load_config()
  18
  19 now = time.time()
  20
  21 version = '0.8'
  22
  23 WEEK = 60 * 60 * 24 * 7
  24
  25 def host(address):
  26         if hasattr(address, 'hostname'):
  27                 return address.hostname
  28         else:
  29                 return address[1].split(':', 1)[0]
  30
  31 def port(address):
  32         if hasattr(address, 'port'):
  33                 return address.port
  34         else:
  35                 port = address[1].split(':', 1)[1:]
  36                 if port:
  37                         return int(port[0])
  38                 else:
  39                         return None
  40
  41 assert port(('http', 'foo:81')) == 81
  42 assert port(urlparse.urlparse('http://foo:81')) == 81
  43
  44 parser = OptionParser(usage="usage: %prog [options] feed.xml")
  45 parser.add_option("-d", "--dependencies", help="also check feeds for dependencies", action='store_true')
  46 parser.add_option("-s", "--skip-archives", help="don't check the archives are OK", action='store_true')
  47 parser.add_option("-v", "--verbose", help="more verbose output", action='count')
  48 parser.add_option("-V", "--version", help="display version information", action='store_true')
  49
  50 (options, args) = parser.parse_args()
  51
  52 if options.version:
  53         print "FeedLint (zero-install) " + version
  54         print "Copyright (C) 2007 Thomas Leonard"
  55         print "This program comes with ABSOLUTELY NO WARRANTY,"
  56         print "to the extent permitted by law."
  57         print "You may redistribute copies of this program"
  58         print "under the terms of the GNU General Public License."
  59         print "For more information about these matters, see the file named COPYING."
  60         sys.exit(0)
  61
  62 if options.verbose:
  63         logger = logging.getLogger()
  64         if options.verbose == 1:
  65                 logger.setLevel(logging.INFO)
  66         else:
  67                 logger.setLevel(logging.DEBUG)
  68
  69 if len(args) < 1:
  70         parser.print_help()
  71         sys.exit(1)
  72
  73 checked = set()
  74
  75 def arg_to_uri(arg):
  76         app = config.app_mgr.lookup_app(arg, missing_ok = True)
  77         if app is not None:
  78                 return app.get_requirements().interface_uri
  79         else:
  80                 return model.canonical_iface_uri(a)
  81
  82 try:
  83         to_check = [arg_to_uri(a) for a in args]
  84 except SafeException, ex:
  85         if options.verbose: raise
  86         print >>sys.stderr, ex
  87         sys.exit(1)
  88
  89 def check_key(feed_url, fingerprint):
  90         for line in os.popen('gpg --with-colons --list-keys %s' % fingerprint):
  91                 if line.startswith('pub:'):
  92                         key_id = line.split(':')[4]
  93                         break
  94         else:
  95                 raise SafeException('Failed to find key with fingerprint %s on your keyring' % fingerprint)
  96
  97         key_url = urlparse.urljoin(feed_url, '%s.gpg' % key_id)
  98
  99         if key_url in checked:
 100                 info("(already checked key URL %s)", key_url)
 101         else:
 102                 checking("Checking key %s" % key_url)
 103                 urllib2.urlopen(key_url).read()
 104                 result('OK')
 105                 checked.add(key_url)
 106
 107 def get_http_size(url, ttl = 3):
 108         address = urlparse.urlparse(url)
 109
 110         if url.lower().startswith('http://'):
 111                 http = httplib.HTTPConnection(host(address), port(address) or 80)
 112         elif url.lower().startswith('https://'):
 113                 http = httplib.HTTPSConnection(host(address), port(address) or 443)
 114         else:
 115                 assert False, url
 116
 117         parts = url.split('/', 3)
 118         if len(parts) == 4:
 119                 path = parts[3]
 120         else:
 121                 path = ''
 122
 123         http.request('HEAD', '/' + path, headers = {'Host': host(address)})
 124         response = http.getresponse()
 125         try:
 126                 if response.status == 200:
 127                         return response.getheader('Content-Length')
 128                 elif response.status in (301, 302, 303):
 129                         new_url_rel = response.getheader('Location') or response.getheader('URI')
 130                         new_url = urlparse.urljoin(url, new_url_rel)
 131                 else:
 132                         raise SafeException("HTTP error: got status code %s" % response.status)
 133         finally:
 134                 response.close()
 135
 136         if ttl:
 137                 result("Moved", 'YELLOW')
 138                 checking("Checking new URL %s" % new_url)
 139                 assert new_url
 140                 return get_http_size(new_url, ttl - 1)
 141         else:
 142                 raise SafeException('Too many redirections.')
 143
 144 def get_ftp_size(url):
 145         address = urlparse.urlparse(url)
 146         ftp = ftplib.FTP(host(address))
 147         try:
 148                 ftp.login()
 149                 ftp.voidcmd('TYPE I')
 150                 return ftp.size(url.split('/', 3)[3])
 151         finally:
 152                 ftp.close()
 153
 154 def get_size(url):
 155         scheme = urlparse.urlparse(url)[0].lower()
 156         if scheme.startswith('http') or scheme.startswith('https'):
 157                 return get_http_size(url)
 158         elif scheme.startswith('ftp'):
 159                 return get_ftp_size(url)
 160         else:
 161                 raise SafeException("Unknown scheme '%s' in '%s'" % (scheme, url))
 162
 163 def check_source(source):
 164         if hasattr(source, 'url'):
 165                 checking("Checking archive %s" % source.url)
 166                 actual_size = get_size(source.url)
 167                 if actual_size is None:
 168                         result("No Content-Length for archive; can't check", 'YELLOW')
 169                 else:
 170                         actual_size = int(actual_size)
 171                         expected_size = source.size + (source.start_offset or 0)
 172                         if actual_size != expected_size:
 173                                 error('Bad length')
 174                                 raise SafeException("Expected archive to have a size of %d, but server says it is %d" %
 175                                                 (expected_size, actual_size))
 176                         result('OK')
 177         elif hasattr(source, 'steps'):
 178                 for step in source.steps:
 179                         check_source(step)
 180
 181 existing_urls = set()
 182 def check_exists(url):
 183         if url in existing_urls: return         # Already checked
 184
 185         checking("Checking URL exists %s" % url)
 186         get_size(url)
 187         result('OK')
 188         existing_urls.add(url)
 189
 190 def scan_implementations(impls, dom):
 191         """Add each implementation in dom to impls. Error if duplicate."""
 192         for elem in dom.childNodes:
 193                 if elem.uri != namespaces.XMLNS_IFACE: continue
 194                 if elem.name == 'implementation':
 195                         impl_id = elem.attrs['id']
 196                         if impl_id in impls:
 197                                 raise SafeException("Duplicate ID {id}!".format(id = impl_id))
 198                         impls[impl_id] = elem
 199                 elif elem.name == 'group':
 200                         scan_implementations(impls, elem)
 201
 202 n_errors = 0
 203
 204 def check_gpg_sig(feed_url, stream):
 205         start = stream.read(5)
 206         if start == '<?xml':
 207                 result('Fetched')
 208         elif start == '-----':
 209                 result('Old sig', colour = 'RED')
 210                 error_new_line('  Feed has an old-style plain GPG signature. Use 0publish --xmlsign.',
 211                                 colour = 'YELLOW')
 212         else:
 213                 result('Fetched')
 214                 error_new_line('  Unknown format. File starts "%s"' % start)
 215
 216         data, sigs = gpg.check_stream(stream)
 217
 218         for s in sigs:
 219                 if isinstance(s, gpg.ValidSig):
 220                         check_key(feed_url, s.fingerprint)
 221                 else:
 222                         raise SafeException("Can't check sig: %s" % s)
 223
 224         return data
 225
 226 while to_check:
 227         feed = to_check.pop()
 228         if feed in checked:
 229                 info("Already checked feed %s", feed)
 230                 continue
 231
 232         checked.add(feed)
 233
 234         checking("Checking " + feed, indent = 0)
 235
 236         try:
 237                 if feed.startswith('/'):
 238                         with open(feed) as stream:
 239                                 dom = qdom.parse(stream)
 240
 241                                 if "uri" in dom.attrs:
 242                                         stream.seek(0)
 243                                         try:
 244                                                 check_gpg_sig(dom.attrs['uri'], stream)
 245                                         except SafeException, ex:
 246                                                 n_errors += 1
 247                                                 error_new_line('  %s' % ex)
 248
 249                                 feed_obj = model.ZeroInstallFeed(dom, local_path = feed)
 250                         result('Local')
 251                 else:
 252                         tmp = tempfile.TemporaryFile(prefix = 'feedlint-')
 253                         try:
 254                                 try:
 255                                         stream = urllib2.urlopen(feed)
 256                                         shutil.copyfileobj(stream, tmp)
 257                                 except Exception as ex:
 258                                         raise SafeException('Failed to fetch feed: {ex}'.format(ex = ex))
 259                                 tmp.seek(0)
 260
 261                                 data = check_gpg_sig(feed, tmp)
 262                                 tmp.seek(0)
 263
 264                                 dom = qdom.parse(data)
 265                                 feed_obj = model.ZeroInstallFeed(dom)
 266
 267                                 if feed_obj.url != feed:
 268                                         raise SafeException('Incorrect URL "%s"' % feed_obj.url)
 269
 270                         finally:
 271                                 tmp.close()
 272
 273                 # Check for duplicate IDs
 274                 scan_implementations({}, dom)
 275
 276                 for f in feed_obj.feeds:
 277                         info("Will check feed %s", f.uri)
 278                         to_check.append(f.uri)
 279
 280                 highest_version = None
 281                 for impl in sorted(feed_obj.implementations.values()):
 282                         if hasattr(impl, 'dependencies'):
 283                                 for r in impl.dependencies.values():
 284                                         if r.interface not in checked:
 285                                                 info("Will check dependency %s", r)
 286                                                 if options.dependencies:
 287                                                         to_check.append(r.interface)
 288                                                 else:
 289                                                         check_exists(r.interface)
 290                         if hasattr(impl, 'download_sources') and not options.skip_archives:
 291                                 for source in impl.download_sources:
 292                                         check_source(source)
 293                         if impl.local_path is None:
 294                                 if not impl.digests:
 295                                         raise SafeException("Version {version} has no digests".format(version = impl.get_version()))
 296                         stability = impl.upstream_stability or model.testing
 297                         if highest_version is None or impl.version > highest_version.version:
 298                                 highest_version = impl
 299                         if stability == model.testing:
 300                                 testing_error = None
 301                                 if not impl.released:
 302                                         if not impl.local_path:
 303                                                 testing_error = "No release date on testing version"
 304                                 else:
 305                                         try:
 306                                                 released = time.strptime(impl.released, '%Y-%m-%d')
 307                                         except ValueError, ex:
 308                                                 testing_error = "Can't parse date"
 309                                         else:
 310                                                 ago = now - time.mktime(released)
 311                                                 if ago < 0:
 312                                                         testing_error = 'Release date is in the future!'
 313                                 if testing_error:
 314                                         raise SafeException("Version %s: %s (released %s)" % (impl.get_version(), testing_error, impl.released))
 315
 316                         # Old Windows versions use 32-bit integers to store versions. Newer versions use 64-bit ones, but in general
 317                         # keeping the numbers small is helpful.
 318                         for i in range(0, len(impl.version), 2):
 319                                 for x in impl.version[i]:
 320                                         if x >= (1 << 31):
 321                                                 raise SafeException("Version %s: component %s won't fit in a 32-bit signed integer" % (impl.get_version(), x))
 322
 323                 if highest_version and (highest_version.upstream_stability or model.testing) is model.testing:
 324                         print highlight('  Highest version (%s) is still "testing"' % highest_version.get_version(), 'YELLOW')
 325
 326                 for homepage in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'homepage'):
 327                         check_exists(homepage.content)
 328
 329                 for icon in feed_obj.get_metadata(namespaces.XMLNS_IFACE, 'icon'):
 330                         check_exists(icon.getAttribute('href'))
 331
 332         except (urllib2.HTTPError, httplib.BadStatusLine, socket.error, ftplib.error_perm), ex:
 333                 err_msg = str(ex).strip() or str(type(ex))
 334                 error_new_line('  ' + err_msg)
 335                 n_errors += 1
 336                 if options.verbose: traceback.print_exc()
 337         except SafeException, ex:
 338                 if options.verbose: raise
 339                 error_new_line('  ' + str(ex))
 340                 n_errors += 1
 341
 342 if n_errors == 0:
 343         print "OK"
 344 else:
 345         print "\nERRORS FOUND:", n_errors
 346         sys.exit(1)