Lib/urlopen.py

   1 # Open an arbitrary URL
   2 #
   3 # See the following document for a tentative description of URLs:
   4 #     Uniform Resource Locators              Tim Berners-Lee
   5 #     INTERNET DRAFT                                    CERN
   6 #     IETF URL Working Group                    14 July 1993
   7 #     draft-ietf-uri-url-01.txt
   8 #
   9 # The object returned by URLopener().open(file) will differ per
  10 # protocol.  All you know is that is has methods read(), readline(),
  11 # readlines(), fileno(), close() and info().  The read*(), fileno()
  12 # and close() methods work like those of open files.
  13 # The info() method returns an rfc822.Message object which can be
  14 # used to query various info about the object, if available.
  15 # (rfc822.Message objects are queried with the getheader() method.)
  16
  17 import socket
  18 import regex
  19
  20
  21 # This really consists of two pieces:
  22 # (1) a class which handles opening of all sorts of URLs
  23 #     (plus assorted utilities etc.)
  24 # (2) a set of functions for parsing URLs
  25 # XXX Should these be separated out into different modules?
  26
  27
  28 # Shortcut for basic usage
  29 _urlopener = None
  30 def urlopen(url):
  31         global _urlopener
  32         if not _urlopener:
  33                 _urlopener = URLopener()
  34         return _urlopener.open(url)
  35 def urlretrieve(url):
  36         global _urlopener
  37         if not _urlopener:
  38                 _urlopener = URLopener()
  39         return _urlopener.retrieve(url)
  40 def urlcleanup():
  41         if _urlopener:
  42                 _urlopener.cleanup()
  43
  44
  45 # Class to open URLs.
  46 # This is a class rather than just a subroutine because we may need
  47 # more than one set of global protocol-specific options.
  48 ftpcache = {}
  49 class URLopener:
  50
  51         # Constructor
  52         def __init__(self):
  53                 self.addheaders = []
  54                 self.tempcache = {}
  55                 self.ftpcache = ftpcache
  56                 # Undocumented feature: you can use a different
  57                 # ftp cache by assigning to the .ftpcache member;
  58                 # in case you want logically independent URL openers
  59
  60         def __del__(self):
  61                 self.close()
  62
  63         def close(self):
  64                 self.cleanup()
  65
  66         def cleanup(self):
  67                 import os
  68                 for url in self.tempcache.keys():
  69                         try:
  70                                 os.unlink(self.tempcache[url][0])
  71                         except os.error:
  72                                 pass
  73                         del self.tempcache[url]
  74
  75         # Add a header to be used by the HTTP interface only
  76         # e.g. u.addheader('Accept', 'sound/basic')
  77         def addheader(self, *args):
  78                 self.addheaders.append(args)
  79
  80         # External interface
  81         # Use URLopener().open(file) instead of open(file, 'r')
  82         def open(self, url):
  83                 type, url = splittype(unwrap(url))
  84                 if not type: type = 'file'
  85                 name = 'open_' + type
  86                 if '-' in name:
  87                         import regsub
  88                         name = regsub.gsub('-', '_', name)
  89                 if not hasattr(self, name):
  90                         raise IOError, ('url error', 'unknown url type', type)
  91                 try:
  92                         return getattr(self, name)(url)
  93                 except socket.error, msg:
  94                         raise IOError, ('socket error', msg)
  95
  96         # External interface
  97         # retrieve(url) returns (filename, None) for a local object
  98         # or (tempfilename, headers) for a remote object
  99         def retrieve(self, url):
 100                 if self.tempcache.has_key(url):
 101                         return self.tempcache[url]
 102                 url1 = unwrap(url)
 103                 if self.tempcache.has_key(url1):
 104                         self.tempcache[url] = self.tempcache[url1]
 105                         return self.tempcache[url1]
 106                 type, url1 = splittype(url1)
 107                 if not type or type == 'file':
 108                         try:
 109                                 fp = self.open_local_file(url1)
 110                                 del fp
 111                                 return splithost(url1)[1], None
 112                         except IOError, msg:
 113                                 pass
 114                 fp = self.open(url)
 115                 headers = fp.info()
 116                 import tempfile
 117                 tfn = tempfile.mktemp()
 118                 self.tempcache[url] = result = tfn, headers
 119                 tfp = open(tfn, 'w')
 120                 bs = 1024*8
 121                 block = fp.read(bs)
 122                 while block:
 123                         tfp.write(block)
 124                         block = fp.read(bs)
 125                 del fp
 126                 del tfp
 127                 return result
 128
 129         # Each method named open_<type> knows how to open that type of URL
 130
 131         # Use HTTP protocol
 132         def open_http(self, url):
 133                 import httplib
 134                 host, selector = splithost(url)
 135                 h = httplib.HTTP(host)
 136                 h.putrequest('GET', selector)
 137                 for args in self.addheaders: apply(h.putheader, args)
 138                 errcode, errmsg, headers = h.getreply()
 139                 if errcode == 200: return addinfo(h.getfile(), headers)
 140                 else: raise IOError, ('http error', errcode, errmsg, headers)
 141
 142         # Use Gopher protocol
 143         def open_gopher(self, url):
 144                 import gopherlib
 145                 host, selector = splithost(url)
 146                 type, selector = splitgophertype(selector)
 147                 selector, query = splitquery(selector)
 148                 if query: fp = gopherlib.send_query(selector, query, host)
 149                 else: fp = gopherlib.send_selector(selector, host)
 150                 return addinfo(fp, noheaders())
 151
 152         # Use local file or FTP depending on form of URL
 153         def open_file(self, url):
 154                 try:
 155                         return self.open_local_file(url)
 156                 except IOError:
 157                         return self.open_ftp(url)
 158
 159         # Use local file
 160         def open_local_file(self, url):
 161                 host, file = splithost(url)
 162                 if not host: return addinfo(open(file, 'r'), noheaders())
 163                 host, port = splitport(host)
 164                 if not port and socket.gethostbyname(host) in (
 165                           localhost(), thishost()):
 166                         return addinfo(open(file, 'r'), noheaders())
 167                 raise IOError, ('local file error', 'not on local host')
 168
 169         # Use FTP protocol
 170         def open_ftp(self, url):
 171                 host, file = splithost(url)
 172                 if not host: raise IOError, ('ftp error', 'no host given')
 173                 host, port = splitport(host)
 174                 host = socket.gethostbyname(host)
 175                 if not port:
 176                         import ftplib
 177                         port = ftplib.FTP_PORT
 178                 key = (host, port)
 179                 try:
 180                         if not self.ftpcache.has_key(key):
 181                                 self.ftpcache[key] = ftpwrapper(host, port)
 182                         return addinfo(self.ftpcache[key].retrfile(file),
 183                                   noheaders())
 184                 except ftperrors(), msg:
 185                         raise IOError, ('ftp error', msg)
 186
 187
 188 # Utility functions
 189
 190 # Return the IP address of the magic hostname 'localhost'
 191 _localhost = None
 192 def localhost():
 193         global _localhost
 194         if not _localhost:
 195                 _localhost = socket.gethostbyname('localhost')
 196         return _localhost
 197
 198 # Return the IP address of the current host
 199 _thishost = None
 200 def thishost():
 201         global _thishost
 202         if not _thishost:
 203                 _thishost = socket.gethostbyname(socket.gethostname())
 204         return _thishost
 205
 206 # Return the set of errors raised by the FTP class
 207 _ftperrors = None
 208 def ftperrors():
 209         global _ftperrors
 210         if not _ftperrors:
 211                 import ftplib
 212                 _ftperrors = (ftplib.error_reply,
 213                               ftplib.error_temp,
 214                               ftplib.error_perm,
 215                               ftplib.error_proto)
 216         return _ftperrors
 217
 218 # Return an empty rfc822.Message object
 219 _noheaders = None
 220 def noheaders():
 221         global _noheaders
 222         if not _noheaders:
 223                 import rfc822
 224                 _noheaders = rfc822.Message(open('/dev/null', 'r'))
 225                 _noheaders.fp.close()   # Recycle file descriptor
 226         return _noheaders
 227
 228
 229 # Utility classes
 230
 231 # Class used by open_ftp() for cache of open FTP connections
 232 class ftpwrapper:
 233         def __init__(self, host, port):
 234                 self.host = host
 235                 self.port = port
 236                 self.init()
 237         def init(self):
 238                 import ftplib
 239                 self.ftp = ftplib.FTP()
 240                 self.ftp.connect(self.host, self.port)
 241                 self.ftp.login()
 242         def retrfile(self, file):
 243                 import ftplib
 244                 try:
 245                         self.ftp.voidcmd('TYPE I')
 246                 except ftplib.all_errors:
 247                         self.init()
 248                         self.ftp.voidcmd('TYPE I')
 249                 conn = None
 250                 if file:
 251                         try:
 252                                 cmd = 'RETR ' + file
 253                                 conn = self.ftp.transfercmd(cmd)
 254                         except ftplib.error_perm, reason:
 255                                 if reason[:3] != '550':
 256                                         raise IOError, ('ftp error', reason)
 257                 if not conn:
 258                         # Try a directory listing
 259                         if file: cmd = 'LIST ' + file
 260                         else: cmd = 'LIST'
 261                         conn = self.ftp.transfercmd(cmd)
 262                 return addclosehook(conn.makefile('r'), self.ftp.voidresp)
 263
 264 # Base class for addinfo and addclosehook
 265 class addbase:
 266         def __init__(self, fp):
 267                 self.fp = fp
 268                 self.read = self.fp.read
 269                 self.readline = self.fp.readline
 270                 self.readlines = self.fp.readlines
 271                 self.fileno = self.fp.fileno
 272         def __repr__(self):
 273                 return '<%s at %s whose fp = %s>' % (
 274                           self.__class__.__name__, `id(self)`, `self.fp`)
 275         def __del__(self):
 276                 self.close()
 277         def close(self):
 278                 self.read = None
 279                 self.readline = None
 280                 self.readlines = None
 281                 self.fileno = None
 282                 self.fp = None
 283
 284 # Class to add a close hook to an open file
 285 class addclosehook(addbase):
 286         def __init__(self, fp, closehook, *hookargs):
 287                 addbase.__init__(self, fp)
 288                 self.closehook = closehook
 289                 self.hookargs = hookargs
 290         def close(self):
 291                 if self.closehook:
 292                         apply(self.closehook, self.hookargs)
 293                         self.closehook = None
 294                         self.hookargs = None
 295                 addbase.close(self)
 296
 297 # class to add an info() method to an open file
 298 class addinfo(addbase):
 299         def __init__(self, fp, headers):
 300                 addbase.__init__(self, fp)
 301                 self.headers = headers
 302         def info(self):
 303                 return self.headers
 304
 305
 306 # Utility to combine a URL with a base URL to form a new URL
 307
 308 def basejoin(base, url):
 309         type, path = splittype(url)
 310         if type: return url
 311         host, path = splithost(path)
 312         basetype, basepath = splittype(base)
 313         basehost, basepath = splithost(basepath)
 314         basepath, basetag = splittag(basepath)
 315         basepath, basequery = splitquery(basepath)
 316         type = basetype or 'file'
 317         if path[:1] != '/':
 318                 import string
 319                 i = string.rfind(basepath, '/')
 320                 if i < 0: basepath = '/'
 321                 else: basepath = basepath[:i+1]
 322                 path = basepath + path
 323         if not host: host = basehost
 324         if host: return type + '://' + host + path
 325         else: return type + ':' + path
 326
 327
 328 # Utilities to parse URLs:
 329 # unwrap('<URL:type//host/path>') --> 'type//host/path'
 330 # splittype('type:opaquestring') --> 'type', 'opaquestring'
 331 # splithost('//host[:port]/path') --> 'host[:port]', '/path'
 332 # splitport('host:port') --> 'host', 'port'
 333 # splitquery('/path?query') --> '/path', 'query'
 334 # splittag('/path#tag') --> '/path', 'tag'
 335 # splitgophertype('/Xselector') --> 'X', 'selector'
 336
 337 def unwrap(url):
 338         import string
 339         url = string.strip(url)
 340         if url[:1] == '<' and url[-1:] == '>':
 341                 url = string.strip(url[1:-1])
 342         if url[:4] == 'URL:': url = string.strip(url[4:])
 343         return url
 344
 345 _typeprog = regex.compile('^\([^/:]+\):\(.*\)$')
 346 def splittype(url):
 347         if _typeprog.match(url) >= 0: return _typeprog.group(1, 2)
 348         return None, url
 349
 350 _hostprog = regex.compile('^//\([^/]+\)\(.*\)$')
 351 def splithost(url):
 352         if _hostprog.match(url) >= 0: return _hostprog.group(1, 2)
 353         return None, url
 354
 355 _portprog = regex.compile('^\(.*\):\([0-9]+\)$')
 356 def splitport(host):
 357         if _portprog.match(host) >= 0: return _portprog.group(1, 2)
 358         return host, None
 359
 360 _queryprog = regex.compile('^\(.*\)\?\([^?]*\)$')
 361 def splitquery(url):
 362         if _queryprog.match(url) >= 0: return _queryprog.group(1, 2)
 363         return url, None
 364
 365 _tagprog = regex.compile('^\(.*\)#\([^#]*\)$')
 366 def splittag(url):
 367         if _tagprog.match(url) >= 0: return _tagprog.group(1, 2)
 368         return url, None
 369
 370 def splitgophertype(selector):
 371         if selector[:1] == '/' and selector[1:2]:
 372                 return selector[1], selector[2:]
 373         return None, selector
 374
 375
 376 # Test program
 377 def test():
 378         import sys
 379         import regsub
 380         args = sys.argv[1:]
 381         if not args:
 382                 args = [
 383                         '/etc/passwd',
 384                         'file:/etc/passwd',
 385                         'file://localhost/etc/passwd',
 386                         'ftp://ftp.cwi.nl/etc/passwd',
 387                         'gopher://gopher.cwi.nl/11/',
 388                         'http://www.cwi.nl/index.html',
 389                         ]
 390         try:
 391                 for url in args:
 392                         print '-'*10, url, '-'*10
 393                         fn, h = urlretrieve(url)
 394                         print fn, h
 395                         if h:
 396                                 print '======'
 397                                 for k in h.keys(): print k + ':', h[k]
 398                                 print '======'
 399                         fp = open(fn, 'r')
 400                         data = fp.read()
 401                         del fp
 402                         print regsub.gsub('\r', '', data)
 403                         fn, h = None, None
 404                 print '-'*40
 405         finally:
 406                 urlcleanup()
 407
 408 # Run test program when run as a script
 409 if __name__ == '__main__':
 410         test()