Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 class HTTPMessage(mimetools.Message):
  97
  98     def addheader(self, key, value):
  99         """Add header for field key handling repeats."""
 100         prev = self.dict.get(key)
 101         if prev is None:
 102             self.dict[key] = value
 103         else:
 104             combined = ", ".join((prev, value))
 105             self.dict[key] = combined
 106
 107     def addcontinue(self, key, more):
 108         """Add more field data from a continuation line."""
 109         prev = self.dict[key]
 110         self.dict[key] = prev + "\n " + more
 111
 112     def readheaders(self):
 113         """Read header lines.
 114
 115         Read header lines up to the entirely blank line that terminates them.
 116         The (normally blank) line that ends the headers is skipped, but not
 117         included in the returned list.  If a non-header line ends the headers,
 118         (which is an error), an attempt is made to backspace over it; it is
 119         never included in the returned list.
 120
 121         The variable self.status is set to the empty string if all went well,
 122         otherwise it is an error message.  The variable self.headers is a
 123         completely uninterpreted list of lines contained in the header (so
 124         printing them will reproduce the header exactly as it appears in the
 125         file).
 126
 127         If multiple header fields with the same name occur, they are combined
 128         according to the rules in RFC 2616 sec 4.2:
 129
 130         Appending each subsequent field-value to the first, each separated
 131         by a comma. The order in which header fields with the same field-name
 132         are received is significant to the interpretation of the combined
 133         field value.
 134         """
 135         # XXX The implementation overrides the readheaders() method of
 136         # rfc822.Message.  The base class design isn't amenable to
 137         # customized behavior here so the method here is a copy of the
 138         # base class code with a few small changes.
 139
 140         self.dict = {}
 141         self.unixfrom = ''
 142         self.headers = list = []
 143         self.status = ''
 144         headerseen = ""
 145         firstline = 1
 146         startofline = unread = tell = None
 147         if hasattr(self.fp, 'unread'):
 148             unread = self.fp.unread
 149         elif self.seekable:
 150             tell = self.fp.tell
 151         while 1:
 152             if tell:
 153                 try:
 154                     startofline = tell()
 155                 except IOError:
 156                     startofline = tell = None
 157                     self.seekable = 0
 158             line = self.fp.readline()
 159             if not line:
 160                 self.status = 'EOF in headers'
 161                 break
 162             # Skip unix From name time lines
 163             if firstline and line.startswith('From '):
 164                 self.unixfrom = self.unixfrom + line
 165                 continue
 166             firstline = 0
 167             if headerseen and line[0] in ' \t':
 168                 # XXX Not sure if continuation lines are handled properly
 169                 # for http and/or for repeating headers
 170                 # It's a continuation line.
 171                 list.append(line)
 172                 self.addcontinue(headerseen, line.strip())
 173                 continue
 174             elif self.iscomment(line):
 175                 # It's a comment.  Ignore it.
 176                 continue
 177             elif self.islast(line):
 178                 # Note! No pushback here!  The delimiter line gets eaten.
 179                 break
 180             headerseen = self.isheader(line)
 181             if headerseen:
 182                 # It's a legal header line, save it.
 183                 list.append(line)
 184                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 185                 continue
 186             else:
 187                 # It's not a header line; throw it back and stop here.
 188                 if not self.dict:
 189                     self.status = 'No headers'
 190                 else:
 191                     self.status = 'Non-header line where header expected'
 192                 # Try to undo the read.
 193                 if unread:
 194                     unread(line)
 195                 elif tell:
 196                     self.fp.seek(startofline)
 197                 else:
 198                     self.status = self.status + '; bad seek'
 199                 break
 200
 201 class HTTPResponse:
 202
 203     # strict: If true, raise BadStatusLine if the status line can't be
 204     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 205     # false because it prevents clients from talking to HTTP/0.9
 206     # servers.  Note that a response with a sufficiently corrupted
 207     # status line will look like an HTTP/0.9 response.
 208
 209     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 210
 211     def __init__(self, sock, debuglevel=0, strict=0):
 212         self.fp = sock.makefile('rb', 0)
 213         self.debuglevel = debuglevel
 214         self.strict = strict
 215
 216         self.msg = None
 217
 218         # from the Status-Line of the response
 219         self.version = _UNKNOWN # HTTP-Version
 220         self.status = _UNKNOWN  # Status-Code
 221         self.reason = _UNKNOWN  # Reason-Phrase
 222
 223         self.chunked = _UNKNOWN         # is "chunked" being used?
 224         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 225         self.length = _UNKNOWN          # number of bytes left in response
 226         self.will_close = _UNKNOWN      # conn will close at end of response
 227
 228     def _read_status(self):
 229         # Initialize with Simple-Response defaults
 230         line = self.fp.readline()
 231         if self.debuglevel > 0:
 232             print "reply:", repr(line)
 233         try:
 234             [version, status, reason] = line.split(None, 2)
 235         except ValueError:
 236             try:
 237                 [version, status] = line.split(None, 1)
 238                 reason = ""
 239             except ValueError:
 240                 # empty version will cause next test to fail and status
 241                 # will be treated as 0.9 response.
 242                 version = ""
 243         if not version.startswith('HTTP/'):
 244             if self.strict:
 245                 self.close()
 246                 raise BadStatusLine(line)
 247             else:
 248                 # assume it's a Simple-Response from an 0.9 server
 249                 self.fp = LineAndFileWrapper(line, self.fp)
 250                 return "HTTP/0.9", 200, ""
 251
 252         # The status code is a three-digit number
 253         try:
 254             status = int(status)
 255             if status < 100 or status > 999:
 256                 raise BadStatusLine(line)
 257         except ValueError:
 258             raise BadStatusLine(line)
 259         return version, status, reason
 260
 261     def begin(self):
 262         if self.msg is not None:
 263             # we've already started reading the response
 264             return
 265
 266         # read until we get a non-100 response
 267         while 1:
 268             version, status, reason = self._read_status()
 269             if status != 100:
 270                 break
 271             # skip the header from the 100 response
 272             while 1:
 273                 skip = self.fp.readline().strip()
 274                 if not skip:
 275                     break
 276                 if self.debuglevel > 0:
 277                     print "header:", skip
 278
 279         self.status = status
 280         self.reason = reason.strip()
 281         if version == 'HTTP/1.0':
 282             self.version = 10
 283         elif version.startswith('HTTP/1.'):
 284             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 285         elif version == 'HTTP/0.9':
 286             self.version = 9
 287         else:
 288             raise UnknownProtocol(version)
 289
 290         if self.version == 9:
 291             self.chunked = 0
 292             self.will_close = 1
 293             self.msg = HTTPMessage(StringIO())
 294             return
 295
 296         self.msg = HTTPMessage(self.fp, 0)
 297         if self.debuglevel > 0:
 298             for hdr in self.msg.headers:
 299                 print "header:", hdr,
 300
 301         # don't let the msg keep an fp
 302         self.msg.fp = None
 303
 304         # are we using the chunked-style of transfer encoding?
 305         tr_enc = self.msg.getheader('transfer-encoding')
 306         if tr_enc and tr_enc.lower() == "chunked":
 307             self.chunked = 1
 308             self.chunk_left = None
 309         else:
 310             self.chunked = 0
 311
 312         # will the connection close at the end of the response?
 313         self.will_close = self._check_close()
 314
 315         # do we have a Content-Length?
 316         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 317         length = self.msg.getheader('content-length')
 318         if length and not self.chunked:
 319             try:
 320                 self.length = int(length)
 321             except ValueError:
 322                 self.length = None
 323         else:
 324             self.length = None
 325
 326         # does the body have a fixed length? (of zero)
 327         if (status == 204 or            # No Content
 328             status == 304 or            # Not Modified
 329             100 <= status < 200):       # 1xx codes
 330             self.length = 0
 331
 332         # if the connection remains open, and we aren't using chunked, and
 333         # a content-length was not provided, then assume that the connection
 334         # WILL close.
 335         if not self.will_close and \
 336            not self.chunked and \
 337            self.length is None:
 338             self.will_close = 1
 339
 340     def _check_close(self):
 341         if self.version == 11:
 342             # An HTTP/1.1 proxy is assumed to stay open unless
 343             # explicitly closed.
 344             conn = self.msg.getheader('connection')
 345             if conn and conn.lower().find("close") >= 0:
 346                 return True
 347             return False
 348
 349         # An HTTP/1.0 response with a Connection header is probably
 350         # the result of a confused proxy.  Ignore it.
 351
 352         # For older HTTP, Keep-Alive indiciates persistent connection.
 353         if self.msg.getheader('keep-alive'):
 354             return False
 355
 356         # Proxy-Connection is a netscape hack.
 357         pconn = self.msg.getheader('proxy-connection')
 358         if pconn and pconn.lower().find("keep-alive") >= 0:
 359             return False
 360
 361         # otherwise, assume it will close
 362         return True
 363
 364     def close(self):
 365         if self.fp:
 366             self.fp.close()
 367             self.fp = None
 368
 369     def isclosed(self):
 370         # NOTE: it is possible that we will not ever call self.close(). This
 371         #       case occurs when will_close is TRUE, length is None, and we
 372         #       read up to the last byte, but NOT past it.
 373         #
 374         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 375         #          called, meaning self.isclosed() is meaningful.
 376         return self.fp is None
 377
 378     def read(self, amt=None):
 379         if self.fp is None:
 380             return ''
 381
 382         if self.chunked:
 383             return self._read_chunked(amt)
 384
 385         if amt is None:
 386             # unbounded read
 387             if self.will_close:
 388                 s = self.fp.read()
 389             else:
 390                 s = self._safe_read(self.length)
 391             self.close()        # we read everything
 392             return s
 393
 394         if self.length is not None:
 395             if amt > self.length:
 396                 # clip the read to the "end of response"
 397                 amt = self.length
 398             self.length -= amt
 399
 400         # we do not use _safe_read() here because this may be a .will_close
 401         # connection, and the user is reading more bytes than will be provided
 402         # (for example, reading in 1k chunks)
 403         s = self.fp.read(amt)
 404
 405         return s
 406
 407     def _read_chunked(self, amt):
 408         assert self.chunked != _UNKNOWN
 409         chunk_left = self.chunk_left
 410         value = ''
 411
 412         # XXX This accumulates chunks by repeated string concatenation,
 413         # which is not efficient as the number or size of chunks gets big.
 414         while 1:
 415             if chunk_left is None:
 416                 line = self.fp.readline()
 417                 i = line.find(';')
 418                 if i >= 0:
 419                     line = line[:i] # strip chunk-extensions
 420                 chunk_left = int(line, 16)
 421                 if chunk_left == 0:
 422                     break
 423             if amt is None:
 424                 value += self._safe_read(chunk_left)
 425             elif amt < chunk_left:
 426                 value += self._safe_read(amt)
 427                 self.chunk_left = chunk_left - amt
 428                 return value
 429             elif amt == chunk_left:
 430                 value += self._safe_read(amt)
 431                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 432                 self.chunk_left = None
 433                 return value
 434             else:
 435                 value += self._safe_read(chunk_left)
 436                 amt -= chunk_left
 437
 438             # we read the whole chunk, get another
 439             self._safe_read(2)      # toss the CRLF at the end of the chunk
 440             chunk_left = None
 441
 442         # read and discard trailer up to the CRLF terminator
 443         ### note: we shouldn't have any trailers!
 444         while 1:
 445             line = self.fp.readline()
 446             if line == '\r\n':
 447                 break
 448
 449         # we read everything; close the "file"
 450         # XXX Shouldn't the client close the file?
 451         self.close()
 452
 453         return value
 454
 455     def _safe_read(self, amt):
 456         """Read the number of bytes requested, compensating for partial reads.
 457
 458         Normally, we have a blocking socket, but a read() can be interrupted
 459         by a signal (resulting in a partial read).
 460
 461         Note that we cannot distinguish between EOF and an interrupt when zero
 462         bytes have been read. IncompleteRead() will be raised in this
 463         situation.
 464
 465         This function should be used when <amt> bytes "should" be present for
 466         reading. If the bytes are truly not available (due to EOF), then the
 467         IncompleteRead exception can be used to detect the problem.
 468         """
 469         s = ''
 470         while amt > 0:
 471             chunk = self.fp.read(amt)
 472             if not chunk:
 473                 raise IncompleteRead(s)
 474             s = s + chunk
 475             amt = amt - len(chunk)
 476         return s
 477
 478     def getheader(self, name, default=None):
 479         if self.msg is None:
 480             raise ResponseNotReady()
 481         return self.msg.getheader(name, default)
 482
 483
 484 class HTTPConnection:
 485
 486     _http_vsn = 11
 487     _http_vsn_str = 'HTTP/1.1'
 488
 489     response_class = HTTPResponse
 490     default_port = HTTP_PORT
 491     auto_open = 1
 492     debuglevel = 0
 493     strict = 0
 494
 495     def __init__(self, host, port=None, strict=None):
 496         self.sock = None
 497         self._buffer = []
 498         self.__response = None
 499         self.__state = _CS_IDLE
 500
 501         self._set_hostport(host, port)
 502         if strict is not None:
 503             self.strict = strict
 504
 505     def _set_hostport(self, host, port):
 506         if port is None:
 507             i = host.find(':')
 508             if i >= 0:
 509                 try:
 510                     port = int(host[i+1:])
 511                 except ValueError:
 512                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 513                 host = host[:i]
 514             else:
 515                 port = self.default_port
 516         self.host = host
 517         self.port = port
 518
 519     def set_debuglevel(self, level):
 520         self.debuglevel = level
 521
 522     def connect(self):
 523         """Connect to the host and port specified in __init__."""
 524         msg = "getaddrinfo returns an empty list"
 525         for res in socket.getaddrinfo(self.host, self.port, 0,
 526                                       socket.SOCK_STREAM):
 527             af, socktype, proto, canonname, sa = res
 528             try:
 529                 self.sock = socket.socket(af, socktype, proto)
 530                 if self.debuglevel > 0:
 531                     print "connect: (%s, %s)" % (self.host, self.port)
 532                 self.sock.connect(sa)
 533             except socket.error, msg:
 534                 if self.debuglevel > 0:
 535                     print 'connect fail:', (self.host, self.port)
 536                 if self.sock:
 537                     self.sock.close()
 538                 self.sock = None
 539                 continue
 540             break
 541         if not self.sock:
 542             raise socket.error, msg
 543
 544     def close(self):
 545         """Close the connection to the HTTP server."""
 546         if self.sock:
 547             self.sock.close()   # close it manually... there may be other refs
 548             self.sock = None
 549         if self.__response:
 550             self.__response.close()
 551             self.__response = None
 552         self.__state = _CS_IDLE
 553
 554     def send(self, str):
 555         """Send `str' to the server."""
 556         if self.sock is None:
 557             if self.auto_open:
 558                 self.connect()
 559             else:
 560                 raise NotConnected()
 561
 562         # send the data to the server. if we get a broken pipe, then close
 563         # the socket. we want to reconnect when somebody tries to send again.
 564         #
 565         # NOTE: we DO propagate the error, though, because we cannot simply
 566         #       ignore the error... the caller will know if they can retry.
 567         if self.debuglevel > 0:
 568             print "send:", repr(str)
 569         try:
 570             self.sock.sendall(str)
 571         except socket.error, v:
 572             if v[0] == 32:      # Broken pipe
 573                 self.close()
 574             raise
 575
 576     def _output(self, s):
 577         """Add a line of output to the current request buffer.
 578
 579         Assumes that the line does *not* end with \\r\\n.
 580         """
 581         self._buffer.append(s)
 582
 583     def _send_output(self):
 584         """Send the currently buffered request and clear the buffer.
 585
 586         Appends an extra \\r\\n to the buffer.
 587         """
 588         self._buffer.extend(("", ""))
 589         msg = "\r\n".join(self._buffer)
 590         del self._buffer[:]
 591         self.send(msg)
 592
 593     def putrequest(self, method, url, skip_host=0):
 594         """Send a request to the server.
 595
 596         `method' specifies an HTTP request method, e.g. 'GET'.
 597         `url' specifies the object being requested, e.g. '/index.html'.
 598         """
 599
 600         # check if a prior response has been completed
 601         # XXX What if it hasn't?
 602         if self.__response and self.__response.isclosed():
 603             self.__response = None
 604
 605         #
 606         # in certain cases, we cannot issue another request on this connection.
 607         # this occurs when:
 608         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 609         #   2) a response to a previous request has signalled that it is going
 610         #      to close the connection upon completion.
 611         #   3) the headers for the previous response have not been read, thus
 612         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 613         #
 614         # if there is no prior response, then we can request at will.
 615         #
 616         # if point (2) is true, then we will have passed the socket to the
 617         # response (effectively meaning, "there is no prior response"), and
 618         # will open a new one when a new request is made.
 619         #
 620         # Note: if a prior response exists, then we *can* start a new request.
 621         #       We are not allowed to begin fetching the response to this new
 622         #       request, however, until that prior response is complete.
 623         #
 624         if self.__state == _CS_IDLE:
 625             self.__state = _CS_REQ_STARTED
 626         else:
 627             raise CannotSendRequest()
 628
 629         if not url:
 630             url = '/'
 631         str = '%s %s %s' % (method, url, self._http_vsn_str)
 632
 633         self._output(str)
 634
 635         if self._http_vsn == 11:
 636             # Issue some standard headers for better HTTP/1.1 compliance
 637
 638             if not skip_host:
 639                 # this header is issued *only* for HTTP/1.1
 640                 # connections. more specifically, this means it is
 641                 # only issued when the client uses the new
 642                 # HTTPConnection() class. backwards-compat clients
 643                 # will be using HTTP/1.0 and those clients may be
 644                 # issuing this header themselves. we should NOT issue
 645                 # it twice; some web servers (such as Apache) barf
 646                 # when they see two Host: headers
 647
 648                 # If we need a non-standard port,include it in the
 649                 # header.  If the request is going through a proxy,
 650                 # but the host of the actual URL, not the host of the
 651                 # proxy.
 652
 653                 netloc = ''
 654                 if url.startswith('http'):
 655                     nil, netloc, nil, nil, nil = urlsplit(url)
 656
 657                 if netloc:
 658                     self.putheader('Host', netloc)
 659                 elif self.port == HTTP_PORT:
 660                     self.putheader('Host', self.host)
 661                 else:
 662                     self.putheader('Host', "%s:%s" % (self.host, self.port))
 663
 664             # note: we are assuming that clients will not attempt to set these
 665             #       headers since *this* library must deal with the
 666             #       consequences. this also means that when the supporting
 667             #       libraries are updated to recognize other forms, then this
 668             #       code should be changed (removed or updated).
 669
 670             # we only want a Content-Encoding of "identity" since we don't
 671             # support encodings such as x-gzip or x-deflate.
 672             self.putheader('Accept-Encoding', 'identity')
 673
 674             # we can accept "chunked" Transfer-Encodings, but no others
 675             # NOTE: no TE header implies *only* "chunked"
 676             #self.putheader('TE', 'chunked')
 677
 678             # if TE is supplied in the header, then it must appear in a
 679             # Connection header.
 680             #self.putheader('Connection', 'TE')
 681
 682         else:
 683             # For HTTP/1.0, the server will assume "not chunked"
 684             pass
 685
 686     def putheader(self, header, value):
 687         """Send a request header line to the server.
 688
 689         For example: h.putheader('Accept', 'text/html')
 690         """
 691         if self.__state != _CS_REQ_STARTED:
 692             raise CannotSendHeader()
 693
 694         str = '%s: %s' % (header, value)
 695         self._output(str)
 696
 697     def endheaders(self):
 698         """Indicate that the last header line has been sent to the server."""
 699
 700         if self.__state == _CS_REQ_STARTED:
 701             self.__state = _CS_REQ_SENT
 702         else:
 703             raise CannotSendHeader()
 704
 705         self._send_output()
 706
 707     def request(self, method, url, body=None, headers={}):
 708         """Send a complete request to the server."""
 709
 710         try:
 711             self._send_request(method, url, body, headers)
 712         except socket.error, v:
 713             # trap 'Broken pipe' if we're allowed to automatically reconnect
 714             if v[0] != 32 or not self.auto_open:
 715                 raise
 716             # try one more time
 717             self._send_request(method, url, body, headers)
 718
 719     def _send_request(self, method, url, body, headers):
 720         # If headers already contains a host header, then define the
 721         # optional skip_host argument to putrequest().  The check is
 722         # harder because field names are case insensitive.
 723         if 'Host' in (headers
 724             or [k for k in headers.iterkeys() if k.lower() == "host"]):
 725             self.putrequest(method, url, skip_host=1)
 726         else:
 727             self.putrequest(method, url)
 728
 729         if body:
 730             self.putheader('Content-Length', str(len(body)))
 731         for hdr, value in headers.items():
 732             self.putheader(hdr, value)
 733         self.endheaders()
 734
 735         if body:
 736             self.send(body)
 737
 738     def getresponse(self):
 739         "Get the response from the server."
 740
 741         # check if a prior response has been completed
 742         if self.__response and self.__response.isclosed():
 743             self.__response = None
 744
 745         #
 746         # if a prior response exists, then it must be completed (otherwise, we
 747         # cannot read this response's header to determine the connection-close
 748         # behavior)
 749         #
 750         # note: if a prior response existed, but was connection-close, then the
 751         # socket and response were made independent of this HTTPConnection
 752         # object since a new request requires that we open a whole new
 753         # connection
 754         #
 755         # this means the prior response had one of two states:
 756         #   1) will_close: this connection was reset and the prior socket and
 757         #                  response operate independently
 758         #   2) persistent: the response was retained and we await its
 759         #                  isclosed() status to become true.
 760         #
 761         if self.__state != _CS_REQ_SENT or self.__response:
 762             raise ResponseNotReady()
 763
 764         if self.debuglevel > 0:
 765             response = self.response_class(self.sock, self.debuglevel,
 766                                            strict=self.strict)
 767         else:
 768             response = self.response_class(self.sock, strict=self.strict)
 769
 770         response.begin()
 771         assert response.will_close != _UNKNOWN
 772         self.__state = _CS_IDLE
 773
 774         if response.will_close:
 775             # this effectively passes the connection to the response
 776             self.close()
 777         else:
 778             # remember this, so we can tell when it is complete
 779             self.__response = response
 780
 781         return response
 782
 783 # The next several classes are used to define FakeSocket,a socket-like
 784 # interface to an SSL connection.
 785
 786 # The primary complexity comes from faking a makefile() method.  The
 787 # standard socket makefile() implementation calls dup() on the socket
 788 # file descriptor.  As a consequence, clients can call close() on the
 789 # parent socket and its makefile children in any order.  The underlying
 790 # socket isn't closed until they are all closed.
 791
 792 # The implementation uses reference counting to keep the socket open
 793 # until the last client calls close().  SharedSocket keeps track of
 794 # the reference counting and SharedSocketClient provides an constructor
 795 # and close() method that call incref() and decref() correctly.
 796
 797 class SharedSocket:
 798
 799     def __init__(self, sock):
 800         self.sock = sock
 801         self._refcnt = 0
 802
 803     def incref(self):
 804         self._refcnt += 1
 805
 806     def decref(self):
 807         self._refcnt -= 1
 808         assert self._refcnt >= 0
 809         if self._refcnt == 0:
 810             self.sock.close()
 811
 812     def __del__(self):
 813         self.sock.close()
 814
 815 class SharedSocketClient:
 816
 817     def __init__(self, shared):
 818         self._closed = 0
 819         self._shared = shared
 820         self._shared.incref()
 821         self._sock = shared.sock
 822
 823     def close(self):
 824         if not self._closed:
 825             self._shared.decref()
 826             self._closed = 1
 827             self._shared = None
 828
 829 class SSLFile(SharedSocketClient):
 830     """File-like object wrapping an SSL socket."""
 831
 832     BUFSIZE = 8192
 833
 834     def __init__(self, sock, ssl, bufsize=None):
 835         SharedSocketClient.__init__(self, sock)
 836         self._ssl = ssl
 837         self._buf = ''
 838         self._bufsize = bufsize or self.__class__.BUFSIZE
 839
 840     def _read(self):
 841         buf = ''
 842         # put in a loop so that we retry on transient errors
 843         while 1:
 844             try:
 845                 buf = self._ssl.read(self._bufsize)
 846             except socket.sslerror, err:
 847                 if (err[0] == socket.SSL_ERROR_WANT_READ
 848                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
 849                     continue
 850                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
 851                     or err[0] == socket.SSL_ERROR_EOF):
 852                     break
 853                 raise
 854             except socket.error, err:
 855                 if err[0] == errno.EINTR:
 856                     continue
 857                 if err[0] == errno.EBADF:
 858                     # XXX socket was closed?
 859                     break
 860                 raise
 861             else:
 862                 break
 863         return buf
 864
 865     def read(self, size=None):
 866         L = [self._buf]
 867         avail = len(self._buf)
 868         while size is None or avail < size:
 869             s = self._read()
 870             if s == '':
 871                 break
 872             L.append(s)
 873             avail += len(s)
 874         all = "".join(L)
 875         if size is None:
 876             self._buf = ''
 877             return all
 878         else:
 879             self._buf = all[size:]
 880             return all[:size]
 881
 882     def readline(self):
 883         L = [self._buf]
 884         self._buf = ''
 885         while 1:
 886             i = L[-1].find("\n")
 887             if i >= 0:
 888                 break
 889             s = self._read()
 890             if s == '':
 891                 break
 892             L.append(s)
 893         if i == -1:
 894             # loop exited because there is no more data
 895             return "".join(L)
 896         else:
 897             all = "".join(L)
 898             # XXX could do enough bookkeeping not to do a 2nd search
 899             i = all.find("\n") + 1
 900             line = all[:i]
 901             self._buf = all[i:]
 902             return line
 903
 904 class FakeSocket(SharedSocketClient):
 905
 906     class _closedsocket:
 907         def __getattr__(self, name):
 908             raise error(9, 'Bad file descriptor')
 909
 910     def __init__(self, sock, ssl):
 911         sock = SharedSocket(sock)
 912         SharedSocketClient.__init__(self, sock)
 913         self._ssl = ssl
 914
 915     def close(self):
 916         SharedSocketClient.close(self)
 917         self._sock = self.__class__._closedsocket()
 918
 919     def makefile(self, mode, bufsize=None):
 920         if mode != 'r' and mode != 'rb':
 921             raise UnimplementedFileMode()
 922         return SSLFile(self._shared, self._ssl, bufsize)
 923
 924     def send(self, stuff, flags = 0):
 925         return self._ssl.write(stuff)
 926
 927     sendall = send
 928
 929     def recv(self, len = 1024, flags = 0):
 930         return self._ssl.read(len)
 931
 932     def __getattr__(self, attr):
 933         return getattr(self._sock, attr)
 934
 935
 936 class HTTPSConnection(HTTPConnection):
 937     "This class allows communication via SSL."
 938
 939     default_port = HTTPS_PORT
 940
 941     def __init__(self, host, port=None, key_file=None, cert_file=None,
 942                  strict=None):
 943         HTTPConnection.__init__(self, host, port, strict)
 944         self.key_file = key_file
 945         self.cert_file = cert_file
 946
 947     def connect(self):
 948         "Connect to a host on a given (SSL) port."
 949
 950         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 951         sock.connect((self.host, self.port))
 952         realsock = sock
 953         if hasattr(sock, "_sock"):
 954             realsock = sock._sock
 955         ssl = socket.ssl(realsock, self.key_file, self.cert_file)
 956         self.sock = FakeSocket(sock, ssl)
 957
 958
 959 class HTTP:
 960     "Compatibility class with httplib.py from 1.5."
 961
 962     _http_vsn = 10
 963     _http_vsn_str = 'HTTP/1.0'
 964
 965     debuglevel = 0
 966
 967     _connection_class = HTTPConnection
 968
 969     def __init__(self, host='', port=None, strict=None):
 970         "Provide a default host, since the superclass requires one."
 971
 972         # some joker passed 0 explicitly, meaning default port
 973         if port == 0:
 974             port = None
 975
 976         # Note that we may pass an empty string as the host; this will throw
 977         # an error when we attempt to connect. Presumably, the client code
 978         # will call connect before then, with a proper host.
 979         self._setup(self._connection_class(host, port, strict))
 980
 981     def _setup(self, conn):
 982         self._conn = conn
 983
 984         # set up delegation to flesh out interface
 985         self.send = conn.send
 986         self.putrequest = conn.putrequest
 987         self.endheaders = conn.endheaders
 988         self.set_debuglevel = conn.set_debuglevel
 989
 990         conn._http_vsn = self._http_vsn
 991         conn._http_vsn_str = self._http_vsn_str
 992
 993         self.file = None
 994
 995     def connect(self, host=None, port=None):
 996         "Accept arguments to set the host/port, since the superclass doesn't."
 997
 998         if host is not None:
 999             self._conn._set_hostport(host, port)
1000         self._conn.connect()
1001
1002     def getfile(self):
1003         "Provide a getfile, since the superclass' does not use this concept."
1004         return self.file
1005
1006     def putheader(self, header, *values):
1007         "The superclass allows only one value argument."
1008         self._conn.putheader(header, '\r\n\t'.join(values))
1009
1010     def getreply(self):
1011         """Compat definition since superclass does not define it.
1012
1013         Returns a tuple consisting of:
1014         - server status code (e.g. '200' if all goes well)
1015         - server "reason" corresponding to status code
1016         - any RFC822 headers in the response from the server
1017         """
1018         try:
1019             response = self._conn.getresponse()
1020         except BadStatusLine, e:
1021             ### hmm. if getresponse() ever closes the socket on a bad request,
1022             ### then we are going to have problems with self.sock
1023
1024             ### should we keep this behavior? do people use it?
1025             # keep the socket open (as a file), and return it
1026             self.file = self._conn.sock.makefile('rb', 0)
1027
1028             # close our socket -- we want to restart after any protocol error
1029             self.close()
1030
1031             self.headers = None
1032             return -1, e.line, None
1033
1034         self.headers = response.msg
1035         self.file = response.fp
1036         return response.status, response.reason, response.msg
1037
1038     def close(self):
1039         self._conn.close()
1040
1041         # note that self.file == response.fp, which gets closed by the
1042         # superclass. just clear the object ref here.
1043         ### hmm. messy. if status==-1, then self.file is owned by us.
1044         ### well... we aren't explicitly closing, but losing this ref will
1045         ### do it
1046         self.file = None
1047
1048 if hasattr(socket, 'ssl'):
1049     class HTTPS(HTTP):
1050         """Compatibility with 1.5 httplib interface
1051
1052         Python 1.5.2 did not have an HTTPS class, but it defined an
1053         interface for sending http requests that is also useful for
1054         https.
1055         """
1056
1057         _connection_class = HTTPSConnection
1058
1059         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1060                      strict=None):
1061             # provide a default host, pass the X509 cert info
1062
1063             # urf. compensate for bad input.
1064             if port == 0:
1065                 port = None
1066             self._setup(self._connection_class(host, port, key_file,
1067                                                cert_file, strict))
1068
1069             # we never actually use these for anything, but we keep them
1070             # here for compatibility with post-1.5.2 CVS.
1071             self.key_file = key_file
1072             self.cert_file = cert_file
1073
1074
1075 class HTTPException(Exception):
1076     # Subclasses that define an __init__ must call Exception.__init__
1077     # or define self.args.  Otherwise, str() will fail.
1078     pass
1079
1080 class NotConnected(HTTPException):
1081     pass
1082
1083 class InvalidURL(HTTPException):
1084     pass
1085
1086 class UnknownProtocol(HTTPException):
1087     def __init__(self, version):
1088         self.args = version,
1089         self.version = version
1090
1091 class UnknownTransferEncoding(HTTPException):
1092     pass
1093
1094 class UnimplementedFileMode(HTTPException):
1095     pass
1096
1097 class IncompleteRead(HTTPException):
1098     def __init__(self, partial):
1099         self.args = partial,
1100         self.partial = partial
1101
1102 class ImproperConnectionState(HTTPException):
1103     pass
1104
1105 class CannotSendRequest(ImproperConnectionState):
1106     pass
1107
1108 class CannotSendHeader(ImproperConnectionState):
1109     pass
1110
1111 class ResponseNotReady(ImproperConnectionState):
1112     pass
1113
1114 class BadStatusLine(HTTPException):
1115     def __init__(self, line):
1116         self.args = line,
1117         self.line = line
1118
1119 # for backwards compatibility
1120 error = HTTPException
1121
1122 class LineAndFileWrapper:
1123     """A limited file-like object for HTTP/0.9 responses."""
1124
1125     # The status-line parsing code calls readline(), which normally
1126     # get the HTTP status line.  For a 0.9 response, however, this is
1127     # actually the first line of the body!  Clients need to get a
1128     # readable file object that contains that line.
1129
1130     def __init__(self, line, file):
1131         self._line = line
1132         self._file = file
1133         self._line_consumed = 0
1134         self._line_offset = 0
1135         self._line_left = len(line)
1136
1137     def __getattr__(self, attr):
1138         return getattr(self._file, attr)
1139
1140     def _done(self):
1141         # called when the last byte is read from the line.  After the
1142         # call, all read methods are delegated to the underlying file
1143         # obhect.
1144         self._line_consumed = 1
1145         self.read = self._file.read
1146         self.readline = self._file.readline
1147         self.readlines = self._file.readlines
1148
1149     def read(self, amt=None):
1150         assert not self._line_consumed and self._line_left
1151         if amt is None or amt > self._line_left:
1152             s = self._line[self._line_offset:]
1153             self._done()
1154             if amt is None:
1155                 return s + self._file.read()
1156             else:
1157                 return s + self._file.read(amt - len(s))
1158         else:
1159             assert amt <= self._line_left
1160             i = self._line_offset
1161             j = i + amt
1162             s = self._line[i:j]
1163             self._line_offset = j
1164             self._line_left -= amt
1165             if self._line_left == 0:
1166                 self._done()
1167             return s
1168
1169     def readline(self):
1170         s = self._line[self._line_offset:]
1171         self._done()
1172         return s
1173
1174     def readlines(self, size=None):
1175         L = [self._line[self._line_offset:]]
1176         self._done()
1177         if size is None:
1178             return L + self._file.readlines()
1179         else:
1180             return L + self._file.readlines(size)
1181
1182 def test():
1183     """Test this module.
1184
1185     A hodge podge of tests collected here, because they have too many
1186     external dependencies for the regular test suite.
1187     """
1188
1189     import sys
1190     import getopt
1191     opts, args = getopt.getopt(sys.argv[1:], 'd')
1192     dl = 0
1193     for o, a in opts:
1194         if o == '-d': dl = dl + 1
1195     host = 'www.python.org'
1196     selector = '/'
1197     if args[0:]: host = args[0]
1198     if args[1:]: selector = args[1]
1199     h = HTTP()
1200     h.set_debuglevel(dl)
1201     h.connect(host)
1202     h.putrequest('GET', selector)
1203     h.endheaders()
1204     status, reason, headers = h.getreply()
1205     print 'status =', status
1206     print 'reason =', reason
1207     print "read", len(h.getfile().read())
1208     print
1209     if headers:
1210         for header in headers.headers: print header.strip()
1211     print
1212
1213     # minimal test that code to extract host from url works
1214     class HTTP11(HTTP):
1215         _http_vsn = 11
1216         _http_vsn_str = 'HTTP/1.1'
1217
1218     h = HTTP11('www.python.org')
1219     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1220     h.endheaders()
1221     h.getreply()
1222     h.close()
1223
1224     if hasattr(socket, 'ssl'):
1225
1226         for host, selector in (('sourceforge.net', '/projects/python'),
1227                                ):
1228             print "https://%s%s" % (host, selector)
1229             hs = HTTPS()
1230             hs.set_debuglevel(dl)
1231             hs.connect(host)
1232             hs.putrequest('GET', selector)
1233             hs.endheaders()
1234             status, reason, headers = hs.getreply()
1235             print 'status =', status
1236             print 'reason =', reason
1237             print "read", len(hs.getfile().read())
1238             print
1239             if headers:
1240                 for header in headers.headers: print header.strip()
1241             print
1242
1243 if __name__ == '__main__':
1244     test()