Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 class HTTPMessage(mimetools.Message):
  97
  98     def addheader(self, key, value):
  99         """Add header for field key handling repeats."""
 100         prev = self.dict.get(key)
 101         if prev is None:
 102             self.dict[key] = value
 103         else:
 104             combined = ", ".join((prev, value))
 105             self.dict[key] = combined
 106
 107     def addcontinue(self, key, more):
 108         """Add more field data from a continuation line."""
 109         prev = self.dict[key]
 110         self.dict[key] = prev + "\n " + more
 111
 112     def readheaders(self):
 113         """Read header lines.
 114
 115         Read header lines up to the entirely blank line that terminates them.
 116         The (normally blank) line that ends the headers is skipped, but not
 117         included in the returned list.  If a non-header line ends the headers,
 118         (which is an error), an attempt is made to backspace over it; it is
 119         never included in the returned list.
 120
 121         The variable self.status is set to the empty string if all went well,
 122         otherwise it is an error message.  The variable self.headers is a
 123         completely uninterpreted list of lines contained in the header (so
 124         printing them will reproduce the header exactly as it appears in the
 125         file).
 126
 127         If multiple header fields with the same name occur, they are combined
 128         according to the rules in RFC 2616 sec 4.2:
 129
 130         Appending each subsequent field-value to the first, each separated
 131         by a comma. The order in which header fields with the same field-name
 132         are received is significant to the interpretation of the combined
 133         field value.
 134         """
 135         # XXX The implementation overrides the readheaders() method of
 136         # rfc822.Message.  The base class design isn't amenable to
 137         # customized behavior here so the method here is a copy of the
 138         # base class code with a few small changes.
 139
 140         self.dict = {}
 141         self.unixfrom = ''
 142         self.headers = hlist = []
 143         self.status = ''
 144         headerseen = ""
 145         firstline = 1
 146         startofline = unread = tell = None
 147         if hasattr(self.fp, 'unread'):
 148             unread = self.fp.unread
 149         elif self.seekable:
 150             tell = self.fp.tell
 151         while True:
 152             if tell:
 153                 try:
 154                     startofline = tell()
 155                 except IOError:
 156                     startofline = tell = None
 157                     self.seekable = 0
 158             line = self.fp.readline()
 159             if not line:
 160                 self.status = 'EOF in headers'
 161                 break
 162             # Skip unix From name time lines
 163             if firstline and line.startswith('From '):
 164                 self.unixfrom = self.unixfrom + line
 165                 continue
 166             firstline = 0
 167             if headerseen and line[0] in ' \t':
 168                 # XXX Not sure if continuation lines are handled properly
 169                 # for http and/or for repeating headers
 170                 # It's a continuation line.
 171                 hlist.append(line)
 172                 self.addcontinue(headerseen, line.strip())
 173                 continue
 174             elif self.iscomment(line):
 175                 # It's a comment.  Ignore it.
 176                 continue
 177             elif self.islast(line):
 178                 # Note! No pushback here!  The delimiter line gets eaten.
 179                 break
 180             headerseen = self.isheader(line)
 181             if headerseen:
 182                 # It's a legal header line, save it.
 183                 hlist.append(line)
 184                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 185                 continue
 186             else:
 187                 # It's not a header line; throw it back and stop here.
 188                 if not self.dict:
 189                     self.status = 'No headers'
 190                 else:
 191                     self.status = 'Non-header line where header expected'
 192                 # Try to undo the read.
 193                 if unread:
 194                     unread(line)
 195                 elif tell:
 196                     self.fp.seek(startofline)
 197                 else:
 198                     self.status = self.status + '; bad seek'
 199                 break
 200
 201 class HTTPResponse:
 202
 203     # strict: If true, raise BadStatusLine if the status line can't be
 204     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 205     # false because it prevents clients from talking to HTTP/0.9
 206     # servers.  Note that a response with a sufficiently corrupted
 207     # status line will look like an HTTP/0.9 response.
 208
 209     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 210
 211     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 212         self.fp = sock.makefile('rb', 0)
 213         self.debuglevel = debuglevel
 214         self.strict = strict
 215         self._method = method
 216
 217         self.msg = None
 218
 219         # from the Status-Line of the response
 220         self.version = _UNKNOWN # HTTP-Version
 221         self.status = _UNKNOWN  # Status-Code
 222         self.reason = _UNKNOWN  # Reason-Phrase
 223
 224         self.chunked = _UNKNOWN         # is "chunked" being used?
 225         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 226         self.length = _UNKNOWN          # number of bytes left in response
 227         self.will_close = _UNKNOWN      # conn will close at end of response
 228
 229     def _read_status(self):
 230         # Initialize with Simple-Response defaults
 231         line = self.fp.readline()
 232         if self.debuglevel > 0:
 233             print "reply:", repr(line)
 234         if not line:
 235             # Presumably, the server closed the connection before
 236             # sending a valid response.
 237             raise BadStatusLine(line)
 238         try:
 239             [version, status, reason] = line.split(None, 2)
 240         except ValueError:
 241             try:
 242                 [version, status] = line.split(None, 1)
 243                 reason = ""
 244             except ValueError:
 245                 # empty version will cause next test to fail and status
 246                 # will be treated as 0.9 response.
 247                 version = ""
 248         if not version.startswith('HTTP/'):
 249             if self.strict:
 250                 self.close()
 251                 raise BadStatusLine(line)
 252             else:
 253                 # assume it's a Simple-Response from an 0.9 server
 254                 self.fp = LineAndFileWrapper(line, self.fp)
 255                 return "HTTP/0.9", 200, ""
 256
 257         # The status code is a three-digit number
 258         try:
 259             status = int(status)
 260             if status < 100 or status > 999:
 261                 raise BadStatusLine(line)
 262         except ValueError:
 263             raise BadStatusLine(line)
 264         return version, status, reason
 265
 266     def begin(self):
 267         if self.msg is not None:
 268             # we've already started reading the response
 269             return
 270
 271         # read until we get a non-100 response
 272         while True:
 273             version, status, reason = self._read_status()
 274             if status != 100:
 275                 break
 276             # skip the header from the 100 response
 277             while True:
 278                 skip = self.fp.readline().strip()
 279                 if not skip:
 280                     break
 281                 if self.debuglevel > 0:
 282                     print "header:", skip
 283
 284         self.status = status
 285         self.reason = reason.strip()
 286         if version == 'HTTP/1.0':
 287             self.version = 10
 288         elif version.startswith('HTTP/1.'):
 289             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 290         elif version == 'HTTP/0.9':
 291             self.version = 9
 292         else:
 293             raise UnknownProtocol(version)
 294
 295         if self.version == 9:
 296             self.chunked = 0
 297             self.will_close = 1
 298             self.msg = HTTPMessage(StringIO())
 299             return
 300
 301         self.msg = HTTPMessage(self.fp, 0)
 302         if self.debuglevel > 0:
 303             for hdr in self.msg.headers:
 304                 print "header:", hdr,
 305
 306         # don't let the msg keep an fp
 307         self.msg.fp = None
 308
 309         # are we using the chunked-style of transfer encoding?
 310         tr_enc = self.msg.getheader('transfer-encoding')
 311         if tr_enc and tr_enc.lower() == "chunked":
 312             self.chunked = 1
 313             self.chunk_left = None
 314         else:
 315             self.chunked = 0
 316
 317         # will the connection close at the end of the response?
 318         self.will_close = self._check_close()
 319
 320         # do we have a Content-Length?
 321         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 322         length = self.msg.getheader('content-length')
 323         if length and not self.chunked:
 324             try:
 325                 self.length = int(length)
 326             except ValueError:
 327                 self.length = None
 328         else:
 329             self.length = None
 330
 331         # does the body have a fixed length? (of zero)
 332         if (status == 204 or            # No Content
 333             status == 304 or            # Not Modified
 334             100 <= status < 200 or      # 1xx codes
 335             self._method == 'HEAD'):
 336             self.length = 0
 337
 338         # if the connection remains open, and we aren't using chunked, and
 339         # a content-length was not provided, then assume that the connection
 340         # WILL close.
 341         if not self.will_close and \
 342            not self.chunked and \
 343            self.length is None:
 344             self.will_close = 1
 345
 346     def _check_close(self):
 347         if self.version == 11:
 348             # An HTTP/1.1 proxy is assumed to stay open unless
 349             # explicitly closed.
 350             conn = self.msg.getheader('connection')
 351             if conn and conn.lower().find("close") >= 0:
 352                 return True
 353             return False
 354
 355         # An HTTP/1.0 response with a Connection header is probably
 356         # the result of a confused proxy.  Ignore it.
 357
 358         # For older HTTP, Keep-Alive indiciates persistent connection.
 359         if self.msg.getheader('keep-alive'):
 360             return False
 361
 362         # Proxy-Connection is a netscape hack.
 363         pconn = self.msg.getheader('proxy-connection')
 364         if pconn and pconn.lower().find("keep-alive") >= 0:
 365             return False
 366
 367         # otherwise, assume it will close
 368         return True
 369
 370     def close(self):
 371         if self.fp:
 372             self.fp.close()
 373             self.fp = None
 374
 375     def isclosed(self):
 376         # NOTE: it is possible that we will not ever call self.close(). This
 377         #       case occurs when will_close is TRUE, length is None, and we
 378         #       read up to the last byte, but NOT past it.
 379         #
 380         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 381         #          called, meaning self.isclosed() is meaningful.
 382         return self.fp is None
 383
 384     def read(self, amt=None):
 385         if self.fp is None:
 386             return ''
 387
 388         if self.chunked:
 389             return self._read_chunked(amt)
 390
 391         if amt is None:
 392             # unbounded read
 393             if self.will_close:
 394                 s = self.fp.read()
 395             else:
 396                 s = self._safe_read(self.length)
 397             self.close()        # we read everything
 398             return s
 399
 400         if self.length is not None:
 401             if amt > self.length:
 402                 # clip the read to the "end of response"
 403                 amt = self.length
 404             self.length -= amt
 405
 406         # we do not use _safe_read() here because this may be a .will_close
 407         # connection, and the user is reading more bytes than will be provided
 408         # (for example, reading in 1k chunks)
 409         s = self.fp.read(amt)
 410
 411         return s
 412
 413     def _read_chunked(self, amt):
 414         assert self.chunked != _UNKNOWN
 415         chunk_left = self.chunk_left
 416         value = ''
 417
 418         # XXX This accumulates chunks by repeated string concatenation,
 419         # which is not efficient as the number or size of chunks gets big.
 420         while True:
 421             if chunk_left is None:
 422                 line = self.fp.readline()
 423                 i = line.find(';')
 424                 if i >= 0:
 425                     line = line[:i] # strip chunk-extensions
 426                 chunk_left = int(line, 16)
 427                 if chunk_left == 0:
 428                     break
 429             if amt is None:
 430                 value += self._safe_read(chunk_left)
 431             elif amt < chunk_left:
 432                 value += self._safe_read(amt)
 433                 self.chunk_left = chunk_left - amt
 434                 return value
 435             elif amt == chunk_left:
 436                 value += self._safe_read(amt)
 437                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 438                 self.chunk_left = None
 439                 return value
 440             else:
 441                 value += self._safe_read(chunk_left)
 442                 amt -= chunk_left
 443
 444             # we read the whole chunk, get another
 445             self._safe_read(2)      # toss the CRLF at the end of the chunk
 446             chunk_left = None
 447
 448         # read and discard trailer up to the CRLF terminator
 449         ### note: we shouldn't have any trailers!
 450         while True:
 451             line = self.fp.readline()
 452             if line == '\r\n':
 453                 break
 454
 455         # we read everything; close the "file"
 456         self.close()
 457
 458         return value
 459
 460     def _safe_read(self, amt):
 461         """Read the number of bytes requested, compensating for partial reads.
 462
 463         Normally, we have a blocking socket, but a read() can be interrupted
 464         by a signal (resulting in a partial read).
 465
 466         Note that we cannot distinguish between EOF and an interrupt when zero
 467         bytes have been read. IncompleteRead() will be raised in this
 468         situation.
 469
 470         This function should be used when <amt> bytes "should" be present for
 471         reading. If the bytes are truly not available (due to EOF), then the
 472         IncompleteRead exception can be used to detect the problem.
 473         """
 474         s = ''
 475         while amt > 0:
 476             chunk = self.fp.read(amt)
 477             if not chunk:
 478                 raise IncompleteRead(s)
 479             s += chunk
 480             amt -= len(chunk)
 481         return s
 482
 483     def getheader(self, name, default=None):
 484         if self.msg is None:
 485             raise ResponseNotReady()
 486         return self.msg.getheader(name, default)
 487
 488
 489 class HTTPConnection:
 490
 491     _http_vsn = 11
 492     _http_vsn_str = 'HTTP/1.1'
 493
 494     response_class = HTTPResponse
 495     default_port = HTTP_PORT
 496     auto_open = 1
 497     debuglevel = 0
 498     strict = 0
 499
 500     def __init__(self, host, port=None, strict=None):
 501         self.sock = None
 502         self._buffer = []
 503         self.__response = None
 504         self.__state = _CS_IDLE
 505         self._method = None
 506
 507         self._set_hostport(host, port)
 508         if strict is not None:
 509             self.strict = strict
 510
 511     def _set_hostport(self, host, port):
 512         if port is None:
 513             i = host.find(':')
 514             if i >= 0:
 515                 try:
 516                     port = int(host[i+1:])
 517                 except ValueError:
 518                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 519                 host = host[:i]
 520             else:
 521                 port = self.default_port
 522         self.host = host
 523         self.port = port
 524
 525     def set_debuglevel(self, level):
 526         self.debuglevel = level
 527
 528     def connect(self):
 529         """Connect to the host and port specified in __init__."""
 530         msg = "getaddrinfo returns an empty list"
 531         for res in socket.getaddrinfo(self.host, self.port, 0,
 532                                       socket.SOCK_STREAM):
 533             af, socktype, proto, canonname, sa = res
 534             try:
 535                 self.sock = socket.socket(af, socktype, proto)
 536                 if self.debuglevel > 0:
 537                     print "connect: (%s, %s)" % (self.host, self.port)
 538                 self.sock.connect(sa)
 539             except socket.error, msg:
 540                 if self.debuglevel > 0:
 541                     print 'connect fail:', (self.host, self.port)
 542                 if self.sock:
 543                     self.sock.close()
 544                 self.sock = None
 545                 continue
 546             break
 547         if not self.sock:
 548             raise socket.error, msg
 549
 550     def close(self):
 551         """Close the connection to the HTTP server."""
 552         if self.sock:
 553             self.sock.close()   # close it manually... there may be other refs
 554             self.sock = None
 555         if self.__response:
 556             self.__response.close()
 557             self.__response = None
 558         self.__state = _CS_IDLE
 559
 560     def send(self, str):
 561         """Send `str' to the server."""
 562         if self.sock is None:
 563             if self.auto_open:
 564                 self.connect()
 565             else:
 566                 raise NotConnected()
 567
 568         # send the data to the server. if we get a broken pipe, then close
 569         # the socket. we want to reconnect when somebody tries to send again.
 570         #
 571         # NOTE: we DO propagate the error, though, because we cannot simply
 572         #       ignore the error... the caller will know if they can retry.
 573         if self.debuglevel > 0:
 574             print "send:", repr(str)
 575         try:
 576             self.sock.sendall(str)
 577         except socket.error, v:
 578             if v[0] == 32:      # Broken pipe
 579                 self.close()
 580             raise
 581
 582     def _output(self, s):
 583         """Add a line of output to the current request buffer.
 584
 585         Assumes that the line does *not* end with \\r\\n.
 586         """
 587         self._buffer.append(s)
 588
 589     def _send_output(self):
 590         """Send the currently buffered request and clear the buffer.
 591
 592         Appends an extra \\r\\n to the buffer.
 593         """
 594         self._buffer.extend(("", ""))
 595         msg = "\r\n".join(self._buffer)
 596         del self._buffer[:]
 597         self.send(msg)
 598
 599     def putrequest(self, method, url, skip_host=0):
 600         """Send a request to the server.
 601
 602         `method' specifies an HTTP request method, e.g. 'GET'.
 603         `url' specifies the object being requested, e.g. '/index.html'.
 604         """
 605
 606         # if a prior response has been completed, then forget about it.
 607         if self.__response and self.__response.isclosed():
 608             self.__response = None
 609
 610         #
 611         # in certain cases, we cannot issue another request on this connection.
 612         # this occurs when:
 613         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 614         #   2) a response to a previous request has signalled that it is going
 615         #      to close the connection upon completion.
 616         #   3) the headers for the previous response have not been read, thus
 617         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 618         #
 619         # if there is no prior response, then we can request at will.
 620         #
 621         # if point (2) is true, then we will have passed the socket to the
 622         # response (effectively meaning, "there is no prior response"), and
 623         # will open a new one when a new request is made.
 624         #
 625         # Note: if a prior response exists, then we *can* start a new request.
 626         #       We are not allowed to begin fetching the response to this new
 627         #       request, however, until that prior response is complete.
 628         #
 629         if self.__state == _CS_IDLE:
 630             self.__state = _CS_REQ_STARTED
 631         else:
 632             raise CannotSendRequest()
 633
 634         # Save the method we use, we need it later in the response phase
 635         self._method = method
 636         if not url:
 637             url = '/'
 638         str = '%s %s %s' % (method, url, self._http_vsn_str)
 639
 640         self._output(str)
 641
 642         if self._http_vsn == 11:
 643             # Issue some standard headers for better HTTP/1.1 compliance
 644
 645             if not skip_host:
 646                 # this header is issued *only* for HTTP/1.1
 647                 # connections. more specifically, this means it is
 648                 # only issued when the client uses the new
 649                 # HTTPConnection() class. backwards-compat clients
 650                 # will be using HTTP/1.0 and those clients may be
 651                 # issuing this header themselves. we should NOT issue
 652                 # it twice; some web servers (such as Apache) barf
 653                 # when they see two Host: headers
 654
 655                 # If we need a non-standard port,include it in the
 656                 # header.  If the request is going through a proxy,
 657                 # but the host of the actual URL, not the host of the
 658                 # proxy.
 659
 660                 netloc = ''
 661                 if url.startswith('http'):
 662                     nil, netloc, nil, nil, nil = urlsplit(url)
 663
 664                 if netloc:
 665                     self.putheader('Host', netloc.encode("idna"))
 666                 elif self.port == HTTP_PORT:
 667                     self.putheader('Host', self.host.encode("idna"))
 668                 else:
 669                     self.putheader('Host', "%s:%s" % (self.host.encode("idna"), self.port))
 670
 671             # note: we are assuming that clients will not attempt to set these
 672             #       headers since *this* library must deal with the
 673             #       consequences. this also means that when the supporting
 674             #       libraries are updated to recognize other forms, then this
 675             #       code should be changed (removed or updated).
 676
 677             # we only want a Content-Encoding of "identity" since we don't
 678             # support encodings such as x-gzip or x-deflate.
 679             self.putheader('Accept-Encoding', 'identity')
 680
 681             # we can accept "chunked" Transfer-Encodings, but no others
 682             # NOTE: no TE header implies *only* "chunked"
 683             #self.putheader('TE', 'chunked')
 684
 685             # if TE is supplied in the header, then it must appear in a
 686             # Connection header.
 687             #self.putheader('Connection', 'TE')
 688
 689         else:
 690             # For HTTP/1.0, the server will assume "not chunked"
 691             pass
 692
 693     def putheader(self, header, value):
 694         """Send a request header line to the server.
 695
 696         For example: h.putheader('Accept', 'text/html')
 697         """
 698         if self.__state != _CS_REQ_STARTED:
 699             raise CannotSendHeader()
 700
 701         str = '%s: %s' % (header, value)
 702         self._output(str)
 703
 704     def endheaders(self):
 705         """Indicate that the last header line has been sent to the server."""
 706
 707         if self.__state == _CS_REQ_STARTED:
 708             self.__state = _CS_REQ_SENT
 709         else:
 710             raise CannotSendHeader()
 711
 712         self._send_output()
 713
 714     def request(self, method, url, body=None, headers={}):
 715         """Send a complete request to the server."""
 716
 717         try:
 718             self._send_request(method, url, body, headers)
 719         except socket.error, v:
 720             # trap 'Broken pipe' if we're allowed to automatically reconnect
 721             if v[0] != 32 or not self.auto_open:
 722                 raise
 723             # try one more time
 724             self._send_request(method, url, body, headers)
 725
 726     def _send_request(self, method, url, body, headers):
 727         # If headers already contains a host header, then define the
 728         # optional skip_host argument to putrequest().  The check is
 729         # harder because field names are case insensitive.
 730         if 'Host' in (headers
 731             or [k for k in headers.iterkeys() if k.lower() == "host"]):
 732             self.putrequest(method, url, skip_host=1)
 733         else:
 734             self.putrequest(method, url)
 735
 736         if body:
 737             self.putheader('Content-Length', str(len(body)))
 738         for hdr, value in headers.iteritems():
 739             self.putheader(hdr, value)
 740         self.endheaders()
 741
 742         if body:
 743             self.send(body)
 744
 745     def getresponse(self):
 746         "Get the response from the server."
 747
 748         # if a prior response has been completed, then forget about it.
 749         if self.__response and self.__response.isclosed():
 750             self.__response = None
 751
 752         #
 753         # if a prior response exists, then it must be completed (otherwise, we
 754         # cannot read this response's header to determine the connection-close
 755         # behavior)
 756         #
 757         # note: if a prior response existed, but was connection-close, then the
 758         # socket and response were made independent of this HTTPConnection
 759         # object since a new request requires that we open a whole new
 760         # connection
 761         #
 762         # this means the prior response had one of two states:
 763         #   1) will_close: this connection was reset and the prior socket and
 764         #                  response operate independently
 765         #   2) persistent: the response was retained and we await its
 766         #                  isclosed() status to become true.
 767         #
 768         if self.__state != _CS_REQ_SENT or self.__response:
 769             raise ResponseNotReady()
 770
 771         if self.debuglevel > 0:
 772             response = self.response_class(self.sock, self.debuglevel,
 773                                            strict=self.strict,
 774                                            method=self._method)
 775         else:
 776             response = self.response_class(self.sock, strict=self.strict,
 777                                            method=self._method)
 778
 779         response.begin()
 780         assert response.will_close != _UNKNOWN
 781         self.__state = _CS_IDLE
 782
 783         if response.will_close:
 784             # this effectively passes the connection to the response
 785             self.close()
 786         else:
 787             # remember this, so we can tell when it is complete
 788             self.__response = response
 789
 790         return response
 791
 792 # The next several classes are used to define FakeSocket,a socket-like
 793 # interface to an SSL connection.
 794
 795 # The primary complexity comes from faking a makefile() method.  The
 796 # standard socket makefile() implementation calls dup() on the socket
 797 # file descriptor.  As a consequence, clients can call close() on the
 798 # parent socket and its makefile children in any order.  The underlying
 799 # socket isn't closed until they are all closed.
 800
 801 # The implementation uses reference counting to keep the socket open
 802 # until the last client calls close().  SharedSocket keeps track of
 803 # the reference counting and SharedSocketClient provides an constructor
 804 # and close() method that call incref() and decref() correctly.
 805
 806 class SharedSocket:
 807
 808     def __init__(self, sock):
 809         self.sock = sock
 810         self._refcnt = 0
 811
 812     def incref(self):
 813         self._refcnt += 1
 814
 815     def decref(self):
 816         self._refcnt -= 1
 817         assert self._refcnt >= 0
 818         if self._refcnt == 0:
 819             self.sock.close()
 820
 821     def __del__(self):
 822         self.sock.close()
 823
 824 class SharedSocketClient:
 825
 826     def __init__(self, shared):
 827         self._closed = 0
 828         self._shared = shared
 829         self._shared.incref()
 830         self._sock = shared.sock
 831
 832     def close(self):
 833         if not self._closed:
 834             self._shared.decref()
 835             self._closed = 1
 836             self._shared = None
 837
 838 class SSLFile(SharedSocketClient):
 839     """File-like object wrapping an SSL socket."""
 840
 841     BUFSIZE = 8192
 842
 843     def __init__(self, sock, ssl, bufsize=None):
 844         SharedSocketClient.__init__(self, sock)
 845         self._ssl = ssl
 846         self._buf = ''
 847         self._bufsize = bufsize or self.__class__.BUFSIZE
 848
 849     def _read(self):
 850         buf = ''
 851         # put in a loop so that we retry on transient errors
 852         while True:
 853             try:
 854                 buf = self._ssl.read(self._bufsize)
 855             except socket.sslerror, err:
 856                 if (err[0] == socket.SSL_ERROR_WANT_READ
 857                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
 858                     continue
 859                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
 860                     or err[0] == socket.SSL_ERROR_EOF):
 861                     break
 862                 raise
 863             except socket.error, err:
 864                 if err[0] == errno.EINTR:
 865                     continue
 866                 if err[0] == errno.EBADF:
 867                     # XXX socket was closed?
 868                     break
 869                 raise
 870             else:
 871                 break
 872         return buf
 873
 874     def read(self, size=None):
 875         L = [self._buf]
 876         avail = len(self._buf)
 877         while size is None or avail < size:
 878             s = self._read()
 879             if s == '':
 880                 break
 881             L.append(s)
 882             avail += len(s)
 883         all = "".join(L)
 884         if size is None:
 885             self._buf = ''
 886             return all
 887         else:
 888             self._buf = all[size:]
 889             return all[:size]
 890
 891     def readline(self):
 892         L = [self._buf]
 893         self._buf = ''
 894         while 1:
 895             i = L[-1].find("\n")
 896             if i >= 0:
 897                 break
 898             s = self._read()
 899             if s == '':
 900                 break
 901             L.append(s)
 902         if i == -1:
 903             # loop exited because there is no more data
 904             return "".join(L)
 905         else:
 906             all = "".join(L)
 907             # XXX could do enough bookkeeping not to do a 2nd search
 908             i = all.find("\n") + 1
 909             line = all[:i]
 910             self._buf = all[i:]
 911             return line
 912
 913 class FakeSocket(SharedSocketClient):
 914
 915     class _closedsocket:
 916         def __getattr__(self, name):
 917             raise error(9, 'Bad file descriptor')
 918
 919     def __init__(self, sock, ssl):
 920         sock = SharedSocket(sock)
 921         SharedSocketClient.__init__(self, sock)
 922         self._ssl = ssl
 923
 924     def close(self):
 925         SharedSocketClient.close(self)
 926         self._sock = self.__class__._closedsocket()
 927
 928     def makefile(self, mode, bufsize=None):
 929         if mode != 'r' and mode != 'rb':
 930             raise UnimplementedFileMode()
 931         return SSLFile(self._shared, self._ssl, bufsize)
 932
 933     def send(self, stuff, flags = 0):
 934         return self._ssl.write(stuff)
 935
 936     sendall = send
 937
 938     def recv(self, len = 1024, flags = 0):
 939         return self._ssl.read(len)
 940
 941     def __getattr__(self, attr):
 942         return getattr(self._sock, attr)
 943
 944
 945 class HTTPSConnection(HTTPConnection):
 946     "This class allows communication via SSL."
 947
 948     default_port = HTTPS_PORT
 949
 950     def __init__(self, host, port=None, key_file=None, cert_file=None,
 951                  strict=None):
 952         HTTPConnection.__init__(self, host, port, strict)
 953         self.key_file = key_file
 954         self.cert_file = cert_file
 955
 956     def connect(self):
 957         "Connect to a host on a given (SSL) port."
 958
 959         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 960         sock.connect((self.host, self.port))
 961         ssl = socket.ssl(sock, self.key_file, self.cert_file)
 962         self.sock = FakeSocket(sock, ssl)
 963
 964
 965 class HTTP:
 966     "Compatibility class with httplib.py from 1.5."
 967
 968     _http_vsn = 10
 969     _http_vsn_str = 'HTTP/1.0'
 970
 971     debuglevel = 0
 972
 973     _connection_class = HTTPConnection
 974
 975     def __init__(self, host='', port=None, strict=None):
 976         "Provide a default host, since the superclass requires one."
 977
 978         # some joker passed 0 explicitly, meaning default port
 979         if port == 0:
 980             port = None
 981
 982         # Note that we may pass an empty string as the host; this will throw
 983         # an error when we attempt to connect. Presumably, the client code
 984         # will call connect before then, with a proper host.
 985         self._setup(self._connection_class(host, port, strict))
 986
 987     def _setup(self, conn):
 988         self._conn = conn
 989
 990         # set up delegation to flesh out interface
 991         self.send = conn.send
 992         self.putrequest = conn.putrequest
 993         self.endheaders = conn.endheaders
 994         self.set_debuglevel = conn.set_debuglevel
 995
 996         conn._http_vsn = self._http_vsn
 997         conn._http_vsn_str = self._http_vsn_str
 998
 999         self.file = None
1000
1001     def connect(self, host=None, port=None):
1002         "Accept arguments to set the host/port, since the superclass doesn't."
1003
1004         if host is not None:
1005             self._conn._set_hostport(host, port)
1006         self._conn.connect()
1007
1008     def getfile(self):
1009         "Provide a getfile, since the superclass' does not use this concept."
1010         return self.file
1011
1012     def putheader(self, header, *values):
1013         "The superclass allows only one value argument."
1014         self._conn.putheader(header, '\r\n\t'.join(values))
1015
1016     def getreply(self):
1017         """Compat definition since superclass does not define it.
1018
1019         Returns a tuple consisting of:
1020         - server status code (e.g. '200' if all goes well)
1021         - server "reason" corresponding to status code
1022         - any RFC822 headers in the response from the server
1023         """
1024         try:
1025             response = self._conn.getresponse()
1026         except BadStatusLine, e:
1027             ### hmm. if getresponse() ever closes the socket on a bad request,
1028             ### then we are going to have problems with self.sock
1029
1030             ### should we keep this behavior? do people use it?
1031             # keep the socket open (as a file), and return it
1032             self.file = self._conn.sock.makefile('rb', 0)
1033
1034             # close our socket -- we want to restart after any protocol error
1035             self.close()
1036
1037             self.headers = None
1038             return -1, e.line, None
1039
1040         self.headers = response.msg
1041         self.file = response.fp
1042         return response.status, response.reason, response.msg
1043
1044     def close(self):
1045         self._conn.close()
1046
1047         # note that self.file == response.fp, which gets closed by the
1048         # superclass. just clear the object ref here.
1049         ### hmm. messy. if status==-1, then self.file is owned by us.
1050         ### well... we aren't explicitly closing, but losing this ref will
1051         ### do it
1052         self.file = None
1053
1054 if hasattr(socket, 'ssl'):
1055     class HTTPS(HTTP):
1056         """Compatibility with 1.5 httplib interface
1057
1058         Python 1.5.2 did not have an HTTPS class, but it defined an
1059         interface for sending http requests that is also useful for
1060         https.
1061         """
1062
1063         _connection_class = HTTPSConnection
1064
1065         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1066                      strict=None):
1067             # provide a default host, pass the X509 cert info
1068
1069             # urf. compensate for bad input.
1070             if port == 0:
1071                 port = None
1072             self._setup(self._connection_class(host, port, key_file,
1073                                                cert_file, strict))
1074
1075             # we never actually use these for anything, but we keep them
1076             # here for compatibility with post-1.5.2 CVS.
1077             self.key_file = key_file
1078             self.cert_file = cert_file
1079
1080
1081 class HTTPException(Exception):
1082     # Subclasses that define an __init__ must call Exception.__init__
1083     # or define self.args.  Otherwise, str() will fail.
1084     pass
1085
1086 class NotConnected(HTTPException):
1087     pass
1088
1089 class InvalidURL(HTTPException):
1090     pass
1091
1092 class UnknownProtocol(HTTPException):
1093     def __init__(self, version):
1094         self.args = version,
1095         self.version = version
1096
1097 class UnknownTransferEncoding(HTTPException):
1098     pass
1099
1100 class UnimplementedFileMode(HTTPException):
1101     pass
1102
1103 class IncompleteRead(HTTPException):
1104     def __init__(self, partial):
1105         self.args = partial,
1106         self.partial = partial
1107
1108 class ImproperConnectionState(HTTPException):
1109     pass
1110
1111 class CannotSendRequest(ImproperConnectionState):
1112     pass
1113
1114 class CannotSendHeader(ImproperConnectionState):
1115     pass
1116
1117 class ResponseNotReady(ImproperConnectionState):
1118     pass
1119
1120 class BadStatusLine(HTTPException):
1121     def __init__(self, line):
1122         self.args = line,
1123         self.line = line
1124
1125 # for backwards compatibility
1126 error = HTTPException
1127
1128 class LineAndFileWrapper:
1129     """A limited file-like object for HTTP/0.9 responses."""
1130
1131     # The status-line parsing code calls readline(), which normally
1132     # get the HTTP status line.  For a 0.9 response, however, this is
1133     # actually the first line of the body!  Clients need to get a
1134     # readable file object that contains that line.
1135
1136     def __init__(self, line, file):
1137         self._line = line
1138         self._file = file
1139         self._line_consumed = 0
1140         self._line_offset = 0
1141         self._line_left = len(line)
1142
1143     def __getattr__(self, attr):
1144         return getattr(self._file, attr)
1145
1146     def _done(self):
1147         # called when the last byte is read from the line.  After the
1148         # call, all read methods are delegated to the underlying file
1149         # object.
1150         self._line_consumed = 1
1151         self.read = self._file.read
1152         self.readline = self._file.readline
1153         self.readlines = self._file.readlines
1154
1155     def read(self, amt=None):
1156         assert not self._line_consumed and self._line_left
1157         if amt is None or amt > self._line_left:
1158             s = self._line[self._line_offset:]
1159             self._done()
1160             if amt is None:
1161                 return s + self._file.read()
1162             else:
1163                 return s + self._file.read(amt - len(s))
1164         else:
1165             assert amt <= self._line_left
1166             i = self._line_offset
1167             j = i + amt
1168             s = self._line[i:j]
1169             self._line_offset = j
1170             self._line_left -= amt
1171             if self._line_left == 0:
1172                 self._done()
1173             return s
1174
1175     def readline(self):
1176         s = self._line[self._line_offset:]
1177         self._done()
1178         return s
1179
1180     def readlines(self, size=None):
1181         L = [self._line[self._line_offset:]]
1182         self._done()
1183         if size is None:
1184             return L + self._file.readlines()
1185         else:
1186             return L + self._file.readlines(size)
1187
1188 def test():
1189     """Test this module.
1190
1191     A hodge podge of tests collected here, because they have too many
1192     external dependencies for the regular test suite.
1193     """
1194
1195     import sys
1196     import getopt
1197     opts, args = getopt.getopt(sys.argv[1:], 'd')
1198     dl = 0
1199     for o, a in opts:
1200         if o == '-d': dl = dl + 1
1201     host = 'www.python.org'
1202     selector = '/'
1203     if args[0:]: host = args[0]
1204     if args[1:]: selector = args[1]
1205     h = HTTP()
1206     h.set_debuglevel(dl)
1207     h.connect(host)
1208     h.putrequest('GET', selector)
1209     h.endheaders()
1210     status, reason, headers = h.getreply()
1211     print 'status =', status
1212     print 'reason =', reason
1213     print "read", len(h.getfile().read())
1214     print
1215     if headers:
1216         for header in headers.headers: print header.strip()
1217     print
1218
1219     # minimal test that code to extract host from url works
1220     class HTTP11(HTTP):
1221         _http_vsn = 11
1222         _http_vsn_str = 'HTTP/1.1'
1223
1224     h = HTTP11('www.python.org')
1225     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1226     h.endheaders()
1227     h.getreply()
1228     h.close()
1229
1230     if hasattr(socket, 'ssl'):
1231
1232         for host, selector in (('sourceforge.net', '/projects/python'),
1233                                ):
1234             print "https://%s%s" % (host, selector)
1235             hs = HTTPS()
1236             hs.set_debuglevel(dl)
1237             hs.connect(host)
1238             hs.putrequest('GET', selector)
1239             hs.endheaders()
1240             status, reason, headers = hs.getreply()
1241             print 'status =', status
1242             print 'reason =', reason
1243             print "read", len(hs.getfile().read())
1244             print
1245             if headers:
1246                 for header in headers.headers: print header.strip()
1247             print
1248
1249 if __name__ == '__main__':
1250     test()