Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 class HTTPMessage(mimetools.Message):
  97
  98     def addheader(self, key, value):
  99         """Add header for field key handling repeats."""
 100         prev = self.dict.get(key)
 101         if prev is None:
 102             self.dict[key] = value
 103         else:
 104             combined = ", ".join((prev, value))
 105             self.dict[key] = combined
 106
 107     def addcontinue(self, key, more):
 108         """Add more field data from a continuation line."""
 109         prev = self.dict[key]
 110         self.dict[key] = prev + "\n " + more
 111
 112     def readheaders(self):
 113         """Read header lines.
 114
 115         Read header lines up to the entirely blank line that terminates them.
 116         The (normally blank) line that ends the headers is skipped, but not
 117         included in the returned list.  If a non-header line ends the headers,
 118         (which is an error), an attempt is made to backspace over it; it is
 119         never included in the returned list.
 120
 121         The variable self.status is set to the empty string if all went well,
 122         otherwise it is an error message.  The variable self.headers is a
 123         completely uninterpreted list of lines contained in the header (so
 124         printing them will reproduce the header exactly as it appears in the
 125         file).
 126
 127         If multiple header fields with the same name occur, they are combined
 128         according to the rules in RFC 2616 sec 4.2:
 129
 130         Appending each subsequent field-value to the first, each separated
 131         by a comma. The order in which header fields with the same field-name
 132         are received is significant to the interpretation of the combined
 133         field value.
 134         """
 135         # XXX The implementation overrides the readheaders() method of
 136         # rfc822.Message.  The base class design isn't amenable to
 137         # customized behavior here so the method here is a copy of the
 138         # base class code with a few small changes.
 139
 140         self.dict = {}
 141         self.unixfrom = ''
 142         self.headers = hlist = []
 143         self.status = ''
 144         headerseen = ""
 145         firstline = 1
 146         startofline = unread = tell = None
 147         if hasattr(self.fp, 'unread'):
 148             unread = self.fp.unread
 149         elif self.seekable:
 150             tell = self.fp.tell
 151         while True:
 152             if tell:
 153                 try:
 154                     startofline = tell()
 155                 except IOError:
 156                     startofline = tell = None
 157                     self.seekable = 0
 158             line = self.fp.readline()
 159             if not line:
 160                 self.status = 'EOF in headers'
 161                 break
 162             # Skip unix From name time lines
 163             if firstline and line.startswith('From '):
 164                 self.unixfrom = self.unixfrom + line
 165                 continue
 166             firstline = 0
 167             if headerseen and line[0] in ' \t':
 168                 # XXX Not sure if continuation lines are handled properly
 169                 # for http and/or for repeating headers
 170                 # It's a continuation line.
 171                 hlist.append(line)
 172                 self.addcontinue(headerseen, line.strip())
 173                 continue
 174             elif self.iscomment(line):
 175                 # It's a comment.  Ignore it.
 176                 continue
 177             elif self.islast(line):
 178                 # Note! No pushback here!  The delimiter line gets eaten.
 179                 break
 180             headerseen = self.isheader(line)
 181             if headerseen:
 182                 # It's a legal header line, save it.
 183                 hlist.append(line)
 184                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 185                 continue
 186             else:
 187                 # It's not a header line; throw it back and stop here.
 188                 if not self.dict:
 189                     self.status = 'No headers'
 190                 else:
 191                     self.status = 'Non-header line where header expected'
 192                 # Try to undo the read.
 193                 if unread:
 194                     unread(line)
 195                 elif tell:
 196                     self.fp.seek(startofline)
 197                 else:
 198                     self.status = self.status + '; bad seek'
 199                 break
 200
 201 class HTTPResponse:
 202
 203     # strict: If true, raise BadStatusLine if the status line can't be
 204     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 205     # false because it prevents clients from talking to HTTP/0.9
 206     # servers.  Note that a response with a sufficiently corrupted
 207     # status line will look like an HTTP/0.9 response.
 208
 209     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 210
 211     def __init__(self, sock, debuglevel=0, strict=0, method=None):
 212         self.fp = sock.makefile('rb', 0)
 213         self.debuglevel = debuglevel
 214         self.strict = strict
 215         self._method = method
 216
 217         self.msg = None
 218
 219         # from the Status-Line of the response
 220         self.version = _UNKNOWN # HTTP-Version
 221         self.status = _UNKNOWN  # Status-Code
 222         self.reason = _UNKNOWN  # Reason-Phrase
 223
 224         self.chunked = _UNKNOWN         # is "chunked" being used?
 225         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 226         self.length = _UNKNOWN          # number of bytes left in response
 227         self.will_close = _UNKNOWN      # conn will close at end of response
 228
 229     def _read_status(self):
 230         # Initialize with Simple-Response defaults
 231         line = self.fp.readline()
 232         if self.debuglevel > 0:
 233             print "reply:", repr(line)
 234         try:
 235             [version, status, reason] = line.split(None, 2)
 236         except ValueError:
 237             try:
 238                 [version, status] = line.split(None, 1)
 239                 reason = ""
 240             except ValueError:
 241                 # empty version will cause next test to fail and status
 242                 # will be treated as 0.9 response.
 243                 version = ""
 244         if not version.startswith('HTTP/'):
 245             if self.strict:
 246                 self.close()
 247                 raise BadStatusLine(line)
 248             else:
 249                 # assume it's a Simple-Response from an 0.9 server
 250                 self.fp = LineAndFileWrapper(line, self.fp)
 251                 return "HTTP/0.9", 200, ""
 252
 253         # The status code is a three-digit number
 254         try:
 255             status = int(status)
 256             if status < 100 or status > 999:
 257                 raise BadStatusLine(line)
 258         except ValueError:
 259             raise BadStatusLine(line)
 260         return version, status, reason
 261
 262     def begin(self):
 263         if self.msg is not None:
 264             # we've already started reading the response
 265             return
 266
 267         # read until we get a non-100 response
 268         while True:
 269             version, status, reason = self._read_status()
 270             if status != 100:
 271                 break
 272             # skip the header from the 100 response
 273             while True:
 274                 skip = self.fp.readline().strip()
 275                 if not skip:
 276                     break
 277                 if self.debuglevel > 0:
 278                     print "header:", skip
 279
 280         self.status = status
 281         self.reason = reason.strip()
 282         if version == 'HTTP/1.0':
 283             self.version = 10
 284         elif version.startswith('HTTP/1.'):
 285             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 286         elif version == 'HTTP/0.9':
 287             self.version = 9
 288         else:
 289             raise UnknownProtocol(version)
 290
 291         if self.version == 9:
 292             self.chunked = 0
 293             self.will_close = 1
 294             self.msg = HTTPMessage(StringIO())
 295             return
 296
 297         self.msg = HTTPMessage(self.fp, 0)
 298         if self.debuglevel > 0:
 299             for hdr in self.msg.headers:
 300                 print "header:", hdr,
 301
 302         # don't let the msg keep an fp
 303         self.msg.fp = None
 304
 305         # are we using the chunked-style of transfer encoding?
 306         tr_enc = self.msg.getheader('transfer-encoding')
 307         if tr_enc and tr_enc.lower() == "chunked":
 308             self.chunked = 1
 309             self.chunk_left = None
 310         else:
 311             self.chunked = 0
 312
 313         # will the connection close at the end of the response?
 314         self.will_close = self._check_close()
 315
 316         # do we have a Content-Length?
 317         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 318         length = self.msg.getheader('content-length')
 319         if length and not self.chunked:
 320             try:
 321                 self.length = int(length)
 322             except ValueError:
 323                 self.length = None
 324         else:
 325             self.length = None
 326
 327         # does the body have a fixed length? (of zero)
 328         if (status == 204 or            # No Content
 329             status == 304 or            # Not Modified
 330             100 <= status < 200 or      # 1xx codes
 331             self._method == 'HEAD'):
 332             self.length = 0
 333
 334         # if the connection remains open, and we aren't using chunked, and
 335         # a content-length was not provided, then assume that the connection
 336         # WILL close.
 337         if not self.will_close and \
 338            not self.chunked and \
 339            self.length is None:
 340             self.will_close = 1
 341
 342     def _check_close(self):
 343         if self.version == 11:
 344             # An HTTP/1.1 proxy is assumed to stay open unless
 345             # explicitly closed.
 346             conn = self.msg.getheader('connection')
 347             if conn and conn.lower().find("close") >= 0:
 348                 return True
 349             return False
 350
 351         # An HTTP/1.0 response with a Connection header is probably
 352         # the result of a confused proxy.  Ignore it.
 353
 354         # For older HTTP, Keep-Alive indiciates persistent connection.
 355         if self.msg.getheader('keep-alive'):
 356             return False
 357
 358         # Proxy-Connection is a netscape hack.
 359         pconn = self.msg.getheader('proxy-connection')
 360         if pconn and pconn.lower().find("keep-alive") >= 0:
 361             return False
 362
 363         # otherwise, assume it will close
 364         return True
 365
 366     def close(self):
 367         if self.fp:
 368             self.fp.close()
 369             self.fp = None
 370
 371     def isclosed(self):
 372         # NOTE: it is possible that we will not ever call self.close(). This
 373         #       case occurs when will_close is TRUE, length is None, and we
 374         #       read up to the last byte, but NOT past it.
 375         #
 376         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 377         #          called, meaning self.isclosed() is meaningful.
 378         return self.fp is None
 379
 380     def read(self, amt=None):
 381         if self.fp is None:
 382             return ''
 383
 384         if self.chunked:
 385             return self._read_chunked(amt)
 386
 387         if amt is None:
 388             # unbounded read
 389             if self.will_close:
 390                 s = self.fp.read()
 391             else:
 392                 s = self._safe_read(self.length)
 393             self.close()        # we read everything
 394             return s
 395
 396         if self.length is not None:
 397             if amt > self.length:
 398                 # clip the read to the "end of response"
 399                 amt = self.length
 400             self.length -= amt
 401
 402         # we do not use _safe_read() here because this may be a .will_close
 403         # connection, and the user is reading more bytes than will be provided
 404         # (for example, reading in 1k chunks)
 405         s = self.fp.read(amt)
 406
 407         return s
 408
 409     def _read_chunked(self, amt):
 410         assert self.chunked != _UNKNOWN
 411         chunk_left = self.chunk_left
 412         value = ''
 413
 414         # XXX This accumulates chunks by repeated string concatenation,
 415         # which is not efficient as the number or size of chunks gets big.
 416         while True:
 417             if chunk_left is None:
 418                 line = self.fp.readline()
 419                 i = line.find(';')
 420                 if i >= 0:
 421                     line = line[:i] # strip chunk-extensions
 422                 chunk_left = int(line, 16)
 423                 if chunk_left == 0:
 424                     break
 425             if amt is None:
 426                 value += self._safe_read(chunk_left)
 427             elif amt < chunk_left:
 428                 value += self._safe_read(amt)
 429                 self.chunk_left = chunk_left - amt
 430                 return value
 431             elif amt == chunk_left:
 432                 value += self._safe_read(amt)
 433                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 434                 self.chunk_left = None
 435                 return value
 436             else:
 437                 value += self._safe_read(chunk_left)
 438                 amt -= chunk_left
 439
 440             # we read the whole chunk, get another
 441             self._safe_read(2)      # toss the CRLF at the end of the chunk
 442             chunk_left = None
 443
 444         # read and discard trailer up to the CRLF terminator
 445         ### note: we shouldn't have any trailers!
 446         while True:
 447             line = self.fp.readline()
 448             if line == '\r\n':
 449                 break
 450
 451         # we read everything; close the "file"
 452         # XXX Shouldn't the client close the file?
 453         self.close()
 454
 455         return value
 456
 457     def _safe_read(self, amt):
 458         """Read the number of bytes requested, compensating for partial reads.
 459
 460         Normally, we have a blocking socket, but a read() can be interrupted
 461         by a signal (resulting in a partial read).
 462
 463         Note that we cannot distinguish between EOF and an interrupt when zero
 464         bytes have been read. IncompleteRead() will be raised in this
 465         situation.
 466
 467         This function should be used when <amt> bytes "should" be present for
 468         reading. If the bytes are truly not available (due to EOF), then the
 469         IncompleteRead exception can be used to detect the problem.
 470         """
 471         s = ''
 472         while amt > 0:
 473             chunk = self.fp.read(amt)
 474             if not chunk:
 475                 raise IncompleteRead(s)
 476             s += chunk
 477             amt -= len(chunk)
 478         return s
 479
 480     def getheader(self, name, default=None):
 481         if self.msg is None:
 482             raise ResponseNotReady()
 483         return self.msg.getheader(name, default)
 484
 485
 486 class HTTPConnection:
 487
 488     _http_vsn = 11
 489     _http_vsn_str = 'HTTP/1.1'
 490
 491     response_class = HTTPResponse
 492     default_port = HTTP_PORT
 493     auto_open = 1
 494     debuglevel = 0
 495     strict = 0
 496
 497     def __init__(self, host, port=None, strict=None):
 498         self.sock = None
 499         self._buffer = []
 500         self.__response = None
 501         self.__state = _CS_IDLE
 502         self._method = None
 503
 504         self._set_hostport(host, port)
 505         if strict is not None:
 506             self.strict = strict
 507
 508     def _set_hostport(self, host, port):
 509         if port is None:
 510             i = host.find(':')
 511             if i >= 0:
 512                 try:
 513                     port = int(host[i+1:])
 514                 except ValueError:
 515                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 516                 host = host[:i]
 517             else:
 518                 port = self.default_port
 519         self.host = host
 520         self.port = port
 521
 522     def set_debuglevel(self, level):
 523         self.debuglevel = level
 524
 525     def connect(self):
 526         """Connect to the host and port specified in __init__."""
 527         msg = "getaddrinfo returns an empty list"
 528         for res in socket.getaddrinfo(self.host, self.port, 0,
 529                                       socket.SOCK_STREAM):
 530             af, socktype, proto, canonname, sa = res
 531             try:
 532                 self.sock = socket.socket(af, socktype, proto)
 533                 if self.debuglevel > 0:
 534                     print "connect: (%s, %s)" % (self.host, self.port)
 535                 self.sock.connect(sa)
 536             except socket.error, msg:
 537                 if self.debuglevel > 0:
 538                     print 'connect fail:', (self.host, self.port)
 539                 if self.sock:
 540                     self.sock.close()
 541                 self.sock = None
 542                 continue
 543             break
 544         if not self.sock:
 545             raise socket.error, msg
 546
 547     def close(self):
 548         """Close the connection to the HTTP server."""
 549         if self.sock:
 550             self.sock.close()   # close it manually... there may be other refs
 551             self.sock = None
 552         if self.__response:
 553             self.__response.close()
 554             self.__response = None
 555         self.__state = _CS_IDLE
 556
 557     def send(self, str):
 558         """Send `str' to the server."""
 559         if self.sock is None:
 560             if self.auto_open:
 561                 self.connect()
 562             else:
 563                 raise NotConnected()
 564
 565         # send the data to the server. if we get a broken pipe, then close
 566         # the socket. we want to reconnect when somebody tries to send again.
 567         #
 568         # NOTE: we DO propagate the error, though, because we cannot simply
 569         #       ignore the error... the caller will know if they can retry.
 570         if self.debuglevel > 0:
 571             print "send:", repr(str)
 572         try:
 573             self.sock.sendall(str)
 574         except socket.error, v:
 575             if v[0] == 32:      # Broken pipe
 576                 self.close()
 577             raise
 578
 579     def _output(self, s):
 580         """Add a line of output to the current request buffer.
 581
 582         Assumes that the line does *not* end with \\r\\n.
 583         """
 584         self._buffer.append(s)
 585
 586     def _send_output(self):
 587         """Send the currently buffered request and clear the buffer.
 588
 589         Appends an extra \\r\\n to the buffer.
 590         """
 591         self._buffer.extend(("", ""))
 592         msg = "\r\n".join(self._buffer)
 593         del self._buffer[:]
 594         self.send(msg)
 595
 596     def putrequest(self, method, url, skip_host=0):
 597         """Send a request to the server.
 598
 599         `method' specifies an HTTP request method, e.g. 'GET'.
 600         `url' specifies the object being requested, e.g. '/index.html'.
 601         """
 602
 603         # check if a prior response has been completed
 604         # XXX What if it hasn't?
 605         if self.__response and self.__response.isclosed():
 606             self.__response = None
 607
 608         #
 609         # in certain cases, we cannot issue another request on this connection.
 610         # this occurs when:
 611         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 612         #   2) a response to a previous request has signalled that it is going
 613         #      to close the connection upon completion.
 614         #   3) the headers for the previous response have not been read, thus
 615         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 616         #
 617         # if there is no prior response, then we can request at will.
 618         #
 619         # if point (2) is true, then we will have passed the socket to the
 620         # response (effectively meaning, "there is no prior response"), and
 621         # will open a new one when a new request is made.
 622         #
 623         # Note: if a prior response exists, then we *can* start a new request.
 624         #       We are not allowed to begin fetching the response to this new
 625         #       request, however, until that prior response is complete.
 626         #
 627         if self.__state == _CS_IDLE:
 628             self.__state = _CS_REQ_STARTED
 629         else:
 630             raise CannotSendRequest()
 631
 632         # Save the method we use, we need it later in the response phase
 633         self._method = method
 634         if not url:
 635             url = '/'
 636         str = '%s %s %s' % (method, url, self._http_vsn_str)
 637
 638         self._output(str)
 639
 640         if self._http_vsn == 11:
 641             # Issue some standard headers for better HTTP/1.1 compliance
 642
 643             if not skip_host:
 644                 # this header is issued *only* for HTTP/1.1
 645                 # connections. more specifically, this means it is
 646                 # only issued when the client uses the new
 647                 # HTTPConnection() class. backwards-compat clients
 648                 # will be using HTTP/1.0 and those clients may be
 649                 # issuing this header themselves. we should NOT issue
 650                 # it twice; some web servers (such as Apache) barf
 651                 # when they see two Host: headers
 652
 653                 # If we need a non-standard port,include it in the
 654                 # header.  If the request is going through a proxy,
 655                 # but the host of the actual URL, not the host of the
 656                 # proxy.
 657
 658                 netloc = ''
 659                 if url.startswith('http'):
 660                     nil, netloc, nil, nil, nil = urlsplit(url)
 661
 662                 if netloc:
 663                     self.putheader('Host', netloc.encode("idna"))
 664                 elif self.port == HTTP_PORT:
 665                     self.putheader('Host', self.host.encode("idna"))
 666                 else:
 667                     self.putheader('Host', "%s:%s" % (self.host.encode("idna"), self.port))
 668
 669             # note: we are assuming that clients will not attempt to set these
 670             #       headers since *this* library must deal with the
 671             #       consequences. this also means that when the supporting
 672             #       libraries are updated to recognize other forms, then this
 673             #       code should be changed (removed or updated).
 674
 675             # we only want a Content-Encoding of "identity" since we don't
 676             # support encodings such as x-gzip or x-deflate.
 677             self.putheader('Accept-Encoding', 'identity')
 678
 679             # we can accept "chunked" Transfer-Encodings, but no others
 680             # NOTE: no TE header implies *only* "chunked"
 681             #self.putheader('TE', 'chunked')
 682
 683             # if TE is supplied in the header, then it must appear in a
 684             # Connection header.
 685             #self.putheader('Connection', 'TE')
 686
 687         else:
 688             # For HTTP/1.0, the server will assume "not chunked"
 689             pass
 690
 691     def putheader(self, header, value):
 692         """Send a request header line to the server.
 693
 694         For example: h.putheader('Accept', 'text/html')
 695         """
 696         if self.__state != _CS_REQ_STARTED:
 697             raise CannotSendHeader()
 698
 699         str = '%s: %s' % (header, value)
 700         self._output(str)
 701
 702     def endheaders(self):
 703         """Indicate that the last header line has been sent to the server."""
 704
 705         if self.__state == _CS_REQ_STARTED:
 706             self.__state = _CS_REQ_SENT
 707         else:
 708             raise CannotSendHeader()
 709
 710         self._send_output()
 711
 712     def request(self, method, url, body=None, headers={}):
 713         """Send a complete request to the server."""
 714
 715         try:
 716             self._send_request(method, url, body, headers)
 717         except socket.error, v:
 718             # trap 'Broken pipe' if we're allowed to automatically reconnect
 719             if v[0] != 32 or not self.auto_open:
 720                 raise
 721             # try one more time
 722             self._send_request(method, url, body, headers)
 723
 724     def _send_request(self, method, url, body, headers):
 725         # If headers already contains a host header, then define the
 726         # optional skip_host argument to putrequest().  The check is
 727         # harder because field names are case insensitive.
 728         if 'Host' in (headers
 729             or [k for k in headers.iterkeys() if k.lower() == "host"]):
 730             self.putrequest(method, url, skip_host=1)
 731         else:
 732             self.putrequest(method, url)
 733
 734         if body:
 735             self.putheader('Content-Length', str(len(body)))
 736         for hdr, value in headers.iteritems():
 737             self.putheader(hdr, value)
 738         self.endheaders()
 739
 740         if body:
 741             self.send(body)
 742
 743     def getresponse(self):
 744         "Get the response from the server."
 745
 746         # check if a prior response has been completed
 747         if self.__response and self.__response.isclosed():
 748             self.__response = None
 749
 750         #
 751         # if a prior response exists, then it must be completed (otherwise, we
 752         # cannot read this response's header to determine the connection-close
 753         # behavior)
 754         #
 755         # note: if a prior response existed, but was connection-close, then the
 756         # socket and response were made independent of this HTTPConnection
 757         # object since a new request requires that we open a whole new
 758         # connection
 759         #
 760         # this means the prior response had one of two states:
 761         #   1) will_close: this connection was reset and the prior socket and
 762         #                  response operate independently
 763         #   2) persistent: the response was retained and we await its
 764         #                  isclosed() status to become true.
 765         #
 766         if self.__state != _CS_REQ_SENT or self.__response:
 767             raise ResponseNotReady()
 768
 769         if self.debuglevel > 0:
 770             response = self.response_class(self.sock, self.debuglevel,
 771                                            strict=self.strict,
 772                                            method=self._method)
 773         else:
 774             response = self.response_class(self.sock, strict=self.strict,
 775                                            method=self._method)
 776
 777         response.begin()
 778         assert response.will_close != _UNKNOWN
 779         self.__state = _CS_IDLE
 780
 781         if response.will_close:
 782             # this effectively passes the connection to the response
 783             self.close()
 784         else:
 785             # remember this, so we can tell when it is complete
 786             self.__response = response
 787
 788         return response
 789
 790 # The next several classes are used to define FakeSocket,a socket-like
 791 # interface to an SSL connection.
 792
 793 # The primary complexity comes from faking a makefile() method.  The
 794 # standard socket makefile() implementation calls dup() on the socket
 795 # file descriptor.  As a consequence, clients can call close() on the
 796 # parent socket and its makefile children in any order.  The underlying
 797 # socket isn't closed until they are all closed.
 798
 799 # The implementation uses reference counting to keep the socket open
 800 # until the last client calls close().  SharedSocket keeps track of
 801 # the reference counting and SharedSocketClient provides an constructor
 802 # and close() method that call incref() and decref() correctly.
 803
 804 class SharedSocket:
 805
 806     def __init__(self, sock):
 807         self.sock = sock
 808         self._refcnt = 0
 809
 810     def incref(self):
 811         self._refcnt += 1
 812
 813     def decref(self):
 814         self._refcnt -= 1
 815         assert self._refcnt >= 0
 816         if self._refcnt == 0:
 817             self.sock.close()
 818
 819     def __del__(self):
 820         self.sock.close()
 821
 822 class SharedSocketClient:
 823
 824     def __init__(self, shared):
 825         self._closed = 0
 826         self._shared = shared
 827         self._shared.incref()
 828         self._sock = shared.sock
 829
 830     def close(self):
 831         if not self._closed:
 832             self._shared.decref()
 833             self._closed = 1
 834             self._shared = None
 835
 836 class SSLFile(SharedSocketClient):
 837     """File-like object wrapping an SSL socket."""
 838
 839     BUFSIZE = 8192
 840
 841     def __init__(self, sock, ssl, bufsize=None):
 842         SharedSocketClient.__init__(self, sock)
 843         self._ssl = ssl
 844         self._buf = ''
 845         self._bufsize = bufsize or self.__class__.BUFSIZE
 846
 847     def _read(self):
 848         buf = ''
 849         # put in a loop so that we retry on transient errors
 850         while True:
 851             try:
 852                 buf = self._ssl.read(self._bufsize)
 853             except socket.sslerror, err:
 854                 if (err[0] == socket.SSL_ERROR_WANT_READ
 855                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
 856                     continue
 857                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
 858                     or err[0] == socket.SSL_ERROR_EOF):
 859                     break
 860                 raise
 861             except socket.error, err:
 862                 if err[0] == errno.EINTR:
 863                     continue
 864                 if err[0] == errno.EBADF:
 865                     # XXX socket was closed?
 866                     break
 867                 raise
 868             else:
 869                 break
 870         return buf
 871
 872     def read(self, size=None):
 873         L = [self._buf]
 874         avail = len(self._buf)
 875         while size is None or avail < size:
 876             s = self._read()
 877             if s == '':
 878                 break
 879             L.append(s)
 880             avail += len(s)
 881         all = "".join(L)
 882         if size is None:
 883             self._buf = ''
 884             return all
 885         else:
 886             self._buf = all[size:]
 887             return all[:size]
 888
 889     def readline(self):
 890         L = [self._buf]
 891         self._buf = ''
 892         while 1:
 893             i = L[-1].find("\n")
 894             if i >= 0:
 895                 break
 896             s = self._read()
 897             if s == '':
 898                 break
 899             L.append(s)
 900         if i == -1:
 901             # loop exited because there is no more data
 902             return "".join(L)
 903         else:
 904             all = "".join(L)
 905             # XXX could do enough bookkeeping not to do a 2nd search
 906             i = all.find("\n") + 1
 907             line = all[:i]
 908             self._buf = all[i:]
 909             return line
 910
 911 class FakeSocket(SharedSocketClient):
 912
 913     class _closedsocket:
 914         def __getattr__(self, name):
 915             raise error(9, 'Bad file descriptor')
 916
 917     def __init__(self, sock, ssl):
 918         sock = SharedSocket(sock)
 919         SharedSocketClient.__init__(self, sock)
 920         self._ssl = ssl
 921
 922     def close(self):
 923         SharedSocketClient.close(self)
 924         self._sock = self.__class__._closedsocket()
 925
 926     def makefile(self, mode, bufsize=None):
 927         if mode != 'r' and mode != 'rb':
 928             raise UnimplementedFileMode()
 929         return SSLFile(self._shared, self._ssl, bufsize)
 930
 931     def send(self, stuff, flags = 0):
 932         return self._ssl.write(stuff)
 933
 934     sendall = send
 935
 936     def recv(self, len = 1024, flags = 0):
 937         return self._ssl.read(len)
 938
 939     def __getattr__(self, attr):
 940         return getattr(self._sock, attr)
 941
 942
 943 class HTTPSConnection(HTTPConnection):
 944     "This class allows communication via SSL."
 945
 946     default_port = HTTPS_PORT
 947
 948     def __init__(self, host, port=None, key_file=None, cert_file=None,
 949                  strict=None):
 950         HTTPConnection.__init__(self, host, port, strict)
 951         self.key_file = key_file
 952         self.cert_file = cert_file
 953
 954     def connect(self):
 955         "Connect to a host on a given (SSL) port."
 956
 957         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 958         sock.connect((self.host, self.port))
 959         realsock = sock
 960         if hasattr(sock, "_sock"):
 961             realsock = sock._sock
 962         ssl = socket.ssl(realsock, self.key_file, self.cert_file)
 963         self.sock = FakeSocket(sock, ssl)
 964
 965
 966 class HTTP:
 967     "Compatibility class with httplib.py from 1.5."
 968
 969     _http_vsn = 10
 970     _http_vsn_str = 'HTTP/1.0'
 971
 972     debuglevel = 0
 973
 974     _connection_class = HTTPConnection
 975
 976     def __init__(self, host='', port=None, strict=None):
 977         "Provide a default host, since the superclass requires one."
 978
 979         # some joker passed 0 explicitly, meaning default port
 980         if port == 0:
 981             port = None
 982
 983         # Note that we may pass an empty string as the host; this will throw
 984         # an error when we attempt to connect. Presumably, the client code
 985         # will call connect before then, with a proper host.
 986         self._setup(self._connection_class(host, port, strict))
 987
 988     def _setup(self, conn):
 989         self._conn = conn
 990
 991         # set up delegation to flesh out interface
 992         self.send = conn.send
 993         self.putrequest = conn.putrequest
 994         self.endheaders = conn.endheaders
 995         self.set_debuglevel = conn.set_debuglevel
 996
 997         conn._http_vsn = self._http_vsn
 998         conn._http_vsn_str = self._http_vsn_str
 999
1000         self.file = None
1001
1002     def connect(self, host=None, port=None):
1003         "Accept arguments to set the host/port, since the superclass doesn't."
1004
1005         if host is not None:
1006             self._conn._set_hostport(host, port)
1007         self._conn.connect()
1008
1009     def getfile(self):
1010         "Provide a getfile, since the superclass' does not use this concept."
1011         return self.file
1012
1013     def putheader(self, header, *values):
1014         "The superclass allows only one value argument."
1015         self._conn.putheader(header, '\r\n\t'.join(values))
1016
1017     def getreply(self):
1018         """Compat definition since superclass does not define it.
1019
1020         Returns a tuple consisting of:
1021         - server status code (e.g. '200' if all goes well)
1022         - server "reason" corresponding to status code
1023         - any RFC822 headers in the response from the server
1024         """
1025         try:
1026             response = self._conn.getresponse()
1027         except BadStatusLine, e:
1028             ### hmm. if getresponse() ever closes the socket on a bad request,
1029             ### then we are going to have problems with self.sock
1030
1031             ### should we keep this behavior? do people use it?
1032             # keep the socket open (as a file), and return it
1033             self.file = self._conn.sock.makefile('rb', 0)
1034
1035             # close our socket -- we want to restart after any protocol error
1036             self.close()
1037
1038             self.headers = None
1039             return -1, e.line, None
1040
1041         self.headers = response.msg
1042         self.file = response.fp
1043         return response.status, response.reason, response.msg
1044
1045     def close(self):
1046         self._conn.close()
1047
1048         # note that self.file == response.fp, which gets closed by the
1049         # superclass. just clear the object ref here.
1050         ### hmm. messy. if status==-1, then self.file is owned by us.
1051         ### well... we aren't explicitly closing, but losing this ref will
1052         ### do it
1053         self.file = None
1054
1055 if hasattr(socket, 'ssl'):
1056     class HTTPS(HTTP):
1057         """Compatibility with 1.5 httplib interface
1058
1059         Python 1.5.2 did not have an HTTPS class, but it defined an
1060         interface for sending http requests that is also useful for
1061         https.
1062         """
1063
1064         _connection_class = HTTPSConnection
1065
1066         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1067                      strict=None):
1068             # provide a default host, pass the X509 cert info
1069
1070             # urf. compensate for bad input.
1071             if port == 0:
1072                 port = None
1073             self._setup(self._connection_class(host, port, key_file,
1074                                                cert_file, strict))
1075
1076             # we never actually use these for anything, but we keep them
1077             # here for compatibility with post-1.5.2 CVS.
1078             self.key_file = key_file
1079             self.cert_file = cert_file
1080
1081
1082 class HTTPException(Exception):
1083     # Subclasses that define an __init__ must call Exception.__init__
1084     # or define self.args.  Otherwise, str() will fail.
1085     pass
1086
1087 class NotConnected(HTTPException):
1088     pass
1089
1090 class InvalidURL(HTTPException):
1091     pass
1092
1093 class UnknownProtocol(HTTPException):
1094     def __init__(self, version):
1095         self.args = version,
1096         self.version = version
1097
1098 class UnknownTransferEncoding(HTTPException):
1099     pass
1100
1101 class UnimplementedFileMode(HTTPException):
1102     pass
1103
1104 class IncompleteRead(HTTPException):
1105     def __init__(self, partial):
1106         self.args = partial,
1107         self.partial = partial
1108
1109 class ImproperConnectionState(HTTPException):
1110     pass
1111
1112 class CannotSendRequest(ImproperConnectionState):
1113     pass
1114
1115 class CannotSendHeader(ImproperConnectionState):
1116     pass
1117
1118 class ResponseNotReady(ImproperConnectionState):
1119     pass
1120
1121 class BadStatusLine(HTTPException):
1122     def __init__(self, line):
1123         self.args = line,
1124         self.line = line
1125
1126 # for backwards compatibility
1127 error = HTTPException
1128
1129 class LineAndFileWrapper:
1130     """A limited file-like object for HTTP/0.9 responses."""
1131
1132     # The status-line parsing code calls readline(), which normally
1133     # get the HTTP status line.  For a 0.9 response, however, this is
1134     # actually the first line of the body!  Clients need to get a
1135     # readable file object that contains that line.
1136
1137     def __init__(self, line, file):
1138         self._line = line
1139         self._file = file
1140         self._line_consumed = 0
1141         self._line_offset = 0
1142         self._line_left = len(line)
1143
1144     def __getattr__(self, attr):
1145         return getattr(self._file, attr)
1146
1147     def _done(self):
1148         # called when the last byte is read from the line.  After the
1149         # call, all read methods are delegated to the underlying file
1150         # object.
1151         self._line_consumed = 1
1152         self.read = self._file.read
1153         self.readline = self._file.readline
1154         self.readlines = self._file.readlines
1155
1156     def read(self, amt=None):
1157         assert not self._line_consumed and self._line_left
1158         if amt is None or amt > self._line_left:
1159             s = self._line[self._line_offset:]
1160             self._done()
1161             if amt is None:
1162                 return s + self._file.read()
1163             else:
1164                 return s + self._file.read(amt - len(s))
1165         else:
1166             assert amt <= self._line_left
1167             i = self._line_offset
1168             j = i + amt
1169             s = self._line[i:j]
1170             self._line_offset = j
1171             self._line_left -= amt
1172             if self._line_left == 0:
1173                 self._done()
1174             return s
1175
1176     def readline(self):
1177         s = self._line[self._line_offset:]
1178         self._done()
1179         return s
1180
1181     def readlines(self, size=None):
1182         L = [self._line[self._line_offset:]]
1183         self._done()
1184         if size is None:
1185             return L + self._file.readlines()
1186         else:
1187             return L + self._file.readlines(size)
1188
1189 def test():
1190     """Test this module.
1191
1192     A hodge podge of tests collected here, because they have too many
1193     external dependencies for the regular test suite.
1194     """
1195
1196     import sys
1197     import getopt
1198     opts, args = getopt.getopt(sys.argv[1:], 'd')
1199     dl = 0
1200     for o, a in opts:
1201         if o == '-d': dl = dl + 1
1202     host = 'www.python.org'
1203     selector = '/'
1204     if args[0:]: host = args[0]
1205     if args[1:]: selector = args[1]
1206     h = HTTP()
1207     h.set_debuglevel(dl)
1208     h.connect(host)
1209     h.putrequest('GET', selector)
1210     h.endheaders()
1211     status, reason, headers = h.getreply()
1212     print 'status =', status
1213     print 'reason =', reason
1214     print "read", len(h.getfile().read())
1215     print
1216     if headers:
1217         for header in headers.headers: print header.strip()
1218     print
1219
1220     # minimal test that code to extract host from url works
1221     class HTTP11(HTTP):
1222         _http_vsn = 11
1223         _http_vsn_str = 'HTTP/1.1'
1224
1225     h = HTTP11('www.python.org')
1226     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1227     h.endheaders()
1228     h.getreply()
1229     h.close()
1230
1231     if hasattr(socket, 'ssl'):
1232
1233         for host, selector in (('sourceforge.net', '/projects/python'),
1234                                ):
1235             print "https://%s%s" % (host, selector)
1236             hs = HTTPS()
1237             hs.set_debuglevel(dl)
1238             hs.connect(host)
1239             hs.putrequest('GET', selector)
1240             hs.endheaders()
1241             status, reason, headers = hs.getreply()
1242             print 'status =', status
1243             print 'reason =', reason
1244             print "read", len(hs.getfile().read())
1245             print
1246             if headers:
1247                 for header in headers.headers: print header.strip()
1248             print
1249
1250 if __name__ == '__main__':
1251     test()