Lib/httplib.py

   1 """HTTP/1.1 client library
   2
   3 <intro stuff goes here>
   4 <other stuff, too>
   5
   6 HTTPConnection go through a number of "states", which defines when a client
   7 may legally make another request or fetch the response for a particular
   8 request. This diagram details these state transitions:
   9
  10     (null)
  11       |
  12       | HTTPConnection()
  13       v
  14     Idle
  15       |
  16       | putrequest()
  17       v
  18     Request-started
  19       |
  20       | ( putheader() )*  endheaders()
  21       v
  22     Request-sent
  23       |
  24       | response = getresponse()
  25       v
  26     Unread-response   [Response-headers-read]
  27       |\____________________
  28       |                     |
  29       | response.read()     | putrequest()
  30       v                     v
  31     Idle                  Req-started-unread-response
  32                      ______/|
  33                    /        |
  34    response.read() |        | ( putheader() )*  endheaders()
  35                    v        v
  36        Request-started    Req-sent-unread-response
  37                             |
  38                             | response.read()
  39                             v
  40                           Request-sent
  41
  42 This diagram presents the following rules:
  43   -- a second request may not be started until {response-headers-read}
  44   -- a response [object] cannot be retrieved until {request-sent}
  45   -- there is no differentiation between an unread response body and a
  46      partially read response body
  47
  48 Note: this enforcement is applied by the HTTPConnection class. The
  49       HTTPResponse class does not enforce this state machine, which
  50       implies sophisticated clients may accelerate the request/response
  51       pipeline. Caution should be taken, though: accelerating the states
  52       beyond the above pattern may imply knowledge of the server's
  53       connection-close behavior for certain requests. For example, it
  54       is impossible to tell whether the server will close the connection
  55       UNTIL the response headers have been read; this means that further
  56       requests cannot be placed into the pipeline until it is known that
  57       the server will NOT be closing the connection.
  58
  59 Logical State                  __state            __response
  60 -------------                  -------            ----------
  61 Idle                           _CS_IDLE           None
  62 Request-started                _CS_REQ_STARTED    None
  63 Request-sent                   _CS_REQ_SENT       None
  64 Unread-response                _CS_IDLE           <response_class>
  65 Req-started-unread-response    _CS_REQ_STARTED    <response_class>
  66 Req-sent-unread-response       _CS_REQ_SENT       <response_class>
  67 """
  68
  69 import errno
  70 import mimetools
  71 import socket
  72 from urlparse import urlsplit
  73
  74 try:
  75     from cStringIO import StringIO
  76 except ImportError:
  77     from StringIO import StringIO
  78
  79 __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",
  80            "HTTPException", "NotConnected", "UnknownProtocol",
  81            "UnknownTransferEncoding", "UnimplementedFileMode",
  82            "IncompleteRead", "InvalidURL", "ImproperConnectionState",
  83            "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",
  84            "BadStatusLine", "error"]
  85
  86 HTTP_PORT = 80
  87 HTTPS_PORT = 443
  88
  89 _UNKNOWN = 'UNKNOWN'
  90
  91 # connection states
  92 _CS_IDLE = 'Idle'
  93 _CS_REQ_STARTED = 'Request-started'
  94 _CS_REQ_SENT = 'Request-sent'
  95
  96 class HTTPMessage(mimetools.Message):
  97
  98     def addheader(self, key, value):
  99         """Add header for field key handling repeats."""
 100         prev = self.dict.get(key)
 101         if prev is None:
 102             self.dict[key] = value
 103         else:
 104             combined = ", ".join((prev, value))
 105             self.dict[key] = combined
 106
 107     def addcontinue(self, key, more):
 108         """Add more field data from a continuation line."""
 109         prev = self.dict[key]
 110         self.dict[key] = prev + "\n " + more
 111
 112     def readheaders(self):
 113         """Read header lines.
 114
 115         Read header lines up to the entirely blank line that terminates them.
 116         The (normally blank) line that ends the headers is skipped, but not
 117         included in the returned list.  If a non-header line ends the headers,
 118         (which is an error), an attempt is made to backspace over it; it is
 119         never included in the returned list.
 120
 121         The variable self.status is set to the empty string if all went well,
 122         otherwise it is an error message.  The variable self.headers is a
 123         completely uninterpreted list of lines contained in the header (so
 124         printing them will reproduce the header exactly as it appears in the
 125         file).
 126
 127         If multiple header fields with the same name occur, they are combined
 128         according to the rules in RFC 2616 sec 4.2:
 129
 130         Appending each subsequent field-value to the first, each separated
 131         by a comma. The order in which header fields with the same field-name
 132         are received is significant to the interpretation of the combined
 133         field value.
 134         """
 135         # XXX The implementation overrides the readheaders() method of
 136         # rfc822.Message.  The base class design isn't amenable to
 137         # customized behavior here so the method here is a copy of the
 138         # base class code with a few small changes.
 139
 140         self.dict = {}
 141         self.unixfrom = ''
 142         self.headers = list = []
 143         self.status = ''
 144         headerseen = ""
 145         firstline = 1
 146         startofline = unread = tell = None
 147         if hasattr(self.fp, 'unread'):
 148             unread = self.fp.unread
 149         elif self.seekable:
 150             tell = self.fp.tell
 151         while 1:
 152             if tell:
 153                 try:
 154                     startofline = tell()
 155                 except IOError:
 156                     startofline = tell = None
 157                     self.seekable = 0
 158             line = self.fp.readline()
 159             if not line:
 160                 self.status = 'EOF in headers'
 161                 break
 162             # Skip unix From name time lines
 163             if firstline and line.startswith('From '):
 164                 self.unixfrom = self.unixfrom + line
 165                 continue
 166             firstline = 0
 167             if headerseen and line[0] in ' \t':
 168                 # XXX Not sure if continuation lines are handled properly
 169                 # for http and/or for repeating headers
 170                 # It's a continuation line.
 171                 list.append(line)
 172                 x = self.dict[headerseen] + "\n " + line.strip()
 173                 self.addcontinue(headerseen, line.strip())
 174                 continue
 175             elif self.iscomment(line):
 176                 # It's a comment.  Ignore it.
 177                 continue
 178             elif self.islast(line):
 179                 # Note! No pushback here!  The delimiter line gets eaten.
 180                 break
 181             headerseen = self.isheader(line)
 182             if headerseen:
 183                 # It's a legal header line, save it.
 184                 list.append(line)
 185                 self.addheader(headerseen, line[len(headerseen)+1:].strip())
 186                 continue
 187             else:
 188                 # It's not a header line; throw it back and stop here.
 189                 if not self.dict:
 190                     self.status = 'No headers'
 191                 else:
 192                     self.status = 'Non-header line where header expected'
 193                 # Try to undo the read.
 194                 if unread:
 195                     unread(line)
 196                 elif tell:
 197                     self.fp.seek(startofline)
 198                 else:
 199                     self.status = self.status + '; bad seek'
 200                 break
 201
 202 class HTTPResponse:
 203
 204     # strict: If true, raise BadStatusLine if the status line can't be
 205     # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is
 206     # false because it prevents clients from talking to HTTP/0.9
 207     # servers.  Note that a response with a sufficiently corrupted
 208     # status line will look like an HTTP/0.9 response.
 209
 210     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.
 211
 212     def __init__(self, sock, debuglevel=0, strict=0):
 213         self.fp = sock.makefile('rb', 0)
 214         self.debuglevel = debuglevel
 215         self.strict = strict
 216
 217         self.msg = None
 218
 219         # from the Status-Line of the response
 220         self.version = _UNKNOWN # HTTP-Version
 221         self.status = _UNKNOWN  # Status-Code
 222         self.reason = _UNKNOWN  # Reason-Phrase
 223
 224         self.chunked = _UNKNOWN         # is "chunked" being used?
 225         self.chunk_left = _UNKNOWN      # bytes left to read in current chunk
 226         self.length = _UNKNOWN          # number of bytes left in response
 227         self.will_close = _UNKNOWN      # conn will close at end of response
 228
 229     def _read_status(self):
 230         # Initialize with Simple-Response defaults
 231         line = self.fp.readline()
 232         if self.debuglevel > 0:
 233             print "reply:", repr(line)
 234         try:
 235             [version, status, reason] = line.split(None, 2)
 236         except ValueError:
 237             try:
 238                 [version, status] = line.split(None, 1)
 239                 reason = ""
 240             except ValueError:
 241                 # empty version will cause next test to fail and status
 242                 # will be treated as 0.9 response.
 243                 version = ""
 244         if not version.startswith('HTTP/'):
 245             if self.strict:
 246                 self.close()
 247                 raise BadStatusLine(line)
 248             else:
 249                 # assume it's a Simple-Response from an 0.9 server
 250                 self.fp = LineAndFileWrapper(line, self.fp)
 251                 return "HTTP/0.9", 200, ""
 252
 253         # The status code is a three-digit number
 254         try:
 255             status = int(status)
 256             if status < 100 or status > 999:
 257                 raise BadStatusLine(line)
 258         except ValueError:
 259             raise BadStatusLine(line)
 260         return version, status, reason
 261
 262     def begin(self):
 263         if self.msg is not None:
 264             # we've already started reading the response
 265             return
 266
 267         # read until we get a non-100 response
 268         while 1:
 269             version, status, reason = self._read_status()
 270             if status != 100:
 271                 break
 272             # skip the header from the 100 response
 273             while 1:
 274                 skip = self.fp.readline().strip()
 275                 if not skip:
 276                     break
 277                 if self.debuglevel > 0:
 278                     print "header:", skip
 279
 280         self.status = status
 281         self.reason = reason.strip()
 282         if version == 'HTTP/1.0':
 283             self.version = 10
 284         elif version.startswith('HTTP/1.'):
 285             self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1
 286         elif version == 'HTTP/0.9':
 287             self.version = 9
 288         else:
 289             raise UnknownProtocol(version)
 290
 291         if self.version == 9:
 292             self.chunked = 0
 293             self.will_close = 1
 294             self.msg = HTTPMessage(StringIO())
 295             return
 296
 297         self.msg = HTTPMessage(self.fp, 0)
 298         if self.debuglevel > 0:
 299             for hdr in self.msg.headers:
 300                 print "header:", hdr,
 301
 302         # don't let the msg keep an fp
 303         self.msg.fp = None
 304
 305         # are we using the chunked-style of transfer encoding?
 306         tr_enc = self.msg.getheader('transfer-encoding')
 307         if tr_enc and tr_enc.lower() == "chunked":
 308             self.chunked = 1
 309             self.chunk_left = None
 310         else:
 311             self.chunked = 0
 312
 313         # will the connection close at the end of the response?
 314         conn = self.msg.getheader('connection')
 315         if conn:
 316             conn = conn.lower()
 317             # a "Connection: close" will always close the connection. if we
 318             # don't see that and this is not HTTP/1.1, then the connection will
 319             # close unless we see a Keep-Alive header.
 320             self.will_close = conn.find('close') != -1 or \
 321                               ( self.version != 11 and \
 322                                 not self.msg.getheader('keep-alive') )
 323         else:
 324             # for HTTP/1.1, the connection will always remain open
 325             # otherwise, it will remain open IFF we see a Keep-Alive header
 326             self.will_close = self.version != 11 and \
 327                               not self.msg.getheader('keep-alive')
 328
 329         # do we have a Content-Length?
 330         # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"
 331         length = self.msg.getheader('content-length')
 332         if length and not self.chunked:
 333             try:
 334                 self.length = int(length)
 335             except ValueError:
 336                 self.length = None
 337         else:
 338             self.length = None
 339
 340         # does the body have a fixed length? (of zero)
 341         if (status == 204 or            # No Content
 342             status == 304 or            # Not Modified
 343             100 <= status < 200):       # 1xx codes
 344             self.length = 0
 345
 346         # if the connection remains open, and we aren't using chunked, and
 347         # a content-length was not provided, then assume that the connection
 348         # WILL close.
 349         if not self.will_close and \
 350            not self.chunked and \
 351            self.length is None:
 352             self.will_close = 1
 353
 354     def close(self):
 355         if self.fp:
 356             self.fp.close()
 357             self.fp = None
 358
 359     def isclosed(self):
 360         # NOTE: it is possible that we will not ever call self.close(). This
 361         #       case occurs when will_close is TRUE, length is None, and we
 362         #       read up to the last byte, but NOT past it.
 363         #
 364         # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be
 365         #          called, meaning self.isclosed() is meaningful.
 366         return self.fp is None
 367
 368     def read(self, amt=None):
 369         if self.fp is None:
 370             return ''
 371
 372         if self.chunked:
 373             return self._read_chunked(amt)
 374
 375         if amt is None:
 376             # unbounded read
 377             if self.will_close:
 378                 s = self.fp.read()
 379             else:
 380                 s = self._safe_read(self.length)
 381             self.close()        # we read everything
 382             return s
 383
 384         if self.length is not None:
 385             if amt > self.length:
 386                 # clip the read to the "end of response"
 387                 amt = self.length
 388             self.length -= amt
 389
 390         # we do not use _safe_read() here because this may be a .will_close
 391         # connection, and the user is reading more bytes than will be provided
 392         # (for example, reading in 1k chunks)
 393         s = self.fp.read(amt)
 394
 395         return s
 396
 397     def _read_chunked(self, amt):
 398         assert self.chunked != _UNKNOWN
 399         chunk_left = self.chunk_left
 400         value = ''
 401
 402         # XXX This accumulates chunks by repeated string concatenation,
 403         # which is not efficient as the number or size of chunks gets big.
 404         while 1:
 405             if chunk_left is None:
 406                 line = self.fp.readline()
 407                 i = line.find(';')
 408                 if i >= 0:
 409                     line = line[:i] # strip chunk-extensions
 410                 chunk_left = int(line, 16)
 411                 if chunk_left == 0:
 412                     break
 413             if amt is None:
 414                 value += self._safe_read(chunk_left)
 415             elif amt < chunk_left:
 416                 value += self._safe_read(amt)
 417                 self.chunk_left = chunk_left - amt
 418                 return value
 419             elif amt == chunk_left:
 420                 value += self._safe_read(amt)
 421                 self._safe_read(2)  # toss the CRLF at the end of the chunk
 422                 self.chunk_left = None
 423                 return value
 424             else:
 425                 value += self._safe_read(chunk_left)
 426                 amt -= chunk_left
 427
 428             # we read the whole chunk, get another
 429             self._safe_read(2)      # toss the CRLF at the end of the chunk
 430             chunk_left = None
 431
 432         # read and discard trailer up to the CRLF terminator
 433         ### note: we shouldn't have any trailers!
 434         while 1:
 435             line = self.fp.readline()
 436             if line == '\r\n':
 437                 break
 438
 439         # we read everything; close the "file"
 440         # XXX Shouldn't the client close the file?
 441         self.close()
 442
 443         return value
 444
 445     def _safe_read(self, amt):
 446         """Read the number of bytes requested, compensating for partial reads.
 447
 448         Normally, we have a blocking socket, but a read() can be interrupted
 449         by a signal (resulting in a partial read).
 450
 451         Note that we cannot distinguish between EOF and an interrupt when zero
 452         bytes have been read. IncompleteRead() will be raised in this
 453         situation.
 454
 455         This function should be used when <amt> bytes "should" be present for
 456         reading. If the bytes are truly not available (due to EOF), then the
 457         IncompleteRead exception can be used to detect the problem.
 458         """
 459         s = ''
 460         while amt > 0:
 461             chunk = self.fp.read(amt)
 462             if not chunk:
 463                 raise IncompleteRead(s)
 464             s = s + chunk
 465             amt = amt - len(chunk)
 466         return s
 467
 468     def getheader(self, name, default=None):
 469         if self.msg is None:
 470             raise ResponseNotReady()
 471         return self.msg.getheader(name, default)
 472
 473
 474 class HTTPConnection:
 475
 476     _http_vsn = 11
 477     _http_vsn_str = 'HTTP/1.1'
 478
 479     response_class = HTTPResponse
 480     default_port = HTTP_PORT
 481     auto_open = 1
 482     debuglevel = 0
 483     strict = 0
 484
 485     def __init__(self, host, port=None, strict=None):
 486         self.sock = None
 487         self._buffer = []
 488         self.__response = None
 489         self.__state = _CS_IDLE
 490
 491         self._set_hostport(host, port)
 492         if strict is not None:
 493             self.strict = strict
 494
 495     def _set_hostport(self, host, port):
 496         if port is None:
 497             i = host.find(':')
 498             if i >= 0:
 499                 try:
 500                     port = int(host[i+1:])
 501                 except ValueError:
 502                     raise InvalidURL("nonnumeric port: '%s'" % host[i+1:])
 503                 host = host[:i]
 504             else:
 505                 port = self.default_port
 506         self.host = host
 507         self.port = port
 508
 509     def set_debuglevel(self, level):
 510         self.debuglevel = level
 511
 512     def connect(self):
 513         """Connect to the host and port specified in __init__."""
 514         msg = "getaddrinfo returns an empty list"
 515         for res in socket.getaddrinfo(self.host, self.port, 0,
 516                                       socket.SOCK_STREAM):
 517             af, socktype, proto, canonname, sa = res
 518             try:
 519                 self.sock = socket.socket(af, socktype, proto)
 520                 if self.debuglevel > 0:
 521                     print "connect: (%s, %s)" % (self.host, self.port)
 522                 self.sock.connect(sa)
 523             except socket.error, msg:
 524                 if self.debuglevel > 0:
 525                     print 'connect fail:', (self.host, self.port)
 526                 if self.sock:
 527                     self.sock.close()
 528                 self.sock = None
 529                 continue
 530             break
 531         if not self.sock:
 532             raise socket.error, msg
 533
 534     def close(self):
 535         """Close the connection to the HTTP server."""
 536         if self.sock:
 537             self.sock.close()   # close it manually... there may be other refs
 538             self.sock = None
 539         if self.__response:
 540             self.__response.close()
 541             self.__response = None
 542         self.__state = _CS_IDLE
 543
 544     def send(self, str):
 545         """Send `str' to the server."""
 546         if self.sock is None:
 547             if self.auto_open:
 548                 self.connect()
 549             else:
 550                 raise NotConnected()
 551
 552         # send the data to the server. if we get a broken pipe, then close
 553         # the socket. we want to reconnect when somebody tries to send again.
 554         #
 555         # NOTE: we DO propagate the error, though, because we cannot simply
 556         #       ignore the error... the caller will know if they can retry.
 557         if self.debuglevel > 0:
 558             print "send:", repr(str)
 559         try:
 560             self.sock.sendall(str)
 561         except socket.error, v:
 562             if v[0] == 32:      # Broken pipe
 563                 self.close()
 564             raise
 565
 566     def _output(self, s):
 567         """Add a line of output to the current request buffer.
 568
 569         Assumes that the line does *not* end with \\r\\n.
 570         """
 571         self._buffer.append(s)
 572
 573     def _send_output(self):
 574         """Send the currently buffered request and clear the buffer.
 575
 576         Appends an extra \\r\\n to the buffer.
 577         """
 578         self._buffer.extend(("", ""))
 579         msg = "\r\n".join(self._buffer)
 580         del self._buffer[:]
 581         self.send(msg)
 582
 583     def putrequest(self, method, url, skip_host=0):
 584         """Send a request to the server.
 585
 586         `method' specifies an HTTP request method, e.g. 'GET'.
 587         `url' specifies the object being requested, e.g. '/index.html'.
 588         """
 589
 590         # check if a prior response has been completed
 591         # XXX What if it hasn't?
 592         if self.__response and self.__response.isclosed():
 593             self.__response = None
 594
 595         #
 596         # in certain cases, we cannot issue another request on this connection.
 597         # this occurs when:
 598         #   1) we are in the process of sending a request.   (_CS_REQ_STARTED)
 599         #   2) a response to a previous request has signalled that it is going
 600         #      to close the connection upon completion.
 601         #   3) the headers for the previous response have not been read, thus
 602         #      we cannot determine whether point (2) is true.   (_CS_REQ_SENT)
 603         #
 604         # if there is no prior response, then we can request at will.
 605         #
 606         # if point (2) is true, then we will have passed the socket to the
 607         # response (effectively meaning, "there is no prior response"), and
 608         # will open a new one when a new request is made.
 609         #
 610         # Note: if a prior response exists, then we *can* start a new request.
 611         #       We are not allowed to begin fetching the response to this new
 612         #       request, however, until that prior response is complete.
 613         #
 614         if self.__state == _CS_IDLE:
 615             self.__state = _CS_REQ_STARTED
 616         else:
 617             raise CannotSendRequest()
 618
 619         if not url:
 620             url = '/'
 621         str = '%s %s %s' % (method, url, self._http_vsn_str)
 622
 623         self._output(str)
 624
 625         if self._http_vsn == 11:
 626             # Issue some standard headers for better HTTP/1.1 compliance
 627
 628             if not skip_host:
 629                 # this header is issued *only* for HTTP/1.1
 630                 # connections. more specifically, this means it is
 631                 # only issued when the client uses the new
 632                 # HTTPConnection() class. backwards-compat clients
 633                 # will be using HTTP/1.0 and those clients may be
 634                 # issuing this header themselves. we should NOT issue
 635                 # it twice; some web servers (such as Apache) barf
 636                 # when they see two Host: headers
 637
 638                 # If we need a non-standard port,include it in the
 639                 # header.  If the request is going through a proxy,
 640                 # but the host of the actual URL, not the host of the
 641                 # proxy.
 642
 643                 netloc = ''
 644                 if url.startswith('http'):
 645                     nil, netloc, nil, nil, nil = urlsplit(url)
 646
 647                 if netloc:
 648                     self.putheader('Host', netloc)
 649                 elif self.port == HTTP_PORT:
 650                     self.putheader('Host', self.host)
 651                 else:
 652                     self.putheader('Host', "%s:%s" % (self.host, self.port))
 653
 654             # note: we are assuming that clients will not attempt to set these
 655             #       headers since *this* library must deal with the
 656             #       consequences. this also means that when the supporting
 657             #       libraries are updated to recognize other forms, then this
 658             #       code should be changed (removed or updated).
 659
 660             # we only want a Content-Encoding of "identity" since we don't
 661             # support encodings such as x-gzip or x-deflate.
 662             self.putheader('Accept-Encoding', 'identity')
 663
 664             # we can accept "chunked" Transfer-Encodings, but no others
 665             # NOTE: no TE header implies *only* "chunked"
 666             #self.putheader('TE', 'chunked')
 667
 668             # if TE is supplied in the header, then it must appear in a
 669             # Connection header.
 670             #self.putheader('Connection', 'TE')
 671
 672         else:
 673             # For HTTP/1.0, the server will assume "not chunked"
 674             pass
 675
 676     def putheader(self, header, value):
 677         """Send a request header line to the server.
 678
 679         For example: h.putheader('Accept', 'text/html')
 680         """
 681         if self.__state != _CS_REQ_STARTED:
 682             raise CannotSendHeader()
 683
 684         str = '%s: %s' % (header, value)
 685         self._output(str)
 686
 687     def endheaders(self):
 688         """Indicate that the last header line has been sent to the server."""
 689
 690         if self.__state == _CS_REQ_STARTED:
 691             self.__state = _CS_REQ_SENT
 692         else:
 693             raise CannotSendHeader()
 694
 695         self._send_output()
 696
 697     def request(self, method, url, body=None, headers={}):
 698         """Send a complete request to the server."""
 699
 700         try:
 701             self._send_request(method, url, body, headers)
 702         except socket.error, v:
 703             # trap 'Broken pipe' if we're allowed to automatically reconnect
 704             if v[0] != 32 or not self.auto_open:
 705                 raise
 706             # try one more time
 707             self._send_request(method, url, body, headers)
 708
 709     def _send_request(self, method, url, body, headers):
 710         # If headers already contains a host header, then define the
 711         # optional skip_host argument to putrequest().  The check is
 712         # harder because field names are case insensitive.
 713         if 'Host' in (headers
 714             or [k for k in headers.iterkeys() if k.lower() == "host"]):
 715             self.putrequest(method, url, skip_host=1)
 716         else:
 717             self.putrequest(method, url)
 718
 719         if body:
 720             self.putheader('Content-Length', str(len(body)))
 721         for hdr, value in headers.items():
 722             self.putheader(hdr, value)
 723         self.endheaders()
 724
 725         if body:
 726             self.send(body)
 727
 728     def getresponse(self):
 729         "Get the response from the server."
 730
 731         # check if a prior response has been completed
 732         if self.__response and self.__response.isclosed():
 733             self.__response = None
 734
 735         #
 736         # if a prior response exists, then it must be completed (otherwise, we
 737         # cannot read this response's header to determine the connection-close
 738         # behavior)
 739         #
 740         # note: if a prior response existed, but was connection-close, then the
 741         # socket and response were made independent of this HTTPConnection
 742         # object since a new request requires that we open a whole new
 743         # connection
 744         #
 745         # this means the prior response had one of two states:
 746         #   1) will_close: this connection was reset and the prior socket and
 747         #                  response operate independently
 748         #   2) persistent: the response was retained and we await its
 749         #                  isclosed() status to become true.
 750         #
 751         if self.__state != _CS_REQ_SENT or self.__response:
 752             raise ResponseNotReady()
 753
 754         if self.debuglevel > 0:
 755             response = self.response_class(self.sock, self.debuglevel,
 756                                            strict=self.strict)
 757         else:
 758             response = self.response_class(self.sock, strict=self.strict)
 759
 760         response.begin()
 761         assert response.will_close != _UNKNOWN
 762         self.__state = _CS_IDLE
 763
 764         if response.will_close:
 765             # this effectively passes the connection to the response
 766             self.close()
 767         else:
 768             # remember this, so we can tell when it is complete
 769             self.__response = response
 770
 771         return response
 772
 773 # The next several classes are used to define FakeSocket,a socket-like
 774 # interface to an SSL connection.
 775
 776 # The primary complexity comes from faking a makefile() method.  The
 777 # standard socket makefile() implementation calls dup() on the socket
 778 # file descriptor.  As a consequence, clients can call close() on the
 779 # parent socket and its makefile children in any order.  The underlying
 780 # socket isn't closed until they are all closed.
 781
 782 # The implementation uses reference counting to keep the socket open
 783 # until the last client calls close().  SharedSocket keeps track of
 784 # the reference counting and SharedSocketClient provides an constructor
 785 # and close() method that call incref() and decref() correctly.
 786
 787 class SharedSocket:
 788
 789     def __init__(self, sock):
 790         self.sock = sock
 791         self._refcnt = 0
 792
 793     def incref(self):
 794         self._refcnt += 1
 795
 796     def decref(self):
 797         self._refcnt -= 1
 798         assert self._refcnt >= 0
 799         if self._refcnt == 0:
 800             self.sock.close()
 801
 802     def __del__(self):
 803         self.sock.close()
 804
 805 class SharedSocketClient:
 806
 807     def __init__(self, shared):
 808         self._closed = 0
 809         self._shared = shared
 810         self._shared.incref()
 811         self._sock = shared.sock
 812
 813     def close(self):
 814         if not self._closed:
 815             self._shared.decref()
 816             self._closed = 1
 817             self._shared = None
 818
 819 class SSLFile(SharedSocketClient):
 820     """File-like object wrapping an SSL socket."""
 821
 822     BUFSIZE = 8192
 823
 824     def __init__(self, sock, ssl, bufsize=None):
 825         SharedSocketClient.__init__(self, sock)
 826         self._ssl = ssl
 827         self._buf = ''
 828         self._bufsize = bufsize or self.__class__.BUFSIZE
 829
 830     def _read(self):
 831         buf = ''
 832         # put in a loop so that we retry on transient errors
 833         while 1:
 834             try:
 835                 buf = self._ssl.read(self._bufsize)
 836             except socket.sslerror, err:
 837                 if (err[0] == socket.SSL_ERROR_WANT_READ
 838                     or err[0] == socket.SSL_ERROR_WANT_WRITE):
 839                     continue
 840                 if (err[0] == socket.SSL_ERROR_ZERO_RETURN
 841                     or err[0] == socket.SSL_ERROR_EOF):
 842                     break
 843                 raise
 844             except socket.error, err:
 845                 if err[0] == errno.EINTR:
 846                     continue
 847                 if err[0] == errno.EBADF:
 848                     # XXX socket was closed?
 849                     break
 850                 raise
 851             else:
 852                 break
 853         return buf
 854
 855     def read(self, size=None):
 856         L = [self._buf]
 857         avail = len(self._buf)
 858         while size is None or avail < size:
 859             s = self._read()
 860             if s == '':
 861                 break
 862             L.append(s)
 863             avail += len(s)
 864         all = "".join(L)
 865         if size is None:
 866             self._buf = ''
 867             return all
 868         else:
 869             self._buf = all[size:]
 870             return all[:size]
 871
 872     def readline(self):
 873         L = [self._buf]
 874         self._buf = ''
 875         while 1:
 876             i = L[-1].find("\n")
 877             if i >= 0:
 878                 break
 879             s = self._read()
 880             if s == '':
 881                 break
 882             L.append(s)
 883         if i == -1:
 884             # loop exited because there is no more data
 885             return "".join(L)
 886         else:
 887             all = "".join(L)
 888             # XXX could do enough bookkeeping not to do a 2nd search
 889             i = all.find("\n") + 1
 890             line = all[:i]
 891             self._buf = all[i:]
 892             return line
 893
 894 class FakeSocket(SharedSocketClient):
 895
 896     class _closedsocket:
 897         def __getattr__(self, name):
 898             raise error(9, 'Bad file descriptor')
 899
 900     def __init__(self, sock, ssl):
 901         sock = SharedSocket(sock)
 902         SharedSocketClient.__init__(self, sock)
 903         self._ssl = ssl
 904
 905     def close(self):
 906         SharedSocketClient.close(self)
 907         self._sock = self.__class__._closedsocket()
 908
 909     def makefile(self, mode, bufsize=None):
 910         if mode != 'r' and mode != 'rb':
 911             raise UnimplementedFileMode()
 912         return SSLFile(self._shared, self._ssl, bufsize)
 913
 914     def send(self, stuff, flags = 0):
 915         return self._ssl.write(stuff)
 916
 917     sendall = send
 918
 919     def recv(self, len = 1024, flags = 0):
 920         return self._ssl.read(len)
 921
 922     def __getattr__(self, attr):
 923         return getattr(self._sock, attr)
 924
 925
 926 class HTTPSConnection(HTTPConnection):
 927     "This class allows communication via SSL."
 928
 929     default_port = HTTPS_PORT
 930
 931     def __init__(self, host, port=None, key_file=None, cert_file=None,
 932                  strict=None):
 933         HTTPConnection.__init__(self, host, port, strict)
 934         self.key_file = key_file
 935         self.cert_file = cert_file
 936
 937     def connect(self):
 938         "Connect to a host on a given (SSL) port."
 939
 940         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 941         sock.connect((self.host, self.port))
 942         realsock = sock
 943         if hasattr(sock, "_sock"):
 944             realsock = sock._sock
 945         ssl = socket.ssl(realsock, self.key_file, self.cert_file)
 946         self.sock = FakeSocket(sock, ssl)
 947
 948
 949 class HTTP:
 950     "Compatibility class with httplib.py from 1.5."
 951
 952     _http_vsn = 10
 953     _http_vsn_str = 'HTTP/1.0'
 954
 955     debuglevel = 0
 956
 957     _connection_class = HTTPConnection
 958
 959     def __init__(self, host='', port=None, strict=None):
 960         "Provide a default host, since the superclass requires one."
 961
 962         # some joker passed 0 explicitly, meaning default port
 963         if port == 0:
 964             port = None
 965
 966         # Note that we may pass an empty string as the host; this will throw
 967         # an error when we attempt to connect. Presumably, the client code
 968         # will call connect before then, with a proper host.
 969         self._setup(self._connection_class(host, port, strict))
 970
 971     def _setup(self, conn):
 972         self._conn = conn
 973
 974         # set up delegation to flesh out interface
 975         self.send = conn.send
 976         self.putrequest = conn.putrequest
 977         self.endheaders = conn.endheaders
 978         self.set_debuglevel = conn.set_debuglevel
 979
 980         conn._http_vsn = self._http_vsn
 981         conn._http_vsn_str = self._http_vsn_str
 982
 983         self.file = None
 984
 985     def connect(self, host=None, port=None):
 986         "Accept arguments to set the host/port, since the superclass doesn't."
 987
 988         if host is not None:
 989             self._conn._set_hostport(host, port)
 990         self._conn.connect()
 991
 992     def getfile(self):
 993         "Provide a getfile, since the superclass' does not use this concept."
 994         return self.file
 995
 996     def putheader(self, header, *values):
 997         "The superclass allows only one value argument."
 998         self._conn.putheader(header, '\r\n\t'.join(values))
 999
1000     def getreply(self):
1001         """Compat definition since superclass does not define it.
1002
1003         Returns a tuple consisting of:
1004         - server status code (e.g. '200' if all goes well)
1005         - server "reason" corresponding to status code
1006         - any RFC822 headers in the response from the server
1007         """
1008         try:
1009             response = self._conn.getresponse()
1010         except BadStatusLine, e:
1011             ### hmm. if getresponse() ever closes the socket on a bad request,
1012             ### then we are going to have problems with self.sock
1013
1014             ### should we keep this behavior? do people use it?
1015             # keep the socket open (as a file), and return it
1016             self.file = self._conn.sock.makefile('rb', 0)
1017
1018             # close our socket -- we want to restart after any protocol error
1019             self.close()
1020
1021             self.headers = None
1022             return -1, e.line, None
1023
1024         self.headers = response.msg
1025         self.file = response.fp
1026         return response.status, response.reason, response.msg
1027
1028     def close(self):
1029         self._conn.close()
1030
1031         # note that self.file == response.fp, which gets closed by the
1032         # superclass. just clear the object ref here.
1033         ### hmm. messy. if status==-1, then self.file is owned by us.
1034         ### well... we aren't explicitly closing, but losing this ref will
1035         ### do it
1036         self.file = None
1037
1038 if hasattr(socket, 'ssl'):
1039     class HTTPS(HTTP):
1040         """Compatibility with 1.5 httplib interface
1041
1042         Python 1.5.2 did not have an HTTPS class, but it defined an
1043         interface for sending http requests that is also useful for
1044         https.
1045         """
1046
1047         _connection_class = HTTPSConnection
1048
1049         def __init__(self, host='', port=None, key_file=None, cert_file=None,
1050                      strict=None):
1051             # provide a default host, pass the X509 cert info
1052
1053             # urf. compensate for bad input.
1054             if port == 0:
1055                 port = None
1056             self._setup(self._connection_class(host, port, key_file,
1057                                                cert_file, strict))
1058
1059             # we never actually use these for anything, but we keep them
1060             # here for compatibility with post-1.5.2 CVS.
1061             self.key_file = key_file
1062             self.cert_file = cert_file
1063
1064
1065 class HTTPException(Exception):
1066     # Subclasses that define an __init__ must call Exception.__init__
1067     # or define self.args.  Otherwise, str() will fail.
1068     pass
1069
1070 class NotConnected(HTTPException):
1071     pass
1072
1073 class InvalidURL(HTTPException):
1074     pass
1075
1076 class UnknownProtocol(HTTPException):
1077     def __init__(self, version):
1078         self.args = version,
1079         self.version = version
1080
1081 class UnknownTransferEncoding(HTTPException):
1082     pass
1083
1084 class UnimplementedFileMode(HTTPException):
1085     pass
1086
1087 class IncompleteRead(HTTPException):
1088     def __init__(self, partial):
1089         self.args = partial,
1090         self.partial = partial
1091
1092 class ImproperConnectionState(HTTPException):
1093     pass
1094
1095 class CannotSendRequest(ImproperConnectionState):
1096     pass
1097
1098 class CannotSendHeader(ImproperConnectionState):
1099     pass
1100
1101 class ResponseNotReady(ImproperConnectionState):
1102     pass
1103
1104 class BadStatusLine(HTTPException):
1105     def __init__(self, line):
1106         self.args = line,
1107         self.line = line
1108
1109 # for backwards compatibility
1110 error = HTTPException
1111
1112 class LineAndFileWrapper:
1113     """A limited file-like object for HTTP/0.9 responses."""
1114
1115     # The status-line parsing code calls readline(), which normally
1116     # get the HTTP status line.  For a 0.9 response, however, this is
1117     # actually the first line of the body!  Clients need to get a
1118     # readable file object that contains that line.
1119
1120     def __init__(self, line, file):
1121         self._line = line
1122         self._file = file
1123         self._line_consumed = 0
1124         self._line_offset = 0
1125         self._line_left = len(line)
1126
1127     def __getattr__(self, attr):
1128         return getattr(self._file, attr)
1129
1130     def _done(self):
1131         # called when the last byte is read from the line.  After the
1132         # call, all read methods are delegated to the underlying file
1133         # obhect.
1134         self._line_consumed = 1
1135         self.read = self._file.read
1136         self.readline = self._file.readline
1137         self.readlines = self._file.readlines
1138
1139     def read(self, amt=None):
1140         assert not self._line_consumed and self._line_left
1141         if amt is None or amt > self._line_left:
1142             s = self._line[self._line_offset:]
1143             self._done()
1144             if amt is None:
1145                 return s + self._file.read()
1146             else:
1147                 return s + self._file.read(amt - len(s))
1148         else:
1149             assert amt <= self._line_left
1150             i = self._line_offset
1151             j = i + amt
1152             s = self._line[i:j]
1153             self._line_offset = j
1154             self._line_left -= amt
1155             if self._line_left == 0:
1156                 self._done()
1157             return s
1158
1159     def readline(self):
1160         s = self._line[self._line_offset:]
1161         self._done()
1162         return s
1163
1164     def readlines(self, size=None):
1165         L = [self._line[self._line_offset:]]
1166         self._done()
1167         if size is None:
1168             return L + self._file.readlines()
1169         else:
1170             return L + self._file.readlines(size)
1171
1172 def test():
1173     """Test this module.
1174
1175     A hodge podge of tests collected here, because they have too many
1176     external dependencies for the regular test suite.
1177     """
1178
1179     import sys
1180     import getopt
1181     opts, args = getopt.getopt(sys.argv[1:], 'd')
1182     dl = 0
1183     for o, a in opts:
1184         if o == '-d': dl = dl + 1
1185     host = 'www.python.org'
1186     selector = '/'
1187     if args[0:]: host = args[0]
1188     if args[1:]: selector = args[1]
1189     h = HTTP()
1190     h.set_debuglevel(dl)
1191     h.connect(host)
1192     h.putrequest('GET', selector)
1193     h.endheaders()
1194     status, reason, headers = h.getreply()
1195     print 'status =', status
1196     print 'reason =', reason
1197     print "read", len(h.getfile().read())
1198     print
1199     if headers:
1200         for header in headers.headers: print header.strip()
1201     print
1202
1203     # minimal test that code to extract host from url works
1204     class HTTP11(HTTP):
1205         _http_vsn = 11
1206         _http_vsn_str = 'HTTP/1.1'
1207
1208     h = HTTP11('www.python.org')
1209     h.putrequest('GET', 'http://www.python.org/~jeremy/')
1210     h.endheaders()
1211     h.getreply()
1212     h.close()
1213
1214     if hasattr(socket, 'ssl'):
1215
1216         for host, selector in (('sourceforge.net', '/projects/python'),
1217                                ('dbserv2.theopalgroup.com', '/mediumfile'),
1218                                ('dbserv2.theopalgroup.com', '/smallfile'),
1219                                ):
1220             print "https://%s%s" % (host, selector)
1221             hs = HTTPS()
1222             hs.set_debuglevel(dl)
1223             hs.connect(host)
1224             hs.putrequest('GET', selector)
1225             hs.endheaders()
1226             status, reason, headers = hs.getreply()
1227             print 'status =', status
1228             print 'reason =', reason
1229             print "read", len(hs.getfile().read())
1230             print
1231             if headers:
1232                 for header in headers.headers: print header.strip()
1233             print
1234
1235     # Test a buggy server -- returns garbled status line.
1236     # http://www.yahoo.com/promotions/mom_com97/supermom.html
1237     c = HTTPConnection("promotions.yahoo.com")
1238     c.set_debuglevel(1)
1239     c.connect()
1240     c.request("GET", "/promotions/mom_com97/supermom.html")
1241     r = c.getresponse()
1242     print r.status, r.version
1243     lines = r.read().split("\n")
1244     print "\n".join(lines[:5])
1245
1246     c = HTTPConnection("promotions.yahoo.com", strict=1)
1247     c.set_debuglevel(1)
1248     c.connect()
1249     c.request("GET", "/promotions/mom_com97/supermom.html")
1250     try:
1251         r = c.getresponse()
1252     except BadStatusLine, err:
1253         print "strict mode failed as expected"
1254         print err
1255     else:
1256         print "XXX strict mode should have failed"
1257
1258     for strict in 0, 1:
1259         h = HTTP(strict=strict)
1260         h.connect("promotions.yahoo.com")
1261         h.putrequest('GET', "/promotions/mom_com97/supermom.html")
1262         h.endheaders()
1263         status, reason, headers = h.getreply()
1264         assert (strict and status == -1) or status == 200, (strict, status)
1265
1266 if __name__ == '__main__':
1267     test()