src/ptclib/httpclnt.cxx

   1 /*
   2  * httpclnt.cxx
   3  *
   4  * HTTP Client class.
   5  *
   6  * Portable Windows Library
   7  *
   8  * Copyright (c) 1993-2002 Equivalence Pty. Ltd.
   9  *
  10  * The contents of this file are subject to the Mozilla Public License
  11  * Version 1.0 (the "License"); you may not use this file except in
  12  * compliance with the License. You may obtain a copy of the License at
  13  * http://www.mozilla.org/MPL/
  14  *
  15  * Software distributed under the License is distributed on an "AS IS"
  16  * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
  17  * the License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * The Original Code is Portable Windows Library.
  21  *
  22  * The Initial Developer of the Original Code is Equivalence Pty. Ltd.
  23  *
  24  * Contributor(s): ______________________________________.
  25  *
  26  * $Log$
  27  * Revision 1.33  2003/04/23 07:00:15  rogerh
  28  * Fix the encoding checking. the find_ip sample program now works again
  29  *
  30  * Revision 1.32  2003/01/28 06:48:35  robertj
  31  * Added https support to PHTTPClient (if #define P_SSL availbel).
  32  *
  33  * Revision 1.31  2002/12/03 22:38:35  robertj
  34  * Removed get document that just returns a content length as the chunked
  35  *   transfer encoding makes this very dangerous.
  36  * Added GetTextDocument() to get a URL content into a PString.
  37  * Added a version pf PostData() that gets the reponse content into a PString.
  38  * Added ReadContentBody() that takes a PString, not just PBYTEArray.
  39  * Fixed bug where conten-encoding must be checked even if there is a
  40  *   full content length MIME field.
  41  *
  42  * Revision 1.30  2002/11/06 22:47:25  robertj
  43  * Fixed header comment (copyright etc)
  44  *
  45  * Revision 1.29  2002/10/10 04:43:44  robertj
  46  * VxWorks port, thanks Martijn Roest
  47  *
  48  * Revision 1.28  2002/05/28 01:41:50  robertj
  49  * Fixed bug in reading chunked data, thanks David Iodice
  50  *
  51  * Revision 1.27  2001/10/30 07:02:28  robertj
  52  * Fixed problem with bad servers causing endless loops in client.
  53  *
  54  * Revision 1.26  2001/10/03 00:26:34  robertj
  55  * Upgraded client to HTTP/1.1 and for chunked mode entity bodies.
  56  *
  57  * Revision 1.25  2001/09/28 08:55:15  robertj
  58  * More changes to support restartable PHTTPClient
  59  *
  60  * Revision 1.24  2001/09/28 00:43:47  robertj
  61  * Added automatic setting of some outward MIME fields.
  62  * Added "user agent" string field for automatic inclusion.
  63  * Added function to read the contents of the HTTP request.
  64  * Added "restarting" of connection if lost persistence.
  65  *
  66  * Revision 1.23  2001/09/11 03:27:46  robertj
  67  * Improved error processing on high level protocol failures, usually
  68  *   caused by unexpected shut down of a socket.
  69  *
  70  * Revision 1.22  2001/09/10 02:51:23  robertj
  71  * Major change to fix problem with error codes being corrupted in a
  72  *   PChannel when have simultaneous reads and writes in threads.
  73  *
  74  * Revision 1.21  2001/02/22 05:27:14  robertj
  75  * Added "nicer" version of GetDocument in HTTP client class.
  76  *
  77  * Revision 1.20  1999/05/13 04:59:24  robertj
  78  * Increased amount of buffering on output request write.
  79  *
  80  * Revision 1.19  1999/05/11 12:23:52  robertj
  81  * Fixed bug introduced in last revision to have arbitrary HTTP commands, missing CRLF.
  82  *
  83  * Revision 1.18  1999/05/04 15:26:01  robertj
  84  * Improved HTTP/1.1 compatibility (pass through user commands).
  85  * Fixed problems with quicktime installer.
  86  *
  87  * Revision 1.17  1998/11/30 04:51:55  robertj
  88  * New directory structure
  89  *
  90  * Revision 1.16  1998/09/23 06:22:09  robertj
  91  * Added open source copyright license.
  92  *
  93  * Revision 1.15  1998/07/24 06:57:21  robertj
  94  * Fixed error returned on illegal URL passed to unopened socket.
  95  * Changed PostData function so just has string for data instead of dictionary.
  96  *
  97  * Revision 1.14  1998/06/16 03:32:56  robertj
  98  * Changed TCP connection shutdown to be parameterised.
  99  *
 100  * Revision 1.13  1998/06/13 15:03:58  robertj
 101  * More conditions for NOT shutting down write.
 102  *
 103  * Revision 1.12  1998/06/13 12:28:04  robertj
 104  * Added shutdown to client command if no content length specified.
 105  *
 106  * Revision 1.11  1998/04/14 03:42:41  robertj
 107  * Fixed error code propagation in HTTP client.
 108  *
 109  * Revision 1.10  1998/02/03 06:27:10  robertj
 110  * Fixed propagation of error codes, especially EOF.
 111  * Fixed writing to some CGI scripts that require CRLF outside of byte count.
 112  *
 113  * Revision 1.9  1998/01/26 00:39:00  robertj
 114  * Added function to allow HTTPClient to automatically connect if URL has hostname.
 115  * Fixed incorrect return values on HTTPClient GetDocument(), Post etc functions.
 116  *
 117  * Revision 1.8  1997/06/12 12:33:35  robertj
 118  * Fixed bug where mising MIME fields is regarded as an eror.
 119  *
 120  * Revision 1.7  1997/03/31 08:26:58  robertj
 121  * GNU compiler compatibilty.
 122  *
 123  * Revision 1.6  1997/03/28 04:40:46  robertj
 124  * Fixed bug in Post function doing wrong command.
 125  *
 126  * Revision 1.5  1997/03/18 22:04:03  robertj
 127  * Fix bug for binary POST commands.
 128  *
 129  * Revision 1.4  1996/12/21 01:26:21  robertj
 130  * Fixed bug in persistent connections when server closes socket during command.
 131  *
 132  * Revision 1.3  1996/12/12 09:24:44  robertj
 133  * Persistent connection support.
 134  *
 135  * Revision 1.2  1996/10/08 13:12:03  robertj
 136  * Fixed bug in HTTP/0.9 response, first 5 character not put back properly.
 137  *
 138  * Revision 1.1  1996/09/14 13:02:18  robertj
 139  * Initial revision
 140  *
 141  * Revision 1.37  1996/08/25 09:37:41  robertj
 142  * Added function to detect "local" host name.
 143  * Fixed printing of trailing '/' in empty URL, is distinction between with and without.
 144  *
 145  * Revision 1.36  1996/08/22 13:22:26  robertj
 146  * Fixed bug in print of URLs, extra @ signs.
 147  *
 148  * Revision 1.35  1996/08/19 13:42:40  robertj
 149  * Fixed errors in URL parsing and display.
 150  * Fixed "Forbidden" problem out of HTTP authorisation system.
 151  * Fixed authorisation so if have no user/password on basic authentication, does not require it.
 152  *
 153  * Revision 1.34  1996/07/27 04:13:47  robertj
 154  * Fixed use of HTTP proxy on non-persistent connections.
 155  *
 156  * Revision 1.33  1996/07/15 10:37:20  robertj
 157  * Improved proxy "self" detection (especially localhost).
 158  *
 159  * Revision 1.32  1996/06/28 13:20:24  robertj
 160  * Modified HTTPAuthority so gets PHTTPReqest (mainly for URL) passed in.
 161  * Moved HTTP form resource to another compilation module.
 162  * Fixed memory leak in POST command.
 163  *
 164  * Revision 1.31  1996/06/10 10:00:00  robertj
 165  * Added global function for query parameters parsing.
 166  *
 167  * Revision 1.30  1996/06/07 13:52:23  robertj
 168  * Added PUT to HTTP proxy FTP. Necessitating redisign of entity body processing.
 169  *
 170  * Revision 1.29  1996/06/05 12:33:04  robertj
 171  * Fixed bug in parsing URL with no path, is NOT absolute!
 172  *
 173  * Revision 1.28  1996/05/30 10:07:26  robertj
 174  * Fixed bug in version number checking of return code compatibility.
 175  *
 176  * Revision 1.27  1996/05/26 03:46:42  robertj
 177  * Compatibility to GNU 2.7.x
 178  *
 179  * Revision 1.26  1996/05/23 10:02:13  robertj
 180  * Added common function for GET and HEAD commands.
 181  * Fixed status codes to be the actual status code instead of sequential enum.
 182  * This fixed some problems with proxy pass through of status codes.
 183  * Fixed bug in URL parsing of username and passwords.
 184  *
 185  * Revision 1.19.1.1  1996/04/17 11:08:22  craigs
 186  * New version by craig pending confirmation by robert
 187  *
 188  * Revision 1.19  1996/04/05 01:46:30  robertj
 189  * Assured PSocket::Write always writes the number of bytes specified, no longer need write loops.
 190  * Added workaraound for NT Netscape Navigator bug with persistent connections.
 191  *
 192  * Revision 1.18  1996/03/31 09:05:07  robertj
 193  * HTTP 1.1 upgrade.
 194  *
 195  * Revision 1.17  1996/03/17 05:48:07  robertj
 196  * Fixed host name print out of URLs.
 197  * Added hit count to PHTTPResource.
 198  *
 199  * Revision 1.16  1996/03/16 05:00:26  robertj
 200  * Added ParseReponse() for splitting reponse line into code and info.
 201  * Added client side support for HTTP socket.
 202  * Added hooks for proxy support in HTTP socket.
 203  * Added translation type to TranslateString() to accommodate query variables.
 204  * Defaulted scheme field in URL to "http".
 205  * Inhibited output of port field on string conversion of URL according to scheme.
 206  *
 207  * Revision 1.15  1996/03/11 10:29:50  robertj
 208  * Fixed bug in help image HTML.
 209  *
 210  * Revision 1.14  1996/03/10 13:15:24  robertj
 211  * Redesign to make resources thread safe.
 212  *
 213  * Revision 1.13  1996/03/02 03:27:37  robertj
 214  * Added function to translate a string to a form suitable for inclusion in a URL.
 215  * Added radio button and selection boxes to HTTP form resource.
 216  * Fixed bug in URL parsing, losing first / if hostname specified.
 217  *
 218  * Revision 1.12  1996/02/25 11:14:24  robertj
 219  * Radio button support for forms.
 220  *
 221  * Revision 1.11  1996/02/25 03:10:34  robertj
 222  * Removed pass through HTTP resource.
 223  * Fixed PHTTPConfig resource to use correct name for config key.
 224  *
 225  * Revision 1.10  1996/02/19 13:48:28  robertj
 226  * Put multiple uses of literal strings into const variables.
 227  * Fixed URL parsing so that the unmangling of strings occurs correctly.
 228  * Moved nested classes from PHTTPForm.
 229  * Added overwrite option to AddResource().
 230  * Added get/set string to PHTTPString resource.
 231  *
 232  * Revision 1.9  1996/02/13 13:09:17  robertj
 233  * Added extra parameters to callback function in PHTTPResources, required
 234  *   by descendants to make informed decisions on data being loaded.
 235  *
 236  * Revision 1.8  1996/02/08 12:26:29  robertj
 237  * Redesign of resource callback mechanism.
 238  * Added new resource types for HTML data entry forms.
 239  *
 240  * Revision 1.7  1996/02/03 11:33:19  robertj
 241  * Changed RadCmd() so can distinguish between I/O error and unknown command.
 242  *
 243  * Revision 1.6  1996/02/03 11:11:49  robertj
 244  * Numerous bug fixes.
 245  * Added expiry date and ismodifiedsince support.
 246  *
 247  * Revision 1.5  1996/01/30 23:32:40  robertj
 248  * Added single .
 249  *
 250  * Revision 1.4  1996/01/28 14:19:09  robertj
 251  * Split HTML into separate source file.
 252  * Beginning of pass through resource type.
 253  * Changed PCharArray in OnLoadData to PString for convenience in mangling data.
 254  * Made PHTTPSpace return standard page on selection of partial path.
 255  *
 256  * Revision 1.3  1996/01/28 02:49:16  robertj
 257  * Further implementation.
 258  *
 259  * Revision 1.2  1996/01/26 02:24:30  robertj
 260  * Further implemetation.
 261  *
 262  * Revision 1.1  1996/01/23 13:04:32  robertj
 263  * Initial revision
 264  *
 265  */
 266
 267 #include <ptlib.h>
 268 #include <ptlib/sockets.h>
 269 #include <ptclib/http.h>
 270
 271 #if P_SSL
 272 #include <ptclib/pssl.h>
 273 #endif
 274
 275 #include <ctype.h>
 276
 277
 278 //////////////////////////////////////////////////////////////////////////////
 279 // PHTTPClient
 280
 281 PHTTPClient::PHTTPClient()
 282 {
 283 }
 284
 285
 286 PHTTPClient::PHTTPClient(const PString & userAgent)
 287   : userAgentName(userAgent)
 288 {
 289 }
 290
 291
 292 int PHTTPClient::ExecuteCommand(Commands cmd,
 293                                 const PURL & url,
 294                                 PMIMEInfo & outMIME,
 295                                 const PString & dataBody,
 296                                 PMIMEInfo & replyMime,
 297                                 BOOL persist)
 298 {
 299   return ExecuteCommand(commandNames[cmd], url, outMIME, dataBody, replyMime, persist);
 300 }
 301
 302
 303 int PHTTPClient::ExecuteCommand(const PString & cmdName,
 304                                 const PURL & url,
 305                                 PMIMEInfo & outMIME,
 306                                 const PString & dataBody,
 307                                 PMIMEInfo & replyMime,
 308                                 BOOL persist)
 309 {
 310   if (!outMIME.Contains(DateTag))
 311     outMIME.SetAt(DateTag, PTime().AsString());
 312
 313   if (!userAgentName && !outMIME.Contains(UserAgentTag))
 314     outMIME.SetAt(UserAgentTag, userAgentName);
 315
 316   if (persist)
 317     outMIME.SetAt(ConnectionTag, KeepAliveTag);
 318
 319   for (PINDEX retry = 0; retry < 3; retry++) {
 320     if (!AssureConnect(url, outMIME))
 321       break;
 322
 323     if (!WriteCommand(cmdName, url.AsString(PURL::URIOnly), outMIME, dataBody)) {
 324       lastResponseCode = -1;
 325       lastResponseInfo = GetErrorText(LastWriteError);
 326       break;
 327     }
 328
 329     // If not persisting need to shut down write so other end stops reading
 330     if (!persist)
 331       Shutdown(ShutdownWrite);
 332
 333     // Await a response, if all OK exit loop
 334     if (ReadResponse(replyMime))
 335       break;
 336
 337     // If not persisting, we have no oppurtunity to write again, just error out
 338     if (!persist)
 339       break;
 340
 341     // If have had a failure to read a response but there was no error then
 342     // we have a shutdown socket probably due to a lack of persistence so ...
 343     if (GetErrorCode(LastReadError) != NoError)
 344       break;
 345
 346     // ... we close the channel and allow AssureConnet() to reopen it.
 347     Close();
 348   }
 349
 350   return lastResponseCode;
 351 }
 352
 353
 354 BOOL PHTTPClient::WriteCommand(Commands cmd,
 355                                const PString & url,
 356                                PMIMEInfo & outMIME,
 357                                const PString & dataBody)
 358 {
 359   return WriteCommand(commandNames[cmd], url, outMIME, dataBody);
 360 }
 361
 362
 363 BOOL PHTTPClient::WriteCommand(const PString & cmdName,
 364                                const PString & url,
 365                                PMIMEInfo & outMIME,
 366                                const PString & dataBody)
 367 {
 368   PINDEX len = dataBody.GetSize()-1;
 369   if (!outMIME.Contains(ContentLengthTag))
 370     outMIME.SetInteger(ContentLengthTag, len);
 371
 372   if (cmdName.IsEmpty())
 373     *this << "GET";
 374   else
 375     *this << cmdName;
 376
 377   *this << ' ' << url << " HTTP/1.1\r\n"
 378         << setfill('\r') << outMIME;
 379
 380   return Write((const char *)dataBody, len);
 381 }
 382
 383
 384 BOOL PHTTPClient::ReadResponse(PMIMEInfo & replyMIME)
 385 {
 386   PString http = ReadString(7);
 387   if (!http) {
 388     UnRead(http);
 389
 390     if (http.Find("HTTP/") == P_MAX_INDEX) {
 391       lastResponseCode = PHTTP::RequestOK;
 392       lastResponseInfo = "HTTP/0.9";
 393       return TRUE;
 394     }
 395
 396     if (http[0] == '\n')
 397       ReadString(1);
 398     else if (http[0] == '\r' &&  http[1] == '\n')
 399       ReadString(2);
 400
 401     if (PHTTP::ReadResponse())
 402       if (replyMIME.Read(*this))
 403         return TRUE;
 404   }
 405
 406   lastResponseCode = -1;
 407   if (GetErrorCode(LastReadError) != NoError)
 408     lastResponseInfo = GetErrorText(LastReadError);
 409   else {
 410     lastResponseInfo = "Premature shutdown";
 411     SetErrorValues(ProtocolFailure, 0, LastReadError);
 412   }
 413
 414   return FALSE;
 415 }
 416
 417
 418 BOOL PHTTPClient::ReadContentBody(PMIMEInfo & replyMIME, PString & body)
 419 {
 420   BOOL ok = InternalReadContentBody(replyMIME, body);
 421   body.SetSize(body.GetSize()+1);
 422   return ok;
 423 }
 424
 425
 426 BOOL PHTTPClient::ReadContentBody(PMIMEInfo & replyMIME, PBYTEArray & body)
 427 {
 428   return InternalReadContentBody(replyMIME, body);
 429 }
 430
 431
 432 BOOL PHTTPClient::InternalReadContentBody(PMIMEInfo & replyMIME, PAbstractArray & body)
 433 {
 434   PCaselessString encoding = replyMIME(TransferEncodingTag);
 435
 436   if (encoding != ChunkedTag) {
 437     if (replyMIME.Contains(ContentLengthTag)) {
 438       PINDEX length = replyMIME.GetInteger(ContentLengthTag);
 439       body.SetSize(length);
 440       return ReadBlock(body.GetPointer(), length);
 441     }
 442
 443     if (!(encoding.IsEmpty())) {
 444       lastResponseCode = -1;
 445       lastResponseInfo = "Unknown Transfer-Encoding extension";
 446       return FALSE;
 447     }
 448
 449     // Must be raw, read to end file variety
 450     static const PINDEX ChunkSize = 2048;
 451     PINDEX bytesRead = 0;
 452     while (ReadBlock((char *)body.GetPointer(bytesRead+ChunkSize)+bytesRead, ChunkSize))
 453       bytesRead += GetLastReadCount();
 454
 455     body.SetSize(bytesRead + GetLastReadCount());
 456     return GetErrorCode(LastReadError) == NoError;
 457   }
 458
 459   // HTTP1.1 chunked format
 460   PINDEX bytesRead = 0;
 461   for (;;) {
 462     // Read chunk length line
 463     PString chunkLengthLine;
 464     if (!ReadLine(chunkLengthLine))
 465       return FALSE;
 466
 467     // A zero length chunk is end of output
 468     PINDEX chunkLength = chunkLengthLine.AsUnsigned(16);
 469     if (chunkLength == 0)
 470       break;
 471
 472     // Read the chunk
 473     if (!ReadBlock((char *)body.GetPointer(bytesRead+chunkLength)+bytesRead, chunkLength))
 474       return FALSE;
 475     bytesRead+= chunkLength;
 476
 477     // Read the trailing CRLF
 478     if (!ReadLine(chunkLengthLine))
 479       return FALSE;
 480   }
 481
 482   // Read the footer
 483   PString footer;
 484   do {
 485     if (!ReadLine(footer))
 486       return FALSE;
 487   } while (replyMIME.AddMIME(footer));
 488
 489   return TRUE;
 490 }
 491
 492
 493 BOOL PHTTPClient::GetTextDocument(const PURL & url,
 494                                   PString & document,
 495                                   BOOL persist)
 496 {
 497   PMIMEInfo outMIME, replyMIME;
 498   if (!GetDocument(url, outMIME, replyMIME, persist))
 499     return FALSE;
 500
 501   return ReadContentBody(replyMIME, document);
 502 }
 503
 504
 505 BOOL PHTTPClient::GetDocument(const PURL & url,
 506                               PMIMEInfo & outMIME,
 507                               PMIMEInfo & replyMIME,
 508                               BOOL persist)
 509 {
 510   return ExecuteCommand(GET, url, outMIME, PString(), replyMIME, persist) == RequestOK;
 511 }
 512
 513
 514 BOOL PHTTPClient::GetHeader(const PURL & url,
 515                             PMIMEInfo & outMIME,
 516                             PMIMEInfo & replyMIME,
 517                             BOOL persist)
 518 {
 519   return ExecuteCommand(HEAD, url, outMIME, PString(), replyMIME, persist) == RequestOK;
 520 }
 521
 522
 523 BOOL PHTTPClient::PostData(const PURL & url,
 524                            PMIMEInfo & outMIME,
 525                            const PString & data,
 526                            PMIMEInfo & replyMIME,
 527                            BOOL persist)
 528 {
 529   PString dataBody = data;
 530   if (!outMIME.Contains(ContentTypeTag)) {
 531     outMIME.SetAt(ContentTypeTag, "application/x-www-form-urlencoded");
 532     dataBody += "\r\n"; // Add CRLF for compatibility with some CGI servers.
 533   }
 534
 535   return ExecuteCommand(POST, url, outMIME, data, replyMIME, persist) == RequestOK;
 536 }
 537
 538
 539 BOOL PHTTPClient::PostData(const PURL & url,
 540                            PMIMEInfo & outMIME,
 541                            const PString & data,
 542                            PMIMEInfo & replyMIME,
 543                            PString & body,
 544                            BOOL persist)
 545 {
 546   if (!PostData(url, outMIME, data, replyMIME, persist))
 547     return FALSE;
 548
 549   return ReadContentBody(replyMIME, body);
 550 }
 551
 552
 553 BOOL PHTTPClient::AssureConnect(const PURL & url, PMIMEInfo & outMIME)
 554 {
 555   PString host = url.GetHostName();
 556
 557   // Is not open or other end shut down, restablish connection
 558   if (!IsOpen()) {
 559     if (host.IsEmpty()) {
 560       lastResponseCode = BadRequest;
 561       lastResponseInfo = "No host specified";
 562       return SetErrorValues(ProtocolFailure, 0, LastReadError);
 563     }
 564
 565 #if P_SSL
 566     if (url.GetScheme() == "https") {
 567       PTCPSocket * tcp = new PTCPSocket(url.GetPort());
 568       tcp->SetReadTimeout(readTimeout);
 569       if (!tcp->Connect(host)) {
 570         lastResponseCode = -2;
 571         lastResponseInfo = tcp->GetErrorText();
 572         delete tcp;
 573         return FALSE;
 574       }
 575
 576       PSSLChannel * ssl = new PSSLChannel;
 577       if (!ssl->Connect(tcp)) {
 578         lastResponseCode = -2;
 579         lastResponseInfo = ssl->GetErrorText();
 580         delete ssl;
 581         return FALSE;
 582       }
 583
 584       if (!Open(ssl)) {
 585         lastResponseCode = -2;
 586         lastResponseInfo = GetErrorText();
 587         return FALSE;
 588       }
 589     }
 590     else
 591 #endif
 592
 593     if (!Connect(host, url.GetPort())) {
 594       lastResponseCode = -2;
 595       lastResponseInfo = GetErrorText();
 596       return FALSE;
 597     }
 598   }
 599
 600   // Have connection, so fill in the required MIME fields
 601   static char HostTag[] = "Host";
 602   if (!outMIME.Contains(HostTag)) {
 603     if (!host)
 604       outMIME.SetAt(HostTag, host);
 605     else {
 606       PIPSocket * sock = GetSocket();
 607       if (sock != NULL)
 608         outMIME.SetAt(HostTag, sock->GetHostName());
 609     }
 610   }
 611
 612   return TRUE;
 613 }
 614
 615
 616 // End Of File ///////////////////////////////////////////////////////////////