src/kits/network/libnetapi/HttpRequest.cpp

   1 /*
   2  * Copyright 2010-2014 Haiku Inc. All rights reserved.
   3  * Distributed under the terms of the MIT License.
   4  *
   5  * Authors:
   6  *              Christophe Huriaux, c.huriaux@gmail.com
   7  *              Niels Sascha Reedijk, niels.reedijk@gmail.com
   8  *              Adrien Destugues, pulkomandy@pulkomandy.tk
   9  */
  10
  11
  12 #include <HttpRequest.h>
  13
  14 #include <arpa/inet.h>
  15 #include <stdio.h>
  16
  17 #include <cstdlib>
  18 #include <deque>
  19 #include <new>
  20
  21 #include <AutoDeleter.h>
  22 #include <Certificate.h>
  23 #include <Debug.h>
  24 #include <DynamicBuffer.h>
  25 #include <File.h>
  26 #include <Socket.h>
  27 #include <SecureSocket.h>
  28 #include <StackOrHeapArray.h>
  29 #include <ZlibCompressionAlgorithm.h>
  30
  31
  32 static const int32 kHttpBufferSize = 4096;
  33
  34
  35 class CheckedSecureSocket: public BSecureSocket
  36 {
  37 public:
  38                                         CheckedSecureSocket(BHttpRequest* request);
  39
  40         bool                    CertificateVerificationFailed(BCertificate& certificate,
  41                                                 const char* message);
  42
  43 private:
  44         BHttpRequest*   fRequest;
  45 };
  46
  47
  48 CheckedSecureSocket::CheckedSecureSocket(BHttpRequest* request)
  49         :
  50         BSecureSocket(),
  51         fRequest(request)
  52 {
  53 }
  54
  55
  56 bool
  57 CheckedSecureSocket::CertificateVerificationFailed(BCertificate& certificate,
  58         const char* message)
  59 {
  60         return fRequest->_CertificateVerificationFailed(certificate, message);
  61 }
  62
  63
  64 BHttpRequest::BHttpRequest(const BUrl& url, bool ssl, const char* protocolName,
  65         BUrlProtocolListener* listener, BUrlContext* context)
  66         :
  67         BNetworkRequest(url, listener, context, "BUrlProtocol.HTTP", protocolName),
  68         fSSL(ssl),
  69         fRequestMethod(B_HTTP_GET),
  70         fHttpVersion(B_HTTP_11),
  71         fResult(url),
  72         fRequestStatus(kRequestInitialState),
  73         fOptHeaders(NULL),
  74         fOptPostFields(NULL),
  75         fOptInputData(NULL),
  76         fOptInputDataSize(-1),
  77         fOptRangeStart(-1),
  78         fOptRangeEnd(-1),
  79         fOptFollowLocation(true)
  80 {
  81         _ResetOptions();
  82         fSocket = NULL;
  83 }
  84
  85
  86 BHttpRequest::BHttpRequest(const BHttpRequest& other)
  87         :
  88         BNetworkRequest(other.Url(), other.fListener, other.fContext,
  89                 "BUrlProtocol.HTTP", other.fSSL ? "HTTPS" : "HTTP"),
  90         fSSL(other.fSSL),
  91         fRequestMethod(other.fRequestMethod),
  92         fHttpVersion(other.fHttpVersion),
  93         fResult(other.fUrl),
  94         fRequestStatus(kRequestInitialState),
  95         fOptHeaders(NULL),
  96         fOptPostFields(NULL),
  97         fOptInputData(NULL),
  98         fOptInputDataSize(-1),
  99         fOptRangeStart(other.fOptRangeStart),
 100         fOptRangeEnd(other.fOptRangeEnd),
 101         fOptFollowLocation(other.fOptFollowLocation)
 102 {
 103         _ResetOptions();
 104                 // FIXME some options may be copied from other instead.
 105         fSocket = NULL;
 106 }
 107
 108
 109 BHttpRequest::~BHttpRequest()
 110 {
 111         Stop();
 112
 113         delete fSocket;
 114
 115         delete fOptInputData;
 116         delete fOptHeaders;
 117         delete fOptPostFields;
 118 }
 119
 120
 121 void
 122 BHttpRequest::SetMethod(const char* const method)
 123 {
 124         fRequestMethod = method;
 125 }
 126
 127
 128 void
 129 BHttpRequest::SetFollowLocation(bool follow)
 130 {
 131         fOptFollowLocation = follow;
 132 }
 133
 134
 135 void
 136 BHttpRequest::SetMaxRedirections(int8 redirections)
 137 {
 138         fOptMaxRedirs = redirections;
 139 }
 140
 141
 142 void
 143 BHttpRequest::SetReferrer(const BString& referrer)
 144 {
 145         fOptReferer = referrer;
 146 }
 147
 148
 149 void
 150 BHttpRequest::SetUserAgent(const BString& agent)
 151 {
 152         fOptUserAgent = agent;
 153 }
 154
 155
 156 void
 157 BHttpRequest::SetDiscardData(bool discard)
 158 {
 159         fOptDiscardData = discard;
 160 }
 161
 162
 163 void
 164 BHttpRequest::SetDisableListener(bool disable)
 165 {
 166         fOptDisableListener = disable;
 167 }
 168
 169
 170 void
 171 BHttpRequest::SetAutoReferrer(bool enable)
 172 {
 173         fOptAutoReferer = enable;
 174 }
 175
 176
 177 void
 178 BHttpRequest::SetHeaders(const BHttpHeaders& headers)
 179 {
 180         AdoptHeaders(new(std::nothrow) BHttpHeaders(headers));
 181 }
 182
 183
 184 void
 185 BHttpRequest::AdoptHeaders(BHttpHeaders* const headers)
 186 {
 187         delete fOptHeaders;
 188         fOptHeaders = headers;
 189 }
 190
 191
 192 void
 193 BHttpRequest::SetPostFields(const BHttpForm& fields)
 194 {
 195         AdoptPostFields(new(std::nothrow) BHttpForm(fields));
 196 }
 197
 198
 199 void
 200 BHttpRequest::AdoptPostFields(BHttpForm* const fields)
 201 {
 202         delete fOptPostFields;
 203         fOptPostFields = fields;
 204
 205         if (fOptPostFields != NULL)
 206                 fRequestMethod = B_HTTP_POST;
 207 }
 208
 209
 210 void
 211 BHttpRequest::AdoptInputData(BDataIO* const data, const ssize_t size)
 212 {
 213         delete fOptInputData;
 214         fOptInputData = data;
 215         fOptInputDataSize = size;
 216 }
 217
 218
 219 void
 220 BHttpRequest::SetUserName(const BString& name)
 221 {
 222         fOptUsername = name;
 223 }
 224
 225
 226 void
 227 BHttpRequest::SetPassword(const BString& password)
 228 {
 229         fOptPassword = password;
 230 }
 231
 232
 233 /*static*/ bool
 234 BHttpRequest::IsInformationalStatusCode(int16 code)
 235 {
 236         return (code >= B_HTTP_STATUS__INFORMATIONAL_BASE)
 237                 && (code <  B_HTTP_STATUS__INFORMATIONAL_END);
 238 }
 239
 240
 241 /*static*/ bool
 242 BHttpRequest::IsSuccessStatusCode(int16 code)
 243 {
 244         return (code >= B_HTTP_STATUS__SUCCESS_BASE)
 245                 && (code <  B_HTTP_STATUS__SUCCESS_END);
 246 }
 247
 248
 249 /*static*/ bool
 250 BHttpRequest::IsRedirectionStatusCode(int16 code)
 251 {
 252         return (code >= B_HTTP_STATUS__REDIRECTION_BASE)
 253                 && (code <  B_HTTP_STATUS__REDIRECTION_END);
 254 }
 255
 256
 257 /*static*/ bool
 258 BHttpRequest::IsClientErrorStatusCode(int16 code)
 259 {
 260         return (code >= B_HTTP_STATUS__CLIENT_ERROR_BASE)
 261                 && (code <  B_HTTP_STATUS__CLIENT_ERROR_END);
 262 }
 263
 264
 265 /*static*/ bool
 266 BHttpRequest::IsServerErrorStatusCode(int16 code)
 267 {
 268         return (code >= B_HTTP_STATUS__SERVER_ERROR_BASE)
 269                 && (code <  B_HTTP_STATUS__SERVER_ERROR_END);
 270 }
 271
 272
 273 /*static*/ int16
 274 BHttpRequest::StatusCodeClass(int16 code)
 275 {
 276         if (BHttpRequest::IsInformationalStatusCode(code))
 277                 return B_HTTP_STATUS_CLASS_INFORMATIONAL;
 278         else if (BHttpRequest::IsSuccessStatusCode(code))
 279                 return B_HTTP_STATUS_CLASS_SUCCESS;
 280         else if (BHttpRequest::IsRedirectionStatusCode(code))
 281                 return B_HTTP_STATUS_CLASS_REDIRECTION;
 282         else if (BHttpRequest::IsClientErrorStatusCode(code))
 283                 return B_HTTP_STATUS_CLASS_CLIENT_ERROR;
 284         else if (BHttpRequest::IsServerErrorStatusCode(code))
 285                 return B_HTTP_STATUS_CLASS_SERVER_ERROR;
 286
 287         return B_HTTP_STATUS_CLASS_INVALID;
 288 }
 289
 290
 291 const BUrlResult&
 292 BHttpRequest::Result() const
 293 {
 294         return fResult;
 295 }
 296
 297
 298 status_t
 299 BHttpRequest::Stop()
 300 {
 301         if (fSocket != NULL) {
 302                 fSocket->Disconnect();
 303                         // Unlock any pending connect, read or write operation.
 304         }
 305         return BNetworkRequest::Stop();
 306 }
 307
 308
 309 void
 310 BHttpRequest::_ResetOptions()
 311 {
 312         delete fOptPostFields;
 313         delete fOptHeaders;
 314
 315         fOptFollowLocation = true;
 316         fOptMaxRedirs = 8;
 317         fOptReferer = "";
 318         fOptUserAgent = "Services Kit (Haiku)";
 319         fOptUsername = "";
 320         fOptPassword = "";
 321         fOptAuthMethods = B_HTTP_AUTHENTICATION_BASIC | B_HTTP_AUTHENTICATION_DIGEST
 322                 | B_HTTP_AUTHENTICATION_IE_DIGEST;
 323         fOptHeaders = NULL;
 324         fOptPostFields = NULL;
 325         fOptSetCookies = true;
 326         fOptDiscardData = false;
 327         fOptDisableListener = false;
 328         fOptAutoReferer = true;
 329 }
 330
 331
 332 status_t
 333 BHttpRequest::_ProtocolLoop()
 334 {
 335         // Initialize the request redirection loop
 336         int8 maxRedirs = fOptMaxRedirs;
 337         bool newRequest;
 338
 339         do {
 340                 newRequest = false;
 341
 342                 // Result reset
 343                 fHeaders.Clear();
 344                 _ResultHeaders().Clear();
 345
 346                 BString host = fUrl.Host();
 347                 int port = fSSL ? 443 : 80;
 348
 349                 if (fUrl.HasPort())
 350                         port = fUrl.Port();
 351
 352                 if (fContext->UseProxy()) {
 353                         host = fContext->GetProxyHost();
 354                         port = fContext->GetProxyPort();
 355                 }
 356
 357                 status_t result = fInputBuffer.InitCheck();
 358                 if (result != B_OK)
 359                         return result;
 360
 361                 if (!_ResolveHostName(host, port)) {
 362                         _EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR,
 363                                 "Unable to resolve hostname (%s), aborting.",
 364                                         fUrl.Host().String());
 365                         return B_SERVER_NOT_FOUND;
 366                 }
 367
 368                 status_t requestStatus = _MakeRequest();
 369                 if (requestStatus != B_OK)
 370                         return requestStatus;
 371
 372                 // Prepare the referer for the next request if needed
 373                 if (fOptAutoReferer)
 374                         fOptReferer = fUrl.UrlString();
 375
 376                 switch (StatusCodeClass(fResult.StatusCode())) {
 377                         case B_HTTP_STATUS_CLASS_INFORMATIONAL:
 378                                 // Header 100:continue should have been
 379                                 // handled in the _MakeRequest read loop
 380                                 break;
 381
 382                         case B_HTTP_STATUS_CLASS_SUCCESS:
 383                                 break;
 384
 385                         case B_HTTP_STATUS_CLASS_REDIRECTION:
 386                         {
 387                                 // Redirection has been explicitly disabled
 388                                 if (!fOptFollowLocation)
 389                                         break;
 390
 391                                 int code = fResult.StatusCode();
 392                                 if (code == B_HTTP_STATUS_MOVED_PERMANENTLY
 393                                         || code == B_HTTP_STATUS_FOUND
 394                                         || code == B_HTTP_STATUS_SEE_OTHER
 395                                         || code == B_HTTP_STATUS_TEMPORARY_REDIRECT) {
 396                                         BString locationUrl = fHeaders["Location"];
 397
 398                                         fUrl = BUrl(fUrl, locationUrl);
 399
 400                                         // 302 and 303 redirections also convert POST requests to GET
 401                                         // (and remove the posted form data)
 402                                         if ((code == B_HTTP_STATUS_FOUND
 403                                                 || code == B_HTTP_STATUS_SEE_OTHER)
 404                                                 && fRequestMethod == B_HTTP_POST) {
 405                                                 SetMethod(B_HTTP_GET);
 406                                                 delete fOptPostFields;
 407                                                 fOptPostFields = NULL;
 408                                                 delete fOptInputData;
 409                                                 fOptInputData = NULL;
 410                                                 fOptInputDataSize = 0;
 411                                         }
 412
 413                                         if (--maxRedirs > 0) {
 414                                                 newRequest = true;
 415
 416                                                 // Redirections may need a switch from http to https.
 417                                                 if (fUrl.Protocol() == "https")
 418                                                         fSSL = true;
 419                                                 else if (fUrl.Protocol() == "http")
 420                                                         fSSL = false;
 421
 422                                                 _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
 423                                                         "Following: %s\n",
 424                                                         fUrl.UrlString().String());
 425                                         }
 426                                 }
 427                                 break;
 428                         }
 429
 430                         case B_HTTP_STATUS_CLASS_CLIENT_ERROR:
 431                                 if (fResult.StatusCode() == B_HTTP_STATUS_UNAUTHORIZED) {
 432                                         BHttpAuthentication* authentication
 433                                                 = &fContext->GetAuthentication(fUrl);
 434                                         status_t status = B_OK;
 435
 436                                         if (authentication->Method() == B_HTTP_AUTHENTICATION_NONE) {
 437                                                 // There is no authentication context for this
 438                                                 // url yet, so let's create one.
 439                                                 BHttpAuthentication newAuth;
 440                                                 newAuth.Initialize(fHeaders["WWW-Authenticate"]);
 441                                                 fContext->AddAuthentication(fUrl, newAuth);
 442
 443                                                 // Get the copy of the authentication we just added.
 444                                                 // That copy is owned by the BUrlContext and won't be
 445                                                 // deleted (unlike the temporary object above)
 446                                                 authentication = &fContext->GetAuthentication(fUrl);
 447                                         }
 448
 449                                         newRequest = false;
 450                                         if (fOptUsername.Length() > 0 && status == B_OK) {
 451                                                 // If we received an username and password, add them
 452                                                 // to the request. This will either change the
 453                                                 // credentials for an existing request, or set them
 454                                                 // for a new one we created just above.
 455                                                 //
 456                                                 // If this request handles HTTP redirections, it will
 457                                                 // also automatically retry connecting and send the
 458                                                 // login information.
 459                                                 authentication->SetUserName(fOptUsername);
 460                                                 authentication->SetPassword(fOptPassword);
 461                                                 newRequest = true;
 462                                         }
 463                                 }
 464                                 break;
 465
 466                         case B_HTTP_STATUS_CLASS_SERVER_ERROR:
 467                                 break;
 468
 469                         default:
 470                         case B_HTTP_STATUS_CLASS_INVALID:
 471                                 break;
 472                 }
 473         } while (newRequest);
 474
 475         _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
 476                 "%ld headers and %ld bytes of data remaining",
 477                 fHeaders.CountHeaders(), fInputBuffer.Size());
 478
 479         if (fResult.StatusCode() == 404)
 480                 return B_RESOURCE_NOT_FOUND;
 481
 482         return B_OK;
 483 }
 484
 485
 486 status_t
 487 BHttpRequest::_MakeRequest()
 488 {
 489         delete fSocket;
 490
 491         if (fSSL)
 492                 fSocket = new(std::nothrow) CheckedSecureSocket(this);
 493         else
 494                 fSocket = new(std::nothrow) BSocket();
 495
 496         if (fSocket == NULL)
 497                 return B_NO_MEMORY;
 498
 499         _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Connection to %s on port %d.",
 500                 fUrl.Authority().String(), fRemoteAddr.Port());
 501         status_t connectError = fSocket->Connect(fRemoteAddr);
 502
 503         if (connectError != B_OK) {
 504                 _EmitDebug(B_URL_PROTOCOL_DEBUG_ERROR, "Socket connection error %s",
 505                         strerror(connectError));
 506                 return connectError;
 507         }
 508
 509         //! ProtocolHook:ConnectionOpened
 510         if (fListener != NULL)
 511                 fListener->ConnectionOpened(this);
 512
 513         _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT,
 514                 "Connection opened, sending request.");
 515
 516         _SendRequest();
 517         _SendHeaders();
 518         fSocket->Write("\r\n", 2);
 519         _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Request sent.");
 520
 521         _SendPostData();
 522         fRequestStatus = kRequestInitialState;
 523
 524
 525
 526         // Receive loop
 527         bool receiveEnd = false;
 528         bool parseEnd = false;
 529         bool readByChunks = false;
 530         bool decompress = false;
 531         status_t readError = B_OK;
 532         ssize_t bytesRead = 0;
 533         ssize_t bytesReceived = 0;
 534         ssize_t bytesTotal = 0;
 535         off_t bytesUnpacked = 0;
 536         char* inputTempBuffer = new(std::nothrow) char[kHttpBufferSize];
 537         ssize_t inputTempSize = kHttpBufferSize;
 538         ssize_t chunkSize = -1;
 539         DynamicBuffer decompressorStorage;
 540         BDataIO* decompressingStream;
 541         ObjectDeleter<BDataIO> decompressingStreamDeleter;
 542
 543         while (!fQuit && !(receiveEnd && parseEnd)) {
 544                 if (!receiveEnd) {
 545                         fSocket->WaitForReadable();
 546                         BStackOrHeapArray<char, 4096> chunk(kHttpBufferSize);
 547                         bytesRead = fSocket->Read(chunk, kHttpBufferSize);
 548
 549                         if (bytesRead < 0) {
 550                                 readError = bytesRead;
 551                                 break;
 552                         } else if (bytesRead == 0)
 553                                 receiveEnd = true;
 554
 555                         fInputBuffer.AppendData(chunk, bytesRead);
 556                 } else
 557                         bytesRead = 0;
 558
 559                 if (fRequestStatus < kRequestStatusReceived) {
 560                         _ParseStatus();
 561
 562                         //! ProtocolHook:ResponseStarted
 563                         if (fRequestStatus >= kRequestStatusReceived && fListener != NULL)
 564                                 fListener->ResponseStarted(this);
 565                 }
 566
 567                 if (fRequestStatus < kRequestHeadersReceived) {
 568                         _ParseHeaders();
 569
 570                         if (fRequestStatus >= kRequestHeadersReceived) {
 571                                 _ResultHeaders() = fHeaders;
 572
 573                                 //! ProtocolHook:HeadersReceived
 574                                 if (fListener != NULL)
 575                                         fListener->HeadersReceived(this);
 576
 577                                 // Parse received cookies
 578                                 if (fContext != NULL) {
 579                                         for (int32 i = 0;  i < fHeaders.CountHeaders(); i++) {
 580                                                 if (fHeaders.HeaderAt(i).NameIs("Set-Cookie")) {
 581                                                         fContext->GetCookieJar().AddCookie(
 582                                                                 fHeaders.HeaderAt(i).Value(), fUrl);
 583                                                 }
 584                                         }
 585                                 }
 586
 587                                 if (BString(fHeaders["Transfer-Encoding"]) == "chunked")
 588                                         readByChunks = true;
 589
 590                                 BString contentEncoding(fHeaders["Content-Encoding"]);
 591                                 // We don't advertise "deflate" support (see above), but we
 592                                 // still try to decompress it, if a server ever sends a deflate
 593                                 // stream despite it not being in our Accept-Encoding list.
 594                                 if (contentEncoding == "gzip"
 595                                                 || contentEncoding == "deflate") {
 596                                         decompress = true;
 597                                         readError = BZlibCompressionAlgorithm()
 598                                                 .CreateDecompressingOutputStream(&decompressorStorage,
 599                                                         NULL, decompressingStream);
 600                                         if (readError != B_OK)
 601                                                 break;
 602
 603                                         decompressingStreamDeleter.SetTo(decompressingStream);
 604                                 }
 605
 606                                 int32 index = fHeaders.HasHeader("Content-Length");
 607                                 if (index != B_ERROR)
 608                                         bytesTotal = atoi(fHeaders.HeaderAt(index).Value());
 609                                 else
 610                                         bytesTotal = -1;
 611                         }
 612                 }
 613
 614                 if (fRequestStatus >= kRequestHeadersReceived) {
 615                         // If Transfer-Encoding is chunked, we should read a complete
 616                         // chunk in buffer before handling it
 617                         if (readByChunks) {
 618                                 if (chunkSize >= 0) {
 619                                         if ((ssize_t)fInputBuffer.Size() >= chunkSize + 2) {
 620                                                         // 2 more bytes to handle the closing CR+LF
 621                                                 bytesRead = chunkSize;
 622                                                 if (inputTempSize < chunkSize + 2) {
 623                                                         delete[] inputTempBuffer;
 624                                                         inputTempSize = chunkSize + 2;
 625                                                         inputTempBuffer
 626                                                                 = new(std::nothrow) char[inputTempSize];
 627                                                 }
 628
 629                                                 if (inputTempBuffer == NULL) {
 630                                                         readError = B_NO_MEMORY;
 631                                                         break;
 632                                                 }
 633
 634                                                 fInputBuffer.RemoveData(inputTempBuffer,
 635                                                         chunkSize + 2);
 636                                                 chunkSize = -1;
 637                                         } else {
 638                                                 // Not enough data, try again later
 639                                                 bytesRead = -1;
 640                                         }
 641                                 } else {
 642                                         BString chunkHeader;
 643                                         if (_GetLine(chunkHeader) == B_ERROR) {
 644                                                 chunkSize = -1;
 645                                                 bytesRead = -1;
 646                                         } else {
 647                                                 // Format of a chunk header:
 648                                                 // <chunk size in hex>[; optional data]
 649                                                 int32 semiColonIndex = chunkHeader.FindFirst(';', 0);
 650
 651                                                 // Cut-off optional data if present
 652                                                 if (semiColonIndex != -1) {
 653                                                         chunkHeader.Remove(semiColonIndex,
 654                                                                 chunkHeader.Length() - semiColonIndex);
 655                                                 }
 656
 657                                                 chunkSize = strtol(chunkHeader.String(), NULL, 16);
 658                                                 PRINT(("BHP[%p] Chunk %s=%ld\n", this,
 659                                                         chunkHeader.String(), chunkSize));
 660                                                 if (chunkSize == 0)
 661                                                         fRequestStatus = kRequestContentReceived;
 662
 663                                                 bytesRead = -1;
 664                                         }
 665                                 }
 666
 667                                 // A chunk of 0 bytes indicates the end of the chunked transfer
 668                                 if (bytesRead == 0)
 669                                         receiveEnd = true;
 670                         } else {
 671                                 bytesRead = fInputBuffer.Size();
 672
 673                                 if (bytesRead > 0) {
 674                                         if (inputTempSize < bytesRead) {
 675                                                 inputTempSize = bytesRead;
 676                                                 delete[] inputTempBuffer;
 677                                                 inputTempBuffer = new(std::nothrow) char[bytesRead];
 678                                         }
 679
 680                                         if (inputTempBuffer == NULL) {
 681                                                 readError = B_NO_MEMORY;
 682                                                 break;
 683                                         }
 684                                         fInputBuffer.RemoveData(inputTempBuffer, bytesRead);
 685                                 }
 686                         }
 687
 688                         if (bytesRead >= 0) {
 689                                 bytesReceived += bytesRead;
 690
 691                                 if (fListener != NULL) {
 692                                         if (decompress) {
 693                                                 readError = decompressingStream->WriteExactly(
 694                                                         inputTempBuffer, bytesRead);
 695                                                 if (readError != B_OK)
 696                                                         break;
 697
 698                                                 ssize_t size = decompressorStorage.Size();
 699                                                 BStackOrHeapArray<char, 4096> buffer(size);
 700                                                 size = decompressorStorage.Read(buffer, size);
 701                                                 if (size > 0) {
 702                                                         fListener->DataReceived(this, buffer, bytesUnpacked,
 703                                                                 size);
 704                                                         bytesUnpacked += size;
 705                                                 }
 706                                         } else if (bytesRead > 0) {
 707                                                 fListener->DataReceived(this, inputTempBuffer,
 708                                                         bytesReceived - bytesRead, bytesRead);
 709                                         }
 710                                         fListener->DownloadProgress(this, bytesReceived,
 711                                                 std::max((ssize_t)0, bytesTotal));
 712                                 }
 713
 714                                 if (bytesTotal >= 0 && bytesReceived >= bytesTotal)
 715                                         receiveEnd = true;
 716
 717                                 if (decompress && receiveEnd) {
 718                                         readError = decompressingStream->Flush();
 719
 720                                         if (readError == B_BUFFER_OVERFLOW)
 721                                                 readError = B_OK;
 722
 723                                         if (readError != B_OK)
 724                                                 break;
 725
 726                                         ssize_t size = decompressorStorage.Size();
 727                                         BStackOrHeapArray<char, 4096> buffer(size);
 728                                         size = decompressorStorage.Read(buffer, size);
 729                                         if (fListener != NULL && size > 0) {
 730                                                 fListener->DataReceived(this, buffer,
 731                                                         bytesUnpacked, size);
 732                                                 bytesUnpacked += size;
 733                                         }
 734                                 }
 735                         }
 736                 }
 737
 738                 parseEnd = (fInputBuffer.Size() == 0);
 739         }
 740
 741         fSocket->Disconnect();
 742         delete[] inputTempBuffer;
 743
 744         if (readError != B_OK)
 745                 return readError;
 746
 747         return fQuit ? B_INTERRUPTED : B_OK;
 748 }
 749
 750
 751 void
 752 BHttpRequest::_ParseStatus()
 753 {
 754         // Status line should be formatted like: HTTP/M.m SSS ...
 755         // With:   M = Major version of the protocol
 756         //         m = Minor version of the protocol
 757         //       SSS = three-digit status code of the response
 758         //       ... = additional text info
 759         BString statusLine;
 760         if (_GetLine(statusLine) == B_ERROR)
 761                 return;
 762
 763         if (statusLine.CountChars() < 12)
 764                 return;
 765
 766         fRequestStatus = kRequestStatusReceived;
 767
 768         BString statusCodeStr;
 769         BString statusText;
 770         statusLine.CopyInto(statusCodeStr, 9, 3);
 771         _SetResultStatusCode(atoi(statusCodeStr.String()));
 772
 773         statusLine.CopyInto(_ResultStatusText(), 13, statusLine.Length() - 13);
 774
 775         _EmitDebug(B_URL_PROTOCOL_DEBUG_TEXT, "Status line received: Code %d (%s)",
 776                 atoi(statusCodeStr.String()), _ResultStatusText().String());
 777 }
 778
 779
 780 void
 781 BHttpRequest::_ParseHeaders()
 782 {
 783         BString currentHeader;
 784         while (_GetLine(currentHeader) != B_ERROR) {
 785                 // An empty line means the end of the header section
 786                 if (currentHeader.Length() == 0) {
 787                         fRequestStatus = kRequestHeadersReceived;
 788                         return;
 789                 }
 790
 791                 _EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_IN, "%s",
 792                         currentHeader.String());
 793                 fHeaders.AddHeader(currentHeader.String());
 794         }
 795 }
 796
 797
 798 void
 799 BHttpRequest::_SendRequest()
 800 {
 801         BString request(fRequestMethod);
 802         request << ' ';
 803
 804         if (fContext->UseProxy()) {
 805                 // When there is a proxy, the request must include the host and port so
 806                 // the proxy knows where to send the request.
 807                 request << Url().Protocol() << "://" << Url().Host();
 808                 if (Url().HasPort())
 809                         request << ':' << Url().Port();
 810         }
 811
 812         if (Url().HasPath())
 813                 request << Url().Path();
 814         else
 815                 request << '/';
 816
 817         if (Url().HasRequest())
 818                 request << '?' << Url().Request();
 819
 820         switch (fHttpVersion) {
 821                 case B_HTTP_11:
 822                         request << " HTTP/1.1\r\n";
 823                         break;
 824
 825                 default:
 826                 case B_HTTP_10:
 827                         request << " HTTP/1.0\r\n";
 828                         break;
 829         }
 830
 831         fSocket->Write(request.String(), request.Length());
 832 }
 833
 834
 835 void
 836 BHttpRequest::_SendHeaders()
 837 {
 838         BHttpHeaders outputHeaders;
 839
 840         // HTTP 1.1 additional headers
 841         if (fHttpVersion == B_HTTP_11) {
 842                 BString host = Url().Host();
 843                 if (Url().HasPort() && !_IsDefaultPort())
 844                         host << ':' << Url().Port();
 845
 846                 outputHeaders.AddHeader("Host", host);
 847
 848                 outputHeaders.AddHeader("Accept", "*/*");
 849                 outputHeaders.AddHeader("Accept-Encoding", "gzip");
 850                         // Allows the server to compress data using the "gzip" format.
 851                         // "deflate" is not supported, because there are two interpretations
 852                         // of what it means (the RFC and Microsoft products), and we don't
 853                         // want to handle this. Very few websites support only deflate,
 854                         // and most of them will send gzip, or at worst, uncompressed data.
 855
 856                 outputHeaders.AddHeader("Connection", "close");
 857                         // Let the remote server close the connection after response since
 858                         // we don't handle multiple request on a single connection
 859         }
 860
 861         // Classic HTTP headers
 862         if (fOptUserAgent.CountChars() > 0)
 863                 outputHeaders.AddHeader("User-Agent", fOptUserAgent.String());
 864
 865         if (fOptReferer.CountChars() > 0)
 866                 outputHeaders.AddHeader("Referer", fOptReferer.String());
 867
 868         // Authentication
 869         if (fContext != NULL) {
 870                 BHttpAuthentication& authentication = fContext->GetAuthentication(fUrl);
 871                 if (authentication.Method() != B_HTTP_AUTHENTICATION_NONE) {
 872                         if (fOptUsername.Length() > 0) {
 873                                 authentication.SetUserName(fOptUsername);
 874                                 authentication.SetPassword(fOptPassword);
 875                         }
 876
 877                         BString request(fRequestMethod);
 878                         outputHeaders.AddHeader("Authorization",
 879                                 authentication.Authorization(fUrl, request));
 880                 }
 881         }
 882
 883         // Required headers for POST data
 884         if (fOptPostFields != NULL && fRequestMethod == B_HTTP_POST) {
 885                 BString contentType;
 886
 887                 switch (fOptPostFields->GetFormType()) {
 888                         case B_HTTP_FORM_MULTIPART:
 889                                 contentType << "multipart/form-data; boundary="
 890                                         << fOptPostFields->GetMultipartBoundary() << "";
 891                                 break;
 892
 893                         case B_HTTP_FORM_URL_ENCODED:
 894                                 contentType << "application/x-www-form-urlencoded";
 895                                 break;
 896                 }
 897
 898                 outputHeaders.AddHeader("Content-Type", contentType);
 899                 outputHeaders.AddHeader("Content-Length",
 900                         fOptPostFields->ContentLength());
 901         } else if (fOptInputData != NULL
 902                         && (fRequestMethod == B_HTTP_POST
 903                         || fRequestMethod == B_HTTP_PUT)) {
 904                 if (fOptInputDataSize >= 0)
 905                         outputHeaders.AddHeader("Content-Length", fOptInputDataSize);
 906                 else
 907                         outputHeaders.AddHeader("Transfer-Encoding", "chunked");
 908         }
 909
 910         // Optional headers specified by the user
 911         if (fOptHeaders != NULL) {
 912                 for (int32 headerIndex = 0; headerIndex < fOptHeaders->CountHeaders();
 913                                 headerIndex++) {
 914                         BHttpHeader& optHeader = (*fOptHeaders)[headerIndex];
 915                         int32 replaceIndex = outputHeaders.HasHeader(optHeader.Name());
 916
 917                         // Add or replace the current option header to the
 918                         // output header list
 919                         if (replaceIndex == -1)
 920                                 outputHeaders.AddHeader(optHeader.Name(), optHeader.Value());
 921                         else
 922                                 outputHeaders[replaceIndex].SetValue(optHeader.Value());
 923                 }
 924         }
 925
 926         // Context cookies
 927         if (fOptSetCookies && fContext != NULL) {
 928                 BString cookieString;
 929
 930                 BNetworkCookieJar::UrlIterator iterator
 931                         = fContext->GetCookieJar().GetUrlIterator(fUrl);
 932                 const BNetworkCookie* cookie = iterator.Next();
 933                 if (cookie != NULL) {
 934                         while (true) {
 935                                 cookieString << cookie->RawCookie(false);
 936                                 cookie = iterator.Next();
 937                                 if (cookie == NULL)
 938                                         break;
 939                                 cookieString << "; ";
 940                         }
 941
 942                         outputHeaders.AddHeader("Cookie", cookieString);
 943                 }
 944         }
 945
 946         // Write output headers to output stream
 947         BString headerData;
 948
 949         for (int32 headerIndex = 0; headerIndex < outputHeaders.CountHeaders();
 950                         headerIndex++) {
 951                 const char* header = outputHeaders.HeaderAt(headerIndex).Header();
 952
 953                 headerData << header;
 954                 headerData << "\r\n";
 955
 956                 _EmitDebug(B_URL_PROTOCOL_DEBUG_HEADER_OUT, "%s", header);
 957         }
 958
 959         fSocket->Write(headerData.String(), headerData.Length());
 960 }
 961
 962
 963 void
 964 BHttpRequest::_SendPostData()
 965 {
 966         if (fRequestMethod == B_HTTP_POST && fOptPostFields != NULL) {
 967                 if (fOptPostFields->GetFormType() != B_HTTP_FORM_MULTIPART) {
 968                         BString outputBuffer = fOptPostFields->RawData();
 969                         _EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
 970                                 "%s", outputBuffer.String());
 971                         fSocket->Write(outputBuffer.String(), outputBuffer.Length());
 972                 } else {
 973                         for (BHttpForm::Iterator it = fOptPostFields->GetIterator();
 974                                 const BHttpFormData* currentField = it.Next();
 975                                 ) {
 976                                 _EmitDebug(B_URL_PROTOCOL_DEBUG_TRANSFER_OUT,
 977                                         it.MultipartHeader().String());
 978                                 fSocket->Write(it.MultipartHeader().String(),
 979                                         it.MultipartHeader().Length());
 980
 981                                 switch (currentField->Type()) {
 982                                         default:
 983                                         case B_HTTPFORM_UNKNOWN:
 984                                                 ASSERT(0);
 985                                                 break;
 986
 987                                         case B_HTTPFORM_STRING:
 988                                                 fSocket->Write(currentField->String().String(),
 989                                                         currentField->String().Length());
 990                                                 break;
 991
 992                                         case B_HTTPFORM_FILE:
 993                                                 {
 994                                                         BFile upFile(currentField->File().Path(),
 995                                                                 B_READ_ONLY);
 996                                                         char readBuffer[kHttpBufferSize];
 997                                                         ssize_t readSize;
 998
 999                                                         readSize = upFile.Read(readBuffer,
1000                                                                 sizeof(readBuffer));
1001                                                         while (readSize > 0) {
1002                                                                 fSocket->Write(readBuffer, readSize);
1003                                                                 readSize = upFile.Read(readBuffer,
1004                                                                         sizeof(readBuffer));
1005                                                         }
1006
1007                                                         break;
1008                                                 }
1009                                         case B_HTTPFORM_BUFFER:
1010                                                 fSocket->Write(currentField->Buffer(),
1011                                                         currentField->BufferSize());
1012                                                 break;
1013                                 }
1014
1015                                 fSocket->Write("\r\n", 2);
1016                         }
1017
1018                         BString footer = fOptPostFields->GetMultipartFooter();
1019                         fSocket->Write(footer.String(), footer.Length());
1020                 }
1021         } else if ((fRequestMethod == B_HTTP_POST || fRequestMethod == B_HTTP_PUT)
1022                 && fOptInputData != NULL) {
1023
1024                 // If the input data is seekable, we rewind it for each new request.
1025                 BPositionIO* seekableData
1026                         = dynamic_cast<BPositionIO*>(fOptInputData);
1027                 if (seekableData)
1028                         seekableData->Seek(0, SEEK_SET);
1029
1030                 for (;;) {
1031                         char outputTempBuffer[kHttpBufferSize];
1032                         ssize_t read = fOptInputData->Read(outputTempBuffer,
1033                                 sizeof(outputTempBuffer));
1034
1035                         if (read <= 0)
1036                                 break;
1037
1038                         if (fOptInputDataSize < 0) {
1039                                 // Chunked transfer
1040                                 char hexSize[16];
1041                                 size_t hexLength = sprintf(hexSize, "%ld", read);
1042
1043                                 fSocket->Write(hexSize, hexLength);
1044                                 fSocket->Write("\r\n", 2);
1045                                 fSocket->Write(outputTempBuffer, read);
1046                                 fSocket->Write("\r\n", 2);
1047                         } else {
1048                                 fSocket->Write(outputTempBuffer, read);
1049                         }
1050                 }
1051
1052                 if (fOptInputDataSize < 0) {
1053                         // Chunked transfer terminating sequence
1054                         fSocket->Write("0\r\n\r\n", 5);
1055                 }
1056         }
1057
1058 }
1059
1060
1061 BHttpHeaders&
1062 BHttpRequest::_ResultHeaders()
1063 {
1064         return fResult.fHeaders;
1065 }
1066
1067
1068 void
1069 BHttpRequest::_SetResultStatusCode(int32 statusCode)
1070 {
1071         fResult.fStatusCode = statusCode;
1072 }
1073
1074
1075 BString&
1076 BHttpRequest::_ResultStatusText()
1077 {
1078         return fResult.fStatusString;
1079 }
1080
1081
1082 bool
1083 BHttpRequest::_CertificateVerificationFailed(BCertificate& certificate,
1084         const char* message)
1085 {
1086         if (fListener != NULL) {
1087                 return fListener->CertificateVerificationFailed(this, certificate,
1088                         message);
1089         }
1090
1091         return false;
1092 }
1093
1094
1095 bool
1096 BHttpRequest::_IsDefaultPort()
1097 {
1098         if (fSSL && Url().Port() == 443)
1099                 return true;
1100         if (!fSSL && Url().Port() == 80)
1101                 return true;
1102         return false;
1103 }
1104
1105