src/http_connection.cpp

   1 /*
   2
   3 Copyright (c) 2007, Arvid Norberg
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions
   8 are met:
   9
  10     * Redistributions of source code must retain the above copyright
  11       notice, this list of conditions and the following disclaimer.
  12     * Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in
  14       the documentation and/or other materials provided with the distribution.
  15     * Neither the name of the author nor the names of its
  16       contributors may be used to endorse or promote products derived
  17       from this software without specific prior written permission.
  18
  19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  23 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29 POSSIBILITY OF SUCH DAMAGE.
  30
  31 */
  32
  33 #include "libtorrent/http_connection.hpp"
  34 #include "libtorrent/escape_string.hpp"
  35 #include "libtorrent/instantiate_connection.hpp"
  36 #include "libtorrent/gzip.hpp"
  37 #include "libtorrent/parse_url.hpp"
  38 #include "libtorrent/socket.hpp"
  39 #include "libtorrent/connection_queue.hpp"
  40
  41 #include <boost/bind.hpp>
  42 #include <boost/lexical_cast.hpp>
  43 #include <string>
  44 #include <algorithm>
  45
  46 using boost::bind;
  47
  48 namespace libtorrent {
  49
  50 enum { max_bottled_buffer = 1024 * 1024 };
  51
  52
  53 void http_connection::get(std::string const& url, time_duration timeout, int prio
  54         , proxy_settings const* ps, int handle_redirects, std::string const& user_agent
  55         , address const& bind_addr)
  56 {
  57         std::string protocol;
  58         std::string auth;
  59         std::string hostname;
  60         std::string path;
  61         char const* error;
  62         int port;
  63
  64         boost::tie(protocol, auth, hostname, port, path, error)
  65                 = parse_url_components(url);
  66
  67         if (error)
  68         {
  69                 callback(asio::error::socket_type_not_supported);
  70                 return;
  71         }
  72
  73         TORRENT_ASSERT(prio >= 0 && prio < 2);
  74
  75         bool ssl = false;
  76         if (protocol == "https") ssl = true;
  77 #ifndef TORRENT_USE_OPENSSL
  78         if (ssl)
  79         {
  80                 callback(asio::error::socket_type_not_supported);
  81                 return;
  82         }
  83 #endif
  84
  85         std::stringstream headers;
  86         if (ps && (ps->type == proxy_settings::http
  87                 || ps->type == proxy_settings::http_pw)
  88                 && !ssl)
  89         {
  90                 // if we're using an http proxy and not an ssl
  91                 // connection, just do a regular http proxy request
  92                 headers << "GET " << url << " HTTP/1.0\r\n";
  93                 if (ps->type == proxy_settings::http_pw)
  94                         headers << "Proxy-Authorization: Basic " << base64encode(
  95                                 ps->username + ":" + ps->password) << "\r\n";
  96                 hostname = ps->hostname;
  97                 port = ps->port;
  98                 ps = 0;
  99         }
 100         else
 101         {
 102                 headers << "GET " << path << " HTTP/1.0\r\n"
 103                         "Host:" << hostname << "\r\n";
 104         }
 105
 106         if (!auth.empty())
 107                 headers << "Authorization: Basic " << base64encode(auth) << "\r\n";
 108
 109         if (!user_agent.empty())
 110                 headers << "User-Agent: " << user_agent << "\r\n";
 111
 112         headers <<
 113                 "Connection: close\r\n"
 114                 "Accept-Encoding: gzip\r\n"
 115                 "\r\n";
 116
 117         sendbuffer = headers.str();
 118         m_url = url;
 119         start(hostname, boost::lexical_cast<std::string>(port), timeout, prio
 120                 , ps, ssl, handle_redirects, bind_addr);
 121 }
 122
 123 void http_connection::start(std::string const& hostname, std::string const& port
 124         , time_duration timeout, int prio, proxy_settings const* ps, bool ssl, int handle_redirects
 125         , address const& bind_addr)
 126 {
 127         TORRENT_ASSERT(prio >= 0 && prio < 2);
 128
 129         m_redirects = handle_redirects;
 130         if (ps) m_proxy = *ps;
 131
 132         m_timeout = timeout;
 133         error_code ec;
 134         m_timer.expires_from_now(m_timeout, ec);
 135         m_timer.async_wait(bind(&http_connection::on_timeout
 136                 , boost::weak_ptr<http_connection>(shared_from_this()), _1));
 137         m_called = false;
 138         m_parser.reset();
 139         m_recvbuffer.clear();
 140         m_read_pos = 0;
 141         m_priority = prio;
 142
 143         if (ec)
 144         {
 145                 callback(ec);
 146                 return;
 147         }
 148
 149         if (m_sock.is_open() && m_hostname == hostname && m_port == port
 150                 && m_ssl == ssl && m_bind_addr == bind_addr)
 151         {
 152                 async_write(m_sock, asio::buffer(sendbuffer)
 153                         , bind(&http_connection::on_write, shared_from_this(), _1));
 154         }
 155         else
 156         {
 157                 m_ssl = ssl;
 158                 m_bind_addr = bind_addr;
 159                 error_code ec;
 160                 m_sock.close(ec);
 161
 162 #ifdef TORRENT_USE_OPENSSL
 163                 if (m_ssl)
 164                 {
 165                         m_sock.instantiate<ssl_stream<socket_type> >(m_resolver.get_io_service());
 166                         ssl_stream<socket_type>& s = m_sock.get<ssl_stream<socket_type> >();
 167                         bool ret = instantiate_connection(m_resolver.get_io_service(), m_proxy, s.next_layer());
 168                         TORRENT_ASSERT(ret);
 169                 }
 170                 else
 171                 {
 172                         m_sock.instantiate<socket_type>(m_resolver.get_io_service());
 173                         bool ret = instantiate_connection(m_resolver.get_io_service()
 174                                 , m_proxy, m_sock.get<socket_type>());
 175                         TORRENT_ASSERT(ret);
 176                 }
 177 #else
 178                 bool ret = instantiate_connection(m_resolver.get_io_service(), m_proxy, m_sock);
 179                 TORRENT_ASSERT(ret);
 180 #endif
 181                 if (m_bind_addr != address_v4::any())
 182                 {
 183                         error_code ec;
 184                         m_sock.bind(tcp::endpoint(m_bind_addr, 0), ec);
 185                         if (ec)
 186                         {
 187                                 callback(ec);
 188                                 return;
 189                         }
 190                 }
 191
 192                 tcp::resolver::query query(hostname, port);
 193                 m_resolver.async_resolve(query, bind(&http_connection::on_resolve
 194                         , shared_from_this(), _1, _2));
 195                 m_hostname = hostname;
 196                 m_port = port;
 197         }
 198 }
 199
 200 void http_connection::on_connect_timeout()
 201 {
 202         TORRENT_ASSERT(m_connection_ticket >= 0);
 203         if (m_connection_ticket > -1) m_cc.done(m_connection_ticket);
 204         m_connection_ticket = -1;
 205
 206         if (!m_endpoints.empty())
 207         {
 208                 m_sock.close();
 209         }
 210         else
 211         {
 212                 callback(asio::error::timed_out);
 213                 close();
 214         }
 215 }
 216
 217 void http_connection::on_timeout(boost::weak_ptr<http_connection> p
 218         , error_code const& e)
 219 {
 220         boost::shared_ptr<http_connection> c = p.lock();
 221         if (!c) return;
 222
 223         if (e == asio::error::operation_aborted) return;
 224
 225         if (c->m_last_receive + c->m_timeout < time_now())
 226         {
 227                 if (c->m_connection_ticket > -1 && !c->m_endpoints.empty())
 228                 {
 229                         c->m_sock.close();
 230                         error_code ec;
 231                         c->m_timer.expires_at(c->m_last_receive + c->m_timeout, ec);
 232                         c->m_timer.async_wait(bind(&http_connection::on_timeout, p, _1));
 233                 }
 234                 else
 235                 {
 236                         c->callback(asio::error::timed_out);
 237                         c->close();
 238                 }
 239                 return;
 240         }
 241
 242         if (!c->m_sock.is_open()) return;
 243         error_code ec;
 244         c->m_timer.expires_at(c->m_last_receive + c->m_timeout, ec);
 245         c->m_timer.async_wait(bind(&http_connection::on_timeout, p, _1));
 246 }
 247
 248 void http_connection::close()
 249 {
 250         error_code ec;
 251         m_timer.cancel(ec);
 252         m_limiter_timer.cancel(ec);
 253         m_sock.close(ec);
 254         m_hostname.clear();
 255         m_port.clear();
 256         m_handler.clear();
 257         m_abort = true;
 258 }
 259
 260 void http_connection::on_resolve(error_code const& e
 261         , tcp::resolver::iterator i)
 262 {
 263         if (e)
 264         {
 265                 callback(e);
 266                 close();
 267                 return;
 268         }
 269         TORRENT_ASSERT(i != tcp::resolver::iterator());
 270
 271         std::transform(i, tcp::resolver::iterator(), std::back_inserter(m_endpoints)
 272                 , boost::bind(&tcp::resolver::iterator::value_type::endpoint, _1));
 273
 274         // The following statement causes msvc to crash (ICE). Since it's not
 275         // necessary in the vast majority of cases, just ignore the endpoint
 276         // order for windows
 277 #if !defined _MSC_VER || _MSC_VER > 1310
 278         // sort the endpoints so that the ones with the same IP version as our
 279         // bound listen socket are first. So that when contacting a tracker,
 280         // we'll talk to it from the same IP that we're listening on
 281         std::partition(m_endpoints.begin(), m_endpoints.end()
 282                 , boost::bind(&address::is_v4, boost::bind(&tcp::endpoint::address, _1)) == m_bind_addr.is_v4());
 283 #endif
 284
 285         queue_connect();
 286 }
 287
 288 void http_connection::queue_connect()
 289 {
 290         TORRENT_ASSERT(!m_endpoints.empty());
 291         tcp::endpoint target = m_endpoints.front();
 292         m_endpoints.pop_front();
 293
 294         m_cc.enqueue(bind(&http_connection::connect, shared_from_this(), _1, target)
 295                 , bind(&http_connection::on_connect_timeout, shared_from_this())
 296                 , m_timeout, m_priority);
 297 }
 298
 299 void http_connection::connect(int ticket, tcp::endpoint target_address)
 300 {
 301         m_connection_ticket = ticket;
 302         m_sock.async_connect(target_address, boost::bind(&http_connection::on_connect
 303                 , shared_from_this(), _1));
 304 }
 305
 306 void http_connection::on_connect(error_code const& e)
 307 {
 308         if (m_connection_ticket >= 0)
 309         {
 310                 m_cc.done(m_connection_ticket);
 311                 m_connection_ticket = -1;
 312         }
 313
 314         m_last_receive = time_now();
 315         if (!e)
 316         {
 317                 if (m_connect_handler) m_connect_handler(*this);
 318                 async_write(m_sock, asio::buffer(sendbuffer)
 319                         , bind(&http_connection::on_write, shared_from_this(), _1));
 320         }
 321         else if (!m_endpoints.empty() && !m_abort)
 322         {
 323                 // The connection failed. Try the next endpoint in the list.
 324                 m_sock.close();
 325                 queue_connect();
 326         }
 327         else
 328         {
 329                 callback(e);
 330                 close();
 331         }
 332 }
 333
 334 void http_connection::callback(error_code const& e, char const* data, int size)
 335 {
 336         if (!m_bottled || !m_called)
 337         {
 338                 std::vector<char> buf;
 339                 if (m_bottled && m_parser.finished())
 340                 {
 341                         std::string const& encoding = m_parser.header("content-encoding");
 342                         if (encoding == "gzip" || encoding == "x-gzip")
 343                         {
 344                                 std::string error;
 345                                 if (inflate_gzip(data, size, buf, max_bottled_buffer, error))
 346                                 {
 347                                         if (m_handler) m_handler(asio::error::fault, m_parser, data, size, *this);
 348                                         close();
 349                                         return;
 350                                 }
 351                                 data = &buf[0];
 352                                 size = int(buf.size());
 353                         }
 354                 }
 355                 m_called = true;
 356                 m_timer.cancel();
 357                 if (m_handler) m_handler(e, m_parser, data, size, *this);
 358         }
 359 }
 360
 361 void http_connection::on_write(error_code const& e)
 362 {
 363         if (e)
 364         {
 365                 callback(e);
 366                 close();
 367                 return;
 368         }
 369
 370         std::string().swap(sendbuffer);
 371         m_recvbuffer.resize(4096);
 372
 373         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 374         if (m_rate_limit > 0 && amount_to_read > m_download_quota)
 375         {
 376                 amount_to_read = m_download_quota;
 377                 if (m_download_quota == 0)
 378                 {
 379                         if (!m_limiter_timer_active)
 380                                 on_assign_bandwidth(error_code());
 381                         return;
 382                 }
 383         }
 384         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 385                 , amount_to_read)
 386                 , bind(&http_connection::on_read
 387                 , shared_from_this(), _1, _2));
 388 }
 389
 390 void http_connection::on_read(error_code const& e
 391         , std::size_t bytes_transferred)
 392 {
 393         if (m_rate_limit)
 394         {
 395                 m_download_quota -= bytes_transferred;
 396                 TORRENT_ASSERT(m_download_quota >= 0);
 397         }
 398
 399         if (e == asio::error::eof)
 400         {
 401                 TORRENT_ASSERT(bytes_transferred == 0);
 402                 char const* data = 0;
 403                 std::size_t size = 0;
 404                 if (m_bottled && m_parser.header_finished())
 405                 {
 406                         data = m_parser.get_body().begin;
 407                         size = m_parser.get_body().left();
 408                 }
 409                 callback(e, data, size);
 410                 close();
 411                 return;
 412         }
 413
 414         if (e)
 415         {
 416                 TORRENT_ASSERT(bytes_transferred == 0);
 417                 callback(e);
 418                 close();
 419                 return;
 420         }
 421
 422         m_read_pos += bytes_transferred;
 423         TORRENT_ASSERT(m_read_pos <= int(m_recvbuffer.size()));
 424
 425         if (m_bottled || !m_parser.header_finished())
 426         {
 427                 libtorrent::buffer::const_interval rcv_buf(&m_recvbuffer[0]
 428                         , &m_recvbuffer[0] + m_read_pos);
 429                 bool error = false;
 430                 m_parser.incoming(rcv_buf, error);
 431                 if (error)
 432                 {
 433                         // HTTP parse error
 434                         error_code ec = asio::error::fault;
 435                         callback(ec, 0, 0);
 436                         return;
 437                 }
 438
 439                 // having a nonempty path means we should handle redirects
 440                 if (m_redirects && m_parser.header_finished())
 441                 {
 442                         int code = m_parser.status_code();
 443
 444                         if (code >= 300 && code < 400)
 445                         {
 446                                 // attempt a redirect
 447                                 std::string const& location = m_parser.header("location");
 448                                 if (location.empty())
 449                                 {
 450                                         // missing location header
 451                                         callback(asio::error::fault);
 452                                         close();
 453                                         return;
 454                                 }
 455
 456                                 error_code ec;
 457                                 m_sock.close(ec);
 458                                 using boost::tuples::ignore;
 459                                 char const* error;
 460                                 boost::tie(ignore, ignore, ignore, ignore, ignore, error)
 461                                         = parse_url_components(location);
 462                                 if (error == 0)
 463                                 {
 464                                         get(location, m_timeout, m_priority, &m_proxy, m_redirects - 1);
 465                                 }
 466                                 else
 467                                 {
 468                                         // some broken web servers send out relative paths
 469                                         // in the location header.
 470                                         std::string url = m_url;
 471                                         // remove the leaf filename
 472                                         std::size_t i = url.find_last_of('/');
 473                                         if (i == std::string::npos)
 474                                         {
 475                                                 url += '/';
 476                                         }
 477                                         else
 478                                         {
 479                                                 url.resize(i + 1);
 480                                         }
 481                                         url += location;
 482
 483                                         get(url, m_timeout, m_priority, &m_proxy, m_redirects - 1);
 484                                 }
 485                                 return;
 486                         }
 487
 488                         m_redirects = 0;
 489                 }
 490
 491                 if (!m_bottled && m_parser.header_finished())
 492                 {
 493                         if (m_read_pos > m_parser.body_start())
 494                                 callback(e, &m_recvbuffer[0] + m_parser.body_start()
 495                                         , m_read_pos - m_parser.body_start());
 496                         m_read_pos = 0;
 497                         m_last_receive = time_now();
 498                 }
 499                 else if (m_bottled && m_parser.finished())
 500                 {
 501                         error_code ec;
 502                         m_timer.cancel(ec);
 503                         callback(e, m_parser.get_body().begin, m_parser.get_body().left());
 504                 }
 505         }
 506         else
 507         {
 508                 TORRENT_ASSERT(!m_bottled);
 509                 callback(e, &m_recvbuffer[0], m_read_pos);
 510                 m_read_pos = 0;
 511                 m_last_receive = time_now();
 512         }
 513
 514         if (int(m_recvbuffer.size()) == m_read_pos)
 515                 m_recvbuffer.resize((std::min)(m_read_pos + 2048, int(max_bottled_buffer)));
 516         if (m_read_pos == max_bottled_buffer)
 517         {
 518                 callback(asio::error::eof);
 519                 close();
 520                 return;
 521         }
 522         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 523         if (m_rate_limit > 0 && amount_to_read > m_download_quota)
 524         {
 525                 amount_to_read = m_download_quota;
 526                 if (m_download_quota == 0)
 527                 {
 528                         if (!m_limiter_timer_active)
 529                                 on_assign_bandwidth(error_code());
 530                         return;
 531                 }
 532         }
 533         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 534                 , amount_to_read)
 535                 , bind(&http_connection::on_read
 536                 , shared_from_this(), _1, _2));
 537 }
 538
 539 void http_connection::on_assign_bandwidth(error_code const& e)
 540 {
 541         if ((e == asio::error::operation_aborted
 542                 && m_limiter_timer_active)
 543                 || !m_sock.is_open())
 544         {
 545                 callback(asio::error::eof);
 546                 return;
 547         }
 548         m_limiter_timer_active = false;
 549         if (e) return;
 550
 551         if (m_download_quota > 0) return;
 552
 553         m_download_quota = m_rate_limit / 4;
 554
 555         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 556         if (amount_to_read > m_download_quota)
 557                 amount_to_read = m_download_quota;
 558
 559         if (!m_sock.is_open()) return;
 560
 561         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 562                 , amount_to_read)
 563                 , bind(&http_connection::on_read
 564                 , shared_from_this(), _1, _2));
 565
 566         error_code ec;
 567         m_limiter_timer_active = true;
 568         m_limiter_timer.expires_from_now(milliseconds(250), ec);
 569         m_limiter_timer.async_wait(bind(&http_connection::on_assign_bandwidth
 570                 , shared_from_this(), _1));
 571 }
 572
 573 void http_connection::rate_limit(int limit)
 574 {
 575         if (!m_sock.is_open()) return;
 576
 577         if (!m_limiter_timer_active)
 578         {
 579                 error_code ec;
 580                 m_limiter_timer_active = true;
 581                 m_limiter_timer.expires_from_now(milliseconds(250), ec);
 582                 m_limiter_timer.async_wait(bind(&http_connection::on_assign_bandwidth
 583                         , shared_from_this(), _1));
 584         }
 585         m_rate_limit = limit;
 586 }
 587
 588 }
 589