http.c

   1 #include "http.h"
   2 #include "pack.h"
   3
   4 int data_received;
   5 int active_requests;
   6 int http_is_verbose;
   7
   8 #ifdef USE_CURL_MULTI
   9 static int max_requests = -1;
  10 static CURLM *curlm;
  11 #endif
  12 #ifndef NO_CURL_EASY_DUPHANDLE
  13 static CURL *curl_default;
  14 #endif
  15
  16 #define PREV_BUF_SIZE 4096
  17 #define RANGE_HEADER_SIZE 30
  18
  19 char curl_errorstr[CURL_ERROR_SIZE];
  20
  21 static int curl_ssl_verify = -1;
  22 static const char *ssl_cert;
  23 #if LIBCURL_VERSION_NUM >= 0x070902
  24 static const char *ssl_key;
  25 #endif
  26 #if LIBCURL_VERSION_NUM >= 0x070908
  27 static const char *ssl_capath;
  28 #endif
  29 static const char *ssl_cainfo;
  30 static long curl_low_speed_limit = -1;
  31 static long curl_low_speed_time = -1;
  32 static int curl_ftp_no_epsv;
  33 static const char *curl_http_proxy;
  34 static char *user_name, *user_pass;
  35
  36 static struct curl_slist *pragma_header;
  37 static struct curl_slist *no_pragma_header;
  38
  39 static struct active_request_slot *active_queue_head;
  40
  41 size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
  42 {
  43         size_t size = eltsize * nmemb;
  44         struct buffer *buffer = buffer_;
  45
  46         if (size > buffer->buf.len - buffer->posn)
  47                 size = buffer->buf.len - buffer->posn;
  48         memcpy(ptr, buffer->buf.buf + buffer->posn, size);
  49         buffer->posn += size;
  50
  51         return size;
  52 }
  53
  54 #ifndef NO_CURL_IOCTL
  55 curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
  56 {
  57         struct buffer *buffer = clientp;
  58
  59         switch (cmd) {
  60         case CURLIOCMD_NOP:
  61                 return CURLIOE_OK;
  62
  63         case CURLIOCMD_RESTARTREAD:
  64                 buffer->posn = 0;
  65                 return CURLIOE_OK;
  66
  67         default:
  68                 return CURLIOE_UNKNOWNCMD;
  69         }
  70 }
  71 #endif
  72
  73 size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
  74 {
  75         size_t size = eltsize * nmemb;
  76         struct strbuf *buffer = buffer_;
  77
  78         strbuf_add(buffer, ptr, size);
  79         data_received++;
  80         return size;
  81 }
  82
  83 size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf)
  84 {
  85         data_received++;
  86         return eltsize * nmemb;
  87 }
  88
  89 static void finish_active_slot(struct active_request_slot *slot);
  90
  91 #ifdef USE_CURL_MULTI
  92 static void process_curl_messages(void)
  93 {
  94         int num_messages;
  95         struct active_request_slot *slot;
  96         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
  97
  98         while (curl_message != NULL) {
  99                 if (curl_message->msg == CURLMSG_DONE) {
 100                         int curl_result = curl_message->data.result;
 101                         slot = active_queue_head;
 102                         while (slot != NULL &&
 103                                slot->curl != curl_message->easy_handle)
 104                                 slot = slot->next;
 105                         if (slot != NULL) {
 106                                 curl_multi_remove_handle(curlm, slot->curl);
 107                                 slot->curl_result = curl_result;
 108                                 finish_active_slot(slot);
 109                         } else {
 110                                 fprintf(stderr, "Received DONE message for unknown request!\n");
 111                         }
 112                 } else {
 113                         fprintf(stderr, "Unknown CURL message received: %d\n",
 114                                 (int)curl_message->msg);
 115                 }
 116                 curl_message = curl_multi_info_read(curlm, &num_messages);
 117         }
 118 }
 119 #endif
 120
 121 static int http_options(const char *var, const char *value, void *cb)
 122 {
 123         if (!strcmp("http.sslverify", var)) {
 124                 curl_ssl_verify = git_config_bool(var, value);
 125                 return 0;
 126         }
 127         if (!strcmp("http.sslcert", var))
 128                 return git_config_string(&ssl_cert, var, value);
 129 #if LIBCURL_VERSION_NUM >= 0x070902
 130         if (!strcmp("http.sslkey", var))
 131                 return git_config_string(&ssl_key, var, value);
 132 #endif
 133 #if LIBCURL_VERSION_NUM >= 0x070908
 134         if (!strcmp("http.sslcapath", var))
 135                 return git_config_string(&ssl_capath, var, value);
 136 #endif
 137         if (!strcmp("http.sslcainfo", var))
 138                 return git_config_string(&ssl_cainfo, var, value);
 139 #ifdef USE_CURL_MULTI
 140         if (!strcmp("http.maxrequests", var)) {
 141                 max_requests = git_config_int(var, value);
 142                 return 0;
 143         }
 144 #endif
 145         if (!strcmp("http.lowspeedlimit", var)) {
 146                 curl_low_speed_limit = (long)git_config_int(var, value);
 147                 return 0;
 148         }
 149         if (!strcmp("http.lowspeedtime", var)) {
 150                 curl_low_speed_time = (long)git_config_int(var, value);
 151                 return 0;
 152         }
 153
 154         if (!strcmp("http.noepsv", var)) {
 155                 curl_ftp_no_epsv = git_config_bool(var, value);
 156                 return 0;
 157         }
 158         if (!strcmp("http.proxy", var))
 159                 return git_config_string(&curl_http_proxy, var, value);
 160
 161         /* Fall back on the default ones */
 162         return git_default_config(var, value, cb);
 163 }
 164
 165 static void init_curl_http_auth(CURL *result)
 166 {
 167         if (user_name) {
 168                 struct strbuf up = STRBUF_INIT;
 169                 if (!user_pass)
 170                         user_pass = xstrdup(getpass("Password: "));
 171                 strbuf_addf(&up, "%s:%s", user_name, user_pass);
 172                 curl_easy_setopt(result, CURLOPT_USERPWD,
 173                                  strbuf_detach(&up, NULL));
 174         }
 175 }
 176
 177 static CURL *get_curl_handle(void)
 178 {
 179         CURL *result = curl_easy_init();
 180
 181         if (!curl_ssl_verify) {
 182                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
 183                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
 184         } else {
 185                 /* Verify authenticity of the peer's certificate */
 186                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
 187                 /* The name in the cert must match whom we tried to connect */
 188                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
 189         }
 190
 191 #if LIBCURL_VERSION_NUM >= 0x070907
 192         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
 193 #endif
 194
 195         init_curl_http_auth(result);
 196
 197         if (ssl_cert != NULL)
 198                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
 199 #if LIBCURL_VERSION_NUM >= 0x070902
 200         if (ssl_key != NULL)
 201                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
 202 #endif
 203 #if LIBCURL_VERSION_NUM >= 0x070908
 204         if (ssl_capath != NULL)
 205                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
 206 #endif
 207         if (ssl_cainfo != NULL)
 208                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
 209         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
 210
 211         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
 212                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
 213                                  curl_low_speed_limit);
 214                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
 215                                  curl_low_speed_time);
 216         }
 217
 218         curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
 219
 220         if (getenv("GIT_CURL_VERBOSE"))
 221                 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
 222
 223         curl_easy_setopt(result, CURLOPT_USERAGENT, GIT_USER_AGENT);
 224
 225         if (curl_ftp_no_epsv)
 226                 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
 227
 228         if (curl_http_proxy)
 229                 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
 230
 231         return result;
 232 }
 233
 234 static void http_auth_init(const char *url)
 235 {
 236         char *at, *colon, *cp, *slash;
 237         int len;
 238
 239         cp = strstr(url, "://");
 240         if (!cp)
 241                 return;
 242
 243         /*
 244          * Ok, the URL looks like "proto://something".  Which one?
 245          * "proto://<user>:<pass>@<host>/...",
 246          * "proto://<user>@<host>/...", or just
 247          * "proto://<host>/..."?
 248          */
 249         cp += 3;
 250         at = strchr(cp, '@');
 251         colon = strchr(cp, ':');
 252         slash = strchrnul(cp, '/');
 253         if (!at || slash <= at)
 254                 return; /* No credentials */
 255         if (!colon || at <= colon) {
 256                 /* Only username */
 257                 len = at - cp;
 258                 user_name = xmalloc(len + 1);
 259                 memcpy(user_name, cp, len);
 260                 user_name[len] = '\0';
 261                 user_pass = NULL;
 262         } else {
 263                 len = colon - cp;
 264                 user_name = xmalloc(len + 1);
 265                 memcpy(user_name, cp, len);
 266                 user_name[len] = '\0';
 267                 len = at - (colon + 1);
 268                 user_pass = xmalloc(len + 1);
 269                 memcpy(user_pass, colon + 1, len);
 270                 user_pass[len] = '\0';
 271         }
 272 }
 273
 274 static void set_from_env(const char **var, const char *envname)
 275 {
 276         const char *val = getenv(envname);
 277         if (val)
 278                 *var = val;
 279 }
 280
 281 void http_init(struct remote *remote)
 282 {
 283         char *low_speed_limit;
 284         char *low_speed_time;
 285
 286         http_is_verbose = 0;
 287
 288         git_config(http_options, NULL);
 289
 290         curl_global_init(CURL_GLOBAL_ALL);
 291
 292         if (remote && remote->http_proxy)
 293                 curl_http_proxy = xstrdup(remote->http_proxy);
 294
 295         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
 296         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
 297
 298 #ifdef USE_CURL_MULTI
 299         {
 300                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
 301                 if (http_max_requests != NULL)
 302                         max_requests = atoi(http_max_requests);
 303         }
 304
 305         curlm = curl_multi_init();
 306         if (curlm == NULL) {
 307                 fprintf(stderr, "Error creating curl multi handle.\n");
 308                 exit(1);
 309         }
 310 #endif
 311
 312         if (getenv("GIT_SSL_NO_VERIFY"))
 313                 curl_ssl_verify = 0;
 314
 315         set_from_env(&ssl_cert, "GIT_SSL_CERT");
 316 #if LIBCURL_VERSION_NUM >= 0x070902
 317         set_from_env(&ssl_key, "GIT_SSL_KEY");
 318 #endif
 319 #if LIBCURL_VERSION_NUM >= 0x070908
 320         set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
 321 #endif
 322         set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
 323
 324         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
 325         if (low_speed_limit != NULL)
 326                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
 327         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
 328         if (low_speed_time != NULL)
 329                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
 330
 331         if (curl_ssl_verify == -1)
 332                 curl_ssl_verify = 1;
 333
 334 #ifdef USE_CURL_MULTI
 335         if (max_requests < 1)
 336                 max_requests = DEFAULT_MAX_REQUESTS;
 337 #endif
 338
 339         if (getenv("GIT_CURL_FTP_NO_EPSV"))
 340                 curl_ftp_no_epsv = 1;
 341
 342         if (remote && remote->url && remote->url[0])
 343                 http_auth_init(remote->url[0]);
 344
 345 #ifndef NO_CURL_EASY_DUPHANDLE
 346         curl_default = get_curl_handle();
 347 #endif
 348 }
 349
 350 void http_cleanup(void)
 351 {
 352         struct active_request_slot *slot = active_queue_head;
 353
 354         while (slot != NULL) {
 355                 struct active_request_slot *next = slot->next;
 356                 if (slot->curl != NULL) {
 357 #ifdef USE_CURL_MULTI
 358                         curl_multi_remove_handle(curlm, slot->curl);
 359 #endif
 360                         curl_easy_cleanup(slot->curl);
 361                 }
 362                 free(slot);
 363                 slot = next;
 364         }
 365         active_queue_head = NULL;
 366
 367 #ifndef NO_CURL_EASY_DUPHANDLE
 368         curl_easy_cleanup(curl_default);
 369 #endif
 370
 371 #ifdef USE_CURL_MULTI
 372         curl_multi_cleanup(curlm);
 373 #endif
 374         curl_global_cleanup();
 375
 376         curl_slist_free_all(pragma_header);
 377         pragma_header = NULL;
 378
 379         curl_slist_free_all(no_pragma_header);
 380         no_pragma_header = NULL;
 381
 382         if (curl_http_proxy) {
 383                 free((void *)curl_http_proxy);
 384                 curl_http_proxy = NULL;
 385         }
 386 }
 387
 388 struct active_request_slot *get_active_slot(void)
 389 {
 390         struct active_request_slot *slot = active_queue_head;
 391         struct active_request_slot *newslot;
 392
 393 #ifdef USE_CURL_MULTI
 394         int num_transfers;
 395
 396         /* Wait for a slot to open up if the queue is full */
 397         while (active_requests >= max_requests) {
 398                 curl_multi_perform(curlm, &num_transfers);
 399                 if (num_transfers < active_requests)
 400                         process_curl_messages();
 401         }
 402 #endif
 403
 404         while (slot != NULL && slot->in_use)
 405                 slot = slot->next;
 406
 407         if (slot == NULL) {
 408                 newslot = xmalloc(sizeof(*newslot));
 409                 newslot->curl = NULL;
 410                 newslot->in_use = 0;
 411                 newslot->next = NULL;
 412
 413                 slot = active_queue_head;
 414                 if (slot == NULL) {
 415                         active_queue_head = newslot;
 416                 } else {
 417                         while (slot->next != NULL)
 418                                 slot = slot->next;
 419                         slot->next = newslot;
 420                 }
 421                 slot = newslot;
 422         }
 423
 424         if (slot->curl == NULL) {
 425 #ifdef NO_CURL_EASY_DUPHANDLE
 426                 slot->curl = get_curl_handle();
 427 #else
 428                 slot->curl = curl_easy_duphandle(curl_default);
 429 #endif
 430         }
 431
 432         active_requests++;
 433         slot->in_use = 1;
 434         slot->local = NULL;
 435         slot->results = NULL;
 436         slot->finished = NULL;
 437         slot->callback_data = NULL;
 438         slot->callback_func = NULL;
 439         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
 440         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
 441         curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
 442         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
 443         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
 444         curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
 445         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 446
 447         return slot;
 448 }
 449
 450 int start_active_slot(struct active_request_slot *slot)
 451 {
 452 #ifdef USE_CURL_MULTI
 453         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
 454         int num_transfers;
 455
 456         if (curlm_result != CURLM_OK &&
 457             curlm_result != CURLM_CALL_MULTI_PERFORM) {
 458                 active_requests--;
 459                 slot->in_use = 0;
 460                 return 0;
 461         }
 462
 463         /*
 464          * We know there must be something to do, since we just added
 465          * something.
 466          */
 467         curl_multi_perform(curlm, &num_transfers);
 468 #endif
 469         return 1;
 470 }
 471
 472 #ifdef USE_CURL_MULTI
 473 struct fill_chain {
 474         void *data;
 475         int (*fill)(void *);
 476         struct fill_chain *next;
 477 };
 478
 479 static struct fill_chain *fill_cfg;
 480
 481 void add_fill_function(void *data, int (*fill)(void *))
 482 {
 483         struct fill_chain *new = xmalloc(sizeof(*new));
 484         struct fill_chain **linkp = &fill_cfg;
 485         new->data = data;
 486         new->fill = fill;
 487         new->next = NULL;
 488         while (*linkp)
 489                 linkp = &(*linkp)->next;
 490         *linkp = new;
 491 }
 492
 493 void fill_active_slots(void)
 494 {
 495         struct active_request_slot *slot = active_queue_head;
 496
 497         while (active_requests < max_requests) {
 498                 struct fill_chain *fill;
 499                 for (fill = fill_cfg; fill; fill = fill->next)
 500                         if (fill->fill(fill->data))
 501                                 break;
 502
 503                 if (!fill)
 504                         break;
 505         }
 506
 507         while (slot != NULL) {
 508                 if (!slot->in_use && slot->curl != NULL) {
 509                         curl_easy_cleanup(slot->curl);
 510                         slot->curl = NULL;
 511                 }
 512                 slot = slot->next;
 513         }
 514 }
 515
 516 void step_active_slots(void)
 517 {
 518         int num_transfers;
 519         CURLMcode curlm_result;
 520
 521         do {
 522                 curlm_result = curl_multi_perform(curlm, &num_transfers);
 523         } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
 524         if (num_transfers < active_requests) {
 525                 process_curl_messages();
 526                 fill_active_slots();
 527         }
 528 }
 529 #endif
 530
 531 void run_active_slot(struct active_request_slot *slot)
 532 {
 533 #ifdef USE_CURL_MULTI
 534         long last_pos = 0;
 535         long current_pos;
 536         fd_set readfds;
 537         fd_set writefds;
 538         fd_set excfds;
 539         int max_fd;
 540         struct timeval select_timeout;
 541         int finished = 0;
 542
 543         slot->finished = &finished;
 544         while (!finished) {
 545                 data_received = 0;
 546                 step_active_slots();
 547
 548                 if (!data_received && slot->local != NULL) {
 549                         current_pos = ftell(slot->local);
 550                         if (current_pos > last_pos)
 551                                 data_received++;
 552                         last_pos = current_pos;
 553                 }
 554
 555                 if (slot->in_use && !data_received) {
 556                         max_fd = 0;
 557                         FD_ZERO(&readfds);
 558                         FD_ZERO(&writefds);
 559                         FD_ZERO(&excfds);
 560                         select_timeout.tv_sec = 0;
 561                         select_timeout.tv_usec = 50000;
 562                         select(max_fd, &readfds, &writefds,
 563                                &excfds, &select_timeout);
 564                 }
 565         }
 566 #else
 567         while (slot->in_use) {
 568                 slot->curl_result = curl_easy_perform(slot->curl);
 569                 finish_active_slot(slot);
 570         }
 571 #endif
 572 }
 573
 574 static void closedown_active_slot(struct active_request_slot *slot)
 575 {
 576         active_requests--;
 577         slot->in_use = 0;
 578 }
 579
 580 void release_active_slot(struct active_request_slot *slot)
 581 {
 582         closedown_active_slot(slot);
 583         if (slot->curl) {
 584 #ifdef USE_CURL_MULTI
 585                 curl_multi_remove_handle(curlm, slot->curl);
 586 #endif
 587                 curl_easy_cleanup(slot->curl);
 588                 slot->curl = NULL;
 589         }
 590 #ifdef USE_CURL_MULTI
 591         fill_active_slots();
 592 #endif
 593 }
 594
 595 static void finish_active_slot(struct active_request_slot *slot)
 596 {
 597         closedown_active_slot(slot);
 598         curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
 599
 600         if (slot->finished != NULL)
 601                 (*slot->finished) = 1;
 602
 603         /* Store slot results so they can be read after the slot is reused */
 604         if (slot->results != NULL) {
 605                 slot->results->curl_result = slot->curl_result;
 606                 slot->results->http_code = slot->http_code;
 607         }
 608
 609         /* Run callback if appropriate */
 610         if (slot->callback_func != NULL)
 611                 slot->callback_func(slot->callback_data);
 612 }
 613
 614 void finish_all_active_slots(void)
 615 {
 616         struct active_request_slot *slot = active_queue_head;
 617
 618         while (slot != NULL)
 619                 if (slot->in_use) {
 620                         run_active_slot(slot);
 621                         slot = active_queue_head;
 622                 } else {
 623                         slot = slot->next;
 624                 }
 625 }
 626
 627 /* Helpers for modifying and creating URLs */
 628 static inline int needs_quote(int ch)
 629 {
 630         if (((ch >= 'A') && (ch <= 'Z'))
 631                         || ((ch >= 'a') && (ch <= 'z'))
 632                         || ((ch >= '0') && (ch <= '9'))
 633                         || (ch == '/')
 634                         || (ch == '-')
 635                         || (ch == '.'))
 636                 return 0;
 637         return 1;
 638 }
 639
 640 static inline int hex(int v)
 641 {
 642         if (v < 10)
 643                 return '0' + v;
 644         else
 645                 return 'A' + v - 10;
 646 }
 647
 648 static void end_url_with_slash(struct strbuf *buf, const char *url)
 649 {
 650         strbuf_addstr(buf, url);
 651         if (buf->len && buf->buf[buf->len - 1] != '/')
 652                 strbuf_addstr(buf, "/");
 653 }
 654
 655 static char *quote_ref_url(const char *base, const char *ref)
 656 {
 657         struct strbuf buf = STRBUF_INIT;
 658         const char *cp;
 659         int ch;
 660
 661         end_url_with_slash(&buf, base);
 662
 663         for (cp = ref; (ch = *cp) != 0; cp++)
 664                 if (needs_quote(ch))
 665                         strbuf_addf(&buf, "%%%02x", ch);
 666                 else
 667                         strbuf_addch(&buf, *cp);
 668
 669         return strbuf_detach(&buf, NULL);
 670 }
 671
 672 void append_remote_object_url(struct strbuf *buf, const char *url,
 673                               const char *hex,
 674                               int only_two_digit_prefix)
 675 {
 676         strbuf_addf(buf, "%s/objects/%.*s/", url, 2, hex);
 677         if (!only_two_digit_prefix)
 678                 strbuf_addf(buf, "%s", hex+2);
 679 }
 680
 681 char *get_remote_object_url(const char *url, const char *hex,
 682                             int only_two_digit_prefix)
 683 {
 684         struct strbuf buf = STRBUF_INIT;
 685         append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
 686         return strbuf_detach(&buf, NULL);
 687 }
 688
 689 /* http_request() targets */
 690 #define HTTP_REQUEST_STRBUF     0
 691 #define HTTP_REQUEST_FILE       1
 692
 693 static int http_request(const char *url, void *result, int target, int options)
 694 {
 695         struct active_request_slot *slot;
 696         struct slot_results results;
 697         struct curl_slist *headers = NULL;
 698         struct strbuf buf = STRBUF_INIT;
 699         int ret;
 700
 701         slot = get_active_slot();
 702         slot->results = &results;
 703         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
 704
 705         if (result == NULL) {
 706                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
 707         } else {
 708                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
 709                 curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
 710
 711                 if (target == HTTP_REQUEST_FILE) {
 712                         long posn = ftell(result);
 713                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 714                                          fwrite);
 715                         if (posn > 0) {
 716                                 strbuf_addf(&buf, "Range: bytes=%ld-", posn);
 717                                 headers = curl_slist_append(headers, buf.buf);
 718                                 strbuf_reset(&buf);
 719                         }
 720                         slot->local = result;
 721                 } else
 722                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
 723                                          fwrite_buffer);
 724         }
 725
 726         strbuf_addstr(&buf, "Pragma:");
 727         if (options & HTTP_NO_CACHE)
 728                 strbuf_addstr(&buf, " no-cache");
 729
 730         headers = curl_slist_append(headers, buf.buf);
 731
 732         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 733         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
 734
 735         if (start_active_slot(slot)) {
 736                 run_active_slot(slot);
 737                 if (results.curl_result == CURLE_OK)
 738                         ret = HTTP_OK;
 739                 else if (missing_target(&results))
 740                         ret = HTTP_MISSING_TARGET;
 741                 else
 742                         ret = HTTP_ERROR;
 743         } else {
 744                 error("Unable to start HTTP request for %s", url);
 745                 ret = HTTP_START_FAILED;
 746         }
 747
 748         slot->local = NULL;
 749         curl_slist_free_all(headers);
 750         strbuf_release(&buf);
 751
 752         return ret;
 753 }
 754
 755 int http_get_strbuf(const char *url, struct strbuf *result, int options)
 756 {
 757         return http_request(url, result, HTTP_REQUEST_STRBUF, options);
 758 }
 759
 760 int http_get_file(const char *url, const char *filename, int options)
 761 {
 762         int ret;
 763         struct strbuf tmpfile = STRBUF_INIT;
 764         FILE *result;
 765
 766         strbuf_addf(&tmpfile, "%s.temp", filename);
 767         result = fopen(tmpfile.buf, "a");
 768         if (! result) {
 769                 error("Unable to open local file %s", tmpfile.buf);
 770                 ret = HTTP_ERROR;
 771                 goto cleanup;
 772         }
 773
 774         ret = http_request(url, result, HTTP_REQUEST_FILE, options);
 775         fclose(result);
 776
 777         if ((ret == HTTP_OK) && move_temp_to_file(tmpfile.buf, filename))
 778                 ret = HTTP_ERROR;
 779 cleanup:
 780         strbuf_release(&tmpfile);
 781         return ret;
 782 }
 783
 784 int http_error(const char *url, int ret)
 785 {
 786         /* http_request has already handled HTTP_START_FAILED. */
 787         if (ret != HTTP_START_FAILED)
 788                 error("%s while accessing %s\n", curl_errorstr, url);
 789
 790         return ret;
 791 }
 792
 793 int http_fetch_ref(const char *base, struct ref *ref)
 794 {
 795         char *url;
 796         struct strbuf buffer = STRBUF_INIT;
 797         int ret = -1;
 798
 799         url = quote_ref_url(base, ref->name);
 800         if (http_get_strbuf(url, &buffer, HTTP_NO_CACHE) == HTTP_OK) {
 801                 strbuf_rtrim(&buffer);
 802                 if (buffer.len == 40)
 803                         ret = get_sha1_hex(buffer.buf, ref->old_sha1);
 804                 else if (!prefixcmp(buffer.buf, "ref: ")) {
 805                         ref->symref = xstrdup(buffer.buf + 5);
 806                         ret = 0;
 807                 }
 808         }
 809
 810         strbuf_release(&buffer);
 811         free(url);
 812         return ret;
 813 }
 814
 815 /* Helpers for fetching packs */
 816 static int fetch_pack_index(unsigned char *sha1, const char *base_url)
 817 {
 818         int ret = 0;
 819         char *hex = xstrdup(sha1_to_hex(sha1));
 820         char *filename;
 821         char *url;
 822         struct strbuf buf = STRBUF_INIT;
 823
 824         /* Don't use the index if the pack isn't there */
 825         end_url_with_slash(&buf, base_url);
 826         strbuf_addf(&buf, "objects/pack/pack-%s.pack", hex);
 827         url = strbuf_detach(&buf, 0);
 828
 829         if (http_get_strbuf(url, NULL, 0)) {
 830                 ret = error("Unable to verify pack %s is available",
 831                             hex);
 832                 goto cleanup;
 833         }
 834
 835         if (has_pack_index(sha1)) {
 836                 ret = 0;
 837                 goto cleanup;
 838         }
 839
 840         if (http_is_verbose)
 841                 fprintf(stderr, "Getting index for pack %s\n", hex);
 842
 843         end_url_with_slash(&buf, base_url);
 844         strbuf_addf(&buf, "objects/pack/pack-%s.idx", hex);
 845         url = strbuf_detach(&buf, NULL);
 846
 847         filename = sha1_pack_index_name(sha1);
 848         if (http_get_file(url, filename, 0) != HTTP_OK)
 849                 ret = error("Unable to get pack index %s\n", url);
 850
 851 cleanup:
 852         free(hex);
 853         free(url);
 854         return ret;
 855 }
 856
 857 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
 858         unsigned char *sha1, const char *base_url)
 859 {
 860         struct packed_git *new_pack;
 861
 862         if (fetch_pack_index(sha1, base_url))
 863                 return -1;
 864
 865         new_pack = parse_pack_index(sha1);
 866         if (!new_pack)
 867                 return -1; /* parse_pack_index() already issued error message */
 868         new_pack->next = *packs_head;
 869         *packs_head = new_pack;
 870         return 0;
 871 }
 872
 873 int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
 874 {
 875         int ret = 0, i = 0;
 876         char *url, *data;
 877         struct strbuf buf = STRBUF_INIT;
 878         unsigned char sha1[20];
 879
 880         end_url_with_slash(&buf, base_url);
 881         strbuf_addstr(&buf, "objects/info/packs");
 882         url = strbuf_detach(&buf, NULL);
 883
 884         ret = http_get_strbuf(url, &buf, HTTP_NO_CACHE);
 885         if (ret != HTTP_OK)
 886                 goto cleanup;
 887
 888         data = buf.buf;
 889         while (i < buf.len) {
 890                 switch (data[i]) {
 891                 case 'P':
 892                         i++;
 893                         if (i + 52 <= buf.len &&
 894                             !prefixcmp(data + i, " pack-") &&
 895                             !prefixcmp(data + i + 46, ".pack\n")) {
 896                                 get_sha1_hex(data + i + 6, sha1);
 897                                 fetch_and_setup_pack_index(packs_head, sha1,
 898                                                       base_url);
 899                                 i += 51;
 900                                 break;
 901                         }
 902                 default:
 903                         while (i < buf.len && data[i] != '\n')
 904                                 i++;
 905                 }
 906                 i++;
 907         }
 908
 909 cleanup:
 910         free(url);
 911         return ret;
 912 }
 913
 914 void release_http_pack_request(struct http_pack_request *preq)
 915 {
 916         if (preq->packfile != NULL) {
 917                 fclose(preq->packfile);
 918                 preq->packfile = NULL;
 919                 preq->slot->local = NULL;
 920         }
 921         if (preq->range_header != NULL) {
 922                 curl_slist_free_all(preq->range_header);
 923                 preq->range_header = NULL;
 924         }
 925         preq->slot = NULL;
 926         free(preq->url);
 927 }
 928
 929 int finish_http_pack_request(struct http_pack_request *preq)
 930 {
 931         int ret;
 932         struct packed_git **lst;
 933
 934         preq->target->pack_size = ftell(preq->packfile);
 935
 936         if (preq->packfile != NULL) {
 937                 fclose(preq->packfile);
 938                 preq->packfile = NULL;
 939                 preq->slot->local = NULL;
 940         }
 941
 942         ret = move_temp_to_file(preq->tmpfile, preq->filename);
 943         if (ret)
 944                 return ret;
 945
 946         lst = preq->lst;
 947         while (*lst != preq->target)
 948                 lst = &((*lst)->next);
 949         *lst = (*lst)->next;
 950
 951         if (verify_pack(preq->target))
 952                 return -1;
 953         install_packed_git(preq->target);
 954
 955         return 0;
 956 }
 957
 958 struct http_pack_request *new_http_pack_request(
 959         struct packed_git *target, const char *base_url)
 960 {
 961         char *url;
 962         char *filename;
 963         long prev_posn = 0;
 964         char range[RANGE_HEADER_SIZE];
 965         struct strbuf buf = STRBUF_INIT;
 966         struct http_pack_request *preq;
 967
 968         preq = xmalloc(sizeof(*preq));
 969         preq->target = target;
 970         preq->range_header = NULL;
 971
 972         end_url_with_slash(&buf, base_url);
 973         strbuf_addf(&buf, "objects/pack/pack-%s.pack",
 974                 sha1_to_hex(target->sha1));
 975         url = strbuf_detach(&buf, NULL);
 976         preq->url = xstrdup(url);
 977
 978         filename = sha1_pack_name(target->sha1);
 979         snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
 980         snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
 981         preq->packfile = fopen(preq->tmpfile, "a");
 982         if (!preq->packfile) {
 983                 error("Unable to open local file %s for pack",
 984                       preq->tmpfile);
 985                 goto abort;
 986         }
 987
 988         preq->slot = get_active_slot();
 989         preq->slot->local = preq->packfile;
 990         curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
 991         curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 992         curl_easy_setopt(preq->slot->curl, CURLOPT_URL, url);
 993         curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
 994                 no_pragma_header);
 995
 996         /*
 997          * If there is data present from a previous transfer attempt,
 998          * resume where it left off
 999          */
1000         prev_posn = ftell(preq->packfile);
1001         if (prev_posn>0) {
1002                 if (http_is_verbose)
1003                         fprintf(stderr,
1004                                 "Resuming fetch of pack %s at byte %ld\n",
1005                                 sha1_to_hex(target->sha1), prev_posn);
1006                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1007                 preq->range_header = curl_slist_append(NULL, range);
1008                 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1009                         preq->range_header);
1010         }
1011
1012         return preq;
1013
1014 abort:
1015         free(filename);
1016         return NULL;
1017 }
1018
1019 /* Helpers for fetching objects (loose) */
1020 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
1021                                void *data)
1022 {
1023         unsigned char expn[4096];
1024         size_t size = eltsize * nmemb;
1025         int posn = 0;
1026         struct http_object_request *freq =
1027                 (struct http_object_request *)data;
1028         do {
1029                 ssize_t retval = xwrite(freq->localfile,
1030                                         (char *) ptr + posn, size - posn);
1031                 if (retval < 0)
1032                         return posn;
1033                 posn += retval;
1034         } while (posn < size);
1035
1036         freq->stream.avail_in = size;
1037         freq->stream.next_in = ptr;
1038         do {
1039                 freq->stream.next_out = expn;
1040                 freq->stream.avail_out = sizeof(expn);
1041                 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
1042                 git_SHA1_Update(&freq->c, expn,
1043                                 sizeof(expn) - freq->stream.avail_out);
1044         } while (freq->stream.avail_in && freq->zret == Z_OK);
1045         data_received++;
1046         return size;
1047 }
1048
1049 struct http_object_request *new_http_object_request(const char *base_url,
1050         unsigned char *sha1)
1051 {
1052         char *hex = sha1_to_hex(sha1);
1053         char *filename;
1054         char prevfile[PATH_MAX];
1055         char *url;
1056         int prevlocal;
1057         unsigned char prev_buf[PREV_BUF_SIZE];
1058         ssize_t prev_read = 0;
1059         long prev_posn = 0;
1060         char range[RANGE_HEADER_SIZE];
1061         struct curl_slist *range_header = NULL;
1062         struct http_object_request *freq;
1063
1064         freq = xmalloc(sizeof(*freq));
1065         hashcpy(freq->sha1, sha1);
1066         freq->localfile = -1;
1067
1068         filename = sha1_file_name(sha1);
1069         snprintf(freq->filename, sizeof(freq->filename), "%s", filename);
1070         snprintf(freq->tmpfile, sizeof(freq->tmpfile),
1071                  "%s.temp", filename);
1072
1073         snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
1074         unlink_or_warn(prevfile);
1075         rename(freq->tmpfile, prevfile);
1076         unlink_or_warn(freq->tmpfile);
1077
1078         if (freq->localfile != -1)
1079                 error("fd leakage in start: %d", freq->localfile);
1080         freq->localfile = open(freq->tmpfile,
1081                                O_WRONLY | O_CREAT | O_EXCL, 0666);
1082         /*
1083          * This could have failed due to the "lazy directory creation";
1084          * try to mkdir the last path component.
1085          */
1086         if (freq->localfile < 0 && errno == ENOENT) {
1087                 char *dir = strrchr(freq->tmpfile, '/');
1088                 if (dir) {
1089                         *dir = 0;
1090                         mkdir(freq->tmpfile, 0777);
1091                         *dir = '/';
1092                 }
1093                 freq->localfile = open(freq->tmpfile,
1094                                        O_WRONLY | O_CREAT | O_EXCL, 0666);
1095         }
1096
1097         if (freq->localfile < 0) {
1098                 error("Couldn't create temporary file %s for %s: %s",
1099                       freq->tmpfile, freq->filename, strerror(errno));
1100                 goto abort;
1101         }
1102
1103         memset(&freq->stream, 0, sizeof(freq->stream));
1104
1105         git_inflate_init(&freq->stream);
1106
1107         git_SHA1_Init(&freq->c);
1108
1109         url = get_remote_object_url(base_url, hex, 0);
1110         freq->url = xstrdup(url);
1111
1112         /*
1113          * If a previous temp file is present, process what was already
1114          * fetched.
1115          */
1116         prevlocal = open(prevfile, O_RDONLY);
1117         if (prevlocal != -1) {
1118                 do {
1119                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1120                         if (prev_read>0) {
1121                                 if (fwrite_sha1_file(prev_buf,
1122                                                      1,
1123                                                      prev_read,
1124                                                      freq) == prev_read) {
1125                                         prev_posn += prev_read;
1126                                 } else {
1127                                         prev_read = -1;
1128                                 }
1129                         }
1130                 } while (prev_read > 0);
1131                 close(prevlocal);
1132         }
1133         unlink_or_warn(prevfile);
1134
1135         /*
1136          * Reset inflate/SHA1 if there was an error reading the previous temp
1137          * file; also rewind to the beginning of the local file.
1138          */
1139         if (prev_read == -1) {
1140                 memset(&freq->stream, 0, sizeof(freq->stream));
1141                 git_inflate_init(&freq->stream);
1142                 git_SHA1_Init(&freq->c);
1143                 if (prev_posn>0) {
1144                         prev_posn = 0;
1145                         lseek(freq->localfile, 0, SEEK_SET);
1146                         ftruncate(freq->localfile, 0);
1147                 }
1148         }
1149
1150         freq->slot = get_active_slot();
1151
1152         curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
1153         curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1154         curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
1155         curl_easy_setopt(freq->slot->curl, CURLOPT_URL, url);
1156         curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1157
1158         /*
1159          * If we have successfully processed data from a previous fetch
1160          * attempt, only fetch the data we don't already have.
1161          */
1162         if (prev_posn>0) {
1163                 if (http_is_verbose)
1164                         fprintf(stderr,
1165                                 "Resuming fetch of object %s at byte %ld\n",
1166                                 hex, prev_posn);
1167                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1168                 range_header = curl_slist_append(range_header, range);
1169                 curl_easy_setopt(freq->slot->curl,
1170                                  CURLOPT_HTTPHEADER, range_header);
1171         }
1172
1173         return freq;
1174
1175         free(url);
1176 abort:
1177         free(filename);
1178         free(freq);
1179         return NULL;
1180 }
1181
1182 void process_http_object_request(struct http_object_request *freq)
1183 {
1184         if (freq->slot == NULL)
1185                 return;
1186         freq->curl_result = freq->slot->curl_result;
1187         freq->http_code = freq->slot->http_code;
1188         freq->slot = NULL;
1189 }
1190
1191 int finish_http_object_request(struct http_object_request *freq)
1192 {
1193         struct stat st;
1194
1195         close(freq->localfile);
1196         freq->localfile = -1;
1197
1198         process_http_object_request(freq);
1199
1200         if (freq->http_code == 416) {
1201                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
1202         } else if (freq->curl_result != CURLE_OK) {
1203                 if (stat(freq->tmpfile, &st) == 0)
1204                         if (st.st_size == 0)
1205                                 unlink_or_warn(freq->tmpfile);
1206                 return -1;
1207         }
1208
1209         git_inflate_end(&freq->stream);
1210         git_SHA1_Final(freq->real_sha1, &freq->c);
1211         if (freq->zret != Z_STREAM_END) {
1212                 unlink_or_warn(freq->tmpfile);
1213                 return -1;
1214         }
1215         if (hashcmp(freq->sha1, freq->real_sha1)) {
1216                 unlink_or_warn(freq->tmpfile);
1217                 return -1;
1218         }
1219         freq->rename =
1220                 move_temp_to_file(freq->tmpfile, freq->filename);
1221
1222         return freq->rename;
1223 }
1224
1225 void abort_http_object_request(struct http_object_request *freq)
1226 {
1227         unlink_or_warn(freq->tmpfile);
1228
1229         release_http_object_request(freq);
1230 }
1231
1232 void release_http_object_request(struct http_object_request *freq)
1233 {
1234         if (freq->localfile != -1) {
1235                 close(freq->localfile);
1236                 freq->localfile = -1;
1237         }
1238         if (freq->url != NULL) {
1239                 free(freq->url);
1240                 freq->url = NULL;
1241         }
1242         freq->slot = NULL;
1243 }