Resync
[CMakeLuaTailorHgBridge.git] / CMakeLua / Utilities / cmcurl-7.19.0 / lib / transfer.c
blob43cd29579a68fbec0a59f4824ee7fc8b4ecef482
1 /***************************************************************************
2 * _ _ ____ _
3 * Project ___| | | | _ \| |
4 * / __| | | | |_) | |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
8 * Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al.
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at http://curl.haxx.se/docs/copyright.html.
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
21 * $Id: transfer.c,v 1.1.1.1 2008-09-23 16:32:05 hoffman Exp $
22 ***************************************************************************/
24 #include "setup.h"
26 /* -- WIN32 approved -- */
27 #include <stdio.h>
28 #include <string.h>
29 #include <stdarg.h>
30 #include <stdlib.h>
31 #include <ctype.h>
32 #include <errno.h>
34 #include "strtoofft.h"
35 #include "strequal.h"
37 #ifdef WIN32
38 #include <time.h>
39 #include <io.h>
40 #else
41 #ifdef HAVE_SYS_SOCKET_H
42 #include <sys/socket.h>
43 #endif
44 #ifdef HAVE_NETINET_IN_H
45 #include <netinet/in.h>
46 #endif
47 #ifdef HAVE_SYS_TIME_H
48 #include <sys/time.h>
49 #endif
50 #ifdef HAVE_UNISTD_H
51 #include <unistd.h>
52 #endif
53 #ifdef HAVE_NETDB_H
54 #include <netdb.h>
55 #endif
56 #ifdef HAVE_ARPA_INET_H
57 #include <arpa/inet.h>
58 #endif
59 #ifdef HAVE_NET_IF_H
60 #include <net/if.h>
61 #endif
62 #ifdef HAVE_SYS_IOCTL_H
63 #include <sys/ioctl.h>
64 #endif
65 #include <signal.h>
67 #ifdef HAVE_SYS_PARAM_H
68 #include <sys/param.h>
69 #endif
71 #ifdef HAVE_SYS_SELECT_H
72 #include <sys/select.h>
73 #endif
75 #ifndef HAVE_SOCKET
76 #error "We can't compile without socket() support!"
77 #endif
79 #endif /* WIN32 */
81 #include "urldata.h"
82 #include <curl/curl.h>
83 #include "netrc.h"
85 #include "content_encoding.h"
86 #include "hostip.h"
87 #include "transfer.h"
88 #include "sendf.h"
89 #include "speedcheck.h"
90 #include "progress.h"
91 #include "http.h"
92 #include "url.h"
93 #include "getinfo.h"
94 #include "sslgen.h"
95 #include "http_digest.h"
96 #include "http_ntlm.h"
97 #include "http_negotiate.h"
98 #include "share.h"
99 #include "memory.h"
100 #include "select.h"
101 #include "multiif.h"
102 #include "easyif.h" /* for Curl_convert_to_network prototype */
104 #define _MPRINTF_REPLACE /* use our functions only */
105 #include <curl/mprintf.h>
107 /* The last #include file should be: */
108 #include "memdebug.h"
110 #define CURL_TIMEOUT_EXPECT_100 1000 /* counting ms here */
113 static CURLcode readwrite_headers(struct SessionHandle *data,
114 struct connectdata *conn,
115 struct SingleRequest *k,
116 ssize_t *nread,
117 bool *stop_reading);
120 * This function will call the read callback to fill our buffer with data
121 * to upload.
123 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
125 struct SessionHandle *data = conn->data;
126 size_t buffersize = (size_t)bytes;
127 int nread;
129 if(data->req.upload_chunky) {
130 /* if chunked Transfer-Encoding */
131 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */
132 data->req.upload_fromhere += 10; /* 32bit hex + CRLF */
135 /* this function returns a size_t, so we typecast to int to prevent warnings
136 with picky compilers */
137 nread = (int)conn->fread_func(data->req.upload_fromhere, 1,
138 buffersize, conn->fread_in);
140 if(nread == CURL_READFUNC_ABORT) {
141 failf(data, "operation aborted by callback");
142 *nreadp = 0;
143 return CURLE_ABORTED_BY_CALLBACK;
145 else if(nread == CURL_READFUNC_PAUSE) {
146 struct SingleRequest *k = &data->req;
147 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
148 k->keepon |= KEEP_WRITE_PAUSE; /* mark socket send as paused */
149 *nreadp = 0;
150 return CURLE_OK; /* nothing was read */
152 else if((size_t)nread > buffersize) {
153 /* the read function returned a too large value */
154 *nreadp = 0;
155 return CURLE_READ_ERROR;
158 if(!data->req.forbidchunk && data->req.upload_chunky) {
159 /* if chunked Transfer-Encoding */
160 char hexbuffer[11];
161 int hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
162 "%x\r\n", nread);
163 /* move buffer pointer */
164 data->req.upload_fromhere -= hexlen;
165 nread += hexlen;
167 /* copy the prefix to the buffer */
168 memcpy(data->req.upload_fromhere, hexbuffer, hexlen);
170 /* always append CRLF to the data */
171 memcpy(data->req.upload_fromhere + nread, "\r\n", 2);
173 if((nread - hexlen) == 0) {
174 /* mark this as done once this chunk is transfered */
175 data->req.upload_done = TRUE;
178 nread+=2; /* for the added CRLF */
181 *nreadp = nread;
183 #ifdef CURL_DOES_CONVERSIONS
184 if(data->set.prefer_ascii) {
185 CURLcode res;
186 res = Curl_convert_to_network(data, data->req.upload_fromhere, nread);
187 /* Curl_convert_to_network calls failf if unsuccessful */
188 if(res != CURLE_OK) {
189 return(res);
192 #endif /* CURL_DOES_CONVERSIONS */
194 return CURLE_OK;
198 * checkhttpprefix()
200 * Returns TRUE if member of the list matches prefix of string
202 static bool
203 checkhttpprefix(struct SessionHandle *data,
204 const char *s)
206 struct curl_slist *head = data->set.http200aliases;
207 bool rc = FALSE;
208 #ifdef CURL_DOES_CONVERSIONS
209 /* convert from the network encoding using a scratch area */
210 char *scratch = calloc(1, strlen(s)+1);
211 if(NULL == scratch) {
212 failf (data, "Failed to calloc memory for conversion!");
213 return FALSE; /* can't return CURLE_OUT_OF_MEMORY so return FALSE */
215 strcpy(scratch, s);
216 if(CURLE_OK != Curl_convert_from_network(data, scratch, strlen(s)+1)) {
217 /* Curl_convert_from_network calls failf if unsuccessful */
218 free(scratch);
219 return FALSE; /* can't return CURLE_foobar so return FALSE */
221 s = scratch;
222 #endif /* CURL_DOES_CONVERSIONS */
224 while(head) {
225 if(checkprefix(head->data, s)) {
226 rc = TRUE;
227 break;
229 head = head->next;
232 if((rc != TRUE) && (checkprefix("HTTP/", s))) {
233 rc = TRUE;
236 #ifdef CURL_DOES_CONVERSIONS
237 free(scratch);
238 #endif /* CURL_DOES_CONVERSIONS */
239 return rc;
243 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP
244 * POST/PUT with multi-pass authentication when a sending was denied and a
245 * resend is necessary.
247 CURLcode Curl_readrewind(struct connectdata *conn)
249 struct SessionHandle *data = conn->data;
251 conn->bits.rewindaftersend = FALSE; /* we rewind now */
253 /* explicitly switch off sending data on this connection now since we are
254 about to restart a new transfer and thus we want to avoid inadvertently
255 sending more data on the existing connection until the next transfer
256 starts */
257 data->req.keepon &= ~KEEP_WRITE;
259 /* We have sent away data. If not using CURLOPT_POSTFIELDS or
260 CURLOPT_HTTPPOST, call app to rewind
262 if(data->set.postfields ||
263 (data->set.httpreq == HTTPREQ_POST_FORM))
264 ; /* do nothing */
265 else {
266 if(data->set.seek_func) {
267 int err;
269 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
270 if(err) {
271 failf(data, "seek callback returned error %d", (int)err);
272 return CURLE_SEND_FAIL_REWIND;
275 else if(data->set.ioctl_func) {
276 curlioerr err;
278 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
279 data->set.ioctl_client);
280 infof(data, "the ioctl callback returned %d\n", (int)err);
282 if(err) {
283 /* FIXME: convert to a human readable error message */
284 failf(data, "ioctl callback returned error %d", (int)err);
285 return CURLE_SEND_FAIL_REWIND;
288 else {
289 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
290 given FILE * stream and we can actually attempt to rewind that
291 ourself with fseek() */
292 if(data->set.fread_func == (curl_read_callback)fread) {
293 if(-1 != fseek(data->set.in, 0, SEEK_SET))
294 /* successful rewind */
295 return CURLE_OK;
298 /* no callback set or failure above, makes us fail at once */
299 failf(data, "necessary data rewind wasn't possible");
300 return CURLE_SEND_FAIL_REWIND;
303 return CURLE_OK;
306 static int data_pending(const struct connectdata *conn)
308 /* in the case of libssh2, we can never be really sure that we have emptied
309 its internal buffers so we MUST always try until we get EAGAIN back */
310 return conn->protocol&(PROT_SCP|PROT_SFTP) ||
311 Curl_ssl_data_pending(conn, FIRSTSOCKET);
314 #ifndef MIN
315 #define MIN(a,b) (a < b ? a : b)
316 #endif
318 static void read_rewind(struct connectdata *conn,
319 size_t thismuch)
321 conn->read_pos -= thismuch;
322 conn->bits.stream_was_rewound = TRUE;
324 #ifdef CURLDEBUG
326 char buf[512 + 1];
327 size_t show;
329 show = MIN(conn->buf_len - conn->read_pos, sizeof(buf)-1);
330 if(conn->master_buffer) {
331 memcpy(buf, conn->master_buffer + conn->read_pos, show);
332 buf[show] = '\0';
334 else {
335 buf[0] = '\0';
338 DEBUGF(infof(conn->data,
339 "Buffer after stream rewind (read_pos = %d): [%s]",
340 conn->read_pos, buf));
342 #endif
347 * Go ahead and do a read if we have a readable socket or if
348 * the stream was rewound (in which case we have data in a
349 * buffer)
351 static CURLcode readwrite_data(struct SessionHandle *data,
352 struct connectdata *conn,
353 struct SingleRequest *k,
354 int *didwhat, bool *done)
356 CURLcode result = CURLE_OK;
357 ssize_t nread; /* number of bytes read */
358 bool is_empty_data = FALSE;
360 /* This is where we loop until we have read everything there is to
361 read or we get a EWOULDBLOCK */
362 do {
363 size_t buffersize = data->set.buffer_size?
364 data->set.buffer_size : BUFSIZE;
365 size_t bytestoread = buffersize;
366 int readrc;
368 if(k->size != -1 && !k->header) {
369 /* make sure we don't read "too much" if we can help it since we
370 might be pipelining and then someone else might want to read what
371 follows! */
372 curl_off_t totalleft = k->size - k->bytecount;
373 if(totalleft < (curl_off_t)bytestoread)
374 bytestoread = (size_t)totalleft;
377 if(bytestoread) {
378 /* receive data from the network! */
379 readrc = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread);
381 /* subzero, this would've blocked */
382 if(0 > readrc)
383 break; /* get out of loop */
385 /* get the CURLcode from the int */
386 result = (CURLcode)readrc;
388 if(result>0)
389 return result;
391 else {
392 /* read nothing but since we wanted nothing we consider this an OK
393 situation to proceed from */
394 nread = 0;
397 if((k->bytecount == 0) && (k->writebytecount == 0)) {
398 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
399 if(k->exp100 > EXP100_SEND_DATA)
400 /* set time stamp to compare with when waiting for the 100 */
401 k->start100 = Curl_tvnow();
404 *didwhat |= KEEP_READ;
405 /* indicates data of zero size, i.e. empty file */
406 is_empty_data = (bool)((nread == 0) && (k->bodywrites == 0));
408 /* NUL terminate, allowing string ops to be used */
409 if(0 < nread || is_empty_data) {
410 k->buf[nread] = 0;
412 else if(0 >= nread) {
413 /* if we receive 0 or less here, the server closed the connection
414 and we bail out from this! */
415 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n"));
416 k->keepon &= ~KEEP_READ;
417 break;
420 /* Default buffer to use when we write the buffer, it may be changed
421 in the flow below before the actual storing is done. */
422 k->str = k->buf;
424 /* Since this is a two-state thing, we check if we are parsing
425 headers at the moment or not. */
426 if(k->header) {
427 /* we are in parse-the-header-mode */
428 bool stop_reading = FALSE;
429 result = readwrite_headers(data, conn, k, &nread, &stop_reading);
430 if(result)
431 return result;
432 if(stop_reading)
433 /* We've stopped dealing with input, get out of the do-while loop */
434 break;
438 /* This is not an 'else if' since it may be a rest from the header
439 parsing, where the beginning of the buffer is headers and the end
440 is non-headers. */
441 if(k->str && !k->header && (nread > 0 || is_empty_data)) {
443 if(0 == k->bodywrites && !is_empty_data) {
444 /* These checks are only made the first time we are about to
445 write a piece of the body */
446 if(conn->protocol&PROT_HTTP) {
447 /* HTTP-only checks */
449 if(data->req.newurl) {
450 if(conn->bits.close) {
451 /* Abort after the headers if "follow Location" is set
452 and we're set to close anyway. */
453 k->keepon &= ~KEEP_READ;
454 *done = TRUE;
455 return CURLE_OK;
457 /* We have a new url to load, but since we want to be able
458 to re-use this connection properly, we read the full
459 response in "ignore more" */
460 k->ignorebody = TRUE;
461 infof(data, "Ignoring the response-body\n");
463 if(data->state.resume_from && !k->content_range &&
464 (data->set.httpreq==HTTPREQ_GET) &&
465 !k->ignorebody) {
466 /* we wanted to resume a download, although the server doesn't
467 * seem to support this and we did this with a GET (if it
468 * wasn't a GET we did a POST or PUT resume) */
469 failf(data, "HTTP server doesn't seem to support "
470 "byte ranges. Cannot resume.");
471 return CURLE_RANGE_ERROR;
474 if(data->set.timecondition && !data->state.range) {
475 /* A time condition has been set AND no ranges have been
476 requested. This seems to be what chapter 13.3.4 of
477 RFC 2616 defines to be the correct action for a
478 HTTP/1.1 client */
479 if((k->timeofdoc > 0) && (data->set.timevalue > 0)) {
480 switch(data->set.timecondition) {
481 case CURL_TIMECOND_IFMODSINCE:
482 default:
483 if(k->timeofdoc < data->set.timevalue) {
484 infof(data,
485 "The requested document is not new enough\n");
486 *done = TRUE;
487 return CURLE_OK;
489 break;
490 case CURL_TIMECOND_IFUNMODSINCE:
491 if(k->timeofdoc > data->set.timevalue) {
492 infof(data,
493 "The requested document is not old enough\n");
494 *done = TRUE;
495 return CURLE_OK;
497 break;
498 } /* switch */
499 } /* two valid time strings */
500 } /* we have a time condition */
502 } /* this is HTTP */
503 } /* this is the first time we write a body part */
504 k->bodywrites++;
506 /* pass data to the debug function before it gets "dechunked" */
507 if(data->set.verbose) {
508 if(k->badheader) {
509 Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff,
510 (size_t)k->hbuflen, conn);
511 if(k->badheader == HEADER_PARTHEADER)
512 Curl_debug(data, CURLINFO_DATA_IN,
513 k->str, (size_t)nread, conn);
515 else
516 Curl_debug(data, CURLINFO_DATA_IN,
517 k->str, (size_t)nread, conn);
520 #ifndef CURL_DISABLE_HTTP
521 if(k->chunk) {
523 * Here comes a chunked transfer flying and we need to decode this
524 * properly. While the name says read, this function both reads
525 * and writes away the data. The returned 'nread' holds the number
526 * of actual data it wrote to the client.
529 CHUNKcode res =
530 Curl_httpchunk_read(conn, k->str, nread, &nread);
532 if(CHUNKE_OK < res) {
533 if(CHUNKE_WRITE_ERROR == res) {
534 failf(data, "Failed writing data");
535 return CURLE_WRITE_ERROR;
537 failf(data, "Received problem %d in the chunky parser", res);
538 return CURLE_RECV_ERROR;
540 else if(CHUNKE_STOP == res) {
541 size_t dataleft;
542 /* we're done reading chunks! */
543 k->keepon &= ~KEEP_READ; /* read no more */
545 /* There are now possibly N number of bytes at the end of the
546 str buffer that weren't written to the client.
548 We DO care about this data if we are pipelining.
549 Push it back to be read on the next pass. */
551 dataleft = conn->chunk.dataleft;
552 if(dataleft != 0) {
553 infof(conn->data, "Leftovers after chunking. "
554 " Rewinding %d bytes\n",dataleft);
555 read_rewind(conn, dataleft);
558 /* If it returned OK, we just keep going */
560 #endif /* CURL_DISABLE_HTTP */
562 if((-1 != k->maxdownload) &&
563 (k->bytecount + nread >= k->maxdownload)) {
564 /* The 'excess' amount below can't be more than BUFSIZE which
565 always will fit in a size_t */
566 size_t excess = (size_t)(k->bytecount + nread - k->maxdownload);
567 if(excess > 0 && !k->ignorebody) {
568 infof(data,
569 "Rewinding stream by : %d"
570 " bytes on url %s (size = %" FORMAT_OFF_T
571 ", maxdownload = %" FORMAT_OFF_T
572 ", bytecount = %" FORMAT_OFF_T ", nread = %d)\n",
573 excess, data->state.path,
574 k->size, k->maxdownload, k->bytecount, nread);
575 read_rewind(conn, excess);
578 nread = (ssize_t) (k->maxdownload - k->bytecount);
579 if(nread < 0 ) /* this should be unusual */
580 nread = 0;
582 k->keepon &= ~KEEP_READ; /* we're done reading */
585 k->bytecount += nread;
587 Curl_pgrsSetDownloadCounter(data, k->bytecount);
589 if(!k->chunk && (nread || k->badheader || is_empty_data)) {
590 /* If this is chunky transfer, it was already written */
592 if(k->badheader && !k->ignorebody) {
593 /* we parsed a piece of data wrongly assuming it was a header
594 and now we output it as body instead */
595 result = Curl_client_write(conn, CLIENTWRITE_BODY,
596 data->state.headerbuff,
597 k->hbuflen);
598 if(result)
599 return result;
601 if(k->badheader < HEADER_ALLBAD) {
602 /* This switch handles various content encodings. If there's an
603 error here, be sure to check over the almost identical code
604 in http_chunks.c.
605 Make sure that ALL_CONTENT_ENCODINGS contains all the
606 encodings handled here. */
607 #ifdef HAVE_LIBZ
608 switch (conn->data->set.http_ce_skip ?
609 IDENTITY : k->content_encoding) {
610 case IDENTITY:
611 #endif
612 /* This is the default when the server sends no
613 Content-Encoding header. See Curl_readwrite_init; the
614 memset() call initializes k->content_encoding to zero. */
615 if(!k->ignorebody)
616 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str,
617 nread);
618 #ifdef HAVE_LIBZ
619 break;
621 case DEFLATE:
622 /* Assume CLIENTWRITE_BODY; headers are not encoded. */
623 if(!k->ignorebody)
624 result = Curl_unencode_deflate_write(conn, k, nread);
625 break;
627 case GZIP:
628 /* Assume CLIENTWRITE_BODY; headers are not encoded. */
629 if(!k->ignorebody)
630 result = Curl_unencode_gzip_write(conn, k, nread);
631 break;
633 case COMPRESS:
634 default:
635 failf (data, "Unrecognized content encoding type. "
636 "libcurl understands `identity', `deflate' and `gzip' "
637 "content encodings.");
638 result = CURLE_BAD_CONTENT_ENCODING;
639 break;
641 #endif
643 k->badheader = HEADER_NORMAL; /* taken care of now */
645 if(result)
646 return result;
649 } /* if(! header and data to read ) */
651 if(is_empty_data) {
652 /* if we received nothing, the server closed the connection and we
653 are done */
654 k->keepon &= ~KEEP_READ;
657 } while(data_pending(conn));
659 if(((k->keepon & (KEEP_READ|KEEP_WRITE)) == KEEP_WRITE) &&
660 conn->bits.close ) {
661 /* When we've read the entire thing and the close bit is set, the server may
662 now close the connection. If there's now any kind of sending going on from
663 our side, we need to stop that immediately. */
664 infof(data, "we are done reading and this is set to close, stop send\n");
665 k->keepon &= ~KEEP_WRITE; /* no writing anymore either */
668 return CURLE_OK;
672 * Read any header lines from the server and pass them to the client app.
674 static CURLcode readwrite_headers(struct SessionHandle *data,
675 struct connectdata *conn,
676 struct SingleRequest *k,
677 ssize_t *nread,
678 bool *stop_reading)
680 CURLcode result;
682 /* header line within buffer loop */
683 do {
684 size_t hbufp_index;
685 size_t rest_length;
686 size_t full_length;
687 int writetype;
689 /* str_start is start of line within buf */
690 k->str_start = k->str;
692 /* data is in network encoding so use 0x0a instead of '\n' */
693 k->end_ptr = memchr(k->str_start, 0x0a, *nread);
695 if(!k->end_ptr) {
696 /* Not a complete header line within buffer, append the data to
697 the end of the headerbuff. */
699 if(k->hbuflen + *nread >= data->state.headersize) {
700 /* We enlarge the header buffer as it is too small */
701 char *newbuff;
702 size_t newsize=CURLMAX((k->hbuflen+*nread)*3/2,
703 data->state.headersize*2);
704 hbufp_index = k->hbufp - data->state.headerbuff;
705 newbuff = (char *)realloc(data->state.headerbuff, newsize);
706 if(!newbuff) {
707 failf (data, "Failed to alloc memory for big header!");
708 return CURLE_OUT_OF_MEMORY;
710 data->state.headersize=newsize;
711 data->state.headerbuff = newbuff;
712 k->hbufp = data->state.headerbuff + hbufp_index;
714 memcpy(k->hbufp, k->str, *nread);
715 k->hbufp += *nread;
716 k->hbuflen += *nread;
717 if(!k->headerline && (k->hbuflen>5)) {
718 /* make a first check that this looks like a HTTP header */
719 if(!checkhttpprefix(data, data->state.headerbuff)) {
720 /* this is not the beginning of a HTTP first header line */
721 k->header = FALSE;
722 k->badheader = HEADER_ALLBAD;
723 break;
727 break; /* read more and try again */
730 /* decrease the size of the remaining (supposed) header line */
731 rest_length = (k->end_ptr - k->str)+1;
732 *nread -= (ssize_t)rest_length;
734 k->str = k->end_ptr + 1; /* move past new line */
736 full_length = k->str - k->str_start;
739 * We're about to copy a chunk of data to the end of the
740 * already received header. We make sure that the full string
741 * fit in the allocated header buffer, or else we enlarge
742 * it.
744 if(k->hbuflen + full_length >=
745 data->state.headersize) {
746 char *newbuff;
747 size_t newsize=CURLMAX((k->hbuflen+full_length)*3/2,
748 data->state.headersize*2);
749 hbufp_index = k->hbufp - data->state.headerbuff;
750 newbuff = (char *)realloc(data->state.headerbuff, newsize);
751 if(!newbuff) {
752 failf (data, "Failed to alloc memory for big header!");
753 return CURLE_OUT_OF_MEMORY;
755 data->state.headersize= newsize;
756 data->state.headerbuff = newbuff;
757 k->hbufp = data->state.headerbuff + hbufp_index;
760 /* copy to end of line */
761 memcpy(k->hbufp, k->str_start, full_length);
762 k->hbufp += full_length;
763 k->hbuflen += full_length;
764 *k->hbufp = 0;
765 k->end_ptr = k->hbufp;
767 k->p = data->state.headerbuff;
769 /****
770 * We now have a FULL header line that p points to
771 *****/
773 if(!k->headerline) {
774 /* the first read header */
775 if((k->hbuflen>5) &&
776 !checkhttpprefix(data, data->state.headerbuff)) {
777 /* this is not the beginning of a HTTP first header line */
778 k->header = FALSE;
779 if(*nread)
780 /* since there's more, this is a partial bad header */
781 k->badheader = HEADER_PARTHEADER;
782 else {
783 /* this was all we read so it's all a bad header */
784 k->badheader = HEADER_ALLBAD;
785 *nread = (ssize_t)rest_length;
787 break;
791 /* headers are in network encoding so
792 use 0x0a and 0x0d instead of '\n' and '\r' */
793 if((0x0a == *k->p) || (0x0d == *k->p)) {
794 size_t headerlen;
795 /* Zero-length header line means end of headers! */
797 #ifdef CURL_DOES_CONVERSIONS
798 if(0x0d == *k->p) {
799 *k->p = '\r'; /* replace with CR in host encoding */
800 k->p++; /* pass the CR byte */
802 if(0x0a == *k->p) {
803 *k->p = '\n'; /* replace with LF in host encoding */
804 k->p++; /* pass the LF byte */
806 #else
807 if('\r' == *k->p)
808 k->p++; /* pass the \r byte */
809 if('\n' == *k->p)
810 k->p++; /* pass the \n byte */
811 #endif /* CURL_DOES_CONVERSIONS */
813 #ifndef CURL_DISABLE_HTTP
814 if(100 <= k->httpcode && 199 >= k->httpcode) {
816 * We have made a HTTP PUT or POST and this is 1.1-lingo
817 * that tells us that the server is OK with this and ready
818 * to receive the data.
819 * However, we'll get more headers now so we must get
820 * back into the header-parsing state!
822 k->header = TRUE;
823 k->headerline = 0; /* restart the header line counter */
825 /* if we did wait for this do enable write now! */
826 if(k->exp100) {
827 k->exp100 = EXP100_SEND_DATA;
828 k->keepon |= KEEP_WRITE;
831 else {
832 k->header = FALSE; /* no more header to parse! */
834 if((k->size == -1) && !k->chunk && !conn->bits.close &&
835 (k->httpversion >= 11) ) {
836 /* On HTTP 1.1, when connection is not to get closed, but no
837 Content-Length nor Content-Encoding chunked have been
838 received, according to RFC2616 section 4.4 point 5, we
839 assume that the server will close the connection to
840 signal the end of the document. */
841 infof(data, "no chunk, no close, no size. Assume close to "
842 "signal end\n");
843 conn->bits.close = TRUE;
847 if(417 == k->httpcode) {
849 * we got: "417 Expectation Failed" this means:
850 * we have made a HTTP call and our Expect Header
851 * seems to cause a problem => abort the write operations
852 * (or prevent them from starting).
854 k->exp100 = EXP100_FAILED;
855 k->keepon &= ~KEEP_WRITE;
859 * When all the headers have been parsed, see if we should give
860 * up and return an error.
862 if(Curl_http_should_fail(conn)) {
863 failf (data, "The requested URL returned error: %d",
864 k->httpcode);
865 return CURLE_HTTP_RETURNED_ERROR;
867 #endif /* CURL_DISABLE_HTTP */
869 /* now, only output this if the header AND body are requested:
871 writetype = CLIENTWRITE_HEADER;
872 if(data->set.include_header)
873 writetype |= CLIENTWRITE_BODY;
875 headerlen = k->p - data->state.headerbuff;
877 result = Curl_client_write(conn, writetype,
878 data->state.headerbuff,
879 headerlen);
880 if(result)
881 return result;
883 data->info.header_size += (long)headerlen;
884 data->req.headerbytecount += (long)headerlen;
886 data->req.deductheadercount =
887 (100 <= k->httpcode && 199 >= k->httpcode)?data->req.headerbytecount:0;
889 if(data->state.resume_from &&
890 (data->set.httpreq==HTTPREQ_GET) &&
891 (k->httpcode == 416)) {
892 /* "Requested Range Not Satisfiable" */
893 *stop_reading = TRUE;
896 #ifndef CURL_DISABLE_HTTP
897 if(!*stop_reading) {
898 /* Curl_http_auth_act() checks what authentication methods
899 * that are available and decides which one (if any) to
900 * use. It will set 'newurl' if an auth method was picked. */
901 result = Curl_http_auth_act(conn);
903 if(result)
904 return result;
906 if(conn->bits.rewindaftersend) {
907 /* We rewind after a complete send, so thus we continue
908 sending now */
909 infof(data, "Keep sending data to get tossed away!\n");
910 k->keepon |= KEEP_WRITE;
913 #endif /* CURL_DISABLE_HTTP */
915 if(!k->header) {
917 * really end-of-headers.
919 * If we requested a "no body", this is a good time to get
920 * out and return home.
922 if(data->set.opt_no_body)
923 *stop_reading = TRUE;
924 else {
925 /* If we know the expected size of this document, we set the
926 maximum download size to the size of the expected
927 document or else, we won't know when to stop reading!
929 Note that we set the download maximum even if we read a
930 "Connection: close" header, to make sure that
931 "Content-Length: 0" still prevents us from attempting to
932 read the (missing) response-body.
934 /* According to RFC2616 section 4.4, we MUST ignore
935 Content-Length: headers if we are now receiving data
936 using chunked Transfer-Encoding.
938 if(k->chunk)
939 k->size=-1;
942 if(-1 != k->size) {
943 /* We do this operation even if no_body is true, since this
944 data might be retrieved later with curl_easy_getinfo()
945 and its CURLINFO_CONTENT_LENGTH_DOWNLOAD option. */
947 Curl_pgrsSetDownloadSize(data, k->size);
948 k->maxdownload = k->size;
950 /* If max download size is *zero* (nothing) we already
951 have nothing and can safely return ok now! */
952 if(0 == k->maxdownload)
953 *stop_reading = TRUE;
955 if(*stop_reading) {
956 /* we make sure that this socket isn't read more now */
957 k->keepon &= ~KEEP_READ;
960 if(data->set.verbose)
961 Curl_debug(data, CURLINFO_HEADER_IN,
962 k->str_start, headerlen, conn);
963 break; /* exit header line loop */
966 /* We continue reading headers, so reset the line-based
967 header parsing variables hbufp && hbuflen */
968 k->hbufp = data->state.headerbuff;
969 k->hbuflen = 0;
970 continue;
973 #ifndef CURL_DISABLE_HTTP
975 * Checks for special headers coming up.
978 if(!k->headerline++) {
979 /* This is the first header, it MUST be the error code line
980 or else we consider this to be the body right away! */
981 int httpversion_major;
982 int nc;
983 #ifdef CURL_DOES_CONVERSIONS
984 #define HEADER1 scratch
985 #define SCRATCHSIZE 21
986 CURLcode res;
987 char scratch[SCRATCHSIZE+1]; /* "HTTP/major.minor 123" */
988 /* We can't really convert this yet because we
989 don't know if it's the 1st header line or the body.
990 So we do a partial conversion into a scratch area,
991 leaving the data at k->p as-is.
993 strncpy(&scratch[0], k->p, SCRATCHSIZE);
994 scratch[SCRATCHSIZE] = 0; /* null terminate */
995 res = Curl_convert_from_network(data,
996 &scratch[0],
997 SCRATCHSIZE);
998 if(CURLE_OK != res) {
999 /* Curl_convert_from_network calls failf if unsuccessful */
1000 return res;
1002 #else
1003 #define HEADER1 k->p /* no conversion needed, just use k->p */
1004 #endif /* CURL_DOES_CONVERSIONS */
1006 nc = sscanf(HEADER1,
1007 " HTTP/%d.%d %3d",
1008 &httpversion_major,
1009 &k->httpversion,
1010 &k->httpcode);
1011 if(nc==3) {
1012 k->httpversion += 10 * httpversion_major;
1014 else {
1015 /* this is the real world, not a Nirvana
1016 NCSA 1.5.x returns this crap when asked for HTTP/1.1
1018 nc=sscanf(HEADER1, " HTTP %3d", &k->httpcode);
1019 k->httpversion = 10;
1021 /* If user has set option HTTP200ALIASES,
1022 compare header line against list of aliases
1024 if(!nc) {
1025 if(checkhttpprefix(data, k->p)) {
1026 nc = 1;
1027 k->httpcode = 200;
1028 k->httpversion = 10;
1033 if(nc) {
1034 data->info.httpcode = k->httpcode;
1035 data->info.httpversion = k->httpversion;
1038 * This code executes as part of processing the header. As a
1039 * result, it's not totally clear how to interpret the
1040 * response code yet as that depends on what other headers may
1041 * be present. 401 and 407 may be errors, but may be OK
1042 * depending on how authentication is working. Other codes
1043 * are definitely errors, so give up here.
1045 if(data->set.http_fail_on_error && (k->httpcode >= 400) &&
1046 ((k->httpcode != 401) || !conn->bits.user_passwd) &&
1047 ((k->httpcode != 407) || !conn->bits.proxy_user_passwd) ) {
1049 if(data->state.resume_from &&
1050 (data->set.httpreq==HTTPREQ_GET) &&
1051 (k->httpcode == 416)) {
1052 /* "Requested Range Not Satisfiable", just proceed and
1053 pretend this is no error */
1055 else {
1056 /* serious error, go home! */
1057 failf (data, "The requested URL returned error: %d",
1058 k->httpcode);
1059 return CURLE_HTTP_RETURNED_ERROR;
1063 if(k->httpversion == 10) {
1064 /* Default action for HTTP/1.0 must be to close, unless
1065 we get one of those fancy headers that tell us the
1066 server keeps it open for us! */
1067 infof(data, "HTTP 1.0, assume close after body\n");
1068 conn->bits.close = TRUE;
1070 else if(k->httpversion >= 11 &&
1071 !conn->bits.close) {
1072 /* If HTTP version is >= 1.1 and connection is persistent
1073 server supports pipelining. */
1074 DEBUGF(infof(data,
1075 "HTTP 1.1 or later with persistent connection, "
1076 "pipelining supported\n"));
1077 conn->server_supports_pipelining = TRUE;
1080 switch(k->httpcode) {
1081 case 204:
1082 /* (quote from RFC2616, section 10.2.5): The server has
1083 * fulfilled the request but does not need to return an
1084 * entity-body ... The 204 response MUST NOT include a
1085 * message-body, and thus is always terminated by the first
1086 * empty line after the header fields. */
1087 /* FALLTHROUGH */
1088 case 416: /* Requested Range Not Satisfiable, it has the
1089 Content-Length: set as the "real" document but no
1090 actual response is sent. */
1091 case 304:
1092 /* (quote from RFC2616, section 10.3.5): The 304 response
1093 * MUST NOT contain a message-body, and thus is always
1094 * terminated by the first empty line after the header
1095 * fields. */
1096 k->size=0;
1097 k->maxdownload=0;
1098 k->ignorecl = TRUE; /* ignore Content-Length headers */
1099 break;
1100 default:
1101 /* nothing */
1102 break;
1105 else {
1106 k->header = FALSE; /* this is not a header line */
1107 break;
1110 #endif /* CURL_DISABLE_HTTP */
1112 #ifdef CURL_DOES_CONVERSIONS
1113 /* convert from the network encoding */
1114 result = Curl_convert_from_network(data, k->p, strlen(k->p));
1115 if(CURLE_OK != result) {
1116 return(result);
1118 /* Curl_convert_from_network calls failf if unsuccessful */
1119 #endif /* CURL_DOES_CONVERSIONS */
1121 #ifndef CURL_DISABLE_HTTP
1122 /* Check for Content-Length: header lines to get size. Ignore
1123 the header completely if we get a 416 response as then we're
1124 resuming a document that we don't get, and this header contains
1125 info about the true size of the document we didn't get now. */
1126 if(!k->ignorecl && !data->set.ignorecl &&
1127 checkprefix("Content-Length:", k->p)) {
1128 curl_off_t contentlength = curlx_strtoofft(k->p+15, NULL, 10);
1129 if(data->set.max_filesize &&
1130 contentlength > data->set.max_filesize) {
1131 failf(data, "Maximum file size exceeded");
1132 return CURLE_FILESIZE_EXCEEDED;
1134 if(contentlength >= 0) {
1135 k->size = contentlength;
1136 k->maxdownload = k->size;
1137 /* we set the progress download size already at this point
1138 just to make it easier for apps/callbacks to extract this
1139 info as soon as possible */
1140 Curl_pgrsSetDownloadSize(data, k->size);
1142 else {
1143 /* Negative Content-Length is really odd, and we know it
1144 happens for example when older Apache servers send large
1145 files */
1146 conn->bits.close = TRUE;
1147 infof(data, "Negative content-length: %" FORMAT_OFF_T
1148 ", closing after transfer\n", contentlength);
1151 /* check for Content-Type: header lines to get the MIME-type */
1152 else if(checkprefix("Content-Type:", k->p)) {
1153 char *contenttype = Curl_copy_header_value(k->p);
1154 if (!contenttype)
1155 return CURLE_OUT_OF_MEMORY;
1156 if (!*contenttype)
1157 /* ignore empty data */
1158 free(contenttype);
1159 else {
1160 Curl_safefree(data->info.contenttype);
1161 data->info.contenttype = contenttype;
1164 else if((k->httpversion == 10) &&
1165 conn->bits.httpproxy &&
1166 Curl_compareheader(k->p,
1167 "Proxy-Connection:", "keep-alive")) {
1169 * When a HTTP/1.0 reply comes when using a proxy, the
1170 * 'Proxy-Connection: keep-alive' line tells us the
1171 * connection will be kept alive for our pleasure.
1172 * Default action for 1.0 is to close.
1174 conn->bits.close = FALSE; /* don't close when done */
1175 infof(data, "HTTP/1.0 proxy connection set to keep alive!\n");
1177 else if((k->httpversion == 11) &&
1178 conn->bits.httpproxy &&
1179 Curl_compareheader(k->p,
1180 "Proxy-Connection:", "close")) {
1182 * We get a HTTP/1.1 response from a proxy and it says it'll
1183 * close down after this transfer.
1185 conn->bits.close = TRUE; /* close when done */
1186 infof(data, "HTTP/1.1 proxy connection set close!\n");
1188 else if((k->httpversion == 10) &&
1189 Curl_compareheader(k->p, "Connection:", "keep-alive")) {
1191 * A HTTP/1.0 reply with the 'Connection: keep-alive' line
1192 * tells us the connection will be kept alive for our
1193 * pleasure. Default action for 1.0 is to close.
1195 * [RFC2068, section 19.7.1] */
1196 conn->bits.close = FALSE; /* don't close when done */
1197 infof(data, "HTTP/1.0 connection set to keep alive!\n");
1199 else if(Curl_compareheader(k->p, "Connection:", "close")) {
1201 * [RFC 2616, section 8.1.2.1]
1202 * "Connection: close" is HTTP/1.1 language and means that
1203 * the connection will close when this request has been
1204 * served.
1206 conn->bits.close = TRUE; /* close when done */
1208 else if(Curl_compareheader(k->p,
1209 "Transfer-Encoding:", "chunked")) {
1211 * [RFC 2616, section 3.6.1] A 'chunked' transfer encoding
1212 * means that the server will send a series of "chunks". Each
1213 * chunk starts with line with info (including size of the
1214 * coming block) (terminated with CRLF), then a block of data
1215 * with the previously mentioned size. There can be any amount
1216 * of chunks, and a chunk-data set to zero signals the
1217 * end-of-chunks. */
1218 k->chunk = TRUE; /* chunks coming our way */
1220 /* init our chunky engine */
1221 Curl_httpchunk_init(conn);
1224 else if(checkprefix("Trailer:", k->p) ||
1225 checkprefix("Trailers:", k->p)) {
1227 * This test helps Curl_httpchunk_read() to determine to look
1228 * for well formed trailers after the zero chunksize record. In
1229 * this case a CRLF is required after the zero chunksize record
1230 * when no trailers are sent, or after the last trailer record.
1232 * It seems both Trailer: and Trailers: occur in the wild.
1234 k->trailerhdrpresent = TRUE;
1237 else if(checkprefix("Content-Encoding:", k->p) &&
1238 data->set.str[STRING_ENCODING]) {
1240 * Process Content-Encoding. Look for the values: identity,
1241 * gzip, deflate, compress, x-gzip and x-compress. x-gzip and
1242 * x-compress are the same as gzip and compress. (Sec 3.5 RFC
1243 * 2616). zlib cannot handle compress. However, errors are
1244 * handled further down when the response body is processed
1246 char *start;
1248 /* Find the first non-space letter */
1249 for(start=k->p+17;
1250 *start && ISSPACE(*start);
1251 start++)
1252 ; /* empty loop */
1254 /* Record the content-encoding for later use */
1255 if(checkprefix("identity", start))
1256 k->content_encoding = IDENTITY;
1257 else if(checkprefix("deflate", start))
1258 k->content_encoding = DEFLATE;
1259 else if(checkprefix("gzip", start)
1260 || checkprefix("x-gzip", start))
1261 k->content_encoding = GZIP;
1262 else if(checkprefix("compress", start)
1263 || checkprefix("x-compress", start))
1264 k->content_encoding = COMPRESS;
1266 else if(checkprefix("Content-Range:", k->p)) {
1267 /* Content-Range: bytes [num]-
1268 Content-Range: bytes: [num]-
1269 Content-Range: [num]-
1271 The second format was added since Sun's webserver
1272 JavaWebServer/1.1.1 obviously sends the header this way!
1273 The third added since some servers use that!
1276 char *ptr = k->p + 14;
1278 /* Move forward until first digit */
1279 while(*ptr && !ISDIGIT(*ptr))
1280 ptr++;
1282 k->offset = curlx_strtoofft(ptr, NULL, 10);
1284 if(data->state.resume_from == k->offset)
1285 /* we asked for a resume and we got it */
1286 k->content_range = TRUE;
1288 #if !defined(CURL_DISABLE_COOKIES)
1289 else if(data->cookies &&
1290 checkprefix("Set-Cookie:", k->p)) {
1291 Curl_share_lock(data, CURL_LOCK_DATA_COOKIE,
1292 CURL_LOCK_ACCESS_SINGLE);
1293 Curl_cookie_add(data,
1294 data->cookies, TRUE, k->p+11,
1295 /* If there is a custom-set Host: name, use it
1296 here, or else use real peer host name. */
1297 conn->allocptr.cookiehost?
1298 conn->allocptr.cookiehost:conn->host.name,
1299 data->state.path);
1300 Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE);
1302 #endif
1303 else if(checkprefix("Last-Modified:", k->p) &&
1304 (data->set.timecondition || data->set.get_filetime) ) {
1305 time_t secs=time(NULL);
1306 k->timeofdoc = curl_getdate(k->p+strlen("Last-Modified:"),
1307 &secs);
1308 if(data->set.get_filetime)
1309 data->info.filetime = (long)k->timeofdoc;
1311 else if((checkprefix("WWW-Authenticate:", k->p) &&
1312 (401 == k->httpcode)) ||
1313 (checkprefix("Proxy-authenticate:", k->p) &&
1314 (407 == k->httpcode))) {
1315 result = Curl_http_input_auth(conn, k->httpcode, k->p);
1316 if(result)
1317 return result;
1319 else if((k->httpcode >= 300 && k->httpcode < 400) &&
1320 checkprefix("Location:", k->p)) {
1321 /* this is the URL that the server advises us to use instead */
1322 char *location = Curl_copy_header_value(k->p);
1323 if (!location)
1324 return CURLE_OUT_OF_MEMORY;
1325 if (!*location)
1326 /* ignore empty data */
1327 free(location);
1328 else {
1329 DEBUGASSERT(!data->req.location);
1330 data->req.location = location;
1332 if(data->set.http_follow_location) {
1333 DEBUGASSERT(!data->req.newurl);
1334 data->req.newurl = strdup(data->req.location); /* clone */
1335 if(!data->req.newurl)
1336 return CURLE_OUT_OF_MEMORY;
1338 /* some cases of POST and PUT etc needs to rewind the data
1339 stream at this point */
1340 result = Curl_http_perhapsrewind(conn);
1341 if(result)
1342 return result;
1346 #endif /* CURL_DISABLE_HTTP */
1349 * End of header-checks. Write them to the client.
1352 writetype = CLIENTWRITE_HEADER;
1353 if(data->set.include_header)
1354 writetype |= CLIENTWRITE_BODY;
1356 if(data->set.verbose)
1357 Curl_debug(data, CURLINFO_HEADER_IN,
1358 k->p, (size_t)k->hbuflen, conn);
1360 result = Curl_client_write(conn, writetype, k->p, k->hbuflen);
1361 if(result)
1362 return result;
1364 data->info.header_size += (long)k->hbuflen;
1365 data->req.headerbytecount += (long)k->hbuflen;
1367 /* reset hbufp pointer && hbuflen */
1368 k->hbufp = data->state.headerbuff;
1369 k->hbuflen = 0;
1371 while(!*stop_reading && *k->str); /* header line within buffer */
1373 /* We might have reached the end of the header part here, but
1374 there might be a non-header part left in the end of the read
1375 buffer. */
1377 return CURLE_OK;
1381 * Send data to upload to the server, when the socket is writable.
1383 static CURLcode readwrite_upload(struct SessionHandle *data,
1384 struct connectdata *conn,
1385 struct SingleRequest *k,
1386 int *didwhat)
1388 ssize_t i, si;
1389 ssize_t bytes_written;
1390 CURLcode result;
1391 ssize_t nread; /* number of bytes read */
1393 if((k->bytecount == 0) && (k->writebytecount == 0))
1394 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
1396 *didwhat |= KEEP_WRITE;
1399 * We loop here to do the READ and SEND loop until we run out of
1400 * data to send or until we get EWOULDBLOCK back
1402 do {
1404 /* only read more data if there's no upload data already
1405 present in the upload buffer */
1406 if(0 == data->req.upload_present) {
1407 /* init the "upload from here" pointer */
1408 data->req.upload_fromhere = k->uploadbuf;
1410 if(!k->upload_done) {
1411 /* HTTP pollution, this should be written nicer to become more
1412 protocol agnostic. */
1413 int fillcount;
1415 if((k->exp100 == EXP100_SENDING_REQUEST) &&
1416 (data->state.proto.http->sending == HTTPSEND_BODY)) {
1417 /* If this call is to send body data, we must take some action:
1418 We have sent off the full HTTP 1.1 request, and we shall now
1419 go into the Expect: 100 state and await such a header */
1420 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
1421 k->keepon &= ~KEEP_WRITE; /* disable writing */
1422 k->start100 = Curl_tvnow(); /* timeout count starts now */
1423 *didwhat &= ~KEEP_WRITE; /* we didn't write anything actually */
1424 break;
1427 result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount);
1428 if(result)
1429 return result;
1431 nread = (ssize_t)fillcount;
1433 else
1434 nread = 0; /* we're done uploading/reading */
1436 if(!nread && (k->keepon & KEEP_WRITE_PAUSE)) {
1437 /* this is a paused transfer */
1438 break;
1440 else if(nread<=0) {
1441 /* done */
1442 k->keepon &= ~KEEP_WRITE; /* we're done writing */
1444 if(conn->bits.rewindaftersend) {
1445 result = Curl_readrewind(conn);
1446 if(result)
1447 return result;
1449 break;
1452 /* store number of bytes available for upload */
1453 data->req.upload_present = nread;
1455 /* convert LF to CRLF if so asked */
1456 #ifdef CURL_DO_LINEEND_CONV
1457 /* always convert if we're FTPing in ASCII mode */
1458 if((data->set.crlf) || (data->set.prefer_ascii))
1459 #else
1460 if(data->set.crlf)
1461 #endif /* CURL_DO_LINEEND_CONV */
1463 if(data->state.scratch == NULL)
1464 data->state.scratch = malloc(2*BUFSIZE);
1465 if(data->state.scratch == NULL) {
1466 failf (data, "Failed to alloc scratch buffer!");
1467 return CURLE_OUT_OF_MEMORY;
1470 * ASCII/EBCDIC Note: This is presumably a text (not binary)
1471 * transfer so the data should already be in ASCII.
1472 * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
1473 * must be used instead of the escape sequences \r & \n.
1475 for(i = 0, si = 0; i < nread; i++, si++) {
1476 if(data->req.upload_fromhere[i] == 0x0a) {
1477 data->state.scratch[si++] = 0x0d;
1478 data->state.scratch[si] = 0x0a;
1479 if(!data->set.crlf) {
1480 /* we're here only because FTP is in ASCII mode...
1481 bump infilesize for the LF we just added */
1482 data->set.infilesize++;
1485 else
1486 data->state.scratch[si] = data->req.upload_fromhere[i];
1488 if(si != nread) {
1489 /* only perform the special operation if we really did replace
1490 anything */
1491 nread = si;
1493 /* upload from the new (replaced) buffer instead */
1494 data->req.upload_fromhere = data->state.scratch;
1496 /* set the new amount too */
1497 data->req.upload_present = nread;
1500 } /* if 0 == data->req.upload_present */
1501 else {
1502 /* We have a partial buffer left from a previous "round". Use
1503 that instead of reading more data */
1506 /* write to socket (send away data) */
1507 result = Curl_write(conn,
1508 conn->writesockfd, /* socket to send to */
1509 data->req.upload_fromhere, /* buffer pointer */
1510 data->req.upload_present, /* buffer size */
1511 &bytes_written); /* actually send away */
1512 if(result)
1513 return result;
1515 if(data->set.verbose)
1516 /* show the data before we change the pointer upload_fromhere */
1517 Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere,
1518 (size_t)bytes_written, conn);
1520 if(data->req.upload_present != bytes_written) {
1521 /* we only wrote a part of the buffer (if anything), deal with it! */
1523 /* store the amount of bytes left in the buffer to write */
1524 data->req.upload_present -= bytes_written;
1526 /* advance the pointer where to find the buffer when the next send
1527 is to happen */
1528 data->req.upload_fromhere += bytes_written;
1530 else {
1531 /* we've uploaded that buffer now */
1532 data->req.upload_fromhere = k->uploadbuf;
1533 data->req.upload_present = 0; /* no more bytes left */
1535 if(k->upload_done) {
1536 /* switch off writing, we're done! */
1537 k->keepon &= ~KEEP_WRITE; /* we're done writing */
1541 k->writebytecount += bytes_written;
1542 Curl_pgrsSetUploadCounter(data, k->writebytecount);
1544 } while(0); /* just to break out from! */
1546 return CURLE_OK;
1550 * Curl_readwrite() is the low-level function to be called when data is to
1551 * be read and written to/from the connection.
1553 CURLcode Curl_readwrite(struct connectdata *conn,
1554 bool *done)
1556 struct SessionHandle *data = conn->data;
1557 struct SingleRequest *k = &data->req;
1558 CURLcode result;
1559 int didwhat=0;
1561 curl_socket_t fd_read;
1562 curl_socket_t fd_write;
1563 int select_res = conn->cselect_bits;
1565 conn->cselect_bits = 0;
1567 /* only use the proper socket if the *_HOLD bit is not set simultaneously as
1568 then we are in rate limiting state in that transfer direction */
1570 if((k->keepon & KEEP_READBITS) == KEEP_READ) {
1571 fd_read = conn->sockfd;
1572 #if defined(USE_LIBSSH2)
1573 if(conn->protocol & (PROT_SCP|PROT_SFTP))
1574 select_res |= CURL_CSELECT_IN;
1575 #endif /* USE_LIBSSH2 */
1576 } else
1577 fd_read = CURL_SOCKET_BAD;
1579 if((k->keepon & KEEP_WRITEBITS) == KEEP_WRITE)
1580 fd_write = conn->writesockfd;
1581 else
1582 fd_write = CURL_SOCKET_BAD;
1584 if(!select_res) { /* Call for select()/poll() only, if read/write/error
1585 status is not known. */
1586 select_res = Curl_socket_ready(fd_read, fd_write, 0);
1589 if(select_res == CURL_CSELECT_ERR) {
1590 failf(data, "select/poll returned error");
1591 return CURLE_SEND_ERROR;
1594 /* We go ahead and do a read if we have a readable socket or if
1595 the stream was rewound (in which case we have data in a
1596 buffer) */
1597 if((k->keepon & KEEP_READ) &&
1598 ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) {
1600 result = readwrite_data(data, conn, k, &didwhat, done);
1601 if(result || *done)
1602 return result;
1605 /* If we still have writing to do, we check if we have a writable socket. */
1606 if((k->keepon & KEEP_WRITE) && (select_res & CURL_CSELECT_OUT)) {
1607 /* write */
1609 result = readwrite_upload(data, conn, k, &didwhat);
1610 if(result)
1611 return result;
1614 k->now = Curl_tvnow();
1615 if(didwhat) {
1616 /* Update read/write counters */
1617 if(k->bytecountp)
1618 *k->bytecountp = k->bytecount; /* read count */
1619 if(k->writebytecountp)
1620 *k->writebytecountp = k->writebytecount; /* write count */
1622 else {
1623 /* no read no write, this is a timeout? */
1624 if(k->exp100 == EXP100_AWAITING_CONTINUE) {
1625 /* This should allow some time for the header to arrive, but only a
1626 very short time as otherwise it'll be too much wasted time too
1627 often. */
1629 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status":
1631 Therefore, when a client sends this header field to an origin server
1632 (possibly via a proxy) from which it has never seen a 100 (Continue)
1633 status, the client SHOULD NOT wait for an indefinite period before
1634 sending the request body.
1638 long ms = Curl_tvdiff(k->now, k->start100);
1639 if(ms > CURL_TIMEOUT_EXPECT_100) {
1640 /* we've waited long enough, continue anyway */
1641 k->exp100 = EXP100_SEND_DATA;
1642 k->keepon |= KEEP_WRITE;
1643 infof(data, "Done waiting for 100-continue\n");
1648 if(Curl_pgrsUpdate(conn))
1649 result = CURLE_ABORTED_BY_CALLBACK;
1650 else
1651 result = Curl_speedcheck(data, k->now);
1652 if(result)
1653 return result;
1655 if(data->set.timeout &&
1656 (Curl_tvdiff(k->now, k->start) >= data->set.timeout)) {
1657 if(k->size != -1) {
1658 failf(data, "Operation timed out after %ld milliseconds with %"
1659 FORMAT_OFF_T " out of %" FORMAT_OFF_T " bytes received",
1660 data->set.timeout, k->bytecount, k->size);
1661 } else {
1662 failf(data, "Operation timed out after %ld milliseconds with %"
1663 FORMAT_OFF_T " bytes received",
1664 data->set.timeout, k->bytecount);
1666 return CURLE_OPERATION_TIMEDOUT;
1669 if(!k->keepon) {
1671 * The transfer has been performed. Just make some general checks before
1672 * returning.
1675 if(!(data->set.opt_no_body) && (k->size != -1) &&
1676 (k->bytecount != k->size) &&
1677 #ifdef CURL_DO_LINEEND_CONV
1678 /* Most FTP servers don't adjust their file SIZE response for CRLFs,
1679 so we'll check to see if the discrepancy can be explained
1680 by the number of CRLFs we've changed to LFs.
1682 (k->bytecount != (k->size + data->state.crlf_conversions)) &&
1683 #endif /* CURL_DO_LINEEND_CONV */
1684 !data->req.newurl) {
1685 failf(data, "transfer closed with %" FORMAT_OFF_T
1686 " bytes remaining to read",
1687 k->size - k->bytecount);
1688 return CURLE_PARTIAL_FILE;
1690 else if(!(data->set.opt_no_body) &&
1691 k->chunk &&
1692 (conn->chunk.state != CHUNK_STOP)) {
1694 * In chunked mode, return an error if the connection is closed prior to
1695 * the empty (terminiating) chunk is read.
1697 * The condition above used to check for
1698 * conn->proto.http->chunk.datasize != 0 which is true after reading
1699 * *any* chunk, not just the empty chunk.
1702 failf(data, "transfer closed with outstanding read data remaining");
1703 return CURLE_PARTIAL_FILE;
1705 if(Curl_pgrsUpdate(conn))
1706 return CURLE_ABORTED_BY_CALLBACK;
1709 /* Now update the "done" boolean we return */
1710 *done = (bool)(0 == (k->keepon&(KEEP_READ|KEEP_WRITE|
1711 KEEP_READ_PAUSE|KEEP_WRITE_PAUSE)));
1713 return CURLE_OK;
1717 * Curl_single_getsock() gets called by the multi interface code when the app
1718 * has requested to get the sockets for the current connection. This function
1719 * will then be called once for every connection that the multi interface
1720 * keeps track of. This function will only be called for connections that are
1721 * in the proper state to have this information available.
1723 int Curl_single_getsock(const struct connectdata *conn,
1724 curl_socket_t *sock, /* points to numsocks number
1725 of sockets */
1726 int numsocks)
1728 const struct SessionHandle *data = conn->data;
1729 int bitmap = GETSOCK_BLANK;
1730 unsigned sockindex = 0;
1732 if(numsocks < 2)
1733 /* simple check but we might need two slots */
1734 return GETSOCK_BLANK;
1736 /* don't include HOLD and PAUSE connections */
1737 if((data->req.keepon & KEEP_READBITS) == KEEP_READ) {
1739 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
1741 bitmap |= GETSOCK_READSOCK(sockindex);
1742 sock[sockindex] = conn->sockfd;
1745 /* don't include HOLD and PAUSE connections */
1746 if((data->req.keepon & KEEP_WRITEBITS) == KEEP_WRITE) {
1748 if((conn->sockfd != conn->writesockfd) ||
1749 !(data->req.keepon & KEEP_READ)) {
1750 /* only if they are not the same socket or we didn't have a readable
1751 one, we increase index */
1752 if(data->req.keepon & KEEP_READ)
1753 sockindex++; /* increase index if we need two entries */
1755 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
1757 sock[sockindex] = conn->writesockfd;
1760 bitmap |= GETSOCK_WRITESOCK(sockindex);
1763 return bitmap;
1768 * Transfer()
1770 * This function is what performs the actual transfer. It is capable of
1771 * doing both ways simultaneously.
1772 * The transfer must already have been setup by a call to Curl_setup_transfer().
1774 * Note that headers are created in a preallocated buffer of a default size.
1775 * That buffer can be enlarged on demand, but it is never shrunken again.
1777 * Parts of this function was once written by the friendly Mark Butler
1778 * <butlerm@xmission.com>.
1781 static CURLcode
1782 Transfer(struct connectdata *conn)
1784 CURLcode result;
1785 struct SessionHandle *data = conn->data;
1786 struct SingleRequest *k = &data->req;
1787 bool done=FALSE;
1789 if((conn->sockfd == CURL_SOCKET_BAD) &&
1790 (conn->writesockfd == CURL_SOCKET_BAD))
1791 /* nothing to read, nothing to write, we're already OK! */
1792 return CURLE_OK;
1794 /* we want header and/or body, if neither then don't do this! */
1795 if(!k->getheader && data->set.opt_no_body)
1796 return CURLE_OK;
1798 while(!done) {
1799 curl_socket_t fd_read;
1800 curl_socket_t fd_write;
1802 /* limit-rate logic: if speed exceeds threshold, then do not include fd in
1803 select set. The current speed is recalculated in each Curl_readwrite()
1804 call */
1805 if((k->keepon & KEEP_WRITE) &&
1806 (!data->set.max_send_speed ||
1807 (data->progress.ulspeed < data->set.max_send_speed) )) {
1808 fd_write = conn->writesockfd;
1809 k->keepon &= ~KEEP_WRITE_HOLD;
1811 else {
1812 fd_write = CURL_SOCKET_BAD;
1813 if(k->keepon & KEEP_WRITE)
1814 k->keepon |= KEEP_WRITE_HOLD; /* hold it */
1817 if((k->keepon & KEEP_READ) &&
1818 (!data->set.max_recv_speed ||
1819 (data->progress.dlspeed < data->set.max_recv_speed)) ) {
1820 fd_read = conn->sockfd;
1821 k->keepon &= ~KEEP_READ_HOLD;
1823 else {
1824 fd_read = CURL_SOCKET_BAD;
1825 if(k->keepon & KEEP_READ)
1826 k->keepon |= KEEP_READ_HOLD; /* hold it */
1829 /* pause logic. Don't check descriptors for paused connections */
1830 if(k->keepon & KEEP_READ_PAUSE)
1831 fd_read = CURL_SOCKET_BAD;
1832 if(k->keepon & KEEP_WRITE_PAUSE)
1833 fd_write = CURL_SOCKET_BAD;
1835 /* The *_HOLD and *_PAUSE logic is necessary since even though there might
1836 be no traffic during the select interval, we still call
1837 Curl_readwrite() for the timeout case and if we limit transfer speed we
1838 must make sure that this function doesn't transfer anything while in
1839 HOLD status. */
1841 switch (Curl_socket_ready(fd_read, fd_write, 1000)) {
1842 case -1: /* select() error, stop reading */
1843 #ifdef EINTR
1844 /* The EINTR is not serious, and it seems you might get this more
1845 often when using the lib in a multi-threaded environment! */
1846 if(SOCKERRNO == EINTR)
1848 else
1849 #endif
1850 done = TRUE; /* no more read or write */
1851 continue;
1852 case 0: /* timeout */
1853 default: /* readable descriptors */
1855 result = Curl_readwrite(conn, &done);
1856 break;
1858 if(result)
1859 return result;
1861 /* "done" signals to us if the transfer(s) are ready */
1864 return CURLE_OK;
1868 * Curl_pretransfer() is called immediately before a transfer starts.
1870 CURLcode Curl_pretransfer(struct SessionHandle *data)
1872 CURLcode res;
1873 if(!data->change.url) {
1874 /* we can't do anything without URL */
1875 failf(data, "No URL set!");
1876 return CURLE_URL_MALFORMAT;
1879 /* Init the SSL session ID cache here. We do it here since we want to do it
1880 after the *_setopt() calls (that could change the size of the cache) but
1881 before any transfer takes place. */
1882 res = Curl_ssl_initsessions(data, data->set.ssl.numsessions);
1883 if(res)
1884 return res;
1886 data->set.followlocation=0; /* reset the location-follow counter */
1887 data->state.this_is_a_follow = FALSE; /* reset this */
1888 data->state.errorbuf = FALSE; /* no error has occurred */
1890 data->state.authproblem = FALSE;
1891 data->state.authhost.want = data->set.httpauth;
1892 data->state.authproxy.want = data->set.proxyauth;
1894 /* If there is a list of cookie files to read, do it now! */
1895 if(data->change.cookielist) {
1896 Curl_cookie_loadfiles(data);
1899 /* Allow data->set.use_port to set which port to use. This needs to be
1900 * disabled for example when we follow Location: headers to URLs using
1901 * different ports! */
1902 data->state.allow_port = TRUE;
1904 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1905 /*************************************************************
1906 * Tell signal handler to ignore SIGPIPE
1907 *************************************************************/
1908 if(!data->set.no_signal)
1909 data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
1910 #endif
1912 Curl_initinfo(data); /* reset session-specific information "variables" */
1913 Curl_pgrsStartNow(data);
1915 return CURLE_OK;
1919 * Curl_posttransfer() is called immediately after a transfer ends
1921 CURLcode Curl_posttransfer(struct SessionHandle *data)
1923 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1924 /* restore the signal handler for SIGPIPE before we get back */
1925 if(!data->set.no_signal)
1926 signal(SIGPIPE, data->state.prev_signal);
1927 #else
1928 (void)data; /* unused parameter */
1929 #endif
1931 if(!(data->progress.flags & PGRS_HIDE) &&
1932 !data->progress.callback)
1933 /* only output if we don't use a progress callback and we're not hidden */
1934 fprintf(data->set.err, "\n");
1936 return CURLE_OK;
1940 * strlen_url() returns the length of the given URL if the spaces within the
1941 * URL were properly URL encoded.
1943 static size_t strlen_url(const char *url)
1945 const char *ptr;
1946 size_t newlen=0;
1947 bool left=TRUE; /* left side of the ? */
1949 for(ptr=url; *ptr; ptr++) {
1950 switch(*ptr) {
1951 case '?':
1952 left=FALSE;
1953 /* fall through */
1954 default:
1955 newlen++;
1956 break;
1957 case ' ':
1958 if(left)
1959 newlen+=3;
1960 else
1961 newlen++;
1962 break;
1965 return newlen;
1968 /* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in
1969 * the source URL accordingly.
1971 static void strcpy_url(char *output, const char *url)
1973 /* we must add this with whitespace-replacing */
1974 bool left=TRUE;
1975 const char *iptr;
1976 char *optr = output;
1977 for(iptr = url; /* read from here */
1978 *iptr; /* until zero byte */
1979 iptr++) {
1980 switch(*iptr) {
1981 case '?':
1982 left=FALSE;
1983 /* fall through */
1984 default:
1985 *optr++=*iptr;
1986 break;
1987 case ' ':
1988 if(left) {
1989 *optr++='%'; /* add a '%' */
1990 *optr++='2'; /* add a '2' */
1991 *optr++='0'; /* add a '0' */
1993 else
1994 *optr++='+'; /* add a '+' here */
1995 break;
1998 *optr=0; /* zero terminate output buffer */
2003 * Returns true if the given URL is absolute (as opposed to relative)
2005 static bool is_absolute_url(const char *url)
2007 char prot[16]; /* URL protocol string storage */
2008 char letter; /* used for a silly sscanf */
2010 return 2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter);
2014 * Concatenate a relative URL to a base URL making it absolute.
2015 * URL-encodes any spaces.
2016 * The returned pointer must be freed by the caller unless NULL
2017 * (returns NULL on out of memory).
2019 static char *concat_url(const char *base, const char *relurl)
2021 /***
2022 TRY to append this new path to the old URL
2023 to the right of the host part. Oh crap, this is doomed to cause
2024 problems in the future...
2026 char *newest;
2027 char *protsep;
2028 char *pathsep;
2029 size_t newlen;
2031 const char *useurl = relurl;
2032 size_t urllen;
2034 /* we must make our own copy of the URL to play with, as it may
2035 point to read-only data */
2036 char *url_clone=strdup(base);
2038 if(!url_clone)
2039 return NULL; /* skip out of this NOW */
2041 /* protsep points to the start of the host name */
2042 protsep=strstr(url_clone, "//");
2043 if(!protsep)
2044 protsep=url_clone;
2045 else
2046 protsep+=2; /* pass the slashes */
2048 if('/' != relurl[0]) {
2049 int level=0;
2051 /* First we need to find out if there's a ?-letter in the URL,
2052 and cut it and the right-side of that off */
2053 pathsep = strchr(protsep, '?');
2054 if(pathsep)
2055 *pathsep=0;
2057 /* we have a relative path to append to the last slash if there's one
2058 available, or if the new URL is just a query string (starts with a
2059 '?') we append the new one at the end of the entire currently worked
2060 out URL */
2061 if(useurl[0] != '?') {
2062 pathsep = strrchr(protsep, '/');
2063 if(pathsep)
2064 *pathsep=0;
2067 /* Check if there's any slash after the host name, and if so, remember
2068 that position instead */
2069 pathsep = strchr(protsep, '/');
2070 if(pathsep)
2071 protsep = pathsep+1;
2072 else
2073 protsep = NULL;
2075 /* now deal with one "./" or any amount of "../" in the newurl
2076 and act accordingly */
2078 if((useurl[0] == '.') && (useurl[1] == '/'))
2079 useurl+=2; /* just skip the "./" */
2081 while((useurl[0] == '.') &&
2082 (useurl[1] == '.') &&
2083 (useurl[2] == '/')) {
2084 level++;
2085 useurl+=3; /* pass the "../" */
2088 if(protsep) {
2089 while(level--) {
2090 /* cut off one more level from the right of the original URL */
2091 pathsep = strrchr(protsep, '/');
2092 if(pathsep)
2093 *pathsep=0;
2094 else {
2095 *protsep=0;
2096 break;
2101 else {
2102 /* We got a new absolute path for this server, cut off from the
2103 first slash */
2104 pathsep = strchr(protsep, '/');
2105 if(pathsep) {
2106 /* When people use badly formatted URLs, such as
2107 "http://www.url.com?dir=/home/daniel" we must not use the first
2108 slash, if there's a ?-letter before it! */
2109 char *sep = strchr(protsep, '?');
2110 if(sep && (sep < pathsep))
2111 pathsep = sep;
2112 *pathsep=0;
2114 else {
2115 /* There was no slash. Now, since we might be operating on a badly
2116 formatted URL, such as "http://www.url.com?id=2380" which doesn't
2117 use a slash separator as it is supposed to, we need to check for a
2118 ?-letter as well! */
2119 pathsep = strchr(protsep, '?');
2120 if(pathsep)
2121 *pathsep=0;
2125 /* If the new part contains a space, this is a mighty stupid redirect
2126 but we still make an effort to do "right". To the left of a '?'
2127 letter we replace each space with %20 while it is replaced with '+'
2128 on the right side of the '?' letter.
2130 newlen = strlen_url(useurl);
2132 urllen = strlen(url_clone);
2134 newest=(char *)malloc( urllen + 1 + /* possible slash */
2135 newlen + 1 /* zero byte */);
2137 if(!newest) {
2138 free(url_clone); /* don't leak this */
2139 return NULL;
2142 /* copy over the root url part */
2143 memcpy(newest, url_clone, urllen);
2145 /* check if we need to append a slash */
2146 if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0]))
2148 else
2149 newest[urllen++]='/';
2151 /* then append the new piece on the right side */
2152 strcpy_url(&newest[urllen], useurl);
2154 free(url_clone);
2156 return newest;
2160 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
2161 * as given by the remote server and set up the new URL to request.
2163 CURLcode Curl_follow(struct SessionHandle *data,
2164 char *newurl, /* this 'newurl' is the Location: string,
2165 and it must be malloc()ed before passed
2166 here */
2167 followtype type) /* see transfer.h */
2169 /* Location: redirect */
2170 bool disallowport = FALSE;
2172 if(type == FOLLOW_REDIR) {
2173 if((data->set.maxredirs != -1) &&
2174 (data->set.followlocation >= data->set.maxredirs)) {
2175 failf(data,"Maximum (%d) redirects followed", data->set.maxredirs);
2176 return CURLE_TOO_MANY_REDIRECTS;
2179 /* mark the next request as a followed location: */
2180 data->state.this_is_a_follow = TRUE;
2182 data->set.followlocation++; /* count location-followers */
2184 if(data->set.http_auto_referer) {
2185 /* We are asked to automatically set the previous URL as the referer
2186 when we get the next URL. We pick the ->url field, which may or may
2187 not be 100% correct */
2189 if(data->change.referer_alloc)
2190 /* If we already have an allocated referer, free this first */
2191 free(data->change.referer);
2193 data->change.referer = strdup(data->change.url);
2194 if (!data->change.referer) {
2195 data->change.referer_alloc = FALSE;
2196 return CURLE_OUT_OF_MEMORY;
2198 data->change.referer_alloc = TRUE; /* yes, free this later */
2202 if(!is_absolute_url(newurl)) {
2203 /***
2204 *DANG* this is an RFC 2068 violation. The URL is supposed
2205 to be absolute and this doesn't seem to be that!
2207 char *absolute = concat_url(data->change.url, newurl);
2208 if (!absolute)
2209 return CURLE_OUT_OF_MEMORY;
2210 free(newurl);
2211 newurl = absolute;
2213 else {
2214 /* This is an absolute URL, don't allow the custom port number */
2215 disallowport = TRUE;
2217 if(strchr(newurl, ' ')) {
2218 /* This new URL contains at least one space, this is a mighty stupid
2219 redirect but we still make an effort to do "right". */
2220 char *newest;
2221 size_t newlen = strlen_url(newurl);
2223 newest = malloc(newlen+1); /* get memory for this */
2224 if (!newest)
2225 return CURLE_OUT_OF_MEMORY;
2226 strcpy_url(newest, newurl); /* create a space-free URL */
2228 free(newurl); /* that was no good */
2229 newurl = newest; /* use this instead now */
2234 if(type == FOLLOW_FAKE) {
2235 /* we're only figuring out the new url if we would've followed locations
2236 but now we're done so we can get out! */
2237 data->info.wouldredirect = newurl;
2238 return CURLE_OK;
2241 if(disallowport)
2242 data->state.allow_port = FALSE;
2244 if(data->change.url_alloc)
2245 free(data->change.url);
2246 else
2247 data->change.url_alloc = TRUE; /* the URL is allocated */
2249 data->change.url = newurl;
2250 newurl = NULL; /* don't free! */
2252 infof(data, "Issue another request to this URL: '%s'\n", data->change.url);
2255 * We get here when the HTTP code is 300-399 (and 401). We need to perform
2256 * differently based on exactly what return code there was.
2258 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on
2259 * a HTTP (proxy-) authentication scheme other than Basic.
2261 switch(data->info.httpcode) {
2262 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the
2263 Authorization: XXXX header in the HTTP request code snippet */
2264 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
2265 Proxy-Authorization: XXXX header in the HTTP request code snippet */
2266 /* 300 - Multiple Choices */
2267 /* 306 - Not used */
2268 /* 307 - Temporary Redirect */
2269 default: /* for all above (and the unknown ones) */
2270 /* Some codes are explicitly mentioned since I've checked RFC2616 and they
2271 * seem to be OK to POST to.
2273 break;
2274 case 301: /* Moved Permanently */
2275 /* (quote from RFC2616, section 10.3.2):
2277 * Note: When automatically redirecting a POST request after receiving a
2278 * 301 status code, some existing HTTP/1.0 user agents will erroneously
2279 * change it into a GET request.
2281 * ----
2283 * Warning: Because most of importants user agents do this obvious RFC2616
2284 * violation, many webservers expect this misbehavior. So these servers
2285 * often answers to a POST request with an error page. To be sure that
2286 * libcurl gets the page that most user agents would get, libcurl has to
2287 * force GET.
2289 * This behaviour can be overridden with CURLOPT_POST301.
2291 if( (data->set.httpreq == HTTPREQ_POST
2292 || data->set.httpreq == HTTPREQ_POST_FORM)
2293 && !data->set.post301) {
2294 infof(data,
2295 "Violate RFC 2616/10.3.2 and switch from POST to GET\n");
2296 data->set.httpreq = HTTPREQ_GET;
2298 break;
2299 case 302: /* Found */
2300 /* (From 10.3.3)
2302 Note: RFC 1945 and RFC 2068 specify that the client is not allowed
2303 to change the method on the redirected request. However, most
2304 existing user agent implementations treat 302 as if it were a 303
2305 response, performing a GET on the Location field-value regardless
2306 of the original request method. The status codes 303 and 307 have
2307 been added for servers that wish to make unambiguously clear which
2308 kind of reaction is expected of the client.
2310 (From 10.3.4)
2312 Note: Many pre-HTTP/1.1 user agents do not understand the 303
2313 status. When interoperability with such clients is a concern, the
2314 302 status code may be used instead, since most user agents react
2315 to a 302 response as described here for 303.
2317 case 303: /* See Other */
2318 /* Disable both types of POSTs, since doing a second POST when
2319 * following isn't what anyone would want! */
2320 if(data->set.httpreq != HTTPREQ_GET) {
2321 data->set.httpreq = HTTPREQ_GET; /* enforce GET request */
2322 infof(data, "Disables POST, goes with %s\n",
2323 data->set.opt_no_body?"HEAD":"GET");
2325 break;
2326 case 304: /* Not Modified */
2327 /* 304 means we did a conditional request and it was "Not modified".
2328 * We shouldn't get any Location: header in this response!
2330 break;
2331 case 305: /* Use Proxy */
2332 /* (quote from RFC2616, section 10.3.6):
2333 * "The requested resource MUST be accessed through the proxy given
2334 * by the Location field. The Location field gives the URI of the
2335 * proxy. The recipient is expected to repeat this single request
2336 * via the proxy. 305 responses MUST only be generated by origin
2337 * servers."
2339 break;
2341 Curl_pgrsTime(data, TIMER_REDIRECT);
2342 Curl_pgrsResetTimes(data);
2344 return CURLE_OK;
2347 static CURLcode
2348 connect_host(struct SessionHandle *data,
2349 struct connectdata **conn)
2351 CURLcode res = CURLE_OK;
2353 bool async;
2354 bool protocol_done=TRUE; /* will be TRUE always since this is only used
2355 within the easy interface */
2356 Curl_pgrsTime(data, TIMER_STARTSINGLE);
2357 res = Curl_connect(data, conn, &async, &protocol_done);
2359 if((CURLE_OK == res) && async) {
2360 /* Now, if async is TRUE here, we need to wait for the name
2361 to resolve */
2362 res = Curl_wait_for_resolv(*conn, NULL);
2363 if(CURLE_OK == res)
2364 /* Resolved, continue with the connection */
2365 res = Curl_async_resolved(*conn, &protocol_done);
2366 else
2367 /* if we can't resolve, we kill this "connection" now */
2368 (void)Curl_disconnect(*conn);
2371 return res;
2374 /* Returns TRUE and sets '*url' if a request retry is wanted.
2376 NOTE: that the *url is malloc()ed. */
2377 bool Curl_retry_request(struct connectdata *conn,
2378 char **url)
2380 bool retry = FALSE;
2381 struct SessionHandle *data = conn->data;
2383 /* if we're talking upload, we can't do the checks below, unless the protocol
2384 is HTTP as when uploading over HTTP we will still get a response */
2385 if(data->set.upload && !(conn->protocol&PROT_HTTP))
2386 return retry;
2388 if((data->req.bytecount +
2389 data->req.headerbytecount == 0) &&
2390 conn->bits.reuse &&
2391 !data->set.opt_no_body) {
2392 /* We got no data, we attempted to re-use a connection and yet we want a
2393 "body". This might happen if the connection was left alive when we were
2394 done using it before, but that was closed when we wanted to read from
2395 it again. Bad luck. Retry the same request on a fresh connect! */
2396 infof(conn->data, "Connection died, retrying a fresh connect\n");
2397 *url = strdup(conn->data->change.url);
2399 conn->bits.close = TRUE; /* close this connection */
2400 conn->bits.retry = TRUE; /* mark this as a connection we're about
2401 to retry. Marking it this way should
2402 prevent i.e HTTP transfers to return
2403 error just because nothing has been
2404 transfered! */
2405 retry = TRUE;
2408 return retry;
2412 * Curl_perform() is the internal high-level function that gets called by the
2413 * external curl_easy_perform() function. It inits, performs and cleans up a
2414 * single file transfer.
2416 CURLcode Curl_perform(struct SessionHandle *data)
2418 CURLcode res;
2419 CURLcode res2;
2420 struct connectdata *conn=NULL;
2421 char *newurl = NULL; /* possibly a new URL to follow to! */
2422 followtype follow = FOLLOW_NONE;
2424 data->state.used_interface = Curl_if_easy;
2426 res = Curl_pretransfer(data);
2427 if(res)
2428 return res;
2431 * It is important that there is NO 'return' from this function at any other
2432 * place than falling down to the end of the function! This is because we
2433 * have cleanup stuff that must be done before we get back, and that is only
2434 * performed after this do-while loop.
2437 do {
2438 res = connect_host(data, &conn); /* primary connection */
2440 if(res == CURLE_OK) {
2441 bool do_done;
2442 if(data->set.connect_only) {
2443 /* keep connection open for application to use the socket */
2444 conn->bits.close = FALSE;
2445 res = Curl_done(&conn, CURLE_OK, FALSE);
2446 break;
2448 res = Curl_do(&conn, &do_done);
2450 if(res == CURLE_OK) {
2451 res = Transfer(conn); /* now fetch that URL please */
2452 if(res == CURLE_OK) {
2453 bool retry = Curl_retry_request(conn, &newurl);
2455 if(retry) {
2456 follow = FOLLOW_RETRY;
2457 if (!newurl)
2458 res = CURLE_OUT_OF_MEMORY;
2460 else {
2462 * We must duplicate the new URL here as the connection data may
2463 * be free()ed in the Curl_done() function. We prefer the newurl
2464 * one since that's used for redirects or just further requests
2465 * for retries or multi-stage HTTP auth methods etc.
2467 if(data->req.newurl) {
2468 follow = FOLLOW_REDIR;
2469 newurl = strdup(data->req.newurl);
2470 if (!newurl)
2471 res = CURLE_OUT_OF_MEMORY;
2473 else if(data->req.location) {
2474 follow = FOLLOW_FAKE;
2475 newurl = strdup(data->req.location);
2476 if (!newurl)
2477 res = CURLE_OUT_OF_MEMORY;
2481 /* in the above cases where 'newurl' gets assigned, we have a fresh
2482 * allocated memory pointed to */
2484 if(res != CURLE_OK) {
2485 /* The transfer phase returned error, we mark the connection to get
2486 * closed to prevent being re-used. This is because we can't
2487 * possibly know if the connection is in a good shape or not now. */
2488 conn->bits.close = TRUE;
2490 if(CURL_SOCKET_BAD != conn->sock[SECONDARYSOCKET]) {
2491 /* if we failed anywhere, we must clean up the secondary socket if
2492 it was used */
2493 sclose(conn->sock[SECONDARYSOCKET]);
2494 conn->sock[SECONDARYSOCKET] = CURL_SOCKET_BAD;
2498 /* Always run Curl_done(), even if some of the previous calls
2499 failed, but return the previous (original) error code */
2500 res2 = Curl_done(&conn, res, FALSE);
2502 if(CURLE_OK == res)
2503 res = res2;
2505 else if(conn)
2506 /* Curl_do() failed, clean up left-overs in the done-call, but note
2507 that at some cases the conn pointer is NULL when Curl_do() failed
2508 and the connection cache is very small so only call Curl_done() if
2509 conn is still "alive".
2511 res2 = Curl_done(&conn, res, FALSE);
2514 * Important: 'conn' cannot be used here, since it may have been closed
2515 * in 'Curl_done' or other functions.
2518 if((res == CURLE_OK) && follow) {
2519 res = Curl_follow(data, newurl, follow);
2520 if(CURLE_OK == res) {
2521 /* if things went fine, Curl_follow() freed or otherwise took
2522 responsibility for the newurl pointer */
2523 newurl = NULL;
2524 if(follow >= FOLLOW_RETRY) {
2525 follow = FOLLOW_NONE;
2526 continue;
2528 /* else we break out of the loop below */
2532 break; /* it only reaches here when this shouldn't loop */
2534 } while(1); /* loop if Location: */
2536 if(newurl)
2537 free(newurl);
2539 if(res && !data->state.errorbuf) {
2541 * As an extra precaution: if no error string has been set and there was
2542 * an error, use the strerror() string or if things are so bad that not
2543 * even that is good, set a bad string that mentions the error code.
2545 const char *str = curl_easy_strerror(res);
2546 if(!str)
2547 failf(data, "unspecified error %d", (int)res);
2548 else
2549 failf(data, "%s", str);
2552 /* run post-transfer unconditionally, but don't clobber the return code if
2553 we already have an error code recorder */
2554 res2 = Curl_posttransfer(data);
2555 if(!res && res2)
2556 res = res2;
2558 return res;
2562 * Curl_setup_transfer() is called to setup some basic properties for the
2563 * upcoming transfer.
2565 CURLcode
2566 Curl_setup_transfer(
2567 struct connectdata *conn, /* connection data */
2568 int sockindex, /* socket index to read from or -1 */
2569 curl_off_t size, /* -1 if unknown at this point */
2570 bool getheader, /* TRUE if header parsing is wanted */
2571 curl_off_t *bytecountp, /* return number of bytes read or NULL */
2572 int writesockindex, /* socket index to write to, it may very well be
2573 the same we read from. -1 disables */
2574 curl_off_t *writecountp /* return number of bytes written or NULL */
2577 struct SessionHandle *data;
2578 struct SingleRequest *k;
2580 DEBUGASSERT(conn != NULL);
2582 data = conn->data;
2583 k = &data->req;
2585 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
2587 /* now copy all input parameters */
2588 conn->sockfd = sockindex == -1 ?
2589 CURL_SOCKET_BAD : conn->sock[sockindex];
2590 conn->writesockfd = writesockindex == -1 ?
2591 CURL_SOCKET_BAD:conn->sock[writesockindex];
2592 k->getheader = getheader;
2594 k->size = size;
2595 k->bytecountp = bytecountp;
2596 k->writebytecountp = writecountp;
2598 /* The code sequence below is placed in this function just because all
2599 necessary input is not always known in do_complete() as this function may
2600 be called after that */
2602 if(!k->getheader) {
2603 k->header = FALSE;
2604 if(size > 0)
2605 Curl_pgrsSetDownloadSize(data, size);
2607 /* we want header and/or body, if neither then don't do this! */
2608 if(k->getheader || !data->set.opt_no_body) {
2610 if(conn->sockfd != CURL_SOCKET_BAD) {
2611 k->keepon |= KEEP_READ;
2614 if(conn->writesockfd != CURL_SOCKET_BAD) {
2615 /* HTTP 1.1 magic:
2617 Even if we require a 100-return code before uploading data, we might
2618 need to write data before that since the REQUEST may not have been
2619 finished sent off just yet.
2621 Thus, we must check if the request has been sent before we set the
2622 state info where we wait for the 100-return code
2624 if((data->state.expect100header) &&
2625 (data->state.proto.http->sending == HTTPSEND_BODY)) {
2626 /* wait with write until we either got 100-continue or a timeout */
2627 k->exp100 = EXP100_AWAITING_CONTINUE;
2628 k->start100 = k->start;
2630 else {
2631 if(data->state.expect100header)
2632 /* when we've sent off the rest of the headers, we must await a
2633 100-continue but first finish sending the request */
2634 k->exp100 = EXP100_SENDING_REQUEST;
2636 /* enable the write bit when we're not waiting for continue */
2637 k->keepon |= KEEP_WRITE;
2639 } /* if(conn->writesockfd != CURL_SOCKET_BAD) */
2640 } /* if(k->getheader || !data->set.opt_no_body) */
2642 return CURLE_OK;