fetch.c

   1 #include <stdio.h>
   2 #include <string.h>
   3 #include <unistd.h>
   4 #include <sys/stat.h>
   5 #include <curl/curl.h>
   6 #include <error.h>
   7 #include <errno.h>
   8
   9 #include <sys/select.h>
  10 #include <sys/time.h>
  11 #include <sys/types.h>
  12
  13 #include "debug.h"
  14 #include "grow.h"
  15 #include "fetch.h"
  16
  17 /*
  18  * TODO:
  19  *      fix connect-timeouts
  20  */
  21
  22
  23 static int cb_write(char *data, size_t size, size_t nmemb, struct grow *gr)
  24 {
  25 /*      printf("zapisuju %d do %d, %d\n", (int) (size * nmemb), gr->ptr, gr->size); */
  26         return grow_append(gr, data, size*nmemb);
  27 }
  28
  29 /* Date: Tue, 23 Oct 2007 23:42:30 GMT */
  30 /* Last-Modified: Mon, 22 Oct 2007 03:25:53 GMT */
  31
  32 #define DATE_HDR "Date: "
  33 #define LASM_HDR "Last-Modified: "
  34 #define CONL_HDR "Content-length: "
  35 #define SCC(s,q) !strncasecmp(s,q,strlen(q))
  36
  37 static int cb_stat(char *data, size_t size, size_t nmemb, struct stat *st)
  38 {
  39         size_t sz = size * nmemb;
  40
  41         /* XXX buffer not ended with 0 */
  42         if (SCC(data, DATE_HDR)) {
  43                 if (!st->st_mtime)
  44                         st->st_mtime = curl_getdate(data+strlen(DATE_HDR), NULL);
  45         } else if (SCC(data, LASM_HDR)) {
  46                 st->st_mtime = curl_getdate(data+strlen(LASM_HDR), NULL);
  47         } else if (SCC(data, CONL_HDR)) {
  48                 st->st_size = (long long) atol(data+strlen(CONL_HDR));
  49         }
  50
  51         return sz;
  52 }
  53
  54 static inline int unixerr(CURLcode ret, long errcode, int data)
  55 {
  56         switch(ret) {
  57                 case CURLE_OK:
  58                         break;
  59                 case CURLE_COULDNT_RESOLVE_HOST:
  60                         return -ENOENT;
  61                 case CURLE_OPERATION_TIMEDOUT:
  62                         return -ETIMEDOUT;
  63 /*              case CURLE_RANGE_ERROR: */
  64 /*                      return -ESPIPE; */
  65                 default:
  66                         if (errcode == 416)
  67                                 return -ESPIPE;
  68                         return -EIO;
  69         }
  70
  71         switch(errcode) {
  72                 case 404: return -ENOENT;
  73                 case 403: return -EPERM;
  74                 case 301: return -EMLINK;
  75                 case 200:
  76                 case 206: return data;
  77                 default: return -EBUSY;
  78         }
  79 }
  80
  81
  82 /* NASTY HACK */
  83 void wait(CURLM *handle)
  84 {
  85         fd_set r, w, e;
  86         int maxfd;
  87         struct timeval tv;
  88
  89         tv.tv_sec = 0;
  90         tv.tv_usec = 10000;
  91
  92         dbg("fetch: waiting\n");
  93         curl_multi_fdset(handle, &r, &w, &e, &maxfd);
  94         select(maxfd, &r, &w, &e, &tv);
  95 }
  96
  97 #define CH(s) { CURLcode _code; _code = s; if (_code != CURLE_OK) { fprintf(stderr, "Error: %s\n%s\n", #s, errbuf); return -EIO; } }
  98
  99 int request(CURL *conn, char *url, int head, struct stat *st, struct grow *data, size_t size, off_t offset)
 100 {
 101         char range[32];
 102         char errbuf[CURL_ERROR_SIZE];
 103         CURLcode ret;
 104         long errcode;
 105
 106         dbg("fetch url:\"%s\" head:%d st:%p data:%p size:%lu off:%ld\n", url, head, st, data, size, offset);
 107
 108 /*      CH(curl_easy_setopt(conn, CURLOPT_VERBOSE, 1)); */
 109         CH(curl_easy_setopt(conn, CURLOPT_ERRORBUFFER, errbuf));
 110         CH(curl_easy_setopt(conn, CURLOPT_URL, url));
 111         CH(curl_easy_setopt(conn, CURLOPT_WRITEFUNCTION, data ? cb_write : NULL));
 112         CH(curl_easy_setopt(conn, CURLOPT_WRITEDATA, data ? data : NULL));
 113         CH(curl_easy_setopt(conn, CURLOPT_HEADERFUNCTION, st ? cb_stat : NULL));
 114         CH(curl_easy_setopt(conn, CURLOPT_HEADERDATA, st ? st : NULL));
 115         CH(curl_easy_setopt(conn, CURLOPT_NOBODY, head));
 116 /*      CH(curl_easy_setopt(conn, CURLOPT_TIMEOUT, 30)); */
 117         if (head) {
 118                 CH(curl_easy_setopt(conn, CURLOPT_CUSTOMREQUEST, "HEAD"));
 119         } else {
 120                 CH(curl_easy_setopt(conn, CURLOPT_CUSTOMREQUEST, NULL));
 121                 CH(curl_easy_setopt(conn, CURLOPT_HTTPGET, 1));
 122         }
 123
 124         if (size > 0) {
 125                 /* FIXME: may overflow when downloading larger files (eg iso images) */
 126                 snprintf(range, 32, "%ld-%ld", (long) offset, (long) (offset+size-1));
 127                 dbg("range: \"%s\"\n", range);
 128                 CH(curl_easy_setopt(conn, CURLOPT_RANGE, range));
 129         } else {
 130                 CH(curl_easy_setopt(conn, CURLOPT_RANGE, NULL));
 131         }
 132
 133         ret = curl_easy_perform(conn);
 134         curl_easy_getinfo(conn, CURLINFO_RESPONSE_CODE, &errcode);
 135
 136         dbg("fetch done, ret:%d code: %ld\n", ret, errcode);
 137         return unixerr(ret, errcode, data?data->ptr:0);
 138 }
 139
 140 /* nebylo by treba storovat tady i CURL */
 141 int request_open_async(struct handle *h, off_t offset, size_t size)
 142 {
 143         char errbuf[CURL_ERROR_SIZE];
 144         CURLM *mult;
 145         CURL *conn;
 146
 147         dbg("setupuju async\n");
 148
 149         mult = curl_multi_init();
 150         if (!mult)
 151                 return -EIO;
 152
 153         conn = curl_easy_init();
 154         if (!conn) {
 155                 curl_multi_cleanup(mult);
 156                 return -EIO;
 157         }
 158
 159         CH(curl_easy_setopt(conn, CURLOPT_ERRORBUFFER, errbuf));
 160         CH(curl_easy_setopt(conn, CURLOPT_VERBOSE, 1));
 161         CH(curl_easy_setopt(conn, CURLOPT_URL, h->url.s));
 162         CH(curl_easy_setopt(conn, CURLOPT_WRITEFUNCTION, cb_write));
 163         CH(curl_easy_setopt(conn, CURLOPT_WRITEDATA, &h->lover));
 164 /*      CH(curl_easy_setopt(conn, CURLOPT_TIMEOUT, 30)); */
 165         if (offset > 0) {
 166                 snprintf(h->range, 32, "%ld-%ld", (long) offset, (long) (offset+size-1));
 167                 dbg("range: \"%s\"\n", h->range);
 168                 CH(curl_easy_setopt(conn, CURLOPT_RANGE, h->range));
 169         }
 170
 171         CH(curl_multi_add_handle(mult, conn));
 172
 173         h->conn = conn;
 174         h->mult = mult;
 175         return 0;
 176 }
 177
 178 void request_close_async(struct handle *h)
 179 {
 180         dbg("async; cleanup\n");
 181         if (!request_done(h)) {
 182                 curl_multi_remove_handle(h->mult, h->conn);
 183                 curl_easy_cleanup(h->conn);
 184                 curl_multi_cleanup(h->mult);
 185         }
 186         h->conn = NULL;
 187         h->mult = NULL;
 188 }
 189
 190 int request_perf_async(struct handle *h, size_t todo, int *done)
 191 {
 192         int left, ret = -1;
 193         long code;
 194         CURLMsg *msg;
 195
 196         dbg("async: todo: %d, ctu %p %p, mam precist %d\n", todo, h->conn, h->mult, h->lover.size);
 197         while (h->lover.ptr < todo) {
 198                 if (ret == 0)
 199                         wait(h->mult);
 200
 201                 ret = curl_multi_perform(h->mult, &left);
 202
 203                 if (ret > 0) {
 204                         dbg("--- MULTI PERFORM ERROR %d ---\n", ret);
 205                         exit(1);
 206                 }
 207
 208                 while((msg = curl_multi_info_read(h->mult, &left))) {
 209                         curl_easy_getinfo(msg->easy_handle, CURLINFO_RESPONSE_CODE, &code);
 210                         if (msg->msg == CURLMSG_DONE) {
 211                                 int ret = unixerr(msg->data.result, code, h->lover.ptr);
 212                                 dbg("async: konec %d %d\n", msg->data.result, code);
 213                                 *done = 1;
 214                                 return ret;
 215                         }
 216                 }
 217         }
 218
 219         dbg("async: vracim %d\n", h->lover.ptr);
 220         return h->lover.ptr;
 221 }