curl/docs/examples/ghiper.c

   1 /*****************************************************************************
   2  *                                  _   _ ____  _
   3  *  Project                     ___| | | |  _ \| |
   4  *                             / __| | | | |_) | |
   5  *                            | (__| |_| |  _ <| |___
   6  *                             \___|\___/|_| \_\_____|
   7  *
   8  * $Id: ghiper.c,v 1.5 2008-05-22 21:20:09 danf Exp $
   9  *
  10  * Example application source code using the multi socket interface to
  11  * download many files at once.
  12  *
  13  * Written by Jeff Pohlmeyer
  14
  15 Requires glib-2.x and a (POSIX?) system that has mkfifo().
  16
  17 This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c"
  18 sample programs, adapted to use glib's g_io_channel in place of libevent.
  19
  20 When running, the program creates the named pipe "hiper.fifo"
  21
  22 Whenever there is input into the fifo, the program reads the input as a list
  23 of URL's and creates some new easy handles to fetch each URL via the
  24 curl_multi "hiper" API.
  25
  26
  27 Thus, you can try a single URL:
  28   % echo http://www.yahoo.com > hiper.fifo
  29
  30 Or a whole bunch of them:
  31   % cat my-url-list > hiper.fifo
  32
  33 The fifo buffer is handled almost instantly, so you can even add more URL's
  34 while the previous requests are still being downloaded.
  35
  36 This is purely a demo app, all retrieved data is simply discarded by the write
  37 callback.
  38
  39 */
  40
  41
  42 #include <glib.h>
  43 #include <sys/stat.h>
  44 #include <unistd.h>
  45 #include <fcntl.h>
  46 #include <stdlib.h>
  47 #include <stdio.h>
  48 #include <errno.h>
  49 #include <curl/curl.h>
  50
  51
  52 #define MSG_OUT g_print   /* Change to "g_error" to write to stderr */
  53 #define SHOW_VERBOSE 0    /* Set to non-zero for libcurl messages */
  54 #define SHOW_PROGRESS 0   /* Set to non-zero to enable progress callback */
  55
  56
  57
  58 /* Global information, common to all connections */
  59 typedef struct _GlobalInfo {
  60   CURLM *multi;
  61   guint timer_event;
  62   int prev_running;
  63   int still_running;
  64   int requested; /* count: curl_easy_init() */
  65   int completed; /* count: curl_easy_cleanup() */
  66 } GlobalInfo;
  67
  68
  69
  70 /* Information associated with a specific easy handle */
  71 typedef struct _ConnInfo {
  72   CURL *easy;
  73   char *url;
  74   GlobalInfo *global;
  75   char error[CURL_ERROR_SIZE];
  76 } ConnInfo;
  77
  78
  79 /* Information associated with a specific socket */
  80 typedef struct _SockInfo {
  81   curl_socket_t sockfd;
  82   CURL *easy;
  83   int action;
  84   long timeout;
  85   GIOChannel *ch;
  86   guint ev;
  87   GlobalInfo *global;
  88 } SockInfo;
  89
  90
  91
  92
  93 /* Die if we get a bad CURLMcode somewhere */
  94 static void mcode_or_die(const char *where, CURLMcode code) {
  95   if ( CURLM_OK != code ) {
  96     const char *s;
  97     switch (code) {
  98       case     CURLM_CALL_MULTI_PERFORM: s="CURLM_CALL_MULTI_PERFORM"; break;
  99       case     CURLM_OK:                 s="CURLM_OK";                 break;
 100       case     CURLM_BAD_HANDLE:         s="CURLM_BAD_HANDLE";         break;
 101       case     CURLM_BAD_EASY_HANDLE:    s="CURLM_BAD_EASY_HANDLE";    break;
 102       case     CURLM_OUT_OF_MEMORY:      s="CURLM_OUT_OF_MEMORY";      break;
 103       case     CURLM_INTERNAL_ERROR:     s="CURLM_INTERNAL_ERROR";     break;
 104       case     CURLM_BAD_SOCKET:         s="CURLM_BAD_SOCKET";         break;
 105       case     CURLM_UNKNOWN_OPTION:     s="CURLM_UNKNOWN_OPTION";     break;
 106       case     CURLM_LAST:               s="CURLM_LAST";               break;
 107       default: s="CURLM_unknown";
 108     }
 109     MSG_OUT("ERROR: %s returns %s\n", where, s);
 110     exit(code);
 111   }
 112 }
 113
 114
 115
 116 /* Check for completed transfers, and remove their easy handles */
 117 static void check_run_count(GlobalInfo *g)
 118 {
 119   if (g->prev_running > g->still_running) {
 120     char *eff_url=NULL;
 121     CURLMsg *msg;
 122     int msgs_left;
 123     ConnInfo *conn=NULL;
 124     CURL*easy;
 125     CURLcode res;
 126
 127     MSG_OUT("REMAINING: %d\n", g->still_running);
 128     /*
 129       I am still uncertain whether it is safe to remove an easy handle
 130       from inside the curl_multi_info_read loop, so here I will search
 131       for completed transfers in the inner "while" loop, and then remove
 132       them in the outer "do-while" loop...
 133    */
 134     do {
 135       easy=NULL;
 136       while ((msg = curl_multi_info_read(g->multi, &msgs_left))) {
 137         if (msg->msg == CURLMSG_DONE) {
 138           easy=msg->easy_handle;
 139           res=msg->data.result;
 140           break;
 141         }
 142       }
 143       if (easy) {
 144           curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
 145           curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
 146           MSG_OUT("DONE: %s => (%d) %s\n", eff_url, res, conn->error);
 147           curl_multi_remove_handle(g->multi, easy);
 148           g_free(conn->url);
 149           curl_easy_cleanup(easy);
 150           g_free(conn);
 151           g->completed++;
 152       }
 153     } while ( easy );
 154     MSG_OUT("Requested: %d Completed:%d\n", g->requested, g->completed);
 155   }
 156   g->prev_running = g->still_running;
 157 }
 158
 159
 160
 161
 162 /* Called by glib when our timeout expires */
 163 static gboolean timer_cb(gpointer data)
 164 {
 165   GlobalInfo *g = (GlobalInfo *)data;
 166   CURLMcode rc;
 167
 168   do {
 169     rc = curl_multi_socket(g->multi, CURL_SOCKET_TIMEOUT, &g->still_running);
 170   } while (rc == CURLM_CALL_MULTI_PERFORM);
 171   mcode_or_die("timer_cb: curl_multi_socket", rc);
 172   check_run_count(g);
 173   return FALSE;
 174 }
 175
 176
 177
 178 /* Update the event timer after curl_multi library calls */
 179 static int update_timeout_cb(CURLM *multi, long timeout_ms, void *userp)
 180 {
 181   struct timeval timeout;
 182   GlobalInfo *g=(GlobalInfo *)userp;
 183   timeout.tv_sec = timeout_ms/1000;
 184   timeout.tv_usec = (timeout_ms%1000)*1000;
 185
 186   MSG_OUT("*** update_timeout_cb %ld => %ld:%ld ***\n",
 187               timeout_ms, timeout.tv_sec, timeout.tv_usec);
 188
 189   g->timer_event = g_timeout_add(timeout_ms, timer_cb, g);
 190   return 0;
 191 }
 192
 193
 194
 195
 196 /* Called by glib when we get action on a multi socket */
 197 static gboolean event_cb(GIOChannel *ch, GIOCondition condition, gpointer data)
 198 {
 199   GlobalInfo *g = (GlobalInfo*) data;
 200   CURLMcode rc;
 201   int fd=g_io_channel_unix_get_fd(ch);
 202   do {
 203     rc = curl_multi_socket(g->multi, fd, &g->still_running);
 204   } while (rc == CURLM_CALL_MULTI_PERFORM);
 205   mcode_or_die("event_cb: curl_multi_socket", rc);
 206   check_run_count(g);
 207   if(g->still_running) {
 208     return TRUE;
 209   } else {
 210     MSG_OUT("last transfer done, kill timeout\n");
 211     if (g->timer_event) { g_source_remove(g->timer_event); }
 212     return FALSE;
 213   }
 214 }
 215
 216
 217
 218 /* Clean up the SockInfo structure */
 219 static void remsock(SockInfo *f)
 220 {
 221   if (!f) { return; }
 222   if (f->ev) { g_source_remove(f->ev); }
 223   g_free(f);
 224 }
 225
 226
 227
 228 /* Assign information to a SockInfo structure */
 229 static void setsock(SockInfo*f, curl_socket_t s, CURL*e, int act, GlobalInfo*g)
 230 {
 231   GIOCondition kind =
 232      (act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0);
 233
 234   f->sockfd = s;
 235   f->action = act;
 236   f->easy = e;
 237   if (f->ev) { g_source_remove(f->ev); }
 238   f->ev=g_io_add_watch(f->ch, kind, event_cb,g);
 239
 240 }
 241
 242
 243
 244 /* Initialize a new SockInfo structure */
 245 static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo *g)
 246 {
 247   SockInfo *fdp = g_malloc0(sizeof(SockInfo));
 248
 249   fdp->global = g;
 250   fdp->ch=g_io_channel_unix_new(s);
 251   setsock(fdp, s, easy, action, g);
 252   curl_multi_assign(g->multi, s, fdp);
 253 }
 254
 255
 256
 257 /* CURLMOPT_SOCKETFUNCTION */
 258 static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp)
 259 {
 260   GlobalInfo *g = (GlobalInfo*) cbp;
 261   SockInfo *fdp = (SockInfo*) sockp;
 262   static const char *whatstr[]={ "none", "IN", "OUT", "INOUT", "REMOVE" };
 263
 264   MSG_OUT("socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]);
 265   if (what == CURL_POLL_REMOVE) {
 266     MSG_OUT("\n");
 267     remsock(fdp);
 268   } else {
 269     if (!fdp) {
 270       MSG_OUT("Adding data: %s%s\n",
 271              what&CURL_POLL_IN?"READ":"",
 272              what&CURL_POLL_OUT?"WRITE":"" );
 273       addsock(s, e, what, g);
 274     }
 275     else {
 276       MSG_OUT(
 277         "Changing action from %d to %d\n", fdp->action, what);
 278       setsock(fdp, s, e, what, g);
 279     }
 280   }
 281   return 0;
 282 }
 283
 284
 285
 286 /* CURLOPT_WRITEFUNCTION */
 287 static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data)
 288 {
 289   size_t realsize = size * nmemb;
 290   ConnInfo *conn = (ConnInfo*) data;
 291   (void)ptr;
 292   (void)conn;
 293   return realsize;
 294 }
 295
 296
 297
 298 /* CURLOPT_PROGRESSFUNCTION */
 299 static int prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)
 300 {
 301   ConnInfo *conn = (ConnInfo *)p;
 302   MSG_OUT("Progress: %s (%g/%g)\n", conn->url, dlnow, dltotal);
 303   return 0;
 304 }
 305
 306
 307
 308 /* Create a new easy handle, and add it to the global curl_multi */
 309 static void new_conn(char *url, GlobalInfo *g )
 310 {
 311   ConnInfo *conn;
 312   CURLMcode rc;
 313
 314   conn = g_malloc0(sizeof(ConnInfo));
 315
 316   conn->error[0]='\0';
 317
 318   conn->easy = curl_easy_init();
 319   if (!conn->easy) {
 320     MSG_OUT("curl_easy_init() failed, exiting!\n");
 321     exit(2);
 322   }
 323   conn->global = g;
 324   conn->url = g_strdup(url);
 325   curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url);
 326   curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb);
 327   curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, &conn);
 328   curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, (long)SHOW_VERBOSE);
 329   curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
 330   curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
 331   curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, SHOW_PROGRESS?0L:1L);
 332   curl_easy_setopt(conn->easy, CURLOPT_PROGRESSFUNCTION, prog_cb);
 333   curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn);
 334   curl_easy_setopt(conn->easy, CURLOPT_FOLLOWLOCATION, 1L);
 335   curl_easy_setopt(conn->easy, CURLOPT_CONNECTTIMEOUT, 30L);
 336   curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_LIMIT, 1L);
 337   curl_easy_setopt(conn->easy, CURLOPT_LOW_SPEED_TIME, 30L);
 338
 339   MSG_OUT("Adding easy %p to multi %p (%s)\n", conn->easy, g->multi, url);
 340   rc =curl_multi_add_handle(g->multi, conn->easy);
 341   mcode_or_die("new_conn: curl_multi_add_handle", rc);
 342   g->requested++;
 343   do {
 344     rc = curl_multi_socket_all(g->multi, &g->still_running);
 345   } while (CURLM_CALL_MULTI_PERFORM == rc);
 346   mcode_or_die("new_conn: curl_multi_socket_all", rc);
 347   check_run_count(g);
 348 }
 349
 350
 351 /* This gets called by glib whenever data is received from the fifo */
 352 static gboolean fifo_cb (GIOChannel *ch, GIOCondition condition, gpointer data)
 353 {
 354   #define BUF_SIZE 1024
 355   gsize len, tp;
 356   gchar *buf, *tmp, *all=NULL;
 357   GIOStatus rv;
 358
 359   do {
 360     GError *err=NULL;
 361     rv = g_io_channel_read_line (ch,&buf,&len,&tp,&err);
 362     if ( buf ) {
 363       if (tp) { buf[tp]='\0'; }
 364       new_conn(buf,(GlobalInfo*)data);
 365       g_free(buf);
 366     } else {
 367       buf = g_malloc(BUF_SIZE+1);
 368       while (TRUE) {
 369         buf[BUF_SIZE]='\0';
 370         g_io_channel_read_chars(ch,buf,BUF_SIZE,&len,&err);
 371         if (len) {
 372           buf[len]='\0';
 373           if (all) {
 374             tmp=all;
 375             all=g_strdup_printf("%s%s", tmp, buf);
 376             g_free(tmp);
 377           } else {
 378             all = g_strdup(buf);
 379           }
 380         } else {
 381            break;
 382         }
 383       }
 384       if (all) {
 385         new_conn(all,(GlobalInfo*)data);
 386         g_free(all);
 387       }
 388       g_free(buf);
 389     }
 390     if ( err ) {
 391       g_error("fifo_cb: %s", err->message);
 392       g_free(err);
 393       break;
 394     }
 395   } while ( (len) && (rv == G_IO_STATUS_NORMAL) );
 396   return TRUE;
 397 }
 398
 399
 400
 401
 402 int init_fifo(void)
 403 {
 404  struct stat st;
 405  const char *fifo = "hiper.fifo";
 406  int socket;
 407
 408  if (lstat (fifo, &st) == 0) {
 409   if ((st.st_mode & S_IFMT) == S_IFREG) {
 410    errno = EEXIST;
 411    perror("lstat");
 412    exit (1);
 413   }
 414  }
 415
 416  unlink (fifo);
 417  if (mkfifo (fifo, 0600) == -1) {
 418   perror("mkfifo");
 419   exit (1);
 420  }
 421
 422  socket = open (fifo, O_RDWR | O_NONBLOCK, 0);
 423
 424  if (socket == -1) {
 425   perror("open");
 426   exit (1);
 427  }
 428  MSG_OUT("Now, pipe some URL's into > %s\n", fifo);
 429
 430  return socket;
 431
 432 }
 433
 434
 435
 436
 437 int main(int argc, char **argv)
 438 {
 439   GlobalInfo *g;
 440   CURLMcode rc;
 441   GMainLoop*gmain;
 442   int fd;
 443   GIOChannel* ch;
 444   g=g_malloc0(sizeof(GlobalInfo));
 445
 446   fd=init_fifo();
 447   ch=g_io_channel_unix_new(fd);
 448   g_io_add_watch(ch,G_IO_IN,fifo_cb,g);
 449   gmain=g_main_loop_new(NULL,FALSE);
 450   g->multi = curl_multi_init();
 451   curl_multi_setopt(g->multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
 452   curl_multi_setopt(g->multi, CURLMOPT_SOCKETDATA, g);
 453   curl_multi_setopt(g->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb);
 454   curl_multi_setopt(g->multi, CURLMOPT_TIMERDATA, g);
 455   do {
 456     rc = curl_multi_socket_all(g->multi, &g->still_running);
 457   } while (CURLM_CALL_MULTI_PERFORM == rc);
 458   g_main_loop_run(gmain);
 459   curl_multi_cleanup(g->multi);
 460   return 0;
 461 }