1 /*****************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
8 * $Id: ghiper.c,v 1.1.1.1 2008-09-23 16:32:05 hoffman Exp $
10 * Example application source code using the multi socket interface to
11 * download many files at once.
13 * Written by Jeff Pohlmeyer
15 Requires glib-2.x and a (POSIX?) system that has mkfifo().
17 This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c"
18 sample programs, adapted to use glib's g_io_channel in place of libevent.
20 When running, the program creates the named pipe "hiper.fifo"
22 Whenever there is input into the fifo, the program reads the input as a list
23 of URL's and creates some new easy handles to fetch each URL via the
24 curl_multi "hiper" API.
27 Thus, you can try a single URL:
28 % echo http://www.yahoo.com > hiper.fifo
30 Or a whole bunch of them:
31 % cat my-url-list > hiper.fifo
33 The fifo buffer is handled almost instantly, so you can even add more URL's
34 while the previous requests are still being downloaded.
36 This is purely a demo app, all retrieved data is simply discarded by the write
49 #include <curl/curl.h>
52 #define MSG_OUT g_print /* Change to "g_error" to write to stderr */
53 #define SHOW_VERBOSE 0 /* Set to non-zero for libcurl messages */
54 #define SHOW_PROGRESS 0 /* Set to non-zero to enable progress callback */
58 /* Global information, common to all connections */
59 typedef struct _GlobalInfo
{
64 int requested
; /* count: curl_easy_init() */
65 int completed
; /* count: curl_easy_cleanup() */
70 /* Information associated with a specific easy handle */
71 typedef struct _ConnInfo
{
75 char error
[CURL_ERROR_SIZE
];
79 /* Information associated with a specific socket */
80 typedef struct _SockInfo
{
93 /* Die if we get a bad CURLMcode somewhere */
94 static void mcode_or_die(const char *where
, CURLMcode code
) {
95 if ( CURLM_OK
!= code
) {
98 case CURLM_CALL_MULTI_PERFORM
: s
="CURLM_CALL_MULTI_PERFORM"; break;
99 case CURLM_OK
: s
="CURLM_OK"; break;
100 case CURLM_BAD_HANDLE
: s
="CURLM_BAD_HANDLE"; break;
101 case CURLM_BAD_EASY_HANDLE
: s
="CURLM_BAD_EASY_HANDLE"; break;
102 case CURLM_OUT_OF_MEMORY
: s
="CURLM_OUT_OF_MEMORY"; break;
103 case CURLM_INTERNAL_ERROR
: s
="CURLM_INTERNAL_ERROR"; break;
104 case CURLM_BAD_SOCKET
: s
="CURLM_BAD_SOCKET"; break;
105 case CURLM_UNKNOWN_OPTION
: s
="CURLM_UNKNOWN_OPTION"; break;
106 case CURLM_LAST
: s
="CURLM_LAST"; break;
107 default: s
="CURLM_unknown";
109 MSG_OUT("ERROR: %s returns %s\n", where
, s
);
116 /* Check for completed transfers, and remove their easy handles */
117 static void check_run_count(GlobalInfo
*g
)
119 if (g
->prev_running
> g
->still_running
) {
127 MSG_OUT("REMAINING: %d\n", g
->still_running
);
129 I am still uncertain whether it is safe to remove an easy handle
130 from inside the curl_multi_info_read loop, so here I will search
131 for completed transfers in the inner "while" loop, and then remove
132 them in the outer "do-while" loop...
136 while ((msg
= curl_multi_info_read(g
->multi
, &msgs_left
))) {
137 if (msg
->msg
== CURLMSG_DONE
) {
138 easy
=msg
->easy_handle
;
139 res
=msg
->data
.result
;
144 curl_easy_getinfo(easy
, CURLINFO_PRIVATE
, &conn
);
145 curl_easy_getinfo(easy
, CURLINFO_EFFECTIVE_URL
, &eff_url
);
146 MSG_OUT("DONE: %s => (%d) %s\n", eff_url
, res
, conn
->error
);
147 curl_multi_remove_handle(g
->multi
, easy
);
149 curl_easy_cleanup(easy
);
154 MSG_OUT("Requested: %d Completed:%d\n", g
->requested
, g
->completed
);
156 g
->prev_running
= g
->still_running
;
162 /* Called by glib when our timeout expires */
163 static gboolean
timer_cb(gpointer data
)
165 GlobalInfo
*g
= (GlobalInfo
*)data
;
169 rc
= curl_multi_socket(g
->multi
, CURL_SOCKET_TIMEOUT
, &g
->still_running
);
170 } while (rc
== CURLM_CALL_MULTI_PERFORM
);
171 mcode_or_die("timer_cb: curl_multi_socket", rc
);
178 /* Update the event timer after curl_multi library calls */
179 static int update_timeout_cb(CURLM
*multi
, long timeout_ms
, void *userp
)
181 struct timeval timeout
;
182 GlobalInfo
*g
=(GlobalInfo
*)userp
;
183 timeout
.tv_sec
= timeout_ms
/1000;
184 timeout
.tv_usec
= (timeout_ms
%1000)*1000;
186 MSG_OUT("*** update_timeout_cb %ld => %ld:%ld ***\n",
187 timeout_ms
, timeout
.tv_sec
, timeout
.tv_usec
);
189 g
->timer_event
= g_timeout_add(timeout_ms
, timer_cb
, g
);
196 /* Called by glib when we get action on a multi socket */
197 static gboolean
event_cb(GIOChannel
*ch
, GIOCondition condition
, gpointer data
)
199 GlobalInfo
*g
= (GlobalInfo
*) data
;
201 int fd
=g_io_channel_unix_get_fd(ch
);
203 rc
= curl_multi_socket(g
->multi
, fd
, &g
->still_running
);
204 } while (rc
== CURLM_CALL_MULTI_PERFORM
);
205 mcode_or_die("event_cb: curl_multi_socket", rc
);
207 if(g
->still_running
) {
210 MSG_OUT("last transfer done, kill timeout\n");
211 if (g
->timer_event
) { g_source_remove(g
->timer_event
); }
218 /* Clean up the SockInfo structure */
219 static void remsock(SockInfo
*f
)
222 if (f
->ev
) { g_source_remove(f
->ev
); }
228 /* Assign information to a SockInfo structure */
229 static void setsock(SockInfo
*f
, curl_socket_t s
, CURL
*e
, int act
, GlobalInfo
*g
)
232 (act
&CURL_POLL_IN
?G_IO_IN
:0)|(act
&CURL_POLL_OUT
?G_IO_OUT
:0);
237 if (f
->ev
) { g_source_remove(f
->ev
); }
238 f
->ev
=g_io_add_watch(f
->ch
, kind
, event_cb
,g
);
244 /* Initialize a new SockInfo structure */
245 static void addsock(curl_socket_t s
, CURL
*easy
, int action
, GlobalInfo
*g
)
247 SockInfo
*fdp
= g_malloc0(sizeof(SockInfo
));
250 fdp
->ch
=g_io_channel_unix_new(s
);
251 setsock(fdp
, s
, easy
, action
, g
);
252 curl_multi_assign(g
->multi
, s
, fdp
);
257 /* CURLMOPT_SOCKETFUNCTION */
258 static int sock_cb(CURL
*e
, curl_socket_t s
, int what
, void *cbp
, void *sockp
)
260 GlobalInfo
*g
= (GlobalInfo
*) cbp
;
261 SockInfo
*fdp
= (SockInfo
*) sockp
;
262 static const char *whatstr
[]={ "none", "IN", "OUT", "INOUT", "REMOVE" };
264 MSG_OUT("socket callback: s=%d e=%p what=%s ", s
, e
, whatstr
[what
]);
265 if (what
== CURL_POLL_REMOVE
) {
270 MSG_OUT("Adding data: %s%s\n",
271 what
&CURL_POLL_IN
?"READ":"",
272 what
&CURL_POLL_OUT
?"WRITE":"" );
273 addsock(s
, e
, what
, g
);
277 "Changing action from %d to %d\n", fdp
->action
, what
);
278 setsock(fdp
, s
, e
, what
, g
);
286 /* CURLOPT_WRITEFUNCTION */
287 static size_t write_cb(void *ptr
, size_t size
, size_t nmemb
, void *data
)
289 size_t realsize
= size
* nmemb
;
290 ConnInfo
*conn
= (ConnInfo
*) data
;
298 /* CURLOPT_PROGRESSFUNCTION */
299 static int prog_cb (void *p
, double dltotal
, double dlnow
, double ult
, double uln
)
301 ConnInfo
*conn
= (ConnInfo
*)p
;
302 MSG_OUT("Progress: %s (%g/%g)\n", conn
->url
, dlnow
, dltotal
);
308 /* Create a new easy handle, and add it to the global curl_multi */
309 static void new_conn(char *url
, GlobalInfo
*g
)
314 conn
= g_malloc0(sizeof(ConnInfo
));
318 conn
->easy
= curl_easy_init();
320 MSG_OUT("curl_easy_init() failed, exiting!\n");
324 conn
->url
= g_strdup(url
);
325 curl_easy_setopt(conn
->easy
, CURLOPT_URL
, conn
->url
);
326 curl_easy_setopt(conn
->easy
, CURLOPT_WRITEFUNCTION
, write_cb
);
327 curl_easy_setopt(conn
->easy
, CURLOPT_WRITEDATA
, &conn
);
328 curl_easy_setopt(conn
->easy
, CURLOPT_VERBOSE
, (long)SHOW_VERBOSE
);
329 curl_easy_setopt(conn
->easy
, CURLOPT_ERRORBUFFER
, conn
->error
);
330 curl_easy_setopt(conn
->easy
, CURLOPT_PRIVATE
, conn
);
331 curl_easy_setopt(conn
->easy
, CURLOPT_NOPROGRESS
, SHOW_PROGRESS
?0L:1L);
332 curl_easy_setopt(conn
->easy
, CURLOPT_PROGRESSFUNCTION
, prog_cb
);
333 curl_easy_setopt(conn
->easy
, CURLOPT_PROGRESSDATA
, conn
);
334 curl_easy_setopt(conn
->easy
, CURLOPT_FOLLOWLOCATION
, 1L);
335 curl_easy_setopt(conn
->easy
, CURLOPT_CONNECTTIMEOUT
, 30L);
336 curl_easy_setopt(conn
->easy
, CURLOPT_LOW_SPEED_LIMIT
, 1L);
337 curl_easy_setopt(conn
->easy
, CURLOPT_LOW_SPEED_TIME
, 30L);
339 MSG_OUT("Adding easy %p to multi %p (%s)\n", conn
->easy
, g
->multi
, url
);
340 rc
=curl_multi_add_handle(g
->multi
, conn
->easy
);
341 mcode_or_die("new_conn: curl_multi_add_handle", rc
);
344 rc
= curl_multi_socket_all(g
->multi
, &g
->still_running
);
345 } while (CURLM_CALL_MULTI_PERFORM
== rc
);
346 mcode_or_die("new_conn: curl_multi_socket_all", rc
);
351 /* This gets called by glib whenever data is received from the fifo */
352 static gboolean
fifo_cb (GIOChannel
*ch
, GIOCondition condition
, gpointer data
)
354 #define BUF_SIZE 1024
356 gchar
*buf
, *tmp
, *all
=NULL
;
361 rv
= g_io_channel_read_line (ch
,&buf
,&len
,&tp
,&err
);
363 if (tp
) { buf
[tp
]='\0'; }
364 new_conn(buf
,(GlobalInfo
*)data
);
367 buf
= g_malloc(BUF_SIZE
+1);
370 g_io_channel_read_chars(ch
,buf
,BUF_SIZE
,&len
,&err
);
375 all
=g_strdup_printf("%s%s", tmp
, buf
);
385 new_conn(all
,(GlobalInfo
*)data
);
391 g_error("fifo_cb: %s", err
->message
);
395 } while ( (len
) && (rv
== G_IO_STATUS_NORMAL
) );
405 const char *fifo
= "hiper.fifo";
408 if (lstat (fifo
, &st
) == 0) {
409 if ((st
.st_mode
& S_IFMT
) == S_IFREG
) {
417 if (mkfifo (fifo
, 0600) == -1) {
422 socket
= open (fifo
, O_RDWR
| O_NONBLOCK
, 0);
428 MSG_OUT("Now, pipe some URL's into > %s\n", fifo
);
437 int main(int argc
, char **argv
)
444 g
=g_malloc0(sizeof(GlobalInfo
));
447 ch
=g_io_channel_unix_new(fd
);
448 g_io_add_watch(ch
,G_IO_IN
,fifo_cb
,g
);
449 gmain
=g_main_loop_new(NULL
,FALSE
);
450 g
->multi
= curl_multi_init();
451 curl_multi_setopt(g
->multi
, CURLMOPT_SOCKETFUNCTION
, sock_cb
);
452 curl_multi_setopt(g
->multi
, CURLMOPT_SOCKETDATA
, g
);
453 curl_multi_setopt(g
->multi
, CURLMOPT_TIMERFUNCTION
, update_timeout_cb
);
454 curl_multi_setopt(g
->multi
, CURLMOPT_TIMERDATA
, g
);
456 rc
= curl_multi_socket_all(g
->multi
, &g
->still_running
);
457 } while (CURLM_CALL_MULTI_PERFORM
== rc
);
458 g_main_loop_run(gmain
);
459 curl_multi_cleanup(g
->multi
);