curl/docs/examples/fopen.c

   1 /*****************************************************************************
   2  *
   3  * This example source code introduces a c library buffered I/O interface to
   4  * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
   5  * rewind(). Supported functions have identical prototypes to their normal c
   6  * lib namesakes and are preceaded by url_ .
   7  *
   8  * Using this code you can replace your program's fopen() with url_fopen()
   9  * and fread() with url_fread() and it become possible to read remote streams
  10  * instead of (only) local files. Local files (ie those that can be directly
  11  * fopened) will drop back to using the underlying clib implementations
  12  *
  13  * See the main() function at the bottom that shows an app that retrives from a
  14  * specified url using fgets() and fread() and saves as two output files.
  15  *
  16  * Coyright (c)2003 Simtec Electronics
  17  *
  18  * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
  19  * reference to original curl example code
  20  *
  21  * Redistribution and use in source and binary forms, with or without
  22  * modification, are permitted provided that the following conditions
  23  * are met:
  24  * 1. Redistributions of source code must retain the above copyright
  25  *    notice, this list of conditions and the following disclaimer.
  26  * 2. Redistributions in binary form must reproduce the above copyright
  27  *    notice, this list of conditions and the following disclaimer in the
  28  *    documentation and/or other materials provided with the distribution.
  29  * 3. The name of the author may not be used to endorse or promote products
  30  *    derived from this software without specific prior written permission.
  31  *
  32  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  33  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  34  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  35  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  36  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  37  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  38  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  39  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  40  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  41  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  42  *
  43  * This example requires libcurl 7.9.7 or later.
  44  */
  45
  46 #include <stdio.h>
  47 #include <string.h>
  48 #ifndef WIN32
  49 #  include <sys/time.h>
  50 #endif
  51 #include <stdlib.h>
  52 #include <errno.h>
  53
  54 #include <curl/curl.h>
  55
  56 enum fcurl_type_e { CFTYPE_NONE=0, CFTYPE_FILE=1, CFTYPE_CURL=2 };
  57
  58 struct fcurl_data
  59 {
  60     enum fcurl_type_e type;     /* type of handle */
  61     union {
  62         CURL *curl;
  63         FILE *file;
  64     } handle;                   /* handle */
  65
  66     char *buffer;               /* buffer to store cached data*/
  67     int buffer_len;             /* currently allocated buffers length */
  68     int buffer_pos;             /* end of data in buffer*/
  69     int still_running;          /* Is background url fetch still in progress */
  70 };
  71
  72 typedef struct fcurl_data URL_FILE;
  73
  74 /* exported functions */
  75 URL_FILE *url_fopen(const char *url,const char *operation);
  76 int url_fclose(URL_FILE *file);
  77 int url_feof(URL_FILE *file);
  78 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
  79 char * url_fgets(char *ptr, int size, URL_FILE *file);
  80 void url_rewind(URL_FILE *file);
  81
  82 /* we use a global one for convenience */
  83 CURLM *multi_handle;
  84
  85 /* curl calls this routine to get more data */
  86 static size_t
  87 write_callback(char *buffer,
  88                size_t size,
  89                size_t nitems,
  90                void *userp)
  91 {
  92     char *newbuff;
  93     int rembuff;
  94
  95     URL_FILE *url = (URL_FILE *)userp;
  96     size *= nitems;
  97
  98     rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
  99
 100     if(size > rembuff)
 101     {
 102         /* not enough space in buffer */
 103         newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
 104         if(newbuff==NULL)
 105         {
 106             fprintf(stderr,"callback buffer grow failed\n");
 107             size=rembuff;
 108         }
 109         else
 110         {
 111             /* realloc suceeded increase buffer size*/
 112             url->buffer_len+=size - rembuff;
 113             url->buffer=newbuff;
 114
 115             /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/
 116         }
 117     }
 118
 119     memcpy(&url->buffer[url->buffer_pos], buffer, size);
 120     url->buffer_pos += size;
 121
 122     /*fprintf(stderr, "callback %d size bytes\n", size);*/
 123
 124     return size;
 125 }
 126
 127 /* use to attempt to fill the read buffer up to requested number of bytes */
 128 static int
 129 fill_buffer(URL_FILE *file,int want,int waittime)
 130 {
 131     fd_set fdread;
 132     fd_set fdwrite;
 133     fd_set fdexcep;
 134     int maxfd;
 135     struct timeval timeout;
 136     int rc;
 137
 138     /* only attempt to fill buffer if transactions still running and buffer
 139      * doesnt exceed required size already
 140      */
 141     if((!file->still_running) || (file->buffer_pos > want))
 142         return 0;
 143
 144     /* attempt to fill buffer */
 145     do
 146     {
 147         FD_ZERO(&fdread);
 148         FD_ZERO(&fdwrite);
 149         FD_ZERO(&fdexcep);
 150
 151         /* set a suitable timeout to fail on */
 152         timeout.tv_sec = 60; /* 1 minute */
 153         timeout.tv_usec = 0;
 154
 155         /* get file descriptors from the transfers */
 156         curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
 157
 158         /* In a real-world program you OF COURSE check the return code of the
 159            function calls, *and* you make sure that maxfd is bigger than -1
 160            so that the call to select() below makes sense! */
 161
 162         rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
 163
 164         switch(rc) {
 165         case -1:
 166             /* select error */
 167             break;
 168
 169         case 0:
 170             break;
 171
 172         default:
 173             /* timeout or readable/writable sockets */
 174             /* note we *could* be more efficient and not wait for
 175              * CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry
 176              * but that gets messy */
 177             while(curl_multi_perform(multi_handle, &file->still_running) ==
 178                   CURLM_CALL_MULTI_PERFORM);
 179
 180             break;
 181         }
 182     } while(file->still_running && (file->buffer_pos < want));
 183     return 1;
 184 }
 185
 186 /* use to remove want bytes from the front of a files buffer */
 187 static int
 188 use_buffer(URL_FILE *file,int want)
 189 {
 190     /* sort out buffer */
 191     if((file->buffer_pos - want) <=0)
 192     {
 193         /* ditch buffer - write will recreate */
 194         if(file->buffer)
 195             free(file->buffer);
 196
 197         file->buffer=NULL;
 198         file->buffer_pos=0;
 199         file->buffer_len=0;
 200     }
 201     else
 202     {
 203         /* move rest down make it available for later */
 204         memmove(file->buffer,
 205                 &file->buffer[want],
 206                 (file->buffer_pos - want));
 207
 208         file->buffer_pos -= want;
 209     }
 210     return 0;
 211 }
 212
 213
 214
 215 URL_FILE *
 216 url_fopen(const char *url,const char *operation)
 217 {
 218     /* this code could check for URLs or types in the 'url' and
 219        basicly use the real fopen() for standard files */
 220
 221     URL_FILE *file;
 222     (void)operation;
 223
 224     file = (URL_FILE *)malloc(sizeof(URL_FILE));
 225     if(!file)
 226         return NULL;
 227
 228     memset(file, 0, sizeof(URL_FILE));
 229
 230     if((file->handle.file=fopen(url,operation)))
 231     {
 232         file->type = CFTYPE_FILE; /* marked as URL */
 233     }
 234     else
 235     {
 236         file->type = CFTYPE_CURL; /* marked as URL */
 237         file->handle.curl = curl_easy_init();
 238
 239         curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
 240         curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
 241         curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
 242         curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
 243
 244         if(!multi_handle)
 245             multi_handle = curl_multi_init();
 246
 247         curl_multi_add_handle(multi_handle, file->handle.curl);
 248
 249         /* lets start the fetch */
 250         while(curl_multi_perform(multi_handle, &file->still_running) ==
 251               CURLM_CALL_MULTI_PERFORM );
 252
 253         if((file->buffer_pos == 0) && (!file->still_running))
 254         {
 255             /* if still_running is 0 now, we should return NULL */
 256
 257             /* make sure the easy handle is not in the multi handle anymore */
 258             curl_multi_remove_handle(multi_handle, file->handle.curl);
 259
 260             /* cleanup */
 261             curl_easy_cleanup(file->handle.curl);
 262
 263             free(file);
 264
 265             file = NULL;
 266         }
 267     }
 268     return file;
 269 }
 270
 271 int
 272 url_fclose(URL_FILE *file)
 273 {
 274     int ret=0;/* default is good return */
 275
 276     switch(file->type)
 277     {
 278     case CFTYPE_FILE:
 279         ret=fclose(file->handle.file); /* passthrough */
 280         break;
 281
 282     case CFTYPE_CURL:
 283         /* make sure the easy handle is not in the multi handle anymore */
 284         curl_multi_remove_handle(multi_handle, file->handle.curl);
 285
 286         /* cleanup */
 287         curl_easy_cleanup(file->handle.curl);
 288         break;
 289
 290     default: /* unknown or supported type - oh dear */
 291         ret=EOF;
 292         errno=EBADF;
 293         break;
 294
 295     }
 296
 297     if(file->buffer)
 298         free(file->buffer);/* free any allocated buffer space */
 299
 300     free(file);
 301
 302     return ret;
 303 }
 304
 305 int
 306 url_feof(URL_FILE *file)
 307 {
 308     int ret=0;
 309
 310     switch(file->type)
 311     {
 312     case CFTYPE_FILE:
 313         ret=feof(file->handle.file);
 314         break;
 315
 316     case CFTYPE_CURL:
 317         if((file->buffer_pos == 0) && (!file->still_running))
 318             ret = 1;
 319         break;
 320     default: /* unknown or supported type - oh dear */
 321         ret=-1;
 322         errno=EBADF;
 323         break;
 324     }
 325     return ret;
 326 }
 327
 328 size_t
 329 url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
 330 {
 331     size_t want;
 332
 333     switch(file->type)
 334     {
 335     case CFTYPE_FILE:
 336         want=fread(ptr,size,nmemb,file->handle.file);
 337         break;
 338
 339     case CFTYPE_CURL:
 340         want = nmemb * size;
 341
 342         fill_buffer(file,want,1);
 343
 344         /* check if theres data in the buffer - if not fill_buffer()
 345          * either errored or EOF */
 346         if(!file->buffer_pos)
 347             return 0;
 348
 349         /* ensure only available data is considered */
 350         if(file->buffer_pos < want)
 351             want = file->buffer_pos;
 352
 353         /* xfer data to caller */
 354         memcpy(ptr, file->buffer, want);
 355
 356         use_buffer(file,want);
 357
 358         want = want / size;     /* number of items - nb correct op - checked
 359                                  * with glibc code*/
 360
 361         /*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
 362         break;
 363
 364     default: /* unknown or supported type - oh dear */
 365         want=0;
 366         errno=EBADF;
 367         break;
 368
 369     }
 370     return want;
 371 }
 372
 373 char *
 374 url_fgets(char *ptr, int size, URL_FILE *file)
 375 {
 376     int want = size - 1;/* always need to leave room for zero termination */
 377     int loop;
 378
 379     switch(file->type)
 380     {
 381     case CFTYPE_FILE:
 382         ptr = fgets(ptr,size,file->handle.file);
 383         break;
 384
 385     case CFTYPE_CURL:
 386         fill_buffer(file,want,1);
 387
 388         /* check if theres data in the buffer - if not fill either errored or
 389          * EOF */
 390         if(!file->buffer_pos)
 391             return NULL;
 392
 393         /* ensure only available data is considered */
 394         if(file->buffer_pos < want)
 395             want = file->buffer_pos;
 396
 397         /*buffer contains data */
 398         /* look for newline or eof */
 399         for(loop=0;loop < want;loop++)
 400         {
 401             if(file->buffer[loop] == '\n')
 402             {
 403                 want=loop+1;/* include newline */
 404                 break;
 405             }
 406         }
 407
 408         /* xfer data to caller */
 409         memcpy(ptr, file->buffer, want);
 410         ptr[want]=0;/* allways null terminate */
 411
 412         use_buffer(file,want);
 413
 414         /*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
 415         break;
 416
 417     default: /* unknown or supported type - oh dear */
 418         ptr=NULL;
 419         errno=EBADF;
 420         break;
 421     }
 422
 423     return ptr;/*success */
 424 }
 425
 426 void
 427 url_rewind(URL_FILE *file)
 428 {
 429     switch(file->type)
 430     {
 431     case CFTYPE_FILE:
 432         rewind(file->handle.file); /* passthrough */
 433         break;
 434
 435     case CFTYPE_CURL:
 436         /* halt transaction */
 437         curl_multi_remove_handle(multi_handle, file->handle.curl);
 438
 439         /* restart */
 440         curl_multi_add_handle(multi_handle, file->handle.curl);
 441
 442         /* ditch buffer - write will recreate - resets stream pos*/
 443         if(file->buffer)
 444             free(file->buffer);
 445
 446         file->buffer=NULL;
 447         file->buffer_pos=0;
 448         file->buffer_len=0;
 449
 450         break;
 451
 452     default: /* unknown or supported type - oh dear */
 453         break;
 454
 455     }
 456
 457 }
 458
 459
 460 /* Small main program to retrive from a url using fgets and fread saving the
 461  * output to two test files (note the fgets method will corrupt binary files if
 462  * they contain 0 chars */
 463 int
 464 main(int argc, char *argv[])
 465 {
 466     URL_FILE *handle;
 467     FILE *outf;
 468
 469     int nread;
 470     char buffer[256];
 471     const char *url;
 472
 473     if(argc < 2)
 474     {
 475         url="http://192.168.7.3/testfile";/* default to testurl */
 476     }
 477     else
 478     {
 479         url=argv[1];/* use passed url */
 480     }
 481
 482     /* copy from url line by line with fgets */
 483     outf=fopen("fgets.test","w+");
 484     if(!outf)
 485     {
 486         perror("couldn't open fgets output file\n");
 487         return 1;
 488     }
 489
 490     handle = url_fopen(url, "r");
 491     if(!handle)
 492     {
 493         printf("couldn't url_fopen() %s\n", url);
 494         fclose(outf);
 495         return 2;
 496     }
 497
 498     while(!url_feof(handle))
 499     {
 500         url_fgets(buffer,sizeof(buffer),handle);
 501         fwrite(buffer,1,strlen(buffer),outf);
 502     }
 503
 504     url_fclose(handle);
 505
 506     fclose(outf);
 507
 508
 509     /* Copy from url with fread */
 510     outf=fopen("fread.test","w+");
 511     if(!outf)
 512     {
 513         perror("couldn't open fread output file\n");
 514         return 1;
 515     }
 516
 517     handle = url_fopen("testfile", "r");
 518     if(!handle) {
 519         printf("couldn't url_fopen() testfile\n");
 520         fclose(outf);
 521         return 2;
 522     }
 523
 524     do {
 525         nread = url_fread(buffer, 1,sizeof(buffer), handle);
 526         fwrite(buffer,1,nread,outf);
 527     } while(nread);
 528
 529     url_fclose(handle);
 530
 531     fclose(outf);
 532
 533
 534     /* Test rewind */
 535     outf=fopen("rewind.test","w+");
 536     if(!outf)
 537     {
 538         perror("couldn't open fread output file\n");
 539         return 1;
 540     }
 541
 542     handle = url_fopen("testfile", "r");
 543     if(!handle) {
 544         printf("couldn't url_fopen() testfile\n");
 545         fclose(outf);
 546         return 2;
 547     }
 548
 549         nread = url_fread(buffer, 1,sizeof(buffer), handle);
 550         fwrite(buffer,1,nread,outf);
 551         url_rewind(handle);
 552
 553         buffer[0]='\n';
 554         fwrite(buffer,1,1,outf);
 555
 556         nread = url_fread(buffer, 1,sizeof(buffer), handle);
 557         fwrite(buffer,1,nread,outf);
 558
 559
 560     url_fclose(handle);
 561
 562     fclose(outf);
 563
 564
 565     return 0;/* all done */
 566 }