lib/libfetch/fetch.c

   1 /*      $NetBSD: fetch.c,v 1.19 2009/08/11 20:48:06 joerg Exp $ */
   2 /*-
   3  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
   4  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer
  12  *    in this position and unchanged.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. The name of the author may not be used to endorse or promote products
  17  *    derived from this software without specific prior written permission
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  *
  30  * $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $
  31  */
  32
  33 #if HAVE_CONFIG_H
  34 #include "config.h"
  35 #endif
  36 #if !defined(NETBSD) && !defined(__minix)
  37 #include <nbcompat.h>
  38 #endif
  39
  40 #include <ctype.h>
  41 #include <errno.h>
  42 #include <stdio.h>
  43 #include <stdlib.h>
  44 #include <string.h>
  45
  46 #include "common.h"
  47
  48 auth_t   fetchAuthMethod;
  49 int      fetchLastErrCode;
  50 char     fetchLastErrString[MAXERRSTRING];
  51 int      fetchTimeout;
  52 volatile int     fetchRestartCalls = 1;
  53 int      fetchDebug;
  54
  55
  56 /*** Local data **************************************************************/
  57
  58 /*
  59  * Error messages for parser errors
  60  */
  61 #define URL_MALFORMED           1
  62 #define URL_BAD_SCHEME          2
  63 #define URL_BAD_PORT            3
  64 static struct fetcherr url_errlist[] = {
  65         { URL_MALFORMED,        FETCH_URL,      "Malformed URL" },
  66         { URL_BAD_SCHEME,       FETCH_URL,      "Invalid URL scheme" },
  67         { URL_BAD_PORT,         FETCH_URL,      "Invalid server port" },
  68         { -1,                   FETCH_UNKNOWN,  "Unknown parser error" }
  69 };
  70
  71
  72 /*** Public API **************************************************************/
  73
  74 /*
  75  * Select the appropriate protocol for the URL scheme, and return a
  76  * read-only stream connected to the document referenced by the URL.
  77  * Also fill out the struct url_stat.
  78  */
  79 fetchIO *
  80 fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
  81 {
  82
  83         if (us != NULL) {
  84                 us->size = -1;
  85                 us->atime = us->mtime = 0;
  86         }
  87         if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
  88                 return (fetchXGetFile(URL, us, flags));
  89         else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
  90                 return (fetchXGetFTP(URL, us, flags));
  91         else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
  92                 return (fetchXGetHTTP(URL, us, flags));
  93         else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
  94                 return (fetchXGetHTTP(URL, us, flags));
  95         url_seterr(URL_BAD_SCHEME);
  96         return (NULL);
  97 }
  98
  99 /*
 100  * Select the appropriate protocol for the URL scheme, and return a
 101  * read-only stream connected to the document referenced by the URL.
 102  */
 103 fetchIO *
 104 fetchGet(struct url *URL, const char *flags)
 105 {
 106         return (fetchXGet(URL, NULL, flags));
 107 }
 108
 109 /*
 110  * Select the appropriate protocol for the URL scheme, and return a
 111  * write-only stream connected to the document referenced by the URL.
 112  */
 113 fetchIO *
 114 fetchPut(struct url *URL, const char *flags)
 115 {
 116
 117         if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 118                 return (fetchPutFile(URL, flags));
 119         else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 120                 return (fetchPutFTP(URL, flags));
 121         else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 122                 return (fetchPutHTTP(URL, flags));
 123         else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 124                 return (fetchPutHTTP(URL, flags));
 125         url_seterr(URL_BAD_SCHEME);
 126         return (NULL);
 127 }
 128
 129 /*
 130  * Select the appropriate protocol for the URL scheme, and return the
 131  * size of the document referenced by the URL if it exists.
 132  */
 133 int
 134 fetchStat(struct url *URL, struct url_stat *us, const char *flags)
 135 {
 136
 137         if (us != NULL) {
 138                 us->size = -1;
 139                 us->atime = us->mtime = 0;
 140         }
 141         if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 142                 return (fetchStatFile(URL, us, flags));
 143         else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 144                 return (fetchStatFTP(URL, us, flags));
 145         else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 146                 return (fetchStatHTTP(URL, us, flags));
 147         else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 148                 return (fetchStatHTTP(URL, us, flags));
 149         url_seterr(URL_BAD_SCHEME);
 150         return (-1);
 151 }
 152
 153 /*
 154  * Select the appropriate protocol for the URL scheme, and return a
 155  * list of files in the directory pointed to by the URL.
 156  */
 157 int
 158 fetchList(struct url_list *ue, struct url *URL, const char *pattern,
 159     const char *flags)
 160 {
 161
 162         if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
 163                 return (fetchListFile(ue, URL, pattern, flags));
 164         else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
 165                 return (fetchListFTP(ue, URL, pattern, flags));
 166         else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
 167                 return (fetchListHTTP(ue, URL, pattern, flags));
 168         else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
 169                 return (fetchListHTTP(ue, URL, pattern, flags));
 170         url_seterr(URL_BAD_SCHEME);
 171         return -1;
 172 }
 173
 174 /*
 175  * Attempt to parse the given URL; if successful, call fetchXGet().
 176  */
 177 fetchIO *
 178 fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
 179 {
 180         struct url *u;
 181         fetchIO *f;
 182
 183         if ((u = fetchParseURL(URL)) == NULL)
 184                 return (NULL);
 185
 186         f = fetchXGet(u, us, flags);
 187
 188         fetchFreeURL(u);
 189         return (f);
 190 }
 191
 192 /*
 193  * Attempt to parse the given URL; if successful, call fetchGet().
 194  */
 195 fetchIO *
 196 fetchGetURL(const char *URL, const char *flags)
 197 {
 198         return (fetchXGetURL(URL, NULL, flags));
 199 }
 200
 201 /*
 202  * Attempt to parse the given URL; if successful, call fetchPut().
 203  */
 204 fetchIO *
 205 fetchPutURL(const char *URL, const char *flags)
 206 {
 207         struct url *u;
 208         fetchIO *f;
 209
 210         if ((u = fetchParseURL(URL)) == NULL)
 211                 return (NULL);
 212
 213         f = fetchPut(u, flags);
 214
 215         fetchFreeURL(u);
 216         return (f);
 217 }
 218
 219 /*
 220  * Attempt to parse the given URL; if successful, call fetchStat().
 221  */
 222 int
 223 fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
 224 {
 225         struct url *u;
 226         int s;
 227
 228         if ((u = fetchParseURL(URL)) == NULL)
 229                 return (-1);
 230
 231         s = fetchStat(u, us, flags);
 232
 233         fetchFreeURL(u);
 234         return (s);
 235 }
 236
 237 /*
 238  * Attempt to parse the given URL; if successful, call fetchList().
 239  */
 240 int
 241 fetchListURL(struct url_list *ue, const char *URL, const char *pattern,
 242     const char *flags)
 243 {
 244         struct url *u;
 245         int rv;
 246
 247         if ((u = fetchParseURL(URL)) == NULL)
 248                 return -1;
 249
 250         rv = fetchList(ue, u, pattern, flags);
 251
 252         fetchFreeURL(u);
 253         return rv;
 254 }
 255
 256 /*
 257  * Make a URL
 258  */
 259 struct url *
 260 fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
 261     const char *user, const char *pwd)
 262 {
 263         struct url *u;
 264
 265         if (!scheme || (!host && !doc)) {
 266                 url_seterr(URL_MALFORMED);
 267                 return (NULL);
 268         }
 269
 270         if (port < 0 || port > 65535) {
 271                 url_seterr(URL_BAD_PORT);
 272                 return (NULL);
 273         }
 274
 275         /* allocate struct url */
 276         if ((u = calloc(1, sizeof(*u))) == NULL) {
 277                 fetch_syserr();
 278                 return (NULL);
 279         }
 280
 281         if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
 282                 fetch_syserr();
 283                 free(u);
 284                 return (NULL);
 285         }
 286
 287 #define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
 288         seturl(scheme);
 289         seturl(host);
 290         seturl(user);
 291         seturl(pwd);
 292 #undef seturl
 293         u->port = port;
 294
 295         return (u);
 296 }
 297
 298 int
 299 fetch_urlpath_safe(char x)
 300 {
 301         if ((x >= '0' && x <= '9') || (x >= 'A' && x <= 'Z') ||
 302             (x >= 'a' && x <= 'z'))
 303                 return 1;
 304
 305         switch (x) {
 306         case '$':
 307         case '-':
 308         case '_':
 309         case '.':
 310         case '+':
 311         case '!':
 312         case '*':
 313         case '\'':
 314         case '(':
 315         case ')':
 316         case ',':
 317         /* The following are allowed in segment and path components: */
 318         case '?':
 319         case ':':
 320         case '@':
 321         case '&':
 322         case '=':
 323         case '/':
 324         case ';':
 325         /* If something is already quoted... */
 326         case '%':
 327                 return 1;
 328         default:
 329                 return 0;
 330         }
 331 }
 332
 333 /*
 334  * Copy an existing URL.
 335  */
 336 struct url *
 337 fetchCopyURL(const struct url *src)
 338 {
 339         struct url *dst;
 340         char *doc;
 341
 342         /* allocate struct url */
 343         if ((dst = malloc(sizeof(*dst))) == NULL) {
 344                 fetch_syserr();
 345                 return (NULL);
 346         }
 347         if ((doc = strdup(src->doc)) == NULL) {
 348                 fetch_syserr();
 349                 free(dst);
 350                 return (NULL);
 351         }
 352         *dst = *src;
 353         dst->doc = doc;
 354
 355         return dst;
 356 }
 357
 358 /*
 359  * Split an URL into components. URL syntax is:
 360  * [method:/][/[user[:pwd]@]host[:port]/][document]
 361  * This almost, but not quite, RFC1738 URL syntax.
 362  */
 363 struct url *
 364 fetchParseURL(const char *URL)
 365 {
 366         const char *p, *q;
 367         struct url *u;
 368         size_t i, count;
 369         int pre_quoted;
 370
 371         /* allocate struct url */
 372         if ((u = calloc(1, sizeof(*u))) == NULL) {
 373                 fetch_syserr();
 374                 return (NULL);
 375         }
 376
 377         if (*URL == '/') {
 378                 pre_quoted = 0;
 379                 strcpy(u->scheme, SCHEME_FILE);
 380                 p = URL;
 381                 goto quote_doc;
 382         }
 383         if (strncmp(URL, "file:", 5) == 0) {
 384                 pre_quoted = 1;
 385                 strcpy(u->scheme, SCHEME_FILE);
 386                 URL += 5;
 387                 if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') {
 388                         url_seterr(URL_MALFORMED);
 389                         goto ouch;
 390                 }
 391                 p = URL + 2;
 392                 goto quote_doc;
 393         }
 394         if (strncmp(URL, "http:", 5) == 0 ||
 395             strncmp(URL, "https:", 6) == 0) {
 396                 pre_quoted = 1;
 397                 if (URL[4] == ':') {
 398                         strcpy(u->scheme, SCHEME_HTTP);
 399                         URL += 5;
 400                 } else {
 401                         strcpy(u->scheme, SCHEME_HTTPS);
 402                         URL += 6;
 403                 }
 404
 405                 if (URL[0] != '/' || URL[1] != '/') {
 406                         url_seterr(URL_MALFORMED);
 407                         goto ouch;
 408                 }
 409                 URL += 2;
 410                 p = URL;
 411                 goto find_user;
 412         }
 413         if (strncmp(URL, "ftp:", 4) == 0) {
 414                 pre_quoted = 1;
 415                 strcpy(u->scheme, SCHEME_FTP);
 416                 URL += 4;
 417                 if (URL[0] != '/' || URL[1] != '/') {
 418                         url_seterr(URL_MALFORMED);
 419                         goto ouch;
 420                 }
 421                 URL += 2;
 422                 p = URL;
 423                 goto find_user;
 424         }
 425
 426         url_seterr(URL_BAD_SCHEME);
 427         goto ouch;
 428
 429 find_user:
 430         p = strpbrk(URL, "/@");
 431         if (p != NULL && *p == '@') {
 432                 /* username */
 433                 for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) {
 434                         if (i < URL_USERLEN)
 435                                 u->user[i++] = *q;
 436                 }
 437
 438                 /* password */
 439                 if (*q == ':') {
 440                         for (q++, i = 0; (*q != '@'); q++)
 441                                 if (i < URL_PWDLEN)
 442                                         u->pwd[i++] = *q;
 443                 }
 444
 445                 p++;
 446         } else {
 447                 p = URL;
 448         }
 449
 450         /* hostname */
 451 #ifdef INET6
 452         if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
 453             (*++q == '\0' || *q == '/' || *q == ':')) {
 454                 if ((i = q - p - 2) > URL_HOSTLEN)
 455                         i = URL_HOSTLEN;
 456                 strncpy(u->host, ++p, i);
 457                 p = q;
 458         } else
 459 #endif
 460                 for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
 461                         if (i < URL_HOSTLEN)
 462                                 u->host[i++] = *p;
 463
 464         /* port */
 465         if (*p == ':') {
 466                 for (q = ++p; *q && (*q != '/'); q++)
 467                         if (isdigit((unsigned char)*q))
 468                                 u->port = u->port * 10 + (*q - '0');
 469                         else {
 470                                 /* invalid port */
 471                                 url_seterr(URL_BAD_PORT);
 472                                 goto ouch;
 473                         }
 474                 p = q;
 475         }
 476
 477         /* document */
 478         if (!*p)
 479                 p = "/";
 480
 481 quote_doc:
 482         count = 1;
 483         for (i = 0; p[i] != '\0'; ++i) {
 484                 if ((!pre_quoted && p[i] == '%') ||
 485                     !fetch_urlpath_safe(p[i]))
 486                         count += 3;
 487                 else
 488                         ++count;
 489         }
 490
 491         if ((u->doc = malloc(count)) == NULL) {
 492                 fetch_syserr();
 493                 goto ouch;
 494         }
 495         for (i = 0; *p != '\0'; ++p) {
 496                 if ((!pre_quoted && *p == '%') ||
 497                     !fetch_urlpath_safe(*p)) {
 498                         u->doc[i++] = '%';
 499                         if ((unsigned char)*p < 160)
 500                                 u->doc[i++] = '0' + ((unsigned char)*p) / 16;
 501                         else
 502                                 u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16;
 503                         if ((unsigned char)*p % 16 < 10)
 504                                 u->doc[i++] = '0' + ((unsigned char)*p) % 16;
 505                         else
 506                                 u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16;
 507                 } else
 508                         u->doc[i++] = *p;
 509         }
 510         u->doc[i] = '\0';
 511
 512         return (u);
 513
 514 ouch:
 515         free(u);
 516         return (NULL);
 517 }
 518
 519 /*
 520  * Free a URL
 521  */
 522 void
 523 fetchFreeURL(struct url *u)
 524 {
 525         free(u->doc);
 526         free(u);
 527 }
 528
 529 static char
 530 xdigit2digit(char digit)
 531 {
 532         digit = tolower((unsigned char)digit);
 533         if (digit >= 'a' && digit <= 'f')
 534                 digit = digit - 'a' + 10;
 535         else
 536                 digit = digit - '0';
 537
 538         return digit;
 539 }
 540
 541 /*
 542  * Unquote whole URL.
 543  * Skips optional parts like query or fragment identifier.
 544  */
 545 char *
 546 fetchUnquotePath(struct url *url)
 547 {
 548         char *unquoted;
 549         const char *iter;
 550         size_t i;
 551
 552         if ((unquoted = malloc(strlen(url->doc) + 1)) == NULL)
 553                 return NULL;
 554
 555         for (i = 0, iter = url->doc; *iter != '\0'; ++iter) {
 556                 if (*iter == '#' || *iter == '?')
 557                         break;
 558                 if (iter[0] != '%' ||
 559                     !isxdigit((unsigned char)iter[1]) ||
 560                     !isxdigit((unsigned char)iter[2])) {
 561                         unquoted[i++] = *iter;
 562                         continue;
 563                 }
 564                 unquoted[i++] = xdigit2digit(iter[1]) * 16 +
 565                     xdigit2digit(iter[2]);
 566                 iter += 2;
 567         }
 568         unquoted[i] = '\0';
 569         return unquoted;
 570 }
 571
 572
 573 /*
 574  * Extract the file name component of a URL.
 575  */
 576 char *
 577 fetchUnquoteFilename(struct url *url)
 578 {
 579         char *unquoted, *filename;
 580         const char *last_slash;
 581
 582         if ((unquoted = fetchUnquotePath(url)) == NULL)
 583                 return NULL;
 584
 585         if ((last_slash = strrchr(unquoted, '/')) == NULL)
 586                 return unquoted;
 587         filename = strdup(last_slash + 1);
 588         free(unquoted);
 589         return filename;
 590 }
 591
 592 char *
 593 fetchStringifyURL(const struct url *url)
 594 {
 595         size_t total;
 596         char *doc;
 597
 598         /* scheme :// user : pwd @ host :port doc */
 599         total = strlen(url->scheme) + 3 + strlen(url->user) + 1 +
 600             strlen(url->pwd) + 1 + strlen(url->host) + 6 + strlen(url->doc) + 1;
 601         if ((doc = malloc(total)) == NULL)
 602                 return NULL;
 603         if (url->port != 0)
 604                 snprintf(doc, total, "%s%s%s%s%s%s%s:%d%s",
 605                     url->scheme,
 606                     url->scheme[0] != '\0' ? "://" : "",
 607                     url->user,
 608                     url->pwd[0] != '\0' ? ":" : "",
 609                     url->pwd,
 610                     url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
 611                     url->host,
 612                     (int)url->port,
 613                     url->doc);
 614         else {
 615                 snprintf(doc, total, "%s%s%s%s%s%s%s%s",
 616                     url->scheme,
 617                     url->scheme[0] != '\0' ? "://" : "",
 618                     url->user,
 619                     url->pwd[0] != '\0' ? ":" : "",
 620                     url->pwd,
 621                     url->user[0] != '\0' || url->pwd[0] != '\0' ? "@" : "",
 622                     url->host,
 623                     url->doc);
 624         }
 625         return doc;
 626 }