external/bsd/fetch/dist/libfetch/common.c

   1 /*      $NetBSD: common.c,v 1.21 2009/10/15 12:36:57 joerg Exp $        */
   2 /*-
   3  * Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
   4  * Copyright (c) 2008 Joerg Sonnenberger <joerg@NetBSD.org>
   5  * All rights reserved.
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  * 1. Redistributions of source code must retain the above copyright
  11  *    notice, this list of conditions and the following disclaimer
  12  *    in this position and unchanged.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. The name of the author may not be used to endorse or promote products
  17  *    derived from this software without specific prior written permission
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  *
  30  * $FreeBSD: common.c,v 1.53 2007/12/19 00:26:36 des Exp $
  31  */
  32
  33 #if HAVE_CONFIG_H
  34 #include "config.h"
  35 #endif
  36 #ifndef NETBSD
  37 #include <nbcompat.h>
  38 #endif
  39
  40 #include <sys/types.h>
  41 #include <sys/socket.h>
  42 #include <sys/time.h>
  43 #include <sys/uio.h>
  44
  45 #include <netinet/in.h>
  46 #include <arpa/inet.h>
  47
  48 #include <ctype.h>
  49 #include <errno.h>
  50 #if defined(HAVE_INTTYPES_H) || defined(NETBSD)
  51 #include <inttypes.h>
  52 #endif
  53 #ifndef NETBSD
  54 #include <nbcompat/netdb.h>
  55 #else
  56 #include <netdb.h>
  57 #endif
  58 #include <pwd.h>
  59 #include <stdarg.h>
  60 #include <stdlib.h>
  61 #include <stdio.h>
  62 #include <string.h>
  63 #include <unistd.h>
  64
  65 #include "fetch.h"
  66 #include "common.h"
  67
  68 #define DECONST(x,y) ((x)(uintptr_t)(y))
  69
  70
  71 /*** Local data **************************************************************/
  72
  73 /*
  74  * Error messages for resolver errors
  75  */
  76 static struct fetcherr netdb_errlist[] = {
  77 #ifdef EAI_NODATA
  78         { EAI_NODATA,   FETCH_RESOLV,   "Host not found" },
  79 #endif
  80         { EAI_AGAIN,    FETCH_TEMP,     "Transient resolver failure" },
  81         { EAI_FAIL,     FETCH_RESOLV,   "Non-recoverable resolver failure" },
  82         { EAI_NONAME,   FETCH_RESOLV,   "No address record" },
  83         { -1,           FETCH_UNKNOWN,  "Unknown resolver error" }
  84 };
  85
  86 /* End-of-Line */
  87 static const char ENDL[2] = "\r\n";
  88
  89
  90 /*** Error-reporting functions ***********************************************/
  91
  92 /*
  93  * Map error code to string
  94  */
  95 static struct fetcherr *
  96 fetch_finderr(struct fetcherr *p, int e)
  97 {
  98         while (p->num != -1 && p->num != e)
  99                 p++;
 100         return (p);
 101 }
 102
 103 /*
 104  * Set error code
 105  */
 106 void
 107 fetch_seterr(struct fetcherr *p, int e)
 108 {
 109         p = fetch_finderr(p, e);
 110         fetchLastErrCode = p->cat;
 111         snprintf(fetchLastErrString, MAXERRSTRING, "%s", p->string);
 112 }
 113
 114 /*
 115  * Set error code according to errno
 116  */
 117 void
 118 fetch_syserr(void)
 119 {
 120         switch (errno) {
 121         case 0:
 122                 fetchLastErrCode = FETCH_OK;
 123                 break;
 124         case EPERM:
 125         case EACCES:
 126         case EROFS:
 127 #ifdef EAUTH
 128         case EAUTH:
 129 #endif
 130 #ifdef ENEEDAUTH
 131         case ENEEDAUTH:
 132 #endif
 133                 fetchLastErrCode = FETCH_AUTH;
 134                 break;
 135         case ENOENT:
 136         case EISDIR: /* XXX */
 137                 fetchLastErrCode = FETCH_UNAVAIL;
 138                 break;
 139         case ENOMEM:
 140                 fetchLastErrCode = FETCH_MEMORY;
 141                 break;
 142         case EBUSY:
 143         case EAGAIN:
 144                 fetchLastErrCode = FETCH_TEMP;
 145                 break;
 146         case EEXIST:
 147                 fetchLastErrCode = FETCH_EXISTS;
 148                 break;
 149         case ENOSPC:
 150                 fetchLastErrCode = FETCH_FULL;
 151                 break;
 152         case EADDRINUSE:
 153         case EADDRNOTAVAIL:
 154         case ENETDOWN:
 155         case ENETUNREACH:
 156         case ENETRESET:
 157         case EHOSTUNREACH:
 158                 fetchLastErrCode = FETCH_NETWORK;
 159                 break;
 160         case ECONNABORTED:
 161         case ECONNRESET:
 162                 fetchLastErrCode = FETCH_ABORT;
 163                 break;
 164         case ETIMEDOUT:
 165                 fetchLastErrCode = FETCH_TIMEOUT;
 166                 break;
 167         case ECONNREFUSED:
 168         case EHOSTDOWN:
 169                 fetchLastErrCode = FETCH_DOWN;
 170                 break;
 171 default:
 172                 fetchLastErrCode = FETCH_UNKNOWN;
 173         }
 174         snprintf(fetchLastErrString, MAXERRSTRING, "%s", strerror(errno));
 175 }
 176
 177
 178 /*
 179  * Emit status message
 180  */
 181 void
 182 fetch_info(const char *fmt, ...)
 183 {
 184         va_list ap;
 185
 186         va_start(ap, fmt);
 187         vfprintf(stderr, fmt, ap);
 188         va_end(ap);
 189         fputc('\n', stderr);
 190 }
 191
 192
 193 /*** Network-related utility functions ***************************************/
 194
 195 /*
 196  * Return the default port for a scheme
 197  */
 198 int
 199 fetch_default_port(const char *scheme)
 200 {
 201         struct servent *se;
 202
 203         if ((se = getservbyname(scheme, "tcp")) != NULL)
 204                 return (ntohs(se->s_port));
 205         if (strcasecmp(scheme, SCHEME_FTP) == 0)
 206                 return (FTP_DEFAULT_PORT);
 207         if (strcasecmp(scheme, SCHEME_HTTP) == 0)
 208                 return (HTTP_DEFAULT_PORT);
 209         return (0);
 210 }
 211
 212 /*
 213  * Return the default proxy port for a scheme
 214  */
 215 int
 216 fetch_default_proxy_port(const char *scheme)
 217 {
 218         if (strcasecmp(scheme, SCHEME_FTP) == 0)
 219                 return (FTP_DEFAULT_PROXY_PORT);
 220         if (strcasecmp(scheme, SCHEME_HTTP) == 0)
 221                 return (HTTP_DEFAULT_PROXY_PORT);
 222         return (0);
 223 }
 224
 225
 226 /*
 227  * Create a connection for an existing descriptor.
 228  */
 229 conn_t *
 230 fetch_reopen(int sd)
 231 {
 232         conn_t *conn;
 233
 234         /* allocate and fill connection structure */
 235         if ((conn = calloc(1, sizeof(*conn))) == NULL)
 236                 return (NULL);
 237         conn->next_buf = NULL;
 238         conn->next_len = 0;
 239         conn->sd = sd;
 240         conn->is_active = 0;
 241         ++conn->ref;
 242         return (conn);
 243 }
 244
 245
 246 /*
 247  * Bump a connection's reference count.
 248  */
 249 conn_t *
 250 fetch_ref(conn_t *conn)
 251 {
 252
 253         ++conn->ref;
 254         return (conn);
 255 }
 256
 257
 258 /*
 259  * Bind a socket to a specific local address
 260  */
 261 int
 262 fetch_bind(int sd, int af, const char *addr)
 263 {
 264         struct addrinfo hints, *res, *res0;
 265
 266         memset(&hints, 0, sizeof(hints));
 267         hints.ai_family = af;
 268         hints.ai_socktype = SOCK_STREAM;
 269         hints.ai_protocol = 0;
 270         if (getaddrinfo(addr, NULL, &hints, &res0))
 271                 return (-1);
 272         for (res = res0; res; res = res->ai_next) {
 273                 if (bind(sd, res->ai_addr, res->ai_addrlen) == 0)
 274                         return (0);
 275         }
 276         return (-1);
 277 }
 278
 279
 280 /*
 281  * Establish a TCP connection to the specified port on the specified host.
 282  */
 283 conn_t *
 284 fetch_connect(const char *host, int port, int af, int verbose)
 285 {
 286         conn_t *conn;
 287         char pbuf[10];
 288         const char *bindaddr;
 289         struct addrinfo hints, *res, *res0;
 290         int sd, error;
 291
 292         if (verbose)
 293                 fetch_info("looking up %s", host);
 294
 295         /* look up host name and set up socket address structure */
 296         snprintf(pbuf, sizeof(pbuf), "%d", port);
 297         memset(&hints, 0, sizeof(hints));
 298         hints.ai_family = af;
 299         hints.ai_socktype = SOCK_STREAM;
 300         hints.ai_protocol = 0;
 301         if ((error = getaddrinfo(host, pbuf, &hints, &res0)) != 0) {
 302                 netdb_seterr(error);
 303                 return (NULL);
 304         }
 305         bindaddr = getenv("FETCH_BIND_ADDRESS");
 306
 307         if (verbose)
 308                 fetch_info("connecting to %s:%d", host, port);
 309
 310         /* try to connect */
 311         for (sd = -1, res = res0; res; sd = -1, res = res->ai_next) {
 312                 if ((sd = socket(res->ai_family, res->ai_socktype,
 313                          res->ai_protocol)) == -1)
 314                         continue;
 315                 if (bindaddr != NULL && *bindaddr != '\0' &&
 316                     fetch_bind(sd, res->ai_family, bindaddr) != 0) {
 317                         fetch_info("failed to bind to '%s'", bindaddr);
 318                         close(sd);
 319                         continue;
 320                 }
 321                 if (connect(sd, res->ai_addr, res->ai_addrlen) == 0)
 322                         break;
 323                 close(sd);
 324         }
 325         freeaddrinfo(res0);
 326         if (sd == -1) {
 327                 fetch_syserr();
 328                 return (NULL);
 329         }
 330
 331         if ((conn = fetch_reopen(sd)) == NULL) {
 332                 fetch_syserr();
 333                 close(sd);
 334         }
 335         return (conn);
 336 }
 337
 338
 339 /*
 340  * Enable SSL on a connection.
 341  */
 342 int
 343 fetch_ssl(conn_t *conn, int verbose)
 344 {
 345
 346 #ifdef WITH_SSL
 347         /* Init the SSL library and context */
 348         if (!SSL_library_init()){
 349                 fprintf(stderr, "SSL library init failed\n");
 350                 return (-1);
 351         }
 352
 353         SSL_load_error_strings();
 354
 355         conn->ssl_meth = SSLv23_client_method();
 356         conn->ssl_ctx = SSL_CTX_new(conn->ssl_meth);
 357         SSL_CTX_set_mode(conn->ssl_ctx, SSL_MODE_AUTO_RETRY);
 358
 359         conn->ssl = SSL_new(conn->ssl_ctx);
 360         if (conn->ssl == NULL){
 361                 fprintf(stderr, "SSL context creation failed\n");
 362                 return (-1);
 363         }
 364         SSL_set_fd(conn->ssl, conn->sd);
 365         if (SSL_connect(conn->ssl) == -1){
 366                 ERR_print_errors_fp(stderr);
 367                 return (-1);
 368         }
 369
 370         if (verbose) {
 371                 X509_NAME *name;
 372                 char *str;
 373
 374                 fprintf(stderr, "SSL connection established using %s\n",
 375                     SSL_get_cipher(conn->ssl));
 376                 conn->ssl_cert = SSL_get_peer_certificate(conn->ssl);
 377                 name = X509_get_subject_name(conn->ssl_cert);
 378                 str = X509_NAME_oneline(name, 0, 0);
 379                 printf("Certificate subject: %s\n", str);
 380                 free(str);
 381                 name = X509_get_issuer_name(conn->ssl_cert);
 382                 str = X509_NAME_oneline(name, 0, 0);
 383                 printf("Certificate issuer: %s\n", str);
 384                 free(str);
 385         }
 386
 387         return (0);
 388 #else
 389         (void)conn;
 390         (void)verbose;
 391         fprintf(stderr, "SSL support disabled\n");
 392         return (-1);
 393 #endif
 394 }
 395
 396
 397 /*
 398  * Read a character from a connection w/ timeout
 399  */
 400 ssize_t
 401 fetch_read(conn_t *conn, char *buf, size_t len)
 402 {
 403         struct timeval now, timeout, waittv;
 404         fd_set readfds;
 405         ssize_t rlen;
 406         int r;
 407
 408         if (len == 0)
 409                 return 0;
 410
 411         if (conn->next_len != 0) {
 412                 if (conn->next_len < len)
 413                         len = conn->next_len;
 414                 memmove(buf, conn->next_buf, len);
 415                 conn->next_len -= len;
 416                 conn->next_buf += len;
 417                 return len;
 418         }
 419
 420         if (fetchTimeout) {
 421                 FD_ZERO(&readfds);
 422                 gettimeofday(&timeout, NULL);
 423                 timeout.tv_sec += fetchTimeout;
 424         }
 425
 426         for (;;) {
 427                 while (fetchTimeout && !FD_ISSET(conn->sd, &readfds)) {
 428                         FD_SET(conn->sd, &readfds);
 429                         gettimeofday(&now, NULL);
 430                         waittv.tv_sec = timeout.tv_sec - now.tv_sec;
 431                         waittv.tv_usec = timeout.tv_usec - now.tv_usec;
 432                         if (waittv.tv_usec < 0) {
 433                                 waittv.tv_usec += 1000000;
 434                                 waittv.tv_sec--;
 435                         }
 436                         if (waittv.tv_sec < 0) {
 437                                 errno = ETIMEDOUT;
 438                                 fetch_syserr();
 439                                 return (-1);
 440                         }
 441                         errno = 0;
 442                         r = select(conn->sd + 1, &readfds, NULL, NULL, &waittv);
 443                         if (r == -1) {
 444                                 if (errno == EINTR && fetchRestartCalls)
 445                                         continue;
 446                                 fetch_syserr();
 447                                 return (-1);
 448                         }
 449                 }
 450 #ifdef WITH_SSL
 451                 if (conn->ssl != NULL)
 452                         rlen = SSL_read(conn->ssl, buf, len);
 453                 else
 454 #endif
 455                         rlen = read(conn->sd, buf, len);
 456                 if (rlen >= 0)
 457                         break;
 458
 459                 if (errno != EINTR || !fetchRestartCalls)
 460                         return (-1);
 461         }
 462         return (rlen);
 463 }
 464
 465
 466 /*
 467  * Read a line of text from a connection w/ timeout
 468  */
 469 #define MIN_BUF_SIZE 1024
 470
 471 int
 472 fetch_getln(conn_t *conn)
 473 {
 474         char *tmp, *next;
 475         size_t tmpsize;
 476         ssize_t len;
 477
 478         if (conn->buf == NULL) {
 479                 if ((conn->buf = malloc(MIN_BUF_SIZE)) == NULL) {
 480                         errno = ENOMEM;
 481                         return (-1);
 482                 }
 483                 conn->bufsize = MIN_BUF_SIZE;
 484         }
 485
 486         conn->buflen = 0;
 487         next = NULL;
 488
 489         do {
 490                 /*
 491                  * conn->bufsize != conn->buflen at this point,
 492                  * so the buffer can be NUL-terminated below for
 493                  * the case of len == 0.
 494                  */
 495                 len = fetch_read(conn, conn->buf + conn->buflen,
 496                     conn->bufsize - conn->buflen);
 497                 if (len == -1)
 498                         return (-1);
 499                 if (len == 0)
 500                         break;
 501                 next = memchr(conn->buf + conn->buflen, '\n', len);
 502                 conn->buflen += len;
 503                 if (conn->buflen == conn->bufsize && next == NULL) {
 504                         tmp = conn->buf;
 505                         tmpsize = conn->bufsize * 2;
 506                         if (tmpsize < conn->bufsize) {
 507                                 errno = ENOMEM;
 508                                 return (-1);
 509                         }
 510                         if ((tmp = realloc(tmp, tmpsize)) == NULL) {
 511                                 errno = ENOMEM;
 512                                 return (-1);
 513                         }
 514                         conn->buf = tmp;
 515                         conn->bufsize = tmpsize;
 516                 }
 517         } while (next == NULL);
 518
 519         if (next != NULL) {
 520                 *next = '\0';
 521                 conn->next_buf = next + 1;
 522                 conn->next_len = conn->buflen - (conn->next_buf - conn->buf);
 523                 conn->buflen = next - conn->buf;
 524         } else {
 525                 conn->buf[conn->buflen] = '\0';
 526                 conn->next_len = 0;
 527         }
 528         return (0);
 529 }
 530
 531
 532 /*
 533  * Write to a connection w/ timeout
 534  */
 535 ssize_t
 536 fetch_write(conn_t *conn, const char *buf, size_t len)
 537 {
 538         struct iovec iov;
 539
 540         iov.iov_base = DECONST(char *, buf);
 541         iov.iov_len = len;
 542         return fetch_writev(conn, &iov, 1);
 543 }
 544
 545 /*
 546  * Write a vector to a connection w/ timeout
 547  * Note: can modify the iovec.
 548  */
 549 ssize_t
 550 fetch_writev(conn_t *conn, struct iovec *iov, int iovcnt)
 551 {
 552         struct timeval now, timeout, waittv;
 553         fd_set writefds;
 554         ssize_t wlen, total;
 555         int r;
 556
 557         if (fetchTimeout) {
 558                 FD_ZERO(&writefds);
 559                 gettimeofday(&timeout, NULL);
 560                 timeout.tv_sec += fetchTimeout;
 561         }
 562
 563         total = 0;
 564         while (iovcnt > 0) {
 565                 while (fetchTimeout && !FD_ISSET(conn->sd, &writefds)) {
 566                         FD_SET(conn->sd, &writefds);
 567                         gettimeofday(&now, NULL);
 568                         waittv.tv_sec = timeout.tv_sec - now.tv_sec;
 569                         waittv.tv_usec = timeout.tv_usec - now.tv_usec;
 570                         if (waittv.tv_usec < 0) {
 571                                 waittv.tv_usec += 1000000;
 572                                 waittv.tv_sec--;
 573                         }
 574                         if (waittv.tv_sec < 0) {
 575                                 errno = ETIMEDOUT;
 576                                 fetch_syserr();
 577                                 return (-1);
 578                         }
 579                         errno = 0;
 580                         r = select(conn->sd + 1, NULL, &writefds, NULL, &waittv);
 581                         if (r == -1) {
 582                                 if (errno == EINTR && fetchRestartCalls)
 583                                         continue;
 584                                 return (-1);
 585                         }
 586                 }
 587                 errno = 0;
 588 #ifdef WITH_SSL
 589                 if (conn->ssl != NULL)
 590                         wlen = SSL_write(conn->ssl,
 591                             iov->iov_base, iov->iov_len);
 592                 else
 593 #endif
 594                         wlen = writev(conn->sd, iov, iovcnt);
 595                 if (wlen == 0) {
 596                         /* we consider a short write a failure */
 597                         errno = EPIPE;
 598                         fetch_syserr();
 599                         return (-1);
 600                 }
 601                 if (wlen < 0) {
 602                         if (errno == EINTR && fetchRestartCalls)
 603                                 continue;
 604                         return (-1);
 605                 }
 606                 total += wlen;
 607                 while (iovcnt > 0 && wlen >= (ssize_t)iov->iov_len) {
 608                         wlen -= iov->iov_len;
 609                         iov++;
 610                         iovcnt--;
 611                 }
 612                 if (iovcnt > 0) {
 613                         iov->iov_len -= wlen;
 614                         iov->iov_base = DECONST(char *, iov->iov_base) + wlen;
 615                 }
 616         }
 617         return (total);
 618 }
 619
 620
 621 /*
 622  * Write a line of text to a connection w/ timeout
 623  */
 624 int
 625 fetch_putln(conn_t *conn, const char *str, size_t len)
 626 {
 627         struct iovec iov[2];
 628         ssize_t ret;
 629
 630         iov[0].iov_base = DECONST(char *, str);
 631         iov[0].iov_len = len;
 632         iov[1].iov_base = DECONST(char *, ENDL);
 633         iov[1].iov_len = sizeof(ENDL);
 634         if (len == 0)
 635                 ret = fetch_writev(conn, &iov[1], 1);
 636         else
 637                 ret = fetch_writev(conn, iov, 2);
 638         if (ret == -1)
 639                 return (-1);
 640         return (0);
 641 }
 642
 643
 644 /*
 645  * Close connection
 646  */
 647 int
 648 fetch_close(conn_t *conn)
 649 {
 650         int ret;
 651
 652         if (--conn->ref > 0)
 653                 return (0);
 654         ret = close(conn->sd);
 655         free(conn->buf);
 656         free(conn);
 657         return (ret);
 658 }
 659
 660
 661 /*** Directory-related utility functions *************************************/
 662
 663 int
 664 fetch_add_entry(struct url_list *ue, struct url *base, const char *name,
 665     int pre_quoted)
 666 {
 667         struct url *tmp;
 668         char *tmp_name;
 669         size_t base_doc_len, name_len, i;
 670         unsigned char c;
 671
 672         if (strchr(name, '/') != NULL ||
 673             strcmp(name, "..") == 0 ||
 674             strcmp(name, ".") == 0)
 675                 return 0;
 676
 677         if (strcmp(base->doc, "/") == 0)
 678                 base_doc_len = 0;
 679         else
 680                 base_doc_len = strlen(base->doc);
 681
 682         name_len = 1;
 683         for (i = 0; name[i] != '\0'; ++i) {
 684                 if ((!pre_quoted && name[i] == '%') ||
 685                     !fetch_urlpath_safe(name[i]))
 686                         name_len += 3;
 687                 else
 688                         ++name_len;
 689         }
 690
 691         tmp_name = malloc( base_doc_len + name_len + 1);
 692         if (tmp_name == NULL) {
 693                 errno = ENOMEM;
 694                 fetch_syserr();
 695                 return (-1);
 696         }
 697
 698         if (ue->length + 1 >= ue->alloc_size) {
 699                 tmp = realloc(ue->urls, (ue->alloc_size * 2 + 1) * sizeof(*tmp));
 700                 if (tmp == NULL) {
 701                         free(tmp_name);
 702                         errno = ENOMEM;
 703                         fetch_syserr();
 704                         return (-1);
 705                 }
 706                 ue->alloc_size = ue->alloc_size * 2 + 1;
 707                 ue->urls = tmp;
 708         }
 709
 710         tmp = ue->urls + ue->length;
 711         strcpy(tmp->scheme, base->scheme);
 712         strcpy(tmp->user, base->user);
 713         strcpy(tmp->pwd, base->pwd);
 714         strcpy(tmp->host, base->host);
 715         tmp->port = base->port;
 716         tmp->doc = tmp_name;
 717         memcpy(tmp->doc, base->doc, base_doc_len);
 718         tmp->doc[base_doc_len] = '/';
 719
 720         for (i = base_doc_len + 1; *name != '\0'; ++name) {
 721                 if ((!pre_quoted && *name == '%') ||
 722                     !fetch_urlpath_safe(*name)) {
 723                         tmp->doc[i++] = '%';
 724                         c = (unsigned char)*name / 16;
 725                         if (c < 10)
 726                                 tmp->doc[i++] = '0' + c;
 727                         else
 728                                 tmp->doc[i++] = 'a' - 10 + c;
 729                         c = (unsigned char)*name % 16;
 730                         if (c < 10)
 731                                 tmp->doc[i++] = '0' + c;
 732                         else
 733                                 tmp->doc[i++] = 'a' - 10 + c;
 734                 } else {
 735                         tmp->doc[i++] = *name;
 736                 }
 737         }
 738         tmp->doc[i] = '\0';
 739
 740         tmp->offset = 0;
 741         tmp->length = 0;
 742         tmp->last_modified = -1;
 743
 744         ++ue->length;
 745
 746         return (0);
 747 }
 748
 749 void
 750 fetchInitURLList(struct url_list *ue)
 751 {
 752         ue->length = ue->alloc_size = 0;
 753         ue->urls = NULL;
 754 }
 755
 756 int
 757 fetchAppendURLList(struct url_list *dst, const struct url_list *src)
 758 {
 759         size_t i, j, len;
 760
 761         len = dst->length + src->length;
 762         if (len > dst->alloc_size) {
 763                 struct url *tmp;
 764
 765                 tmp = realloc(dst->urls, len * sizeof(*tmp));
 766                 if (tmp == NULL) {
 767                         errno = ENOMEM;
 768                         fetch_syserr();
 769                         return (-1);
 770                 }
 771                 dst->alloc_size = len;
 772                 dst->urls = tmp;
 773         }
 774
 775         for (i = 0, j = dst->length; i < src->length; ++i, ++j) {
 776                 dst->urls[j] = src->urls[i];
 777                 dst->urls[j].doc = strdup(src->urls[i].doc);
 778                 if (dst->urls[j].doc == NULL) {
 779                         while (i-- > 0)
 780                                 free(dst->urls[j].doc);
 781                         fetch_syserr();
 782                         return -1;
 783                 }
 784         }
 785         dst->length = len;
 786
 787         return 0;
 788 }
 789
 790 void
 791 fetchFreeURLList(struct url_list *ue)
 792 {
 793         size_t i;
 794
 795         for (i = 0; i < ue->length; ++i)
 796                 free(ue->urls[i].doc);
 797         free(ue->urls);
 798         ue->length = ue->alloc_size = 0;
 799 }
 800
 801
 802 /*** Authentication-related utility functions ********************************/
 803
 804 static const char *
 805 fetch_read_word(FILE *f)
 806 {
 807         static char word[1024];
 808
 809         if (fscanf(f, " %1023s ", word) != 1)
 810                 return (NULL);
 811         return (word);
 812 }
 813
 814 /*
 815  * Get authentication data for a URL from .netrc
 816  */
 817 int
 818 fetch_netrc_auth(struct url *url)
 819 {
 820         char fn[PATH_MAX];
 821         const char *word;
 822         char *p;
 823         FILE *f;
 824
 825         if ((p = getenv("NETRC")) != NULL) {
 826                 if (snprintf(fn, sizeof(fn), "%s", p) >= (int)sizeof(fn)) {
 827                         fetch_info("$NETRC specifies a file name "
 828                             "longer than PATH_MAX");
 829                         return (-1);
 830                 }
 831         } else {
 832                 if ((p = getenv("HOME")) != NULL) {
 833                         struct passwd *pwd;
 834
 835                         if ((pwd = getpwuid(getuid())) == NULL ||
 836                             (p = pwd->pw_dir) == NULL)
 837                                 return (-1);
 838                 }
 839                 if (snprintf(fn, sizeof(fn), "%s/.netrc", p) >= (int)sizeof(fn))
 840                         return (-1);
 841         }
 842
 843         if ((f = fopen(fn, "r")) == NULL)
 844                 return (-1);
 845         while ((word = fetch_read_word(f)) != NULL) {
 846                 if (strcmp(word, "default") == 0)
 847                         break;
 848                 if (strcmp(word, "machine") == 0 &&
 849                     (word = fetch_read_word(f)) != NULL &&
 850                     strcasecmp(word, url->host) == 0) {
 851                         break;
 852                 }
 853         }
 854         if (word == NULL)
 855                 goto ferr;
 856         while ((word = fetch_read_word(f)) != NULL) {
 857                 if (strcmp(word, "login") == 0) {
 858                         if ((word = fetch_read_word(f)) == NULL)
 859                                 goto ferr;
 860                         if (snprintf(url->user, sizeof(url->user),
 861                                 "%s", word) > (int)sizeof(url->user)) {
 862                                 fetch_info("login name in .netrc is too long");
 863                                 url->user[0] = '\0';
 864                         }
 865                 } else if (strcmp(word, "password") == 0) {
 866                         if ((word = fetch_read_word(f)) == NULL)
 867                                 goto ferr;
 868                         if (snprintf(url->pwd, sizeof(url->pwd),
 869                                 "%s", word) > (int)sizeof(url->pwd)) {
 870                                 fetch_info("password in .netrc is too long");
 871                                 url->pwd[0] = '\0';
 872                         }
 873                 } else if (strcmp(word, "account") == 0) {
 874                         if ((word = fetch_read_word(f)) == NULL)
 875                                 goto ferr;
 876                         /* XXX not supported! */
 877                 } else {
 878                         break;
 879                 }
 880         }
 881         fclose(f);
 882         return (0);
 883  ferr:
 884         fclose(f);
 885         return (-1);
 886 }
 887
 888 /*
 889  * The no_proxy environment variable specifies a set of domains for
 890  * which the proxy should not be consulted; the contents is a comma-,
 891  * or space-separated list of domain names.  A single asterisk will
 892  * override all proxy variables and no transactions will be proxied
 893  * (for compatability with lynx and curl, see the discussion at
 894  * <http://curl.haxx.se/mail/archive_pre_oct_99/0009.html>).
 895  */
 896 int
 897 fetch_no_proxy_match(const char *host)
 898 {
 899         const char *no_proxy, *p, *q;
 900         size_t h_len, d_len;
 901
 902         if ((no_proxy = getenv("NO_PROXY")) == NULL &&
 903             (no_proxy = getenv("no_proxy")) == NULL)
 904                 return (0);
 905
 906         /* asterisk matches any hostname */
 907         if (strcmp(no_proxy, "*") == 0)
 908                 return (1);
 909
 910         h_len = strlen(host);
 911         p = no_proxy;
 912         do {
 913                 /* position p at the beginning of a domain suffix */
 914                 while (*p == ',' || isspace((unsigned char)*p))
 915                         p++;
 916
 917                 /* position q at the first separator character */
 918                 for (q = p; *q; ++q)
 919                         if (*q == ',' || isspace((unsigned char)*q))
 920                                 break;
 921
 922                 d_len = q - p;
 923                 if (d_len > 0 && h_len > d_len &&
 924                     strncasecmp(host + h_len - d_len,
 925                         p, d_len) == 0) {
 926                         /* domain name matches */
 927                         return (1);
 928                 }
 929
 930                 p = q + 1;
 931         } while (*q);
 932
 933         return (0);
 934 }
 935
 936 struct fetchIO {
 937         void *io_cookie;
 938         ssize_t (*io_read)(void *, void *, size_t);
 939         ssize_t (*io_write)(void *, const void *, size_t);
 940         void (*io_close)(void *);
 941 };
 942
 943 void
 944 fetchIO_close(fetchIO *f)
 945 {
 946         if (f->io_close != NULL)
 947                 (*f->io_close)(f->io_cookie);
 948
 949         free(f);
 950 }
 951
 952 fetchIO *
 953 fetchIO_unopen(void *io_cookie, ssize_t (*io_read)(void *, void *, size_t),
 954     ssize_t (*io_write)(void *, const void *, size_t),
 955     void (*io_close)(void *))
 956 {
 957         fetchIO *f;
 958
 959         f = malloc(sizeof(*f));
 960         if (f == NULL)
 961                 return f;
 962
 963         f->io_cookie = io_cookie;
 964         f->io_read = io_read;
 965         f->io_write = io_write;
 966         f->io_close = io_close;
 967
 968         return f;
 969 }
 970
 971 ssize_t
 972 fetchIO_read(fetchIO *f, void *buf, size_t len)
 973 {
 974         if (f->io_read == NULL)
 975                 return EBADF;
 976         return (*f->io_read)(f->io_cookie, buf, len);
 977 }
 978
 979 ssize_t
 980 fetchIO_write(fetchIO *f, const void *buf, size_t len)
 981 {
 982         if (f->io_read == NULL)
 983                 return EBADF;
 984         return (*f->io_write)(f->io_cookie, buf, len);
 985 }