diskcache.c

   1 /*
   2 Copyright (c) 2003-2010 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheWriteoutOnClose = (64 * 1024);
  42
  43 int maxDiskCacheEntrySize = -1;
  44
  45 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  46 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  47 int diskCacheTruncateSize =  1024 * 1024;
  48 int preciseExpiry = 0;
  49
  50 static DiskCacheEntryRec negativeEntry = {
  51     NULL, NULL,
  52     -1, -1, -1, -1, 0, 0, NULL, NULL
  53 };
  54
  55 #ifndef LOCAL_ROOT
  56 #define LOCAL_ROOT "/usr/share/polipo/www/"
  57 #endif
  58
  59 #ifndef DISK_CACHE_ROOT
  60 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  61 #endif
  62
  63 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  64 static int atomSetterFlush(ConfigVariablePtr, void*);
  65 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  66
  67 void
  68 preinitDiskcache()
  69 {
  70     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  71     localDocumentRoot = internAtom(LOCAL_ROOT);
  72
  73     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  74                              configIntSetter,
  75                              "Access rights for new directories.");
  76     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  77                              configIntSetter,
  78                              "Access rights for new cache files.");
  79     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  80                              configIntSetter,
  81                              "Number of bytes to write out eagerly.");
  82     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  83                              "Root of the disk cache.");
  84     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  85                              "Root of the local tree.");
  86     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  87                     "File descriptors used by the on-disk cache.");
  88     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  89                     "Time after which on-disk objects are removed.");
  90     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  91                     "Time after which on-disk objects are truncated.");
  92     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  93                     "Size to which on-disk objects are truncated.");
  94     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  95                     "Whether to consider all files for purging.");
  96     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  97                              configIntSetter,
  98                              "Maximum size of objects cached on disk.");
  99 }
 100
 101 static int
 102 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 103 {
 104     int i;
 105     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 106     i = *(int*)value;
 107     if(i < 0 || i > 1000000)
 108         return -3;
 109     maxDiskEntries = i;
 110     while(numDiskEntries > maxDiskEntries)
 111         destroyDiskEntry(diskEntriesLast->object, 0);
 112     return 1;
 113 }
 114
 115 static int
 116 atomSetterFlush(ConfigVariablePtr var, void *value)
 117 {
 118     discardObjects(1, 0);
 119     return configAtomSetter(var, value);
 120 }
 121
 122 static int
 123 checkRoot(AtomPtr root)
 124 {
 125     struct stat ss;
 126     int rc;
 127
 128     if(!root || root->length == 0)
 129         return 0;
 130
 131     if(root->string[0] != '/') {
 132         return -2;
 133     }
 134
 135     rc = stat(root->string, &ss);
 136     if(rc < 0)
 137         return -1;
 138     else if(!S_ISDIR(ss.st_mode)) {
 139         errno = ENOTDIR;
 140         return -1;
 141     }
 142     return 1;
 143 }
 144
 145 static AtomPtr
 146 maybeAddSlash(AtomPtr atom)
 147 {
 148     AtomPtr newAtom = NULL;
 149     if(!atom) return NULL;
 150     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 151         newAtom = atomCat(atom, "/");
 152         releaseAtom(atom);
 153         return newAtom;
 154     }
 155     return atom;
 156 }
 157
 158 void
 159 initDiskcache()
 160 {
 161     int rc;
 162
 163     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 164     rc = checkRoot(diskCacheRoot);
 165     if(rc <= 0) {
 166         switch(rc) {
 167         case 0: break;
 168         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 169         case -2:
 170             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 171                    diskCacheRoot->string);
 172             break;
 173         default: abort();
 174         }
 175         releaseAtom(diskCacheRoot);
 176         diskCacheRoot = NULL;
 177     }
 178
 179     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 180     rc = checkRoot(localDocumentRoot);
 181     if(rc <= 0) {
 182         switch(rc) {
 183         case 0: break;
 184         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 185         case -2:
 186             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 187             break;
 188         default: abort();
 189         }
 190         releaseAtom(localDocumentRoot);
 191         localDocumentRoot = NULL;
 192     }
 193 }
 194
 195 #ifdef DEBUG_DISK_CACHE
 196 #define CHECK_ENTRY(entry) check_entry((entry))
 197 static void
 198 check_entry(DiskCacheEntryPtr entry)
 199 {
 200     if(entry && entry->fd < 0)
 201         assert(entry == &negativeEntry);
 202     if(entry && entry->fd >= 0) {
 203         assert((!entry->previous) == (entry == diskEntries));
 204         assert((!entry->next) == (entry == diskEntriesLast));
 205         if(entry->size >= 0)
 206             assert(entry->size + entry->body_offset >= entry->offset);
 207         assert(entry->body_offset >= 0);
 208         if(entry->offset >= 0) {
 209             off_t offset;
 210             offset = lseek(entry->fd, 0, SEEK_CUR);
 211             assert(offset == entry->offset);
 212         }
 213         if(entry->size >= 0) {
 214             int rc;
 215             struct stat ss;
 216             rc = fstat(entry->fd, &ss);
 217             assert(rc >= 0);
 218             assert(ss.st_size == entry->size + entry->body_offset);
 219         }
 220     }
 221 }
 222 #else
 223 #define CHECK_ENTRY(entry) do {} while(0)
 224 #endif
 225
 226 int
 227 diskEntrySize(ObjectPtr object)
 228 {
 229     struct stat buf;
 230     int rc;
 231     DiskCacheEntryPtr entry = object->disk_entry;
 232
 233     if(!entry || entry == &negativeEntry)
 234         return -1;
 235
 236     if(entry->size >= 0)
 237         return entry->size;
 238
 239     rc = fstat(entry->fd, &buf);
 240     if(rc < 0) {
 241         do_log_error(L_ERROR, errno, "Couldn't stat");
 242         return -1;
 243     }
 244
 245     if(buf.st_size <= entry->body_offset)
 246         entry->size = 0;
 247     else
 248         entry->size =  buf.st_size - entry->body_offset;
 249     CHECK_ENTRY(entry);
 250     if(object->length >= 0 && entry->size == object->length)
 251         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 252     return entry->size;
 253 }
 254
 255 static int
 256 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 257 {
 258     off_t rc;
 259
 260     CHECK_ENTRY(entry);
 261     assert(entry != &negativeEntry);
 262     if(entry->offset == offset)
 263         return 1;
 264     if(offset > entry->body_offset) {
 265         /* Avoid extending the file by mistake */
 266         if(entry->size < 0)
 267             diskEntrySize(entry->object);
 268         if(entry->size < 0)
 269             return -1;
 270         if(entry->size + entry->body_offset < offset)
 271             return -1;
 272     }
 273     rc = lseek(entry->fd, offset, SEEK_SET);
 274     if(rc < 0) {
 275         do_log_error(L_ERROR, errno, "Couldn't seek");
 276         entry->offset = -1;
 277         return -1;
 278     }
 279     entry->offset = offset;
 280     return 1;
 281 }
 282
 283 /* Given a local URL, constructs the filename where it can be found. */
 284
 285 int
 286 localFilename(char *buf, int n, char *key, int len)
 287 {
 288     int i, j;
 289     if(len <= 0 || key[0] != '/') return -1;
 290
 291     if(urlIsSpecial(key, len)) return -1;
 292
 293     if(localDocumentRoot == NULL ||
 294        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 295         return -1;
 296
 297     if(n <= localDocumentRoot->length)
 298         return -1;
 299
 300     i = 0;
 301     if(key[i] != '/')
 302         return -1;
 303
 304     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 305     j = localDocumentRoot->length;
 306     if(buf[j - 1] == '/')
 307         j--;
 308
 309     while(i < len) {
 310         if(j >= n - 1)
 311             return -1;
 312         if(key[i] == '/' && i < len - 2)
 313             if(key[i + 1] == '.' &&
 314                (key[i + 2] == '.' || key[i + 2] == '/'))
 315                 return -1;
 316         buf[j++] = key[i++];
 317     }
 318
 319     if(buf[j - 1] == '/') {
 320         if(j >= n - 11)
 321             return -1;
 322         memcpy(buf + j, "index.html", 10);
 323         j += 10;
 324     }
 325
 326     buf[j] = '\0';
 327     return j;
 328 }
 329
 330 static void
 331 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 332 {
 333     static MD5_CTX ctx;
 334     MD5Init(&ctx);
 335     MD5Update(&ctx, key, len);
 336     MD5Final(&ctx);
 337     memcpy(dst, ctx.digest, 16);
 338 }
 339
 340 /* Check whether a character can be stored in a filename.  This is
 341    needed since we want to support deficient file systems. */
 342 static int
 343 fssafe(char c)
 344 {
 345     if(c <= 31 || c >= 127)
 346         return 0;
 347     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 348        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 349         return 1;
 350     return 0;
 351 }
 352
 353 /* Given a URL, returns the directory name within which all files
 354    starting with this URL can be found. */
 355 static int
 356 urlDirname(char *buf, int n, const char *url, int len)
 357 {
 358     int i, j;
 359     if(len < 8)
 360         return -1;
 361     if(memcmp(url, "http://", 7) != 0)
 362         return -1;
 363
 364     if(diskCacheRoot == NULL ||
 365        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 366         return -1;
 367
 368     if(n <= diskCacheRoot->length)
 369         return -1;
 370
 371     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 372     j = diskCacheRoot->length;
 373
 374     if(buf[j - 1] != '/')
 375         buf[j++] = '/';
 376
 377     for(i = 7; i < len; i++) {
 378         if(i >= len || url[i] == '/')
 379             break;
 380         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 381             return -1;
 382         if(url[i] == '%' || !fssafe(url[i])) {
 383             if(j + 3 >= n) return -1;
 384             buf[j++] = '%';
 385             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 386             buf[j++] = i2h(url[i] & 0x0F);
 387         } else {
 388             buf[j++] = url[i]; if(j >= n) return -1;
 389         }
 390     }
 391     buf[j++] = '/'; if(j >= n) return -1;
 392     buf[j] = '\0';
 393     return j;
 394 }
 395
 396 /* Given a URL, returns the filename where the cached data can be
 397    found. */
 398 static int
 399 urlFilename(char *restrict buf, int n, const char *url, int len)
 400 {
 401     int j;
 402     unsigned char md5buf[18];
 403     j = urlDirname(buf, n, url, len);
 404     if(j < 0 || j + 24 >= n)
 405         return -1;
 406     md5((unsigned char*)url, len, md5buf);
 407     b64cpy(buf + j, (char*)md5buf, 16, 1);
 408     buf[j + 24] = '\0';
 409     return j + 24;
 410 }
 411
 412 static char *
 413 dirnameUrl(char *url, int n, char *name, int len)
 414 {
 415     int i, j, k, c1, c2;
 416     k = diskCacheRoot->length;
 417     if(len < k)
 418         return NULL;
 419     if(memcmp(name, diskCacheRoot->string, k) != 0)
 420         return NULL;
 421     if(n < 8)
 422         return NULL;
 423     memcpy(url, "http://", 7);
 424     if(name[len - 1] == '/')
 425         len --;
 426     j = 7;
 427     for(i = k; i < len; i++) {
 428         if(name[i] == '%') {
 429             if(i >= len - 2)
 430                 return NULL;
 431             c1 = h2i(name[i + 1]);
 432             c2 = h2i(name[i + 2]);
 433             if(c1 < 0 || c2 < 0)
 434                 return NULL;
 435             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 436             i += 2;             /* skip extra digits */
 437         } else if(i < len - 1 &&
 438                   name[i] == '.' && name[i + 1] == '/') {
 439                 return NULL;
 440         } else if(i == len - 1 && name[i] == '.') {
 441             return NULL;
 442         } else {
 443             url[j++] = name[i]; if(j >= n) goto fail;
 444         }
 445     }
 446     url[j++] = '/'; if(j >= n) goto fail;
 447     url[j] = '\0';
 448     return url;
 449
 450  fail:
 451     return NULL;
 452 }
 453
 454 /* Create a file and all intermediate directories. */
 455 static int
 456 createFile(const char *name, int path_start)
 457 {
 458     int fd;
 459     char buf[1024];
 460     int n;
 461     int rc;
 462
 463     if(name[path_start] == '/')
 464         path_start++;
 465
 466     if(path_start < 2 || name[path_start - 1] != '/' ) {
 467         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 468         return -1;
 469     }
 470
 471     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 472               diskCacheFilePermissions);
 473     if(fd >= 0)
 474         return fd;
 475     if(errno != ENOENT) {
 476         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 477         return -1;
 478     }
 479
 480     n = path_start;
 481     while(name[n] != '\0' && n < 1024) {
 482         while(name[n] != '/' && name[n] != '\0' && n < 512)
 483             n++;
 484         if(name[n] != '/' || n >= 1024)
 485             break;
 486         memcpy(buf, name, n + 1);
 487         buf[n + 1] = '\0';
 488         rc = mkdir(buf, diskCacheDirectoryPermissions);
 489         if(rc < 0 && errno != EEXIST) {
 490             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 491             return -1;
 492         }
 493         n++;
 494     }
 495     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 496               diskCacheFilePermissions);
 497     if(fd < 0) {
 498         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 499         return -1;
 500     }
 501
 502     return fd;
 503 }
 504
 505 static int
 506 chooseBodyOffset(int n, ObjectPtr object)
 507 {
 508     int length = MAX(object->size, object->length);
 509     int body_offset;
 510
 511     if(object->length >= 0 && object->length + n < 4096 - 4)
 512         return -1;              /* no gap for small objects */
 513
 514     if(n <= 128)
 515         body_offset = 256;
 516     else if(n <= 192)
 517         body_offset = 384;
 518     else if(n <= 256)
 519         body_offset = 512;
 520     else if(n <= 384)
 521         body_offset = 768;
 522     else if(n <= 512)
 523         body_offset = 1024;
 524     else if(n <= 1024)
 525         body_offset = 2048;
 526     else if(n < 2048)
 527         body_offset = 4096;
 528     else
 529         body_offset = ((n + 32 + 4095) / 4096 + 1) * 4096;
 530
 531     /* Tweak the gap so that we don't use up a full disk block for
 532        a small tail */
 533     if(object->length >= 0 && object->length < 64 * 1024) {
 534         int last = (body_offset + object->length) % 4096;
 535         int gap = body_offset - n - 32;
 536         if(last < gap / 2)
 537             body_offset -= last;
 538     }
 539
 540     /* Rewriting large objects is expensive -- don't use small gaps.
 541        This has the additional benefit of block-aligning large bodies. */
 542     if(length >= 64 * 1024) {
 543         int min_gap, min_offset;
 544         if(length >= 512 * 1024)
 545             min_gap = 4096;
 546         else if(length >= 256 * 1024)
 547             min_gap = 2048;
 548         else
 549             min_gap = 1024;
 550
 551         min_offset = ((n + 32 + min_gap - 1) / min_gap + 1) * min_gap;
 552         body_offset = MAX(body_offset, min_offset);
 553     }
 554
 555     return body_offset;
 556 }
 557
 558 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 559    otherwise the offset at which the file descriptor is left. */
 560 /* If chunk is not null, it should be the first chunk of the object,
 561    and will be written out in the same operation if possible. */
 562 static int
 563 writeHeaders(int fd, int *body_offset_return,
 564              ObjectPtr object, char *chunk, int chunk_len)
 565 {
 566     int n, rc, error = -1;
 567     int body_offset = *body_offset_return;
 568     char *buf = NULL;
 569     int buf_is_chunk = 0;
 570     int bufsize = 0;
 571
 572     if(object->flags & OBJECT_LOCAL)
 573         return -1;
 574
 575     if(body_offset > CHUNK_SIZE)
 576         goto overflow;
 577
 578     /* get_chunk might trigger object expiry */
 579     bufsize = CHUNK_SIZE;
 580     buf_is_chunk = 1;
 581     buf = maybe_get_chunk();
 582     if(!buf) {
 583         bufsize = 2048;
 584         buf_is_chunk = 0;
 585         buf = malloc(2048);
 586         if(buf == NULL) {
 587             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 588             return -1;
 589         }
 590     }
 591
 592  format_again:
 593     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 594                   object->code, object->message->string);
 595
 596     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 597     if(n < 0)
 598         goto overflow;
 599
 600     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 601     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 602
 603     if(object->age >= 0 && object->age != object->date) {
 604         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 605         n = format_time(buf, n, bufsize, object->age);
 606     }
 607
 608     if(object->atime >= 0) {
 609         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 610         n = format_time(buf, n, bufsize, object->atime);
 611     }
 612
 613     if(n < 0)
 614         goto overflow;
 615
 616     if(body_offset < 0)
 617         body_offset = chooseBodyOffset(n, object);
 618
 619     if(body_offset > bufsize)
 620         goto overflow;
 621
 622     if(body_offset > 0 && body_offset != n + 4)
 623         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 624                       body_offset);
 625
 626     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 627     if(n < 0)
 628         goto overflow;
 629
 630     if(body_offset < 0)
 631         body_offset = n;
 632     if(n > body_offset) {
 633         error = -2;
 634         goto fail;
 635     }
 636
 637     if(n < body_offset)
 638         memset(buf + n, 0, body_offset - n);
 639
 640  again:
 641 #ifdef HAVE_READV_WRITEV
 642     if(chunk_len > 0) {
 643         struct iovec iov[2];
 644         iov[0].iov_base = buf;
 645         iov[0].iov_len = body_offset;
 646         iov[1].iov_base = chunk;
 647         iov[1].iov_len = chunk_len;
 648         rc = writev(fd, iov, 2);
 649     } else
 650 #endif
 651         rc = write(fd, buf, body_offset);
 652
 653     if(rc < 0 && errno == EINTR)
 654         goto again;
 655
 656     if(rc < body_offset)
 657         goto fail;
 658     if(object->length >= 0 &&
 659        rc - body_offset >= object->length)
 660         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 661
 662     *body_offset_return = body_offset;
 663     if(buf_is_chunk)
 664         dispose_chunk(buf);
 665     else
 666         free(buf);
 667     return rc;
 668
 669  overflow:
 670     if(bufsize < bigBufferSize) {
 671         char *oldbuf = buf;
 672         buf = malloc(bigBufferSize);
 673         if(!buf) {
 674             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 675             goto fail;
 676         }
 677         bufsize = bigBufferSize;
 678         if(oldbuf) {
 679             if(buf_is_chunk)
 680                 dispose_chunk(oldbuf);
 681             else
 682                 free(oldbuf);
 683         }
 684         buf_is_chunk = 0;
 685         goto format_again;
 686     }
 687     /* fall through */
 688
 689  fail:
 690     if(buf_is_chunk)
 691         dispose_chunk(buf);
 692     else
 693         free(buf);
 694     return error;
 695 }
 696
 697 typedef struct _MimeEntry {
 698     char *extension;
 699     char *mime;
 700 } MimeEntryRec;
 701
 702 static const MimeEntryRec mimeEntries[] = {
 703     { "html", "text/html" },
 704     { "htm", "text/html" },
 705     { "text", "text/plain" },
 706     { "txt", "text/plain" },
 707     { "png", "image/png" },
 708     { "gif", "image/gif" },
 709     { "jpeg", "image/jpeg" },
 710     { "jpg", "image/jpeg" },
 711     { "ico", "image/x-icon" },
 712     { "pdf", "application/pdf" },
 713     { "ps", "application/postscript" },
 714     { "tar", "application/x-tar" },
 715     { "pac", "application/x-ns-proxy-autoconfig" },
 716     { "css", "text/css" },
 717     { "js",  "application/x-javascript" },
 718     { "xml", "text/xml" },
 719     { "swf", "application/x-shockwave-flash" },
 720 };
 721
 722 static char*
 723 localObjectMimeType(ObjectPtr object, char **encoding_return)
 724 {
 725     char *name = object->key;
 726     int nlen = object->key_size;
 727     int i;
 728
 729     assert(nlen >= 1);
 730
 731     if(name[nlen - 1] == '/') {
 732         *encoding_return = NULL;
 733         return "text/html";
 734     }
 735
 736     if(nlen < 3) {
 737         *encoding_return = NULL;
 738         return "application/octet-stream";
 739     }
 740
 741     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 742         *encoding_return = "x-gzip";
 743         nlen -= 3;
 744     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 745         *encoding_return = "x-compress";
 746         nlen -= 2;
 747     } else {
 748         *encoding_return = NULL;
 749     }
 750
 751     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 752         int len = strlen(mimeEntries[i].extension);
 753         if(nlen > len &&
 754            name[nlen - len - 1] == '.' &&
 755            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 756             return mimeEntries[i].mime;
 757     }
 758
 759     return "application/octet-stream";
 760 }
 761
 762 /* Same interface as validateEntry -- see below */
 763 int
 764 validateLocalEntry(ObjectPtr object, int fd,
 765                    int *body_offset_return, off_t *offset_return)
 766 {
 767     struct stat ss;
 768     char buf[512];
 769     int n, rc;
 770     char *encoding;
 771
 772     rc = fstat(fd, &ss);
 773     if(rc < 0) {
 774         do_log_error(L_ERROR, errno, "Couldn't stat");
 775         return -1;
 776     }
 777
 778     if(S_ISREG(ss.st_mode)) {
 779         if(!(ss.st_mode & S_IROTH) ||
 780            (object->length >= 0 && object->length != ss.st_size) ||
 781            (object->last_modified >= 0 &&
 782             object->last_modified != ss.st_mtime))
 783             return -1;
 784     } else {
 785         notifyObject(object);
 786         return -1;
 787     }
 788
 789     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 790                   (unsigned long)ss.st_ino,
 791                   (unsigned long)ss.st_size,
 792                   (unsigned long)ss.st_mtime);
 793     if(n >= 512)
 794         n = -1;
 795
 796     if(n > 0 && object->etag) {
 797         if(strlen(object->etag) != n ||
 798            memcmp(object->etag, buf, n) != 0)
 799             return -1;
 800     }
 801
 802     if(!(object->flags & OBJECT_INITIAL)) {
 803         if(!object->last_modified && !object->etag)
 804             return -1;
 805     }
 806
 807     if(object->flags & OBJECT_INITIAL) {
 808         object->length = ss.st_size;
 809         object->last_modified = ss.st_mtime;
 810         object->date = current_time.tv_sec;
 811         object->age = current_time.tv_sec;
 812         object->code = 200;
 813         if(n > 0)
 814             object->etag = strdup(buf); /* okay if fails */
 815         object->message = internAtom("Okay");
 816         n = snnprintf(buf, 0, 512,
 817                       "\r\nServer: Polipo"
 818                       "\r\nContent-Type: %s",
 819                       localObjectMimeType(object, &encoding));
 820         if(encoding != NULL)
 821             n = snnprintf(buf, n, 512,
 822                           "\r\nContent-Encoding: %s", encoding);
 823         if(n < 0)
 824             return -1;
 825         object->headers = internAtomN(buf, n);
 826         if(object->headers == NULL)
 827             return -1;
 828         object->flags &= ~OBJECT_INITIAL;
 829     }
 830
 831     if(body_offset_return)
 832         *body_offset_return = 0;
 833     if(offset_return)
 834         *offset_return = 0;
 835     return 0;
 836 }
 837
 838 /* Assumes fd is at offset 0.
 839    Returns -1 if not valid, 1 if metadata should be written out, 0
 840    otherwise. */
 841 int
 842 validateEntry(ObjectPtr object, int fd,
 843               int *body_offset_return, off_t *offset_return)
 844 {
 845     char *buf;
 846     int buf_is_chunk, bufsize;
 847     int rc, n;
 848     int dummy;
 849     int code;
 850     AtomPtr headers;
 851     time_t date, last_modified, expires, polipo_age, polipo_access;
 852     int length;
 853     off_t offset = -1;
 854     int body_offset;
 855     char *etag;
 856     AtomPtr via;
 857     CacheControlRec cache_control;
 858     char *location;
 859     AtomPtr message;
 860     int dirty = 0;
 861
 862     if(object->flags & OBJECT_LOCAL)
 863         return validateLocalEntry(object, fd,
 864                                   body_offset_return, offset_return);
 865
 866     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 867         return 0;
 868
 869     /* get_chunk might trigger object expiry */
 870     bufsize = CHUNK_SIZE;
 871     buf_is_chunk = 1;
 872     buf = maybe_get_chunk();
 873     if(!buf) {
 874         bufsize = 2048;
 875         buf_is_chunk = 0;
 876         buf = malloc(2048);
 877         if(buf == NULL) {
 878             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 879             return -1;
 880         }
 881     }
 882
 883  again:
 884     rc = read(fd, buf, bufsize);
 885     if(rc < 0) {
 886         if(errno == EINTR)
 887             goto again;
 888         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 889         goto fail;
 890     }
 891     offset = rc;
 892
 893  parse_again:
 894     n = findEndOfHeaders(buf, 0, rc, &dummy);
 895     if(n < 0) {
 896         char *oldbuf = buf;
 897         if(bufsize < bigBufferSize) {
 898             buf = malloc(bigBufferSize);
 899             if(!buf) {
 900                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 901                 goto fail;
 902             }
 903             bufsize = bigBufferSize;
 904             memcpy(buf, oldbuf, offset);
 905             if(buf_is_chunk)
 906                 dispose_chunk(oldbuf);
 907             else
 908                 free(oldbuf);
 909             buf_is_chunk = 0;
 910         again2:
 911             rc = read(fd, buf + offset, bufsize - offset);
 912             if(rc < 0) {
 913                 if(errno == EINTR)
 914                     goto again2;
 915                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 916                 goto fail;
 917             }
 918             offset += rc;
 919             goto parse_again;
 920         }
 921         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 922         goto fail;
 923     }
 924
 925     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 926     if(rc < 0) {
 927         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 928         goto fail;
 929     }
 930
 931     if(object->code != 0 && object->code != code) {
 932         releaseAtom(message);
 933         goto fail;
 934     }
 935
 936     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 937                           &headers, &length, &cache_control, NULL, NULL,
 938                           &date, &last_modified, &expires, &polipo_age,
 939                           &polipo_access, &body_offset,
 940                           NULL, &etag, NULL,
 941                           NULL, NULL, &location, &via, NULL);
 942     if(rc < 0) {
 943         releaseAtom(message);
 944         goto fail;
 945     }
 946     if(body_offset < 0)
 947         body_offset = n;
 948
 949     if(!location || strlen(location) != object->key_size ||
 950        memcmp(location, object->key, object->key_size) != 0) {
 951         do_log(L_ERROR, "Inconsistent cache file for %s.\n", scrub(location));
 952         goto invalid;
 953     }
 954
 955     if(polipo_age < 0)
 956         polipo_age = date;
 957
 958     if(polipo_age < 0) {
 959         do_log(L_ERROR, "Undated disk entry for %s.\n", scrub(location));
 960         goto invalid;
 961     }
 962
 963     if(!(object->flags & OBJECT_INITIAL)) {
 964         if((last_modified >= 0) != (object->last_modified >= 0))
 965             goto invalid;
 966
 967         if((object->cache_control & CACHE_MISMATCH) ||
 968            (cache_control.flags & CACHE_MISMATCH))
 969             goto invalid;
 970
 971         if(last_modified >= 0 && object->last_modified >= 0 &&
 972            last_modified != object->last_modified)
 973             goto invalid;
 974
 975         if(length >= 0 && object->length >= 0)
 976             if(length != object->length)
 977                 goto invalid;
 978
 979         if(!!etag != !!object->etag)
 980             goto invalid;
 981
 982         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 983             goto invalid;
 984
 985         /* If we don't have a usable ETag, and either CACHE_VARY or we
 986            don't have a last-modified date, we validate disk entries by
 987            using their date. */
 988         if(!(etag && object->etag) &&
 989            (!(last_modified >= 0 && object->last_modified >= 0) ||
 990             ((cache_control.flags & CACHE_VARY) ||
 991              (object->cache_control & CACHE_VARY)))) {
 992             if(date >= 0 && date != object->date)
 993                 goto invalid;
 994             if(polipo_age >= 0 && polipo_age != object->age)
 995                 goto invalid;
 996         }
 997         if((object->cache_control & CACHE_VARY) && dontTrustVaryETag >= 1) {
 998             /* Check content-type to work around mod_gzip bugs */
 999             if(!httpHeaderMatch(atomContentType, object->headers, headers) ||
1000                !httpHeaderMatch(atomContentEncoding, object->headers, headers))
1001                 goto invalid;
1002         }
1003     }
1004
1005     if(location)
1006         free(location);
1007
1008     if(headers) {
1009         if(!object->headers)
1010             object->headers = headers;
1011         else
1012             releaseAtom(headers);
1013     }
1014
1015     if(object->code == 0) {
1016         object->code = code;
1017         object->message = retainAtom(message);
1018     }
1019     if(object->date <= date)
1020         object->date = date;
1021     else
1022         dirty = 1;
1023     if(object->last_modified < 0)
1024         object->last_modified = last_modified;
1025     if(object->expires < 0)
1026         object->expires = expires;
1027     else if(object->expires > expires)
1028         dirty = 1;
1029     if(object->age < 0)
1030         object->age = polipo_age;
1031     else if(object->age > polipo_age)
1032         dirty = 1;
1033     if(object->atime <= polipo_access)
1034         object->atime = polipo_access;
1035     else
1036         dirty = 1;
1037
1038     object->cache_control |= cache_control.flags;
1039
1040     if(object->age < 0) object->age = object->date;
1041     if(object->age < 0) object->age = 0; /* a long time ago */
1042     if(object->length < 0) object->length = length;
1043     if(!object->etag)
1044         object->etag = etag;
1045     else {
1046         if(etag)
1047             free(etag);
1048     }
1049     releaseAtom(message);
1050
1051     if(object->flags & OBJECT_INITIAL) object->via = via;
1052     object->flags &= ~OBJECT_INITIAL;
1053     if(offset > body_offset) {
1054         /* We need to make sure we don't invoke object expiry recursively */
1055         objectSetChunks(object, 1);
1056         if(object->numchunks >= 1) {
1057             if(object->chunks[0].data == NULL)
1058                 object->chunks[0].data = maybe_get_chunk();
1059             if(object->chunks[0].data)
1060                 objectAddData(object, buf + body_offset,
1061                               0, MIN(offset - body_offset, CHUNK_SIZE));
1062         }
1063     }
1064
1065     httpTweakCachability(object);
1066
1067     if(buf_is_chunk)
1068         dispose_chunk(buf);
1069     else
1070         free(buf);
1071     if(body_offset_return) *body_offset_return = body_offset;
1072     if(offset_return) *offset_return = offset;
1073     return dirty;
1074
1075  invalid:
1076     releaseAtom(message);
1077     if(etag) free(etag);
1078     if(location) free(location);
1079     if(via) releaseAtom(via);
1080     /* fall through */
1081
1082  fail:
1083     if(buf_is_chunk)
1084         dispose_chunk(buf);
1085     else
1086         free(buf);
1087     return -1;
1088 }
1089
1090 void
1091 dirtyDiskEntry(ObjectPtr object)
1092 {
1093     DiskCacheEntryPtr entry = object->disk_entry;
1094     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1095 }
1096
1097 int
1098 revalidateDiskEntry(ObjectPtr object)
1099 {
1100     DiskCacheEntryPtr entry = object->disk_entry;
1101     int rc;
1102     int body_offset;
1103
1104     if(!entry || entry == &negativeEntry)
1105         return 1;
1106
1107     CHECK_ENTRY(entry);
1108     rc = entrySeek(entry, 0);
1109     if(rc < 0) return 0;
1110
1111     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1112     if(rc < 0) {
1113         destroyDiskEntry(object, 0);
1114         return 0;
1115     }
1116     if(body_offset != entry->body_offset) {
1117         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1118                body_offset, entry->body_offset);
1119         destroyDiskEntry(object, 0);
1120         return 0;
1121     }
1122
1123     entry->metadataDirty |= !!rc;
1124     CHECK_ENTRY(entry);
1125     return 1;
1126 }
1127
1128 static inline int
1129 objectHasDiskEntry(ObjectPtr object)
1130 {
1131     return object->disk_entry && object->disk_entry != &negativeEntry;
1132 }
1133
1134 static DiskCacheEntryPtr
1135 makeDiskEntry(ObjectPtr object, int create)
1136 {
1137     DiskCacheEntryPtr entry = NULL;
1138     char buf[1024];
1139     int fd = -1;
1140     int negative = 0, size = -1, name_len = -1;
1141     char *name = NULL;
1142     off_t offset = -1;
1143     int body_offset = -1;
1144     int rc;
1145     int local = (object->flags & OBJECT_LOCAL) != 0;
1146     int dirty = 0;
1147
1148    if(local && create)
1149        return NULL;
1150
1151     if(!local && !(object->flags & OBJECT_PUBLIC))
1152         return NULL;
1153
1154     if(maxDiskCacheEntrySize >= 0) {
1155         if(object->length > 0) {
1156             if(object->length > maxDiskCacheEntrySize)
1157                 return NULL;
1158         } else {
1159             if(object->size > maxDiskCacheEntrySize)
1160                 return NULL;
1161         }
1162     }
1163
1164     if(object->disk_entry) {
1165         entry = object->disk_entry;
1166         CHECK_ENTRY(entry);
1167         if(entry != &negativeEntry) {
1168             /* We'll keep the entry -- put it at the front. */
1169             if(entry != diskEntries && entry != &negativeEntry) {
1170                 entry->previous->next = entry->next;
1171                 if(entry->next)
1172                     entry->next->previous = entry->previous;
1173                 else
1174                     diskEntriesLast = entry->previous;
1175                 entry->next = diskEntries;
1176                 diskEntries->previous = entry;
1177                 entry->previous = NULL;
1178                 diskEntries = entry;
1179             }
1180             return entry;
1181         } else {
1182             if(entry == &negativeEntry) {
1183                 negative = 1;
1184                 if(!create) return NULL;
1185                 object->disk_entry = NULL;
1186             }
1187             entry = NULL;
1188             destroyDiskEntry(object, 0);
1189         }
1190     }
1191
1192     if(numDiskEntries > maxDiskEntries)
1193         destroyDiskEntry(diskEntriesLast->object, 0);
1194
1195     if(!local) {
1196         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1197             return NULL;
1198         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1199         if(name_len < 0) return NULL;
1200         if(!negative)
1201             fd = open(buf, O_RDWR | O_BINARY);
1202         if(fd >= 0) {
1203             rc = validateEntry(object, fd, &body_offset, &offset);
1204             if(rc >= 0) {
1205                 dirty = rc;
1206             } else {
1207                 close(fd);
1208                 fd = -1;
1209                 rc = unlink(buf);
1210                 if(rc < 0 && errno != ENOENT) {
1211                     do_log_error(L_WARN,  errno,
1212                                  "Couldn't unlink stale disk entry %s",
1213                                  scrub(buf));
1214                     /* But continue -- it's okay to have stale entries. */
1215                 }
1216             }
1217         }
1218
1219         if(fd < 0 && create && name_len > 0 &&
1220            !(object->flags & OBJECT_INITIAL)) {
1221             fd = createFile(buf, diskCacheRoot->length);
1222             if(fd < 0)
1223                 return NULL;
1224
1225             if(fd >= 0) {
1226                 char *data = NULL;
1227                 int dsize = 0;
1228                 if(object->numchunks > 0) {
1229                     data = object->chunks[0].data;
1230                     dsize = object->chunks[0].size;
1231                 }
1232                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1233                 if(rc < 0) {
1234                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1235                     rc = unlink(buf);
1236                     if(rc < 0 && errno != ENOENT)
1237                         do_log_error(L_ERROR, errno,
1238                                      "Couldn't unlink truncated entry %s",
1239                                      scrub(buf));
1240                     close(fd);
1241                     return NULL;
1242                 }
1243                 assert(rc >= body_offset);
1244                 size = rc - body_offset;
1245                 offset = rc;
1246                 dirty = 0;
1247             }
1248         }
1249     } else {
1250         /* local */
1251         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1252             return NULL;
1253
1254         name_len =
1255             localFilename(buf, 1024, object->key, object->key_size);
1256         if(name_len < 0)
1257             return NULL;
1258         fd = open(buf, O_RDONLY | O_BINARY);
1259         if(fd >= 0) {
1260             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1261                 close(fd);
1262                 fd = -1;
1263             }
1264         }
1265         offset = 0;
1266     }
1267
1268     if(fd < 0) {
1269         object->disk_entry = &negativeEntry;
1270         return NULL;
1271     }
1272     assert(body_offset >= 0);
1273
1274     name = strdup_n(buf, name_len);
1275     if(name == NULL) {
1276         do_log(L_ERROR, "Couldn't allocate name.\n");
1277         close(fd);
1278         return NULL;
1279     }
1280
1281     entry = malloc(sizeof(DiskCacheEntryRec));
1282     if(entry == NULL) {
1283         do_log(L_ERROR, "Couldn't allocate entry.\n");
1284         free(name);
1285         close(fd);
1286         return NULL;
1287     }
1288
1289     entry->filename = name;
1290     entry->object = object;
1291     entry->fd = fd;
1292     entry->body_offset = body_offset;
1293     entry->local = local;
1294     entry->offset = offset;
1295     entry->size = size;
1296     entry->metadataDirty = dirty;
1297
1298     entry->next = diskEntries;
1299     if(diskEntries)
1300         diskEntries->previous = entry;
1301     diskEntries = entry;
1302     if(diskEntriesLast == NULL)
1303         diskEntriesLast = entry;
1304     entry->previous = NULL;
1305     numDiskEntries++;
1306
1307     object->disk_entry = entry;
1308
1309     CHECK_ENTRY(entry);
1310     return entry;
1311 }
1312
1313 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1314 static int
1315 rewriteEntry(ObjectPtr object)
1316 {
1317     int old_body_offset = object->disk_entry->body_offset;
1318     int fd, rc, n;
1319     DiskCacheEntryPtr entry;
1320     char* buf;
1321     int buf_is_chunk, bufsize;
1322     int offset;
1323
1324     fd = dup(object->disk_entry->fd);
1325     if(fd < 0) {
1326         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1327         return -1;
1328     }
1329
1330     rc = destroyDiskEntry(object, 1);
1331     if(rc < 0) {
1332         close(fd);
1333         return -1;
1334     }
1335     entry = makeDiskEntry(object, 1);
1336     if(!entry) {
1337         close(fd);
1338         return -1;
1339     }
1340
1341     offset = diskEntrySize(object);
1342     if(offset < 0) {
1343         close(fd);
1344         return -1;
1345     }
1346
1347     bufsize = CHUNK_SIZE;
1348     buf_is_chunk = 1;
1349     buf = maybe_get_chunk();
1350     if(!buf) {
1351         bufsize = 2048;
1352         buf_is_chunk = 0;
1353         buf = malloc(2048);
1354         if(buf == NULL) {
1355             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1356             close(fd);
1357             return -1;
1358         }
1359     }
1360
1361     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1362     if(rc < 0)
1363         goto done;
1364
1365     while(1) {
1366         CHECK_ENTRY(entry);
1367         n = read(fd, buf, bufsize);
1368         if(n < 0 && errno == EINTR)
1369             continue;
1370         if(n <= 0)
1371             goto done;
1372         rc = entrySeek(entry, entry->body_offset + offset);
1373         if(rc < 0)
1374             goto done;
1375     write_again:
1376         rc = write(entry->fd, buf, n);
1377         if(rc >= 0) {
1378             entry->offset += rc;
1379             entry->size += rc;
1380         } else if(errno == EINTR) {
1381             goto write_again;
1382         }
1383         if(rc < n)
1384             goto done;
1385     }
1386
1387  done:
1388     CHECK_ENTRY(entry);
1389     if(object->length >= 0 && entry->size == object->length)
1390         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1391     close(fd);
1392     if(buf_is_chunk)
1393         dispose_chunk(buf);
1394     else
1395         free(buf);
1396     return 1;
1397 }
1398
1399 int
1400 destroyDiskEntry(ObjectPtr object, int d)
1401 {
1402     DiskCacheEntryPtr entry = object->disk_entry;
1403     int rc, urc = 1;
1404
1405     assert(!entry || !entry->local || !d);
1406
1407     if(d && !entry)
1408         entry = makeDiskEntry(object, 0);
1409
1410     CHECK_ENTRY(entry);
1411
1412     if(!entry || entry == &negativeEntry) {
1413         return 1;
1414     }
1415
1416     assert(entry->object == object);
1417
1418     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1419         /* See writeoutToDisk */
1420         d = 1;
1421     }
1422
1423     if(d) {
1424         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1425         if(entry->filename) {
1426             urc = unlink(entry->filename);
1427             if(urc < 0)
1428                 do_log_error(L_WARN, errno,
1429                              "Couldn't unlink %s", scrub(entry->filename));
1430         }
1431     } else {
1432         if(entry && entry->metadataDirty)
1433             writeoutMetadata(object);
1434         makeDiskEntry(object, 0);
1435         /* rewriteDiskEntry may change the disk entry */
1436         entry = object->disk_entry;
1437         if(entry == NULL || entry == &negativeEntry)
1438             return 0;
1439         if(diskCacheWriteoutOnClose > 0)
1440             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1441     }
1442  again:
1443     rc = close(entry->fd);
1444     if(rc < 0 && errno == EINTR)
1445         goto again;
1446
1447     entry->fd = -1;
1448
1449     if(entry->filename)
1450         free(entry->filename);
1451     entry->filename = NULL;
1452
1453     if(entry->previous)
1454         entry->previous->next = entry->next;
1455     else
1456         diskEntries = entry->next;
1457     if(entry->next)
1458         entry->next->previous = entry->previous;
1459     else
1460         diskEntriesLast = entry->previous;
1461
1462     numDiskEntries--;
1463     assert(numDiskEntries >= 0);
1464
1465     free(entry);
1466     object->disk_entry = NULL;
1467     if(urc < 0)
1468         return -1;
1469     else
1470         return 1;
1471 }
1472
1473 ObjectPtr
1474 objectGetFromDisk(ObjectPtr object)
1475 {
1476     DiskCacheEntryPtr entry = makeDiskEntry(object, 0);
1477     if(!entry) return NULL;
1478     return object;
1479 }
1480
1481
1482 int
1483 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1484 {
1485     DiskCacheEntryPtr entry;
1486     int rc, result;
1487     int i, j, k;
1488     int complete;
1489
1490     if(object->type != OBJECT_HTTP)
1491         return 0;
1492
1493     if(object->flags & OBJECT_LINEAR)
1494         return 0;
1495
1496     if(object->length >= 0) {
1497         chunks = MIN(chunks,
1498                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1499     }
1500
1501     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1502     if(rc < 0)
1503         return 0;
1504
1505     complete = 1;
1506     if(object->flags & OBJECT_INITIAL) {
1507         complete = 0;
1508     } else if((object->length < 0 || object->size < object->length) &&
1509               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1510         complete = 0;
1511     } else {
1512         for(k = 0; k < chunks; k++) {
1513             int s;
1514             i = offset / CHUNK_SIZE + k;
1515             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1516             if(object->chunks[i].size < s) {
1517                 complete = 0;
1518                 break;
1519             }
1520         }
1521     }
1522
1523     if(complete)
1524         return 1;
1525
1526     /* This has the side-effect of revalidating the entry, which is
1527        what makes HEAD requests work. */
1528     entry = makeDiskEntry(object, 0);
1529     if(!entry)
1530         return 0;
1531
1532     for(k = 0; k < chunks; k++) {
1533         i = offset / CHUNK_SIZE + k;
1534         if(!object->chunks[i].data)
1535             object->chunks[i].data = get_chunk();
1536         if(!object->chunks[i].data) {
1537             chunks = k;
1538             break;
1539         }
1540         lockChunk(object, i);
1541     }
1542
1543     result = 0;
1544
1545     for(k = 0; k < chunks; k++) {
1546         int o;
1547         i = offset / CHUNK_SIZE + k;
1548         j = object->chunks[i].size;
1549         o = i * CHUNK_SIZE + j;
1550
1551         if(object->chunks[i].size == CHUNK_SIZE)
1552             continue;
1553
1554         if(entry->size >= 0 && entry->size <= o)
1555             break;
1556
1557         if(entry->offset != entry->body_offset + o) {
1558             rc = entrySeek(entry, entry->body_offset + o);
1559             if(rc < 0) {
1560                 result = 0;
1561                 break;
1562             }
1563         }
1564
1565         CHECK_ENTRY(entry);
1566         again:
1567         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1568         if(rc < 0) {
1569             if(errno == EINTR)
1570                 goto again;
1571             entry->offset = -1;
1572             do_log_error(L_ERROR, errno, "Couldn't read");
1573             break;
1574         }
1575
1576         entry->offset += rc;
1577         object->chunks[i].size += rc;
1578         if(object->size < o + rc)
1579             object->size = o + rc;
1580
1581         if(entry->object->length >= 0 && entry->size < 0 &&
1582            entry->offset - entry->body_offset == entry->object->length)
1583             entry->size = entry->object->length;
1584
1585         if(rc < CHUNK_SIZE - j) {
1586             /* Paranoia: the read may have been interrupted half-way. */
1587             if(entry->size < 0) {
1588                 if(rc == 0 ||
1589                    (entry->object->length >= 0 &&
1590                     entry->object->length ==
1591                     entry->offset - entry->body_offset))
1592                     entry->size = entry->offset - entry->body_offset;
1593                 break;
1594             } else if(entry->size != entry->offset - entry->body_offset) {
1595                 if(rc == 0 ||
1596                    entry->size < entry->offset - entry->body_offset) {
1597                     do_log(L_WARN,
1598                            "Disk entry size changed behind our back: "
1599                            "%ld -> %ld (%d).\n",
1600                            (long)entry->size,
1601                            (long)entry->offset - entry->body_offset,
1602                            object->size);
1603                     entry->size = -1;
1604                 }
1605             }
1606             break;
1607         }
1608
1609         CHECK_ENTRY(entry);
1610         result = 1;
1611     }
1612
1613     CHECK_ENTRY(object->disk_entry);
1614     for(k = 0; k < chunks; k++) {
1615         i = offset / CHUNK_SIZE + k;
1616         unlockChunk(object, i);
1617     }
1618
1619     if(result > 0) {
1620         notifyObject(object);
1621         return 1;
1622     } else {
1623         return 0;
1624     }
1625 }
1626
1627 int
1628 writeoutToDisk(ObjectPtr object, int upto, int max)
1629 {
1630     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1631         /* An object was created with an unknown length, and then grew
1632            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1633         destroyDiskEntry(object, 1);
1634         return 0;
1635     }
1636
1637     return reallyWriteoutToDisk(object, upto, max);
1638 }
1639
1640 static int
1641 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1642 {
1643     DiskCacheEntryPtr entry;
1644     int rc;
1645     int i, j;
1646     int offset;
1647     int bytes = 0;
1648
1649     if(upto < 0)
1650         upto = object->size;
1651
1652     if((object->cache_control & CACHE_NO_STORE) ||
1653        (object->flags & OBJECT_LOCAL))
1654         return 0;
1655
1656     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1657         return 0;
1658
1659     entry = makeDiskEntry(object, 1);
1660     if(!entry) return 0;
1661
1662     assert(!entry->local);
1663
1664     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1665         goto done;
1666
1667     diskEntrySize(object);
1668     if(entry->size < 0)
1669         return 0;
1670
1671     if(object->length >= 0 && entry->size >= object->length) {
1672         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1673         goto done;
1674     }
1675
1676     if(entry->size >= upto)
1677         goto done;
1678
1679     offset = entry->size;
1680
1681     /* Avoid a seek in case we start writing at the beginning */
1682     if(offset == 0 && entry->metadataDirty) {
1683         writeoutMetadata(object);
1684         /* rewriteDiskEntry may change the entry */
1685         entry = makeDiskEntry(object, 0);
1686         if(entry == NULL)
1687             return 0;
1688     }
1689
1690     rc = entrySeek(entry, offset + entry->body_offset);
1691     if(rc < 0) return 0;
1692
1693     do {
1694         if(max >= 0 && bytes >= max)
1695             break;
1696         CHECK_ENTRY(entry);
1697         assert(entry->offset == offset + entry->body_offset);
1698         i = offset / CHUNK_SIZE;
1699         j = offset % CHUNK_SIZE;
1700         if(i >= object->numchunks)
1701             break;
1702         if(object->chunks[i].size <= j)
1703             break;
1704     again:
1705         rc = write(entry->fd, object->chunks[i].data + j,
1706                    object->chunks[i].size - j);
1707         if(rc < 0) {
1708             if(errno == EINTR)
1709                 goto again;
1710             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1711             break;
1712         }
1713         entry->offset += rc;
1714         offset += rc;
1715         bytes += rc;
1716         if(entry->size < offset)
1717             entry->size = offset;
1718     } while(j + rc >= CHUNK_SIZE);
1719
1720  done:
1721     CHECK_ENTRY(entry);
1722     if(entry->metadataDirty)
1723         writeoutMetadata(object);
1724
1725     return bytes;
1726 }
1727
1728 int
1729 writeoutMetadata(ObjectPtr object)
1730 {
1731     DiskCacheEntryPtr entry;
1732     int rc;
1733
1734     if((object->cache_control & CACHE_NO_STORE) ||
1735        (object->flags & OBJECT_LOCAL))
1736         return 0;
1737
1738     entry = makeDiskEntry(object, 0);
1739     if(entry == NULL || entry == &negativeEntry)
1740         goto fail;
1741
1742     assert(!entry->local);
1743
1744     rc = entrySeek(entry, 0);
1745     if(rc < 0) goto fail;
1746
1747     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1748     if(rc == -2) {
1749         rc = rewriteEntry(object);
1750         if(rc < 0) return 0;
1751         return 1;
1752     }
1753     if(rc < 0) goto fail;
1754     entry->offset = rc;
1755     entry->metadataDirty = 0;
1756     return 1;
1757
1758  fail:
1759     /* We need this in order to avoid trying to write this entry out
1760        multiple times. */
1761     if(entry && entry != &negativeEntry)
1762         entry->metadataDirty = 0;
1763     return 0;
1764 }
1765
1766 static void
1767 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1768 {
1769     if(dst->filename == NULL) {
1770         dst->filename = src->filename;
1771         dst->body_offset = src->body_offset;
1772     } else
1773         free(src->filename);
1774     free(src->location);
1775     if(dst->length < 0)
1776         dst->length = src->length;
1777     if(dst->size < 0)
1778         dst->size = src->size;
1779     if(dst->age < 0)
1780         dst->age = src->age;
1781     if(dst->date < 0)
1782         dst->date = src->date;
1783     if(dst->last_modified < 0)
1784         dst->last_modified = src->last_modified;
1785     free(src);
1786 }
1787
1788 DiskObjectPtr
1789 readDiskObject(char *filename, struct stat *sb)
1790 {
1791     int fd, rc, n, dummy, code;
1792     int length, size;
1793     time_t date, last_modified, age, atime, expires;
1794     char *location = NULL, *fn = NULL;
1795     DiskObjectPtr dobject;
1796     char *buf;
1797     int buf_is_chunk, bufsize;
1798     int body_offset;
1799     struct stat ss;
1800
1801     fd = -1;
1802
1803     if(sb == NULL) {
1804         rc = stat(filename, &ss);
1805         if(rc < 0) {
1806             do_log_error(L_WARN, errno, "Couldn't stat %s", scrub(filename));
1807             return NULL;
1808         }
1809         sb = &ss;
1810     }
1811
1812     buf_is_chunk = 1;
1813     bufsize = CHUNK_SIZE;
1814     buf = get_chunk();
1815     if(buf == NULL) {
1816         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1817         return NULL;
1818     }
1819
1820     if(S_ISREG(sb->st_mode)) {
1821         fd = open(filename, O_RDONLY | O_BINARY);
1822         if(fd < 0)
1823             goto fail;
1824     again:
1825         rc = read(fd, buf, bufsize);
1826         if(rc < 0)
1827             goto fail;
1828
1829         n = findEndOfHeaders(buf, 0, rc, &dummy);
1830         if(n < 0) {
1831             long lrc;
1832             if(buf_is_chunk) {
1833                 dispose_chunk(buf);
1834                 buf_is_chunk = 0;
1835                 bufsize = bigBufferSize;
1836                 buf = malloc(bigBufferSize);
1837                 if(buf == NULL)
1838                     goto fail2;
1839                 lrc = lseek(fd, 0, SEEK_SET);
1840                 if(lrc < 0)
1841                     goto fail;
1842                 goto again;
1843             }
1844             goto fail;
1845         }
1846
1847         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1848         if(rc < 0)
1849             goto fail;
1850
1851         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1852                               NULL, &length, NULL, NULL, NULL,
1853                               &date, &last_modified, &expires, &age,
1854                               &atime, &body_offset, NULL,
1855                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1856         if(rc < 0 || location == NULL)
1857             goto fail;
1858         if(body_offset < 0)
1859             body_offset = n;
1860
1861         size = sb->st_size - body_offset;
1862         if(size < 0)
1863             size = 0;
1864     } else if(S_ISDIR(sb->st_mode)) {
1865         char *n;
1866         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1867         if(n == NULL)
1868             goto fail;
1869         location = strdup(n);
1870         if(location == NULL)
1871             goto fail;
1872         length = -1;
1873         size = -1;
1874         body_offset = -1;
1875         age = -1;
1876         atime = -1;
1877         date = -1;
1878         last_modified = -1;
1879         expires = -1;
1880     } else {
1881         goto fail;
1882     }
1883
1884     dobject = malloc(sizeof(DiskObjectRec));
1885     if(!dobject)
1886         goto fail;
1887
1888     fn = strdup(filename);
1889     if(!fn)
1890         goto fail;
1891
1892     if(buf_is_chunk)
1893         dispose_chunk(buf);
1894     else
1895         free(buf);
1896
1897     dobject->location = location;
1898     dobject->filename = fn;
1899     dobject->length = length;
1900     dobject->body_offset = body_offset;
1901     dobject->size = size;
1902     dobject->age = age;
1903     dobject->access = atime;
1904     dobject->date = date;
1905     dobject->last_modified = last_modified;
1906     dobject->expires = expires;
1907     if(fd >= 0) close(fd);
1908     return dobject;
1909
1910  fail:
1911     if(buf_is_chunk)
1912         dispose_chunk(buf);
1913     else
1914         free(buf);
1915  fail2:
1916     if(fd >= 0) close(fd);
1917     if(location) free(location);
1918     return NULL;
1919 }
1920
1921
1922 DiskObjectPtr
1923 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1924 {
1925     DiskObjectPtr dobject = NULL;
1926     int c = 0;
1927
1928     dobject = readDiskObject((char*)filename, sb);
1929     if(dobject == NULL)
1930         return 0;
1931
1932     if(!dobjects ||
1933        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1934         if(dobjects && c == 0) {
1935             mergeDobjects(dobjects, dobject);
1936         } else {
1937             dobject->next = dobjects;
1938             dobjects = dobject;
1939         }
1940     } else {
1941         DiskObjectPtr other = dobjects;
1942         while(other->next) {
1943             c = strcmp(dobject->location, other->next->location);
1944             if(c < 0)
1945                 break;
1946             other = other->next;
1947         }
1948         if(strcmp(dobject->location, other->location) == 0) {
1949             mergeDobjects(other, dobject);
1950         } else {
1951             dobject->next = other->next;
1952             other->next = dobject;
1953         }
1954     }
1955     return dobjects;
1956 }
1957
1958 /* Determine whether p is below root */
1959 static int
1960 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1961 {
1962     char *cp;
1963     int m = strlen(p->location);
1964     if(m < n)
1965         return 0;
1966     if(memcmp(root, p->location, n) != 0)
1967         return 0;
1968     if(recursive)
1969         return 1;
1970     if(m == 0 || p->location[m - 1] == '/')
1971         return 1;
1972     cp = strchr(p->location + n, '/');
1973     if(cp && cp - p->location != m - 1)
1974         return 0;
1975     return 1;
1976 }
1977
1978 /* Filter out all disk objects that are not under root */
1979 DiskObjectPtr
1980 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
1981 {
1982     int n = strlen(root);
1983     DiskObjectPtr p, q;
1984
1985     while(from && !filter(from, root, n, recursive)) {
1986         p = from;
1987         from = p->next;
1988         free(p->location);
1989         free(p);
1990     }
1991
1992     p = from;
1993     while(p && p->next) {
1994         if(!filter(p->next, root, n, recursive)) {
1995             q = p->next;
1996             p->next = q->next;
1997             free(q->location);
1998             free(q);
1999         } else {
2000             p = p->next;
2001         }
2002     }
2003     return from;
2004 }
2005
2006 DiskObjectPtr
2007 insertRoot(DiskObjectPtr from, const char *root)
2008 {
2009     DiskObjectPtr p;
2010
2011     p = from;
2012     while(p) {
2013         if(strcmp(root, p->location) == 0)
2014             return from;
2015         p = p->next;
2016     }
2017
2018     p = malloc(sizeof(DiskObjectRec));
2019     if(!p) return from;
2020     p->location = strdup(root);
2021     if(p->location == NULL) {
2022         free(p);
2023         return from;
2024     }
2025     p->filename = NULL;
2026     p->length = -1;
2027     p->size = -1;
2028     p->age = -1;
2029     p->access = -1;
2030     p->last_modified = -1;
2031     p->expires = -1;
2032     p->next = from;
2033     return p;
2034 }
2035
2036 /* Insert all missing directories in a sorted list of dobjects */
2037 DiskObjectPtr
2038 insertDirs(DiskObjectPtr from)
2039 {
2040     DiskObjectPtr p, q, new;
2041     int n, m;
2042     char *cp;
2043
2044     p = NULL; q = from;
2045     while(q) {
2046         n = strlen(q->location);
2047         if(n > 0 && q->location[n - 1] != '/') {
2048             cp = strrchr(q->location, '/');
2049             m = cp - q->location + 1;
2050             if(cp && (!p || strlen(p->location) < m ||
2051                       memcmp(p->location, q->location, m) != 0)) {
2052                 new = malloc(sizeof(DiskObjectRec));
2053                 if(!new) break;
2054                 new->location = strdup_n(q->location, m);
2055                 if(new->location == NULL) {
2056                     free(new);
2057                     break;
2058                 }
2059                 new->filename = NULL;
2060                 new->length = -1;
2061                 new->size = -1;
2062                 new->age = -1;
2063                 new->access = -1;
2064                 new->last_modified = -1;
2065                 new->expires = -1;
2066                 new->next = q;
2067                 if(p)
2068                     p->next = new;
2069                 else
2070                     from = new;
2071             }
2072         }
2073         p = q;
2074         q = q->next;
2075     }
2076     return from;
2077 }
2078
2079 void
2080 indexDiskObjects(FILE *out, const char *root, int recursive)
2081 {
2082     int n, i, isdir;
2083     DIR *dir;
2084     struct dirent *dirent;
2085     char buf[1024];
2086     char *fts_argv[2];
2087     FTS *fts;
2088     FTSENT *fe;
2089     DiskObjectPtr dobjects = NULL;
2090     char *of = root[0] == '\0' ? "" : " of ";
2091
2092     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2093             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2094             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2095             "<html><head>\n"
2096             "<title>%s%s%s</title>\n"
2097             "</head><body>\n"
2098             "<h1>%s%s%s</h1>\n",
2099             recursive ? "Recursive index" : "Index", of, root,
2100             recursive ? "Recursive index" : "Index", of, root);
2101
2102     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2103         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2104         goto trailer;
2105     }
2106
2107     if(diskCacheRoot->length >= 1024) {
2108         fprintf(out,
2109                 "<p>The value of <tt>diskCacheRoot</tt> is "
2110                 "too long (%d).</p>\n",
2111                 diskCacheRoot->length);
2112         goto trailer;
2113     }
2114
2115     if(strlen(root) < 8) {
2116         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2117         buf[diskCacheRoot->length] = '\0';
2118         n = diskCacheRoot->length;
2119     } else {
2120         n = urlDirname(buf, 1024, root, strlen(root));
2121     }
2122     if(n > 0) {
2123         if(recursive) {
2124             dir = NULL;
2125             fts_argv[0] = buf;
2126             fts_argv[1] = NULL;
2127             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2128             if(fts) {
2129                 while(1) {
2130                     fe = fts_read(fts);
2131                     if(!fe) break;
2132                     if(fe->fts_info != FTS_DP)
2133                         dobjects =
2134                             processObject(dobjects,
2135                                           fe->fts_path,
2136                                           fe->fts_info == FTS_NS ||
2137                                           fe->fts_info == FTS_NSOK ?
2138                                           fe->fts_statp : NULL);
2139                 }
2140                 fts_close(fts);
2141             }
2142         } else {
2143             dir = opendir(buf);
2144             if(dir) {
2145                 while(1) {
2146                     dirent = readdir(dir);
2147                     if(!dirent) break;
2148                     if(n + strlen(dirent->d_name) < 1024) {
2149                         strcpy(buf + n, dirent->d_name);
2150                     } else {
2151                         continue;
2152                     }
2153                     dobjects = processObject(dobjects, buf, NULL);
2154                 }
2155                 closedir(dir);
2156             } else {
2157                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2158                         strerror(errno), errno);
2159                 goto trailer;
2160             }
2161         }
2162     }
2163
2164     if(dobjects) {
2165         DiskObjectPtr dobject;
2166         int entryno;
2167         dobjects = insertRoot(dobjects, root);
2168         dobjects = insertDirs(dobjects);
2169         dobjects = filterDiskObjects(dobjects, root, recursive);
2170         buf[0] = '\0';
2171         alternatingHttpStyle(out, "diskcachelist");
2172         fprintf(out, "<table id=diskcachelist>\n");
2173         fprintf(out, "<tbody>\n");
2174         entryno = 0;
2175         while(dobjects) {
2176             dobject = dobjects;
2177             i = strlen(dobject->location);
2178             isdir = (i == 0 || dobject->location[i - 1] == '/');
2179             if(entryno % 2)
2180                 fprintf(out, "<tr class=odd>");
2181             else
2182                 fprintf(out, "<tr class=even>");
2183             if(dobject->size >= 0) {
2184                 fprintf(out, "<td><a href=\"%s\"><tt>",
2185                         dobject->location);
2186                 htmlPrint(out,
2187                           dobject->location, strlen(dobject->location));
2188                 fprintf(out, "</tt></a></td> ");
2189                 if(dobject->length >= 0) {
2190                     if(dobject->size == dobject->length)
2191                         fprintf(out, "<td>%d</td> ", dobject->length);
2192                     else
2193                         fprintf(out, "<td>%d/%d</td> ",
2194                                dobject->size, dobject->length);
2195                 } else {
2196                     /* Avoid a trigraph. */
2197                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2198                 }
2199                 if(dobject->last_modified >= 0) {
2200                     struct tm *tm = gmtime(&dobject->last_modified);
2201                     if(tm == NULL)
2202                         n = -1;
2203                     else
2204                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2205                 } else
2206                     n = -1;
2207                 if(n > 0) {
2208                     buf[n] = '\0';
2209                     fprintf(out, "<td>%s</td> ", buf);
2210                 } else {
2211                     fprintf(out, "<td></td>");
2212                 }
2213
2214                 if(dobject->date >= 0) {
2215                     struct tm *tm = gmtime(&dobject->date);
2216                     if(tm == NULL)
2217                         n = -1;
2218                     else
2219                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2220                 } else
2221                     n = -1;
2222                 if(n > 0) {
2223                     buf[n] = '\0';
2224                     fprintf(out, "<td>%s</td>", buf);
2225                 } else {
2226                     fprintf(out, "<td></td>");
2227                 }
2228             } else {
2229                 fprintf(out, "<td><tt>");
2230                 htmlPrint(out, dobject->location,
2231                           strlen(dobject->location));
2232                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2233             }
2234             if(isdir) {
2235                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2236                         "<td><a href=\"/polipo/recursive-index?%s\">"
2237                         "recursive</a></td>",
2238                         dobject->location, dobject->location);
2239             }
2240             fprintf(out, "</tr>\n");
2241             entryno++;
2242             dobjects = dobject->next;
2243             free(dobject->location);
2244             free(dobject->filename);
2245             free(dobject);
2246         }
2247         fprintf(out, "</tbody>\n");
2248         fprintf(out, "</table>\n");
2249     }
2250
2251  trailer:
2252     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2253     fprintf(out, "</body></html>\n");
2254     return;
2255 }
2256
2257 static int
2258 checkForZeroes(char *buf, int n)
2259 {
2260     int i, j;
2261     unsigned long *lbuf = (unsigned long *)buf;
2262     assert(n % sizeof(unsigned long) == 0);
2263
2264     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2265         if(lbuf[i] != 0L)
2266             return i * sizeof(unsigned long);
2267     }
2268     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2269         if(buf[i * sizeof(unsigned long) + j] != 0)
2270             break;
2271     }
2272
2273     return i * sizeof(unsigned long) + j;
2274 }
2275
2276 static int
2277 copyFile(int from, char *filename, int n)
2278 {
2279     char *buf;
2280     int to, offset, nread, nzeroes, rc;
2281
2282     buf = malloc(CHUNK_SIZE);
2283     if(buf == NULL)
2284         return -1;
2285
2286     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2287               diskCacheFilePermissions);
2288     if(to < 0) {
2289         free(buf);
2290         return -1;
2291     }
2292
2293     offset = 0;
2294     while(offset < n) {
2295         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2296         if(nread <= 0)
2297             break;
2298         nzeroes = checkForZeroes(buf, nread & -8);
2299         if(nzeroes > 0) {
2300             /* I like holes */
2301             rc = lseek(to, nzeroes, SEEK_CUR);
2302             if(rc != offset + nzeroes) {
2303                 if(rc < 0)
2304                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2305                 else
2306                     do_log(L_ERROR,
2307                            "Couldn't extend file: "
2308                            "unexpected offset %d != %d + %d.\n",
2309                            rc, offset, nread);
2310                 break;
2311             }
2312         }
2313         if(nread > nzeroes) {
2314             rc = write(to, buf + nzeroes, nread - nzeroes);
2315             if(rc != nread - nzeroes) {
2316                 if(rc < 0)
2317                     do_log_error(L_ERROR, errno, "Couldn't write");
2318                 else
2319                     do_log(L_ERROR, "Short write.\n");
2320                 break;
2321             }
2322         }
2323         offset += nread;
2324     }
2325     free(buf);
2326     close(to);
2327     if(offset <= 0)
2328         unlink(filename);       /* something went wrong straight away */
2329     return 1;
2330 }
2331
2332 static long int
2333 expireFile(char *filename, struct stat *sb,
2334            int *considered, int *unlinked, int *truncated)
2335 {
2336     DiskObjectPtr dobject = NULL;
2337     time_t t;
2338     int fd, rc;
2339     long int ret = sb->st_size;
2340
2341     if(!preciseExpiry) {
2342         t = sb->st_mtime;
2343         if(t > current_time.tv_sec + 1) {
2344             do_log(L_WARN, "File %s has access time in the future.\n",
2345                    filename);
2346             t = current_time.tv_sec;
2347         }
2348
2349         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2350            (sb->st_size < diskCacheTruncateSize ||
2351             t > current_time.tv_sec - diskCacheTruncateTime))
2352             return ret;
2353     }
2354
2355     (*considered)++;
2356
2357     dobject = readDiskObject(filename, sb);
2358     if(!dobject) {
2359         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n",
2360                scrub(filename));
2361         rc = unlink(filename);
2362         if(rc < 0) {
2363             do_log_error(L_ERROR, errno,
2364                          "Couldn't unlink %s", scrub(filename));
2365             return ret;
2366         } else {
2367             (*unlinked)++;
2368             return 0;
2369         }
2370     }
2371
2372     t = dobject->access;
2373     if(t < 0) t = dobject->age;
2374     if(t < 0) t = dobject->date;
2375
2376     if(t > current_time.tv_sec)
2377         do_log(L_WARN,
2378                "Disk entry %s (%s) has access time in the future.\n",
2379                scrub(dobject->location), scrub(dobject->filename));
2380
2381     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2382         rc = unlink(dobject->filename);
2383         if(rc < 0) {
2384             do_log_error(L_ERROR, errno, "Couldn't unlink %s",
2385                          scrub(filename));
2386         } else {
2387             (*unlinked)++;
2388             ret = 0;
2389         }
2390     } else if(dobject->size >
2391               diskCacheTruncateSize + 4 * dobject->body_offset &&
2392               t < current_time.tv_sec - diskCacheTruncateTime) {
2393         /* We need to copy rather than simply truncate in place: the
2394            latter would confuse a running polipo. */
2395         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2396         rc = unlink(dobject->filename);
2397         if(rc < 0) {
2398             do_log_error(L_ERROR, errno, "Couldn't unlink %s",
2399                          scrub(filename));
2400         } else {
2401             (*unlinked)++;
2402             copyFile(fd, dobject->filename,
2403                      dobject->body_offset + diskCacheTruncateSize);
2404             (*unlinked)--;
2405             (*truncated)++;
2406             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2407         }
2408         close(fd);
2409     }
2410     free(dobject->location);
2411     free(dobject->filename);
2412     free(dobject);
2413     return ret;
2414 }
2415
2416 void
2417 expireDiskObjects()
2418 {
2419     int rc;
2420     char *fts_argv[2];
2421     FTS *fts;
2422     FTSENT *fe;
2423     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2424     int dirs = 0, rmdirs = 0;
2425     long left = 0, total = 0;
2426
2427     if(diskCacheRoot == NULL ||
2428        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2429         return;
2430
2431     fts_argv[0] = diskCacheRoot->string;
2432     fts_argv[1] = NULL;
2433     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2434     if(fts == NULL) {
2435         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2436     } else {
2437         while(1) {
2438             gettimeofday(&current_time, NULL);
2439
2440             fe = fts_read(fts);
2441             if(!fe) break;
2442
2443             if(fe->fts_info == FTS_D)
2444                 continue;
2445
2446             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2447                fe->fts_info == FTS_DNR) {
2448                 if(fe->fts_accpath[0] == '/' &&
2449                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2450                     continue;
2451                 dirs++;
2452                 rc = rmdir(fe->fts_accpath);
2453                 if(rc >= 0)
2454                     rmdirs++;
2455                 else if(errno != ENOTEMPTY && errno != EEXIST)
2456                     do_log_error(L_ERROR, errno,
2457                                  "Couldn't remove directory %s",
2458                                  scrub(fe->fts_accpath));
2459                 continue;
2460             } else if(fe->fts_info == FTS_NS) {
2461                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2462                              scrub(fe->fts_accpath));
2463                 continue;
2464             } else if(fe->fts_info == FTS_ERR) {
2465                 do_log_error(L_ERROR, fe->fts_errno,
2466                              "Couldn't fts_read disk cache");
2467                 break;
2468             }
2469
2470             if(!S_ISREG(fe->fts_statp->st_mode)) {
2471                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2472                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2473                 continue;
2474             }
2475
2476             files++;
2477             left += expireFile(fe->fts_accpath, fe->fts_statp,
2478                                &considered, &unlinked, &truncated);
2479             total += fe->fts_statp->st_size;
2480         }
2481         fts_close(fts);
2482     }
2483
2484     printf("Disk cache purged.\n");
2485     printf("%d files, %d considered, %d removed, %d truncated "
2486            "(%ldkB -> %ldkB).\n",
2487            files, considered, unlinked, truncated, total/1024, left/1024);
2488     printf("%d directories, %d removed.\n", dirs, rmdirs);
2489     return;
2490 }
2491
2492 #else
2493
2494 void
2495 preinitDiskcache()
2496 {
2497     return;
2498 }
2499
2500 void
2501 initDiskcache()
2502 {
2503     return;
2504 }
2505
2506 int
2507 writeoutToDisk(ObjectPtr object, int upto, int max)
2508 {
2509     return 0;
2510 }
2511
2512 int
2513 destroyDiskEntry(ObjectPtr object, int d)
2514 {
2515     return 0;
2516 }
2517
2518 ObjectPtr
2519 objectGetFromDisk(ObjectPtr object)
2520 {
2521     return NULL;
2522 }
2523
2524 int
2525 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2526 {
2527     return 0;
2528 }
2529
2530 int
2531 revalidateDiskEntry(ObjectPtr object)
2532 {
2533     return 0;
2534 }
2535
2536 void
2537 dirtyDiskEntry(ObjectPtr object)
2538 {
2539     return;
2540 }
2541
2542 void
2543 expireDiskObjects()
2544 {
2545     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2546 }
2547
2548 int
2549 diskEntrySize(ObjectPtr object)
2550 {
2551     return -1;
2552 }
2553 #endif