diskcache.c

   1 /*
   2 Copyright (c) 2003-2006 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheWriteoutOnClose = (32 * 1024);
  42
  43 int maxDiskCacheEntrySize = -1;
  44
  45 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  46 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  47 int diskCacheTruncateSize =  1024 * 1024;
  48 int preciseExpiry = 0;
  49
  50 static DiskCacheEntryRec negativeEntry = {
  51     NULL, NULL,
  52     -1, -1, -1, -1, 0, 0, 0, NULL, NULL
  53 };
  54
  55 #ifndef LOCAL_ROOT
  56 #define LOCAL_ROOT "/usr/share/polipo/www/"
  57 #endif
  58
  59 #ifndef DISK_CACHE_ROOT
  60 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  61 #endif
  62
  63 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  64 static int atomSetterFlush(ConfigVariablePtr, void*);
  65 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  66
  67 void
  68 preinitDiskcache()
  69 {
  70     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  71     localDocumentRoot = internAtom(LOCAL_ROOT);
  72
  73     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  74                              configIntSetter,
  75                              "Access rights for new directories.");
  76     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  77                              configIntSetter,
  78                              "Access rights for new cache files.");
  79     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  80                              configIntSetter,
  81                              "Number of bytes to write out eagerly.");
  82     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  83                              "Root of the disk cache.");
  84     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  85                              "Root of the local tree.");
  86     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  87                     "File descriptors used by the on-disk cache.");
  88     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  89                     "Time after which on-disk objects are removed.");
  90     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  91                     "Time after which on-disk objects are truncated.");
  92     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  93                     "Size to which on-disk objects are truncated.");
  94     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  95                     "Whether to consider all files for purging.");
  96     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  97                              configIntSetter,
  98                              "Maximum size of objects cached on disk.");
  99 }
 100
 101 static int
 102 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 103 {
 104     int i;
 105     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 106     i = *(int*)value;
 107     if(i < 0 || i > 1000000)
 108         return -3;
 109     maxDiskEntries = i;
 110     while(numDiskEntries > maxDiskEntries)
 111         destroyDiskEntry(diskEntriesLast->object, 0);
 112     return 1;
 113 }
 114
 115 static int
 116 atomSetterFlush(ConfigVariablePtr var, void *value)
 117 {
 118     discardObjects(1, 0);
 119     return configAtomSetter(var, value);
 120 }
 121
 122 static int
 123 checkRoot(AtomPtr root)
 124 {
 125     struct stat ss;
 126     int rc;
 127
 128     if(!root || root->length == 0)
 129         return 0;
 130
 131     if(root->string[0] != '/') {
 132         return -2;
 133     }
 134
 135     rc = stat(root->string, &ss);
 136     if(rc < 0)
 137         return -1;
 138     else if(!S_ISDIR(ss.st_mode)) {
 139         errno = ENOTDIR;
 140         return -1;
 141     }
 142     return 1;
 143 }
 144
 145 static AtomPtr
 146 maybeAddSlash(AtomPtr atom)
 147 {
 148     AtomPtr newAtom = NULL;
 149     if(!atom) return NULL;
 150     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 151         newAtom = atomCat(atom, "/");
 152         releaseAtom(atom);
 153         return newAtom;
 154     }
 155     return atom;
 156 }
 157
 158 void
 159 initDiskcache()
 160 {
 161     int rc;
 162
 163     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 164     rc = checkRoot(diskCacheRoot);
 165     if(rc <= 0) {
 166         switch(rc) {
 167         case 0: break;
 168         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 169         case -2:
 170             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 171                    diskCacheRoot->string);
 172             break;
 173         default: abort();
 174         }
 175         releaseAtom(diskCacheRoot);
 176         diskCacheRoot = NULL;
 177     }
 178
 179     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 180     rc = checkRoot(localDocumentRoot);
 181     if(rc <= 0) {
 182         switch(rc) {
 183         case 0: break;
 184         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 185         case -2:
 186             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 187             break;
 188         default: abort();
 189         }
 190         releaseAtom(localDocumentRoot);
 191         localDocumentRoot = NULL;
 192     }
 193 }
 194
 195 #ifdef DEBUG_DISK_CACHE
 196 #define CHECK_ENTRY(entry) check_entry((entry))
 197 static void
 198 check_entry(DiskCacheEntryPtr entry)
 199 {
 200     if(entry && entry->fd < 0)
 201         assert(entry == &negativeEntry);
 202     if(entry && entry->fd >= 0) {
 203         assert((!entry->previous) == (entry == diskEntries));
 204         assert((!entry->next) == (entry == diskEntriesLast));
 205         if(entry->size >= 0)
 206             assert(entry->size + entry->body_offset >= entry->offset);
 207         assert(entry->body_offset >= 0);
 208         if(entry->offset >= 0) {
 209             off_t offset;
 210             offset = lseek(entry->fd, 0, SEEK_CUR);
 211             assert(offset == entry->offset);
 212         }
 213         if(entry->size >= 0) {
 214             int rc;
 215             struct stat ss;
 216             rc = fstat(entry->fd, &ss);
 217             assert(rc >= 0);
 218             assert(ss.st_size == entry->size + entry->body_offset);
 219         }
 220     }
 221 }
 222 #else
 223 #define CHECK_ENTRY(entry) do {} while(0)
 224 #endif
 225
 226 int
 227 diskEntrySize(ObjectPtr object)
 228 {
 229     struct stat buf;
 230     int rc;
 231     DiskCacheEntryPtr entry = object->disk_entry;
 232
 233     if(!entry || entry == &negativeEntry)
 234         return -1;
 235
 236     if(entry->size >= 0)
 237         return entry->size;
 238
 239     rc = fstat(entry->fd, &buf);
 240     if(rc < 0) {
 241         do_log_error(L_ERROR, errno, "Couldn't stat");
 242         return -1;
 243     }
 244
 245     if(buf.st_size <= entry->body_offset)
 246         entry->size = 0;
 247     else
 248         entry->size =  buf.st_size - entry->body_offset;
 249     CHECK_ENTRY(entry);
 250     if(object->length >= 0 && entry->size == object->length)
 251         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 252     return entry->size;
 253 }
 254
 255 static int
 256 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 257 {
 258     off_t rc;
 259
 260     CHECK_ENTRY(entry);
 261     assert(entry != &negativeEntry);
 262     if(entry->offset == offset)
 263         return 1;
 264     if(offset > entry->body_offset) {
 265         /* Avoid extending the file by mistake */
 266         if(entry->size < 0)
 267             diskEntrySize(entry->object);
 268         if(entry->size < 0)
 269             return -1;
 270         if(entry->size + entry->body_offset < offset)
 271             return -1;
 272     }
 273     rc = lseek(entry->fd, offset, SEEK_SET);
 274     if(rc < 0) {
 275         do_log_error(L_ERROR, errno, "Couldn't seek");
 276         entry->offset = -1;
 277         return -1;
 278     }
 279     entry->offset = offset;
 280     return 1;
 281 }
 282
 283 /* Given a local URL, constructs the filename where it can be found. */
 284
 285 int
 286 localFilename(char *buf, int n, char *key, int len)
 287 {
 288     int i, j;
 289     if(len <= 0 || key[0] != '/') return -1;
 290
 291     if(urlIsSpecial(key, len)) return -1;
 292
 293     if(localDocumentRoot == NULL ||
 294        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 295         return -1;
 296
 297     if(n <= localDocumentRoot->length)
 298         return -1;
 299
 300     i = 0;
 301     if(key[i] != '/')
 302         return -1;
 303
 304     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 305     j = localDocumentRoot->length;
 306     if(buf[j - 1] == '/')
 307         j--;
 308
 309     while(i < len) {
 310         if(j >= n - 1)
 311             return -1;
 312         if(key[i] == '/' && i < len - 2)
 313             if(key[i + 1] == '.' &&
 314                (key[i + 2] == '.' || key[i + 2] == '/'))
 315                 return -1;
 316         buf[j++] = key[i++];
 317     }
 318
 319     if(buf[j - 1] == '/') {
 320         if(j >= n - 11)
 321             return -1;
 322         memcpy(buf + j, "index.html", 10);
 323         j += 10;
 324     }
 325
 326     buf[j] = '\0';
 327     return j;
 328 }
 329
 330 static void
 331 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 332 {
 333     static MD5_CTX ctx;
 334     MD5Init(&ctx);
 335     MD5Update(&ctx, key, len);
 336     MD5Final(&ctx);
 337     memcpy(dst, ctx.digest, 16);
 338 }
 339
 340 /* Check whether a character can be stored in a filename.  This is
 341    needed since we want to support deficient file systems. */
 342 static int
 343 fssafe(char c)
 344 {
 345     if(c <= 31 || c >= 127)
 346         return 0;
 347     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 348        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 349         return 1;
 350     return 0;
 351 }
 352
 353 /* Given a URL, returns the directory name within which all files
 354    starting with this URL can be found. */
 355 static int
 356 urlDirname(char *buf, int n, const char *url, int len)
 357 {
 358     int i, j;
 359     if(len < 8)
 360         return -1;
 361     if(memcmp(url, "http://", 7) != 0)
 362         return -1;
 363
 364     if(diskCacheRoot == NULL ||
 365        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 366         return -1;
 367
 368     if(n <= diskCacheRoot->length)
 369         return -1;
 370
 371     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 372     j = diskCacheRoot->length;
 373
 374     if(buf[j - 1] != '/')
 375         buf[j++] = '/';
 376
 377     for(i = 7; i < len; i++) {
 378         if(i >= len || url[i] == '/')
 379             break;
 380         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 381             return -1;
 382         if(url[i] == '%' || !fssafe(url[i])) {
 383             if(j + 3 >= n) return -1;
 384             buf[j++] = '%';
 385             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 386             buf[j++] = i2h(url[i] & 0x0F);
 387         } else {
 388             buf[j++] = url[i]; if(j >= n) return -1;
 389         }
 390     }
 391     buf[j++] = '/'; if(j >= n) return -1;
 392     buf[j] = '\0';
 393     return j;
 394 }
 395
 396 /* Given a URL, returns the filename where the cached data can be
 397    found. */
 398 static int
 399 urlFilename(char *restrict buf, int n, const char *url, int len)
 400 {
 401     int j;
 402     unsigned char md5buf[18];
 403     j = urlDirname(buf, n, url, len);
 404     if(j < 0 || j + 24 >= n)
 405         return -1;
 406     md5((unsigned char*)url, len, md5buf);
 407     b64cpy(buf + j, (char*)md5buf, 16, 1);
 408     buf[j + 24] = '\0';
 409     return j + 24;
 410 }
 411
 412 static char *
 413 dirnameUrl(char *url, int n, char *name, int len)
 414 {
 415     int i, j, k, c1, c2;
 416     k = diskCacheRoot->length;
 417     if(len < k)
 418         return NULL;
 419     if(memcmp(name, diskCacheRoot->string, k) != 0)
 420         return NULL;
 421     if(n < 8)
 422         return NULL;
 423     memcpy(url, "http://", 7);
 424     if(name[len - 1] == '/')
 425         len --;
 426     j = 7;
 427     for(i = k; i < len; i++) {
 428         if(name[i] == '%') {
 429             if(i >= len - 2)
 430                 return NULL;
 431             c1 = h2i(name[i + 1]);
 432             c2 = h2i(name[i + 2]);
 433             if(c1 < 0 || c2 < 0)
 434                 return NULL;
 435             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 436             i += 2;             /* skip extra digits */
 437         } else if(i < len - 1 &&
 438                   name[i] == '.' && name[i + 1] == '/') {
 439                 return NULL;
 440         } else if(i == len - 1 && name[i] == '.') {
 441             return NULL;
 442         } else {
 443             url[j++] = name[i]; if(j >= n) goto fail;
 444         }
 445     }
 446     url[j++] = '/'; if(j >= n) goto fail;
 447     url[j] = '\0';
 448     return url;
 449
 450  fail:
 451     return NULL;
 452 }
 453
 454 /* Create a file and all intermediate directories. */
 455 static int
 456 createFile(const char *name, int path_start)
 457 {
 458     int fd;
 459     char buf[1024];
 460     int n;
 461     int rc;
 462
 463     if(name[path_start] == '/')
 464         path_start++;
 465
 466     if(path_start < 2 || name[path_start - 1] != '/' ) {
 467         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 468         return -1;
 469     }
 470
 471     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 472               diskCacheFilePermissions);
 473     if(fd >= 0)
 474         return fd;
 475     if(errno != ENOENT) {
 476         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 477         return -1;
 478     }
 479
 480     n = path_start;
 481     while(name[n] != '\0' && n < 1024) {
 482         while(name[n] != '/' && name[n] != '\0' && n < 512)
 483             n++;
 484         if(name[n] != '/' || n >= 1024)
 485             break;
 486         memcpy(buf, name, n + 1);
 487         buf[n + 1] = '\0';
 488         rc = mkdir(buf, diskCacheDirectoryPermissions);
 489         if(rc < 0 && errno != EEXIST) {
 490             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 491             return -1;
 492         }
 493         n++;
 494     }
 495     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 496               diskCacheFilePermissions);
 497     if(fd < 0) {
 498         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 499         return -1;
 500     }
 501
 502     return fd;
 503 }
 504
 505 static int
 506 chooseBodyOffset(int n, ObjectPtr object)
 507 {
 508     int length = MAX(object->size, object->length);
 509     int body_offset;
 510
 511     if(object->length >= 0 && object->length + n < 4096 - 4)
 512         return -1;              /* no gap for small objects */
 513
 514     if(n <= 128)
 515         body_offset = 256;
 516     else if(n <= 192)
 517         body_offset = 384;
 518     else if(n <= 256)
 519         body_offset = 512;
 520     else if(n <= 384)
 521         body_offset = 768;
 522     else if(n <= 512)
 523         body_offset = 1024;
 524     else if(n <= 1024)
 525         body_offset = 2048;
 526     else if(n < 2048)
 527         body_offset = 4096;
 528     else
 529         body_offset = ((n + 32 + 4095) / 4096 + 1) * 4096;
 530
 531     /* Tweak the gap so that we don't use up a full disk block for
 532        a small tail */
 533     if(object->length >= 0 && object->length < 64 * 1024) {
 534         int last = (body_offset + object->length) % 4096;
 535         int gap = body_offset - n - 32;
 536         if(last < gap / 2)
 537             body_offset -= last;
 538     }
 539
 540     /* Rewriting large objects is expensive -- don't use small gaps.
 541        This has the additional benefit of block-aligning large bodies. */
 542     if(length >= 64 * 1024) {
 543         int min_gap, min_offset;
 544         if(length >= 512 * 1024)
 545             min_gap = 4096;
 546         else if(length >= 256 * 1024)
 547             min_gap = 2048;
 548         else
 549             min_gap = 1024;
 550
 551         min_offset = ((n + 32 + min_gap - 1) / min_gap + 1) * min_gap;
 552         body_offset = MAX(body_offset, min_offset);
 553     }
 554
 555     return body_offset;
 556 }
 557
 558 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 559    otherwise the offset at which the file descriptor is left. */
 560 /* If chunk is not null, it should be the first chunk of the object,
 561    and will be written out in the same operation if possible. */
 562 static int
 563 writeHeaders(int fd, int *body_offset_return,
 564              ObjectPtr object, char *chunk, int chunk_len)
 565 {
 566     int n;
 567     int rc;
 568     int body_offset = *body_offset_return;
 569     char *buf = NULL;
 570     int buf_is_chunk = 0;
 571     int bufsize = 0;
 572
 573     if(object->flags & OBJECT_LOCAL)
 574         return -1;
 575
 576     if(body_offset > CHUNK_SIZE)
 577         goto overflow;
 578
 579     /* get_chunk might trigger object expiry */
 580     bufsize = CHUNK_SIZE;
 581     buf_is_chunk = 1;
 582     buf = maybe_get_chunk();
 583     if(!buf) {
 584         bufsize = 2048;
 585         buf_is_chunk = 0;
 586         buf = malloc(2048);
 587         if(buf == NULL) {
 588             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 589             return -1;
 590         }
 591     }
 592
 593  format_again:
 594     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 595                   object->code, object->message->string);
 596
 597     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 598     if(n < 0)
 599         goto overflow;
 600
 601     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 602     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 603
 604     if(object->age >= 0 && object->age != object->date) {
 605         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 606         n = format_time(buf, n, bufsize, object->age);
 607     }
 608
 609     if(object->atime >= 0) {
 610         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 611         n = format_time(buf, n, bufsize, object->atime);
 612     }
 613
 614     if(n < 0)
 615         goto overflow;
 616
 617     if(body_offset < 0)
 618         body_offset = chooseBodyOffset(n, object);
 619
 620     if(body_offset > bufsize)
 621         goto overflow;
 622
 623     if(body_offset > 0 && body_offset != n + 4)
 624         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 625                       body_offset);
 626
 627     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 628     if(n < 0)
 629         goto overflow;
 630
 631     if(body_offset < 0)
 632         body_offset = n;
 633     if(n > body_offset)
 634         goto fail;
 635
 636     if(n < body_offset)
 637         memset(buf + n, 0, body_offset - n);
 638
 639  again:
 640 #ifdef HAVE_READV_WRITEV
 641     if(chunk_len > 0) {
 642         struct iovec iov[2];
 643         iov[0].iov_base = buf;
 644         iov[0].iov_len = body_offset;
 645         iov[1].iov_base = chunk;
 646         iov[1].iov_len = chunk_len;
 647         rc = writev(fd, iov, 2);
 648     } else
 649 #endif
 650         rc = write(fd, buf, body_offset);
 651
 652     if(rc < 0 && errno == EINTR)
 653         goto again;
 654
 655     if(rc < body_offset)
 656         goto fail;
 657     if(object->length >= 0 &&
 658        rc - body_offset >= object->length)
 659         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 660
 661     *body_offset_return = body_offset;
 662     if(buf_is_chunk)
 663         dispose_chunk(buf);
 664     else
 665         free(buf);
 666     return rc;
 667
 668  overflow:
 669     if(bufsize < bigBufferSize) {
 670         char *oldbuf = buf;
 671         buf = malloc(bigBufferSize);
 672         if(!buf) {
 673             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 674             goto fail;
 675         }
 676         bufsize = bigBufferSize;
 677         if(oldbuf) {
 678             if(buf_is_chunk)
 679                 dispose_chunk(oldbuf);
 680             else
 681                 free(oldbuf);
 682         }
 683         buf_is_chunk = 0;
 684         goto format_again;
 685     }
 686     /* fall through */
 687
 688  fail:
 689     if(buf_is_chunk)
 690         dispose_chunk(buf);
 691     else
 692         free(buf);
 693     return -1;
 694 }
 695
 696 typedef struct _MimeEntry {
 697     char *extension;
 698     char *mime;
 699 } MimeEntryRec;
 700
 701 static const MimeEntryRec mimeEntries[] = {
 702     { "html", "text/html" },
 703     { "htm", "text/html" },
 704     { "text", "text/plain" },
 705     { "txt", "text/plain" },
 706     { "png", "image/png" },
 707     { "gif", "image/gif" },
 708     { "jpeg", "image/jpeg" },
 709     { "jpg", "image/jpeg" },
 710     { "ico", "image/x-icon" },
 711     { "pdf", "application/pdf" },
 712     { "ps", "application/postscript" },
 713     { "tar", "application/x-tar" },
 714     { "pac", "application/x-ns-proxy-autoconfig" },
 715     { "css", "text/css" },
 716     { "js",  "application/x-javascript" },
 717     { "xml", "text/xml" },
 718     { "swf", "application/x-shockwave-flash" },
 719 };
 720
 721 static char*
 722 localObjectMimeType(ObjectPtr object, char **encoding_return)
 723 {
 724     char *name = object->key;
 725     int nlen = object->key_size;
 726     int i;
 727
 728     assert(nlen >= 1);
 729
 730     if(name[nlen - 1] == '/') {
 731         *encoding_return = NULL;
 732         return "text/html";
 733     }
 734
 735     if(nlen < 3) {
 736         *encoding_return = NULL;
 737         return "application/octet-stream";
 738     }
 739
 740     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 741         *encoding_return = "x-gzip";
 742         nlen -= 3;
 743     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 744         *encoding_return = "x-compress";
 745         nlen -= 2;
 746     } else {
 747         *encoding_return = NULL;
 748     }
 749
 750     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 751         int len = strlen(mimeEntries[i].extension);
 752         if(nlen > len &&
 753            name[nlen - len - 1] == '.' &&
 754            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 755             return mimeEntries[i].mime;
 756     }
 757
 758     return "application/octet-stream";
 759 }
 760
 761 /* Same interface as validateEntry -- see below */
 762 int
 763 validateLocalEntry(ObjectPtr object, int fd,
 764                    int *body_offset_return, off_t *offset_return)
 765 {
 766     struct stat ss;
 767     char buf[512];
 768     int n, rc;
 769     char *encoding;
 770
 771     rc = fstat(fd, &ss);
 772     if(rc < 0) {
 773         do_log_error(L_ERROR, errno, "Couldn't stat");
 774         return -1;
 775     }
 776
 777     if(S_ISREG(ss.st_mode)) {
 778         if(!(ss.st_mode & S_IROTH) ||
 779            (object->length >= 0 && object->length != ss.st_size) ||
 780            (object->last_modified >= 0 &&
 781             object->last_modified != ss.st_mtime))
 782             return -1;
 783     } else {
 784         notifyObject(object);
 785         return -1;
 786     }
 787
 788     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 789                   (unsigned long)ss.st_ino,
 790                   (unsigned long)ss.st_size,
 791                   (unsigned long)ss.st_mtime);
 792     if(n >= 512)
 793         n = -1;
 794
 795     if(n > 0 && object->etag) {
 796         if(strlen(object->etag) != n ||
 797            memcmp(object->etag, buf, n) != 0)
 798             return -1;
 799     }
 800
 801     if(!(object->flags & OBJECT_INITIAL)) {
 802         if(!object->last_modified && !object->etag)
 803             return -1;
 804     }
 805
 806     if(object->flags & OBJECT_INITIAL) {
 807         object->length = ss.st_size;
 808         object->last_modified = ss.st_mtime;
 809         object->date = current_time.tv_sec;
 810         object->age = current_time.tv_sec;
 811         object->code = 200;
 812         if(n > 0)
 813             object->etag = strdup(buf); /* okay if fails */
 814         object->message = internAtom("Okay");
 815         n = snnprintf(buf, 0, 512,
 816                       "\r\nServer: Polipo"
 817                       "\r\nContent-Type: %s",
 818                       localObjectMimeType(object, &encoding));
 819         if(encoding != NULL)
 820             n = snnprintf(buf, n, 512,
 821                           "\r\nContent-Encoding: %s", encoding);
 822         if(n < 0)
 823             return -1;
 824         object->headers = internAtomN(buf, n);
 825         if(object->headers == NULL)
 826             return -1;
 827         object->flags &= ~OBJECT_INITIAL;
 828     }
 829
 830     if(body_offset_return)
 831         *body_offset_return = 0;
 832     if(offset_return)
 833         *offset_return = 0;
 834     return 0;
 835 }
 836
 837 /* Assumes fd is at offset 0.
 838    Returns -1 if not valid, 1 if metadata should be written out, 0
 839    otherwise. */
 840 int
 841 validateEntry(ObjectPtr object, int fd,
 842               int *body_offset_return, off_t *offset_return)
 843 {
 844     char *buf;
 845     int buf_is_chunk, bufsize;
 846     int rc, n;
 847     int dummy;
 848     int code;
 849     AtomPtr headers;
 850     time_t date, last_modified, expires, polipo_age, polipo_access;
 851     int length;
 852     off_t offset = -1;
 853     int body_offset;
 854     char *etag;
 855     AtomPtr via;
 856     CacheControlRec cache_control;
 857     char *location;
 858     AtomPtr message;
 859     int dirty = 0;
 860
 861     if(object->flags & OBJECT_LOCAL)
 862         return validateLocalEntry(object, fd,
 863                                   body_offset_return, offset_return);
 864
 865     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 866         return 0;
 867
 868     /* get_chunk might trigger object expiry */
 869     bufsize = CHUNK_SIZE;
 870     buf_is_chunk = 1;
 871     buf = maybe_get_chunk();
 872     if(!buf) {
 873         bufsize = 2048;
 874         buf_is_chunk = 0;
 875         buf = malloc(2048);
 876         if(buf == NULL) {
 877             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 878             return -1;
 879         }
 880     }
 881
 882  again:
 883     rc = read(fd, buf, bufsize);
 884     if(rc < 0) {
 885         if(errno == EINTR)
 886             goto again;
 887         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 888         goto fail;
 889     }
 890     offset = rc;
 891
 892  parse_again:
 893     n = findEndOfHeaders(buf, 0, rc, &dummy);
 894     if(n < 0) {
 895         char *oldbuf = buf;
 896         if(bufsize < bigBufferSize) {
 897             buf = malloc(bigBufferSize);
 898             if(!buf) {
 899                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 900                 goto fail;
 901             }
 902             bufsize = bigBufferSize;
 903             memcpy(buf, oldbuf, offset);
 904             if(buf_is_chunk)
 905                 dispose_chunk(oldbuf);
 906             else
 907                 free(oldbuf);
 908             buf_is_chunk = 0;
 909         again2:
 910             rc = read(fd, buf + offset, bufsize - offset);
 911             if(rc < 0) {
 912                 if(errno == EINTR)
 913                     goto again2;
 914                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 915                 goto fail;
 916             }
 917             offset += rc;
 918             goto parse_again;
 919         }
 920         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 921         goto fail;
 922     }
 923
 924     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 925     if(rc < 0) {
 926         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 927         goto fail;
 928     }
 929
 930     if(object->code != 0 && object->code != code) {
 931         releaseAtom(message);
 932         goto fail;
 933     }
 934
 935     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 936                           &headers, &length, &cache_control, NULL, NULL,
 937                           &date, &last_modified, &expires, &polipo_age,
 938                           &polipo_access, &body_offset,
 939                           NULL, &etag, NULL,
 940                           NULL, NULL, &location, &via, NULL);
 941     if(rc < 0) {
 942         releaseAtom(message);
 943         goto fail;
 944     }
 945     if(body_offset < 0)
 946         body_offset = n;
 947
 948     if(!location || strlen(location) != object->key_size ||
 949        memcmp(location, object->key, object->key_size) != 0) {
 950         do_log(L_ERROR, "Inconsistent cache file for %s.\n", location);
 951         goto invalid;
 952     }
 953
 954     if(polipo_age < 0)
 955         polipo_age = date;
 956
 957     if(polipo_age < 0) {
 958         do_log(L_ERROR, "Undated disk entry for %s.\n", location);
 959         goto invalid;
 960     }
 961
 962     if(!(object->flags & OBJECT_INITIAL)) {
 963         if((last_modified >= 0) != (object->last_modified >= 0))
 964             goto invalid;
 965
 966         if((object->cache_control & CACHE_MISMATCH) ||
 967            (cache_control.flags & CACHE_MISMATCH))
 968             goto invalid;
 969
 970         if(last_modified >= 0 && object->last_modified >= 0 &&
 971            last_modified != object->last_modified)
 972             goto invalid;
 973
 974         if(length >= 0 && object->length >= 0)
 975             if(length != object->length)
 976                 goto invalid;
 977
 978         if(!!etag != !!object->etag)
 979             goto invalid;
 980
 981         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 982             goto invalid;
 983
 984         /* If we don't have a usable ETag, and either CACHE_VARY or we
 985            don't have a last-modified date, we validate disk entries by
 986            using their date. */
 987         if(!(etag && object->etag) &&
 988            (!(last_modified >= 0 && object->last_modified >= 0) ||
 989             ((cache_control.flags & CACHE_VARY) ||
 990              (object->cache_control & CACHE_VARY)))) {
 991             if(date >= 0 && date != object->date)
 992                 goto invalid;
 993             if(polipo_age >= 0 && polipo_age != object->age)
 994                 goto invalid;
 995         }
 996         if((object->cache_control & CACHE_VARY) && dontTrustVaryETag >= 1) {
 997             /* Check content-type to work around mod_gzip bugs */
 998             if(!httpHeaderMatch(atomContentType, object->headers, headers) ||
 999                !httpHeaderMatch(atomContentEncoding, object->headers, headers))
1000                 goto invalid;
1001         }
1002     }
1003
1004     if(location)
1005         free(location);
1006
1007     if(headers) {
1008         if(!object->headers)
1009             object->headers = headers;
1010         else
1011             releaseAtom(headers);
1012     }
1013
1014     if(object->code == 0) {
1015         object->code = code;
1016         object->message = retainAtom(message);
1017     }
1018     if(object->date <= date)
1019         object->date = date;
1020     else
1021         dirty = 1;
1022     if(object->last_modified < 0)
1023         object->last_modified = last_modified;
1024     if(object->expires < 0)
1025         object->expires = expires;
1026     else if(object->expires > expires)
1027         dirty = 1;
1028     if(object->age < 0)
1029         object->age = polipo_age;
1030     else if(object->age > polipo_age)
1031         dirty = 1;
1032     if(object->atime <= polipo_access)
1033         object->atime = polipo_access;
1034     else
1035         dirty = 1;
1036
1037     object->cache_control |= cache_control.flags;
1038
1039     if(object->age < 0) object->age = object->date;
1040     if(object->age < 0) object->age = 0; /* a long time ago */
1041     if(object->length < 0) object->length = length;
1042     if(!object->etag)
1043         object->etag = etag;
1044     else {
1045         if(etag)
1046             free(etag);
1047     }
1048     releaseAtom(message);
1049
1050     if(object->flags & OBJECT_INITIAL) object->via = via;
1051     object->flags &= ~OBJECT_INITIAL;
1052     if(offset > body_offset) {
1053         /* We need to make sure we don't invoke object expiry recursively */
1054         objectSetChunks(object, 1);
1055         if(object->numchunks >= 1) {
1056             if(object->chunks[0].data == NULL)
1057                 object->chunks[0].data = maybe_get_chunk();
1058             if(object->chunks[0].data)
1059                 objectAddData(object, buf + body_offset,
1060                               0, MIN(offset - body_offset, CHUNK_SIZE));
1061         }
1062     }
1063
1064     httpTweakCachability(object);
1065
1066     if(buf_is_chunk)
1067         dispose_chunk(buf);
1068     else
1069         free(buf);
1070     if(body_offset_return) *body_offset_return = body_offset;
1071     if(offset_return) *offset_return = offset;
1072     return dirty;
1073
1074  invalid:
1075     releaseAtom(message);
1076     if(etag) free(etag);
1077     if(location) free(location);
1078     if(via) releaseAtom(via);
1079     /* fall through */
1080
1081  fail:
1082     if(buf_is_chunk)
1083         dispose_chunk(buf);
1084     else
1085         free(buf);
1086     return -1;
1087 }
1088
1089 void
1090 dirtyDiskEntry(ObjectPtr object)
1091 {
1092     DiskCacheEntryPtr entry = object->disk_entry;
1093     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1094 }
1095
1096 int
1097 revalidateDiskEntry(ObjectPtr object)
1098 {
1099     DiskCacheEntryPtr entry = object->disk_entry;
1100     int rc;
1101     int body_offset;
1102
1103     if(!entry || entry == &negativeEntry)
1104         return 1;
1105
1106     CHECK_ENTRY(entry);
1107     rc = entrySeek(entry, 0);
1108     if(rc < 0) return 0;
1109
1110     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1111     if(rc < 0) {
1112         destroyDiskEntry(object, 0);
1113         return 0;
1114     }
1115     if(body_offset != entry->body_offset) {
1116         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1117                body_offset, entry->body_offset);
1118         destroyDiskEntry(object, 0);
1119         return 0;
1120     }
1121
1122     entry->metadataDirty |= !!rc;
1123     CHECK_ENTRY(entry);
1124     return 1;
1125 }
1126
1127 static inline int
1128 objectHasDiskEntry(ObjectPtr object)
1129 {
1130     return object->disk_entry && object->disk_entry != &negativeEntry;
1131 }
1132
1133 static DiskCacheEntryPtr
1134 makeDiskEntry(ObjectPtr object, int writeable, int create)
1135 {
1136     DiskCacheEntryPtr entry = NULL;
1137     char buf[1024];
1138     int fd = -1;
1139     int negative = 0, isWriteable = 0, size = -1, name_len = -1;
1140     char *name = NULL;
1141     off_t offset = -1;
1142     int body_offset = -1;
1143     int rc;
1144     int local = (object->flags & OBJECT_LOCAL) != 0;
1145     int dirty = 0;
1146
1147     if(local && (writeable || create))
1148         return NULL;
1149
1150     if(!local && !(object->flags & OBJECT_PUBLIC))
1151         return NULL;
1152
1153     if(maxDiskCacheEntrySize >= 0) {
1154         if(object->length > 0) {
1155             if(object->length > maxDiskCacheEntrySize)
1156                 return NULL;
1157         } else {
1158             if(object->size > maxDiskCacheEntrySize)
1159                 return NULL;
1160         }
1161     }
1162
1163     if(object->disk_entry) {
1164         entry = object->disk_entry;
1165         CHECK_ENTRY(entry);
1166         if(entry != &negativeEntry && (!writeable || entry->writeable)) {
1167             /* We'll keep the entry -- put it at the front. */
1168             if(entry != diskEntries && entry != &negativeEntry) {
1169                 entry->previous->next = entry->next;
1170                 if(entry->next)
1171                     entry->next->previous = entry->previous;
1172                 else
1173                     diskEntriesLast = entry->previous;
1174                 entry->next = diskEntries;
1175                 diskEntries->previous = entry;
1176                 entry->previous = NULL;
1177                 diskEntries = entry;
1178             }
1179             return entry;
1180         } else {
1181             if(entry == &negativeEntry) {
1182                 negative = 1;
1183                 if(!create) return NULL;
1184                 object->disk_entry = NULL;
1185             }
1186             entry = NULL;
1187             destroyDiskEntry(object, 0);
1188         }
1189     }
1190
1191     if(numDiskEntries > maxDiskEntries)
1192         destroyDiskEntry(diskEntriesLast->object, 0);
1193
1194     if(!local) {
1195         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1196             return NULL;
1197         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1198         if(name_len < 0) return NULL;
1199         if(!negative) {
1200             isWriteable = 1;
1201             fd = open(buf, O_RDWR | O_BINARY);
1202             if(fd < 0 && !writeable && errno == EACCES) {
1203                 writeable = 0;
1204                 fd = open(buf, O_RDONLY | O_BINARY);
1205             }
1206         }
1207         if(fd >= 0) {
1208             rc = validateEntry(object, fd, &body_offset, &offset);
1209             if(rc >= 0) {
1210                 dirty = rc;
1211             } else {
1212                 close(fd);
1213                 fd = -1;
1214                 rc = unlink(buf);
1215                 if(rc < 0 && errno != ENOENT) {
1216                     do_log_error(L_WARN,  errno,
1217                                  "Couldn't unlink stale disk entry %s",
1218                                  buf);
1219                     /* But continue -- it's okay to have stale entries. */
1220                 }
1221             }
1222         }
1223
1224         if(fd < 0 && create && name_len > 0 &&
1225            !(object->flags & OBJECT_INITIAL)) {
1226             isWriteable = 1;
1227             fd = createFile(buf, diskCacheRoot->length);
1228             if(fd < 0)
1229                 return NULL;
1230
1231             if(fd >= 0) {
1232                 char *data = NULL;
1233                 int dsize = 0;
1234                 if(object->numchunks > 0) {
1235                     data = object->chunks[0].data;
1236                     dsize = object->chunks[0].size;
1237                 }
1238                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1239                 if(rc < 0) {
1240                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1241                     rc = unlink(buf);
1242                     if(rc < 0 && errno != ENOENT)
1243                         do_log_error(L_ERROR, errno,
1244                                      "Couldn't unlink truncated entry %s",
1245                                      buf);
1246                     close(fd);
1247                     return NULL;
1248                 }
1249                 assert(rc >= body_offset);
1250                 size = rc - body_offset;
1251                 offset = rc;
1252                 dirty = 0;
1253             }
1254         }
1255     } else {
1256         /* local */
1257         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1258             return NULL;
1259
1260         name_len =
1261             localFilename(buf, 1024, object->key, object->key_size);
1262         if(name_len < 0)
1263             return NULL;
1264         isWriteable = 0;
1265         fd = open(buf, O_RDONLY | O_BINARY);
1266         if(fd >= 0) {
1267             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1268                 close(fd);
1269                 fd = -1;
1270             }
1271         }
1272         offset = 0;
1273     }
1274
1275     if(fd < 0) {
1276         object->disk_entry = &negativeEntry;
1277         return NULL;
1278     }
1279     assert(body_offset >= 0);
1280
1281     name = strdup_n(buf, name_len);
1282     if(name == NULL) {
1283         do_log(L_ERROR, "Couldn't allocate name.\n");
1284         close(fd);
1285         fd = -1;
1286         return NULL;
1287     }
1288
1289     entry = malloc(sizeof(DiskCacheEntryRec));
1290     if(entry == NULL) {
1291         do_log(L_ERROR, "Couldn't allocate entry.\n");
1292         free(name);
1293         close(fd);
1294         return NULL;
1295     }
1296
1297     entry->filename = name;
1298     entry->object = object;
1299     entry->fd = fd;
1300     entry->body_offset = body_offset;
1301     entry->local = local;
1302     entry->offset = offset;
1303     entry->size = size;
1304     entry->metadataDirty = dirty;
1305     entry->writeable = isWriteable;
1306
1307     entry->next = diskEntries;
1308     if(diskEntries)
1309         diskEntries->previous = entry;
1310     diskEntries = entry;
1311     if(diskEntriesLast == NULL)
1312         diskEntriesLast = entry;
1313     entry->previous = NULL;
1314     numDiskEntries++;
1315
1316     object->disk_entry = entry;
1317
1318     CHECK_ENTRY(entry);
1319     return entry;
1320 }
1321
1322 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1323 static int
1324 rewriteEntry(ObjectPtr object)
1325 {
1326     int old_body_offset = object->disk_entry->body_offset;
1327     int fd, rc, n;
1328     DiskCacheEntryPtr entry;
1329     char* buf;
1330     int buf_is_chunk, bufsize;
1331     int offset;
1332
1333     fd = dup(object->disk_entry->fd);
1334     if(fd < 0) {
1335         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1336         return -1;
1337     }
1338
1339     rc = destroyDiskEntry(object, 1);
1340     if(rc < 0) {
1341         close(fd);
1342         return -1;
1343     }
1344     entry = makeDiskEntry(object, 1, 1);
1345     if(!entry) {
1346         close(fd);
1347         return -1;
1348     }
1349
1350     offset = diskEntrySize(object);
1351     if(offset < 0) {
1352         close(fd);
1353         return -1;
1354     }
1355
1356     bufsize = CHUNK_SIZE;
1357     buf_is_chunk = 1;
1358     buf = maybe_get_chunk();
1359     if(!buf) {
1360         bufsize = 2048;
1361         buf_is_chunk = 0;
1362         buf = malloc(2048);
1363         if(buf == NULL) {
1364             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1365             close(fd);
1366             return -1;
1367         }
1368     }
1369
1370     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1371     if(rc < 0)
1372         goto done;
1373
1374     while(1) {
1375         CHECK_ENTRY(entry);
1376         n = read(fd, buf, bufsize);
1377         if(n <= 0)
1378             goto done;
1379         rc = entrySeek(entry, entry->body_offset + offset);
1380         if(rc < 0)
1381             goto done;
1382         rc = write(entry->fd, buf, n);
1383         if(rc >= 0) {
1384             entry->offset += rc;
1385             entry->size += rc;
1386         }
1387         if(rc < n)
1388             goto done;
1389     }
1390
1391  done:
1392     CHECK_ENTRY(entry);
1393     if(object->length >= 0 && entry->size == object->length)
1394         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1395     close(fd);
1396     if(buf_is_chunk)
1397         dispose_chunk(buf);
1398     else
1399         free(buf);
1400     return 1;
1401 }
1402
1403 int
1404 destroyDiskEntry(ObjectPtr object, int d)
1405 {
1406     DiskCacheEntryPtr entry = object->disk_entry;
1407     int rc, urc = 1;
1408
1409     assert(!entry || !entry->local || !d);
1410
1411     if(d && !entry)
1412         entry = makeDiskEntry(object, 1, 0);
1413
1414     CHECK_ENTRY(entry);
1415
1416     if(!entry || entry == &negativeEntry) {
1417         return 1;
1418     }
1419
1420     assert(entry->object == object);
1421
1422     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1423         /* See writeoutToDisk */
1424         d = 1;
1425     }
1426
1427     if(d) {
1428         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1429         if(entry->filename) {
1430             urc = unlink(entry->filename);
1431             if(urc < 0)
1432                 do_log_error(L_WARN, errno,
1433                              "Couldn't unlink %s", entry->filename);
1434         }
1435     } else {
1436         if(entry && entry->metadataDirty)
1437             writeoutMetadata(object);
1438         makeDiskEntry(object, 1, 0);
1439         /* rewriteDiskEntry may change the disk entry */
1440         entry = object->disk_entry;
1441         if(entry == NULL || entry == &negativeEntry)
1442             return 0;
1443         if(entry->writeable && diskCacheWriteoutOnClose > 0)
1444             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1445     }
1446  again:
1447     rc = close(entry->fd);
1448     if(rc < 0 && errno == EINTR)
1449         goto again;
1450
1451     entry->fd = -1;
1452
1453     if(entry->filename)
1454         free(entry->filename);
1455     entry->filename = NULL;
1456
1457     if(entry->previous)
1458         entry->previous->next = entry->next;
1459     else
1460         diskEntries = entry->next;
1461     if(entry->next)
1462         entry->next->previous = entry->previous;
1463     else
1464         diskEntriesLast = entry->previous;
1465
1466     numDiskEntries--;
1467     assert(numDiskEntries >= 0);
1468
1469     free(entry);
1470     object->disk_entry = NULL;
1471     if(urc < 0)
1472         return -1;
1473     else
1474         return 1;
1475 }
1476
1477 ObjectPtr
1478 objectGetFromDisk(ObjectPtr object)
1479 {
1480     DiskCacheEntryPtr entry = makeDiskEntry(object, 0, 0);
1481     if(!entry) return NULL;
1482     return object;
1483 }
1484
1485
1486 int
1487 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1488 {
1489     DiskCacheEntryPtr entry;
1490     int rc, result;
1491     int i, j, k;
1492     int complete;
1493
1494     if(object->type != OBJECT_HTTP)
1495         return 0;
1496
1497     if(object->flags & OBJECT_LINEAR)
1498         return 0;
1499
1500     if(object->length >= 0) {
1501         chunks = MIN(chunks,
1502                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1503     }
1504
1505     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1506     if(rc < 0)
1507         return 0;
1508
1509     complete = 1;
1510     if(object->flags & OBJECT_INITIAL) {
1511         complete = 0;
1512     } else if((object->length < 0 || object->size < object->length) &&
1513               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1514         complete = 0;
1515     } else {
1516         for(k = 0; k < chunks; k++) {
1517             int s;
1518             i = offset / CHUNK_SIZE + k;
1519             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1520             if(object->chunks[i].size < s) {
1521                 complete = 0;
1522                 break;
1523             }
1524         }
1525     }
1526
1527     if(complete)
1528         return 1;
1529
1530     /* This has the side-effect of revalidating the entry, which is
1531        what makes HEAD requests work. */
1532     entry = makeDiskEntry(object, 0, 0);
1533     if(!entry)
1534         return 0;
1535
1536     for(k = 0; k < chunks; k++) {
1537         i = offset / CHUNK_SIZE + k;
1538         if(!object->chunks[i].data)
1539             object->chunks[i].data = get_chunk();
1540         if(!object->chunks[i].data) {
1541             chunks = k;
1542             break;
1543         }
1544         lockChunk(object, i);
1545     }
1546
1547     result = 0;
1548
1549     for(k = 0; k < chunks; k++) {
1550         int o;
1551         i = offset / CHUNK_SIZE + k;
1552         j = object->chunks[i].size;
1553         o = i * CHUNK_SIZE + j;
1554
1555         if(object->chunks[i].size == CHUNK_SIZE)
1556             continue;
1557
1558         if(entry->size >= 0 && entry->size <= o)
1559             break;
1560
1561         if(entry->offset != entry->body_offset + o) {
1562             rc = entrySeek(entry, entry->body_offset + o);
1563             if(rc < 0) {
1564                 result = 0;
1565                 break;
1566             }
1567         }
1568
1569         CHECK_ENTRY(entry);
1570         again:
1571         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1572         if(rc < 0) {
1573             if(errno == EINTR)
1574                 goto again;
1575             entry->offset = -1;
1576             do_log_error(L_ERROR, errno, "Couldn't read");
1577             break;
1578         }
1579
1580         entry->offset += rc;
1581         object->chunks[i].size += rc;
1582         if(object->size < o + rc)
1583             object->size = o + rc;
1584
1585         if(entry->object->length >= 0 && entry->size < 0 &&
1586            entry->offset - entry->body_offset == entry->object->length)
1587             entry->size = entry->object->length;
1588
1589         if(rc < CHUNK_SIZE - j) {
1590             /* Paranoia: the read may have been interrupted half-way. */
1591             if(entry->size < 0) {
1592                 if(rc == 0 ||
1593                    (entry->object->length >= 0 &&
1594                     entry->object->length ==
1595                     entry->offset - entry->body_offset))
1596                     entry->size = entry->offset - entry->body_offset;
1597                 break;
1598             } else if(entry->size != entry->offset - entry->body_offset) {
1599                 if(rc == 0 ||
1600                    entry->size < entry->offset - entry->body_offset) {
1601                     do_log(L_WARN,
1602                            "Disk entry size changed behind our back: "
1603                            "%ld -> %ld (%d).\n",
1604                            (long)entry->size,
1605                            (long)entry->offset - entry->body_offset,
1606                            object->size);
1607                     entry->size = -1;
1608                 }
1609             }
1610             break;
1611         }
1612
1613         CHECK_ENTRY(entry);
1614         result = 1;
1615     }
1616
1617     CHECK_ENTRY(object->disk_entry);
1618     for(k = 0; k < chunks; k++) {
1619         i = offset / CHUNK_SIZE + k;
1620         unlockChunk(object, i);
1621     }
1622
1623     if(result > 0) {
1624         notifyObject(object);
1625         return 1;
1626     } else {
1627         return 0;
1628     }
1629 }
1630
1631 int
1632 writeoutToDisk(ObjectPtr object, int upto, int max)
1633 {
1634     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1635         /* An object was created with an unknown length, and then grew
1636            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1637         destroyDiskEntry(object, 1);
1638         return 0;
1639     }
1640
1641     return reallyWriteoutToDisk(object, upto, max);
1642 }
1643
1644 static int
1645 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1646 {
1647     DiskCacheEntryPtr entry;
1648     int rc;
1649     int i, j;
1650     int offset;
1651     int bytes = 0;
1652
1653     if(upto < 0)
1654         upto = object->size;
1655
1656     if((object->cache_control & CACHE_NO_STORE) ||
1657        (object->flags & OBJECT_LOCAL))
1658         return 0;
1659
1660     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1661         return 0;
1662
1663     entry = makeDiskEntry(object, 1, 1);
1664     if(!entry) return 0;
1665
1666     assert(!entry->local);
1667
1668     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1669         goto done;
1670
1671     diskEntrySize(object);
1672     if(entry->size < 0)
1673         return 0;
1674
1675     if(object->length >= 0 && entry->size >= object->length) {
1676         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1677         goto done;
1678     }
1679
1680     if(entry->size >= upto)
1681         goto done;
1682
1683     if(!entry->writeable) {
1684         entry = makeDiskEntry(object, 1, 1);
1685         if(!entry)
1686             return 0;
1687         if(!entry->writeable)
1688             return 0;
1689         diskEntrySize(object);
1690         if(entry->size < 0)
1691             return 0;
1692     }
1693
1694     offset = entry->size;
1695
1696     /* Avoid a seek in case we start writing at the beginning */
1697     if(offset == 0 && entry->metadataDirty) {
1698         writeoutMetadata(object);
1699         /* rewriteDiskEntry may change the entry */
1700         entry = makeDiskEntry(object, 1, 0);
1701         if(entry == NULL || !entry->writeable)
1702             return 0;
1703     }
1704
1705     rc = entrySeek(entry, offset + entry->body_offset);
1706     if(rc < 0) return 0;
1707
1708     do {
1709         if(max >= 0 && bytes >= max)
1710             break;
1711         CHECK_ENTRY(entry);
1712         assert(entry->offset == offset + entry->body_offset);
1713         i = offset / CHUNK_SIZE;
1714         j = offset % CHUNK_SIZE;
1715         if(i >= object->numchunks)
1716             break;
1717         if(object->chunks[i].size <= j)
1718             break;
1719     again:
1720         rc = write(entry->fd, object->chunks[i].data + j,
1721                    object->chunks[i].size - j);
1722         if(rc < 0) {
1723             if(errno == EINTR)
1724                 goto again;
1725             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1726             break;
1727         }
1728         entry->offset += rc;
1729         offset += rc;
1730         bytes += rc;
1731         if(entry->size < offset)
1732             entry->size = offset;
1733     } while(j + rc >= CHUNK_SIZE);
1734
1735  done:
1736     CHECK_ENTRY(entry);
1737     if(entry->metadataDirty)
1738         writeoutMetadata(object);
1739
1740     return bytes;
1741 }
1742
1743 int
1744 writeoutMetadata(ObjectPtr object)
1745 {
1746     DiskCacheEntryPtr entry;
1747     int rc;
1748
1749     if((object->cache_control & CACHE_NO_STORE) ||
1750        (object->flags & OBJECT_LOCAL))
1751         return 0;
1752
1753     entry = makeDiskEntry(object, 1, 0);
1754     if(entry == NULL || entry == &negativeEntry)
1755         goto fail;
1756
1757     assert(!entry->local);
1758
1759     rc = entrySeek(entry, 0);
1760     if(rc < 0) goto fail;
1761
1762     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1763     if(rc == -2) {
1764         rc = rewriteEntry(object);
1765         if(rc < 0) return 0;
1766         return 1;
1767     }
1768     if(rc < 0) goto fail;
1769     entry->offset = rc;
1770     entry->metadataDirty = 0;
1771     return 1;
1772
1773  fail:
1774     /* We need this in order to avoid trying to write this entry out
1775        multiple times. */
1776     if(entry && entry != &negativeEntry)
1777         entry->metadataDirty = 0;
1778     return 0;
1779 }
1780
1781 static void
1782 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1783 {
1784     if(dst->filename == NULL) {
1785         dst->filename = src->filename;
1786         dst->body_offset = src->body_offset;
1787     } else
1788         free(src->filename);
1789     free(src->location);
1790     if(dst->length < 0)
1791         dst->length = src->length;
1792     if(dst->size < 0)
1793         dst->size = src->size;
1794     if(dst->age < 0)
1795         dst->age = src->age;
1796     if(dst->date < 0)
1797         dst->date = src->date;
1798     if(dst->last_modified < 0)
1799         dst->last_modified = src->last_modified;
1800     free(src);
1801 }
1802
1803 DiskObjectPtr
1804 readDiskObject(char *filename, struct stat *sb)
1805 {
1806     int fd, rc, n, dummy, code;
1807     int length, size;
1808     time_t date, last_modified, age, atime, expires;
1809     char *location = NULL, *fn = NULL;
1810     DiskObjectPtr dobject;
1811     char *buf;
1812     int buf_is_chunk, bufsize;
1813     int body_offset;
1814     struct stat ss;
1815
1816     fd = -1;
1817
1818     if(sb == NULL) {
1819         rc = stat(filename, &ss);
1820         if(rc < 0) {
1821             do_log_error(L_WARN, errno, "Couldn't stat %s", filename);
1822             return NULL;
1823         }
1824         sb = &ss;
1825     }
1826
1827     buf_is_chunk = 1;
1828     bufsize = CHUNK_SIZE;
1829     buf = get_chunk();
1830     if(buf == NULL) {
1831         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1832         return NULL;
1833     }
1834
1835     if(S_ISREG(sb->st_mode)) {
1836         fd = open(filename, O_RDONLY | O_BINARY);
1837         if(fd < 0)
1838             goto fail;
1839     again:
1840         rc = read(fd, buf, bufsize);
1841         if(rc < 0)
1842             goto fail;
1843
1844         n = findEndOfHeaders(buf, 0, rc, &dummy);
1845         if(n < 0) {
1846             long lrc;
1847             if(buf_is_chunk) {
1848                 dispose_chunk(buf);
1849                 buf_is_chunk = 0;
1850                 bufsize = bigBufferSize;
1851                 buf = malloc(bigBufferSize);
1852                 if(buf == NULL)
1853                     goto fail2;
1854                 lrc = lseek(fd, 0, SEEK_SET);
1855                 if(lrc < 0)
1856                     goto fail;
1857                 goto again;
1858             }
1859             goto fail;
1860         }
1861
1862         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1863         if(rc < 0)
1864             goto fail;
1865
1866         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1867                               NULL, &length, NULL, NULL, NULL,
1868                               &date, &last_modified, &expires, &age,
1869                               &atime, &body_offset, NULL,
1870                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1871         if(rc < 0 || location == NULL)
1872             goto fail;
1873         if(body_offset < 0)
1874             body_offset = n;
1875
1876         size = sb->st_size - body_offset;
1877         if(size < 0)
1878             size = 0;
1879     } else if(S_ISDIR(sb->st_mode)) {
1880         char *n;
1881         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1882         if(n == NULL)
1883             goto fail;
1884         location = strdup(n);
1885         if(location == NULL)
1886             goto fail;
1887         length = -1;
1888         size = -1;
1889         body_offset = -1;
1890         age = -1;
1891         atime = -1;
1892         date = -1;
1893         last_modified = -1;
1894     } else {
1895         goto fail;
1896     }
1897
1898     dobject = malloc(sizeof(DiskObjectRec));
1899     if(!dobject)
1900         goto fail;
1901
1902     fn = strdup(filename);
1903     if(!fn)
1904         goto fail;
1905
1906     if(buf_is_chunk)
1907         dispose_chunk(buf);
1908     else
1909         free(buf);
1910
1911     dobject->location = location;
1912     dobject->filename = fn;
1913     dobject->length = length;
1914     dobject->body_offset = body_offset;
1915     dobject->size = size;
1916     dobject->age = age;
1917     dobject->access = atime;
1918     dobject->date = date;
1919     dobject->last_modified = last_modified;
1920     dobject->expires = expires;
1921     if(fd >= 0) close(fd);
1922     return dobject;
1923
1924  fail:
1925     if(buf_is_chunk)
1926         dispose_chunk(buf);
1927     else
1928         free(buf);
1929  fail2:
1930     if(fd >= 0) close(fd);
1931     if(location) free(location);
1932     return NULL;
1933 }
1934
1935
1936 DiskObjectPtr
1937 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1938 {
1939     DiskObjectPtr dobject = NULL;
1940     int c = 0;
1941
1942     dobject = readDiskObject((char*)filename, sb);
1943     if(dobject == NULL)
1944         return 0;
1945
1946     if(!dobjects ||
1947        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1948         if(dobjects && c == 0) {
1949             mergeDobjects(dobjects, dobject);
1950         } else {
1951             dobject->next = dobjects;
1952             dobjects = dobject;
1953         }
1954     } else {
1955         DiskObjectPtr other = dobjects;
1956         while(other->next) {
1957             c = strcmp(dobject->location, other->next->location);
1958             if(c < 0)
1959                 break;
1960             other = other->next;
1961         }
1962         if(strcmp(dobject->location, other->location) == 0) {
1963             mergeDobjects(other, dobject);
1964         } else {
1965             dobject->next = other->next;
1966             other->next = dobject;
1967         }
1968     }
1969     return dobjects;
1970 }
1971
1972 /* Determine whether p is below root */
1973 static int
1974 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1975 {
1976     char *cp;
1977     int m = strlen(p->location);
1978     if(m < n)
1979         return 0;
1980     if(memcmp(root, p->location, n) != 0)
1981         return 0;
1982     if(recursive)
1983         return 1;
1984     if(m == 0 || p->location[m - 1] == '/')
1985         return 1;
1986     cp = strchr(p->location + n, '/');
1987     if(cp && cp - p->location != m - 1)
1988         return 0;
1989     return 1;
1990 }
1991
1992 /* Filter out all disk objects that are not under root */
1993 DiskObjectPtr
1994 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
1995 {
1996     int n = strlen(root);
1997     DiskObjectPtr p, q;
1998
1999     while(from && !filter(from, root, n, recursive)) {
2000         p = from;
2001         from = p->next;
2002         free(p->location);
2003         free(p);
2004     }
2005
2006     p = from;
2007     while(p && p->next) {
2008         if(!filter(p->next, root, n, recursive)) {
2009             q = p->next;
2010             p->next = q->next;
2011             free(q->location);
2012             free(q);
2013         } else {
2014             p = p->next;
2015         }
2016     }
2017     return from;
2018 }
2019
2020 DiskObjectPtr
2021 insertRoot(DiskObjectPtr from, const char *root)
2022 {
2023     DiskObjectPtr p;
2024
2025     p = from;
2026     while(p) {
2027         if(strcmp(root, p->location) == 0)
2028             return from;
2029         p = p->next;
2030     }
2031
2032     p = malloc(sizeof(DiskObjectRec));
2033     if(!p) return from;
2034     p->location = strdup(root);
2035     if(p->location == NULL) {
2036         free(p);
2037         return from;
2038     }
2039     p->filename = NULL;
2040     p->length = -1;
2041     p->size = -1;
2042     p->age = -1;
2043     p->access = -1;
2044     p->last_modified = -1;
2045     p->expires = -1;
2046     p->next = from;
2047     return p;
2048 }
2049
2050 /* Insert all missing directories in a sorted list of dobjects */
2051 DiskObjectPtr
2052 insertDirs(DiskObjectPtr from)
2053 {
2054     DiskObjectPtr p, q, new;
2055     int n, m;
2056     char *cp;
2057
2058     p = NULL; q = from;
2059     while(q) {
2060         n = strlen(q->location);
2061         if(n > 0 && q->location[n - 1] != '/') {
2062             cp = strrchr(q->location, '/');
2063             m = cp - q->location + 1;
2064             if(cp && (!p || strlen(p->location) < m ||
2065                       memcmp(p->location, q->location, m) != 0)) {
2066                 new = malloc(sizeof(DiskObjectRec));
2067                 if(!new) break;
2068                 new->location = strdup_n(q->location, m);
2069                 if(new->location == NULL) {
2070                     free(new);
2071                     break;
2072                 }
2073                 new->filename = NULL;
2074                 new->length = -1;
2075                 new->size = -1;
2076                 new->age = -1;
2077                 new->access = -1;
2078                 new->last_modified = -1;
2079                 new->expires = -1;
2080                 new->next = q;
2081                 if(p)
2082                     p->next = new;
2083                 else
2084                     from = new;
2085             }
2086         }
2087         p = q;
2088         q = q->next;
2089     }
2090     return from;
2091 }
2092
2093 void
2094 indexDiskObjects(FILE *out, const char *root, int recursive)
2095 {
2096     int n, i, isdir;
2097     DIR *dir;
2098     struct dirent *dirent;
2099     char buf[1024];
2100     char *fts_argv[2];
2101     FTS *fts;
2102     FTSENT *fe;
2103     DiskObjectPtr dobjects = NULL;
2104     char *of = root[0] == '\0' ? "" : " of ";
2105
2106     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2107             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2108             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2109             "<html><head>\n"
2110             "<title>%s%s%s</title>\n"
2111             "</head><body>\n"
2112             "<h1>%s%s%s</h1>\n",
2113             recursive ? "Recursive index" : "Index", of, root,
2114             recursive ? "Recursive index" : "Index", of, root);
2115
2116     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2117         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2118         goto trailer;
2119     }
2120
2121     if(diskCacheRoot->length >= 1024) {
2122         fprintf(out,
2123                 "<p>The value of <tt>diskCacheRoot</tt> is "
2124                 "too long (%d).</p>\n",
2125                 diskCacheRoot->length);
2126         goto trailer;
2127     }
2128
2129     if(strlen(root) < 8) {
2130         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2131         buf[diskCacheRoot->length] = '\0';
2132         n = diskCacheRoot->length;
2133     } else {
2134         n = urlDirname(buf, 1024, root, strlen(root));
2135     }
2136     if(n > 0) {
2137         if(recursive) {
2138             dir = NULL;
2139             fts_argv[0] = buf;
2140             fts_argv[1] = NULL;
2141             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2142             if(fts) {
2143                 while(1) {
2144                     fe = fts_read(fts);
2145                     if(!fe) break;
2146                     if(fe->fts_info != FTS_DP)
2147                         dobjects =
2148                             processObject(dobjects,
2149                                           fe->fts_path,
2150                                           fe->fts_info == FTS_NS ||
2151                                           fe->fts_info == FTS_NSOK ?
2152                                           fe->fts_statp : NULL);
2153                 }
2154                 fts_close(fts);
2155             }
2156         } else {
2157             dir = opendir(buf);
2158             if(dir) {
2159                 while(1) {
2160                     dirent = readdir(dir);
2161                     if(!dirent) break;
2162                     if(n + strlen(dirent->d_name) < 1024) {
2163                         strcpy(buf + n, dirent->d_name);
2164                     } else {
2165                         continue;
2166                     }
2167                     dobjects = processObject(dobjects, buf, NULL);
2168                 }
2169                 closedir(dir);
2170             } else {
2171                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2172                         strerror(errno), errno);
2173                 goto trailer;
2174             }
2175         }
2176     }
2177
2178     if(dobjects) {
2179         DiskObjectPtr dobject;
2180         int entryno;
2181         dobjects = insertRoot(dobjects, root);
2182         dobjects = insertDirs(dobjects);
2183         dobjects = filterDiskObjects(dobjects, root, recursive);
2184         dobject = dobjects;
2185         buf[0] = '\0';
2186         alternatingHttpStyle(out, "diskcachelist");
2187         fprintf(out, "<table id=diskcachelist>\n");
2188         fprintf(out, "<tbody>\n");
2189         entryno = 0;
2190         while(dobjects) {
2191             dobject = dobjects;
2192             i = strlen(dobject->location);
2193             isdir = (i == 0 || dobject->location[i - 1] == '/');
2194             if(entryno % 2)
2195                 fprintf(out, "<tr class=odd>");
2196             else
2197                 fprintf(out, "<tr class=even>");
2198             if(dobject->size >= 0) {
2199                 fprintf(out, "<td><a href=\"%s\"><tt>",
2200                         dobject->location);
2201                 htmlPrint(out,
2202                           dobject->location, strlen(dobject->location));
2203                 fprintf(out, "</tt></a></td> ");
2204                 if(dobject->length >= 0) {
2205                     if(dobject->size == dobject->length)
2206                         fprintf(out, "<td>%d</td> ", dobject->length);
2207                     else
2208                         fprintf(out, "<td>%d/%d</td> ",
2209                                dobject->size, dobject->length);
2210                 } else {
2211                     /* Avoid a trigraph. */
2212                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2213                 }
2214                 if(dobject->last_modified >= 0) {
2215                     struct tm *tm = gmtime(&dobject->last_modified);
2216                     if(tm == NULL)
2217                         n = -1;
2218                     else
2219                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2220                 } else
2221                     n = -1;
2222                 if(n > 0) {
2223                     buf[n] = '\0';
2224                     fprintf(out, "<td>%s</td> ", buf);
2225                 } else {
2226                     fprintf(out, "<td></td>");
2227                 }
2228
2229                 if(dobject->date >= 0) {
2230                     struct tm *tm = gmtime(&dobject->date);
2231                     if(tm == NULL)
2232                         n = -1;
2233                     else
2234                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2235                 } else
2236                     n = -1;
2237                 if(n > 0) {
2238                     buf[n] = '\0';
2239                     fprintf(out, "<td>%s</td>", buf);
2240                 } else {
2241                     fprintf(out, "<td></td>");
2242                 }
2243             } else {
2244                 fprintf(out, "<td><tt>");
2245                 htmlPrint(out, dobject->location,
2246                           strlen(dobject->location));
2247                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2248             }
2249             if(isdir) {
2250                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2251                         "<td><a href=\"/polipo/recursive-index?%s\">"
2252                         "recursive</a></td>",
2253                         dobject->location, dobject->location);
2254             }
2255             fprintf(out, "</tr>\n");
2256             entryno++;
2257             dobjects = dobject->next;
2258             free(dobject->location);
2259             free(dobject->filename);
2260             free(dobject);
2261         }
2262         fprintf(out, "</tbody>\n");
2263         fprintf(out, "</table>\n");
2264     }
2265
2266  trailer:
2267     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2268     fprintf(out, "</body></html>\n");
2269     return;
2270 }
2271
2272 static int
2273 checkForZeroes(char *buf, int n)
2274 {
2275     int i, j;
2276     unsigned long *lbuf = (unsigned long *)buf;
2277     assert(n % sizeof(unsigned long) == 0);
2278
2279     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2280         if(lbuf[i] != 0L)
2281             return i * sizeof(unsigned long);
2282     }
2283     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2284         if(buf[i * sizeof(unsigned long) + j] != 0)
2285             break;
2286     }
2287
2288     return i * sizeof(unsigned long) + j;
2289 }
2290
2291 static int
2292 copyFile(int from, char *filename, int n)
2293 {
2294     char *buf;
2295     int to, offset, nread, nzeroes, rc;
2296
2297     buf = malloc(CHUNK_SIZE);
2298     if(buf == NULL)
2299         return -1;
2300
2301     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2302               diskCacheFilePermissions);
2303     if(to < 0) {
2304         free(buf);
2305         return -1;
2306     }
2307
2308     offset = 0;
2309     while(offset < n) {
2310         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2311         if(nread <= 0)
2312             break;
2313         nzeroes = checkForZeroes(buf, nread & -8);
2314         if(nzeroes > 0) {
2315             /* I like holes */
2316             rc = lseek(to, nzeroes, SEEK_CUR);
2317             if(rc != offset + nzeroes) {
2318                 if(rc < 0)
2319                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2320                 else
2321                     do_log(L_ERROR,
2322                            "Couldn't extend file: "
2323                            "unexpected offset %d != %d + %d.\n",
2324                            rc, offset, nread);
2325                 break;
2326             }
2327         }
2328         if(nread > nzeroes) {
2329             rc = write(to, buf + nzeroes, nread - nzeroes);
2330             if(rc != nread - nzeroes) {
2331                 if(rc < 0)
2332                     do_log_error(L_ERROR, errno, "Couldn't write");
2333                 else
2334                     do_log(L_ERROR, "Short write.\n");
2335                 break;
2336             }
2337         }
2338         offset += nread;
2339     }
2340     free(buf);
2341     close(to);
2342     if(offset <= 0)
2343         unlink(filename);       /* something went wrong straight away */
2344     return 1;
2345 }
2346
2347 static long int
2348 expireFile(char *filename, struct stat *sb,
2349            int *considered, int *unlinked, int *truncated)
2350 {
2351     DiskObjectPtr dobject = NULL;
2352     time_t t;
2353     int fd, rc;
2354     long int ret = sb->st_size;
2355
2356     if(!preciseExpiry) {
2357         t = sb->st_mtime;
2358         if(t > current_time.tv_sec + 1) {
2359             do_log(L_WARN, "File %s has access time in the future.\n",
2360                    filename);
2361             t = current_time.tv_sec;
2362         }
2363
2364         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2365            (sb->st_size < diskCacheTruncateSize ||
2366             t > current_time.tv_sec - diskCacheTruncateTime))
2367             return ret;
2368     }
2369
2370     (*considered)++;
2371
2372     dobject = readDiskObject(filename, sb);
2373     if(!dobject) {
2374         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n", filename);
2375         rc = unlink(filename);
2376         if(rc < 0) {
2377             do_log_error(L_ERROR, errno,
2378                          "Couldn't unlink %s", filename);
2379             return ret;
2380         } else {
2381             (*unlinked)++;
2382             return 0;
2383         }
2384     }
2385
2386     t = dobject->access;
2387     if(t < 0) t = dobject->age;
2388     if(t < 0) t = dobject->date;
2389
2390     if(t > current_time.tv_sec)
2391         do_log(L_WARN,
2392                "Disk entry %s (%s) has access time in the future.\n",
2393                dobject->location, dobject->filename);
2394
2395     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2396         rc = unlink(dobject->filename);
2397         if(rc < 0) {
2398             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2399         } else {
2400             (*unlinked)++;
2401             ret = 0;
2402         }
2403     } else if(dobject->size >
2404               diskCacheTruncateSize + 4 * dobject->body_offset &&
2405               t < current_time.tv_sec - diskCacheTruncateTime) {
2406         /* We need to copy rather than simply truncate in place: the
2407            latter would confuse a running polipo. */
2408         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2409         rc = unlink(dobject->filename);
2410         if(rc < 0) {
2411             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2412             close(fd);
2413             fd = -1;
2414         } else {
2415             (*unlinked)++;
2416             copyFile(fd, dobject->filename,
2417                      dobject->body_offset + diskCacheTruncateSize);
2418             close(fd);
2419             (*unlinked)--;
2420             (*truncated)++;
2421             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2422         }
2423     }
2424     free(dobject->location);
2425     free(dobject->filename);
2426     free(dobject);
2427     return ret;
2428 }
2429
2430 void
2431 expireDiskObjects()
2432 {
2433     int rc;
2434     char *fts_argv[2];
2435     FTS *fts;
2436     FTSENT *fe;
2437     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2438     int dirs = 0, rmdirs = 0;
2439     long left = 0, total = 0;
2440
2441     if(diskCacheRoot == NULL ||
2442        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2443         return;
2444
2445     fts_argv[0] = diskCacheRoot->string;
2446     fts_argv[1] = NULL;
2447     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2448     if(fts == NULL) {
2449         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2450     } else {
2451         while(1) {
2452             gettimeofday(&current_time, NULL);
2453
2454             fe = fts_read(fts);
2455             if(!fe) break;
2456
2457             if(fe->fts_info == FTS_D)
2458                 continue;
2459
2460             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2461                fe->fts_info == FTS_DNR) {
2462                 if(fe->fts_accpath[0] == '/' &&
2463                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2464                     continue;
2465                 dirs++;
2466                 rc = rmdir(fe->fts_accpath);
2467                 if(rc >= 0)
2468                     rmdirs++;
2469                 else if(errno != ENOTEMPTY && errno != EEXIST)
2470                     do_log_error(L_ERROR, errno,
2471                                  "Couldn't remove directory %s",
2472                                  fe->fts_accpath);
2473                 continue;
2474             } else if(fe->fts_info == FTS_NS) {
2475                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2476                              fe->fts_accpath);
2477                 continue;
2478             } else if(fe->fts_info == FTS_ERR) {
2479                 do_log_error(L_ERROR, fe->fts_errno,
2480                              "Couldn't fts_read disk cache");
2481                 break;
2482             }
2483
2484             if(!S_ISREG(fe->fts_statp->st_mode)) {
2485                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2486                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2487                 continue;
2488             }
2489
2490             files++;
2491             left += expireFile(fe->fts_accpath, fe->fts_statp,
2492                                &considered, &unlinked, &truncated);
2493             total += fe->fts_statp->st_size;
2494         }
2495         fts_close(fts);
2496     }
2497
2498     printf("Disk cache purged.\n");
2499     printf("%d files, %d considered, %d removed, %d truncated "
2500            "(%ldkB -> %ldkB).\n",
2501            files, considered, unlinked, truncated, total/1024, left/1024);
2502     printf("%d directories, %d removed.\n", dirs, rmdirs);
2503     return;
2504 }
2505
2506 #else
2507
2508 void
2509 preinitDiskcache()
2510 {
2511     return;
2512 }
2513
2514 void
2515 initDiskcache()
2516 {
2517     return;
2518 }
2519
2520 int
2521 writeoutToDisk(ObjectPtr object, int upto, int max)
2522 {
2523     return 0;
2524 }
2525
2526 int
2527 destroyDiskEntry(ObjectPtr object, int d)
2528 {
2529     return 0;
2530 }
2531
2532 ObjectPtr
2533 objectGetFromDisk(ObjectPtr object)
2534 {
2535     return NULL;
2536 }
2537
2538 int
2539 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2540 {
2541     return 0;
2542 }
2543
2544 int
2545 revalidateDiskEntry(ObjectPtr object)
2546 {
2547     return 0;
2548 }
2549
2550 void
2551 dirtyDiskEntry(ObjectPtr object)
2552 {
2553     return;
2554 }
2555
2556 void
2557 expireDiskObjects()
2558 {
2559     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2560 }
2561
2562 int
2563 diskEntrySize(ObjectPtr object)
2564 {
2565     return -1;
2566 }
2567 #endif