diskcache.c

   1 /*
   2 Copyright (c) 2003-2006 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheWriteoutOnClose = (32 * 1024);
  42
  43 int maxDiskCacheEntrySize = -1;
  44
  45 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  46 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  47 int diskCacheTruncateSize =  1024 * 1024;
  48 int preciseExpiry = 0;
  49
  50 static DiskCacheEntryRec negativeEntry = {
  51     NULL, NULL,
  52     -1, -1, -1, -1, 0, 0, 0, NULL, NULL
  53 };
  54
  55 #ifndef LOCAL_ROOT
  56 #define LOCAL_ROOT "/usr/share/polipo/www/"
  57 #endif
  58
  59 #ifndef DISK_CACHE_ROOT
  60 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  61 #endif
  62
  63 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  64 static int atomSetterFlush(ConfigVariablePtr, void*);
  65 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  66
  67 void
  68 preinitDiskcache()
  69 {
  70     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  71     localDocumentRoot = internAtom(LOCAL_ROOT);
  72
  73     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  74                              configIntSetter,
  75                              "Access rights for new directories.");
  76     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  77                              configIntSetter,
  78                              "Access rights for new cache files.");
  79     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  80                              configIntSetter,
  81                              "Number of bytes to write out eagerly.");
  82     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  83                              "Root of the disk cache.");
  84     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  85                              "Root of the local tree.");
  86     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  87                     "File descriptors used by the on-disk cache.");
  88     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  89                     "Time after which on-disk objects are removed.");
  90     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  91                     "Time after which on-disk objects are truncated.");
  92     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  93                     "Size to which on-disk objects are truncated.");
  94     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  95                     "Whether to consider all files for purging.");
  96     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  97                              configIntSetter,
  98                              "Maximum size of objects cached on disk.");
  99 }
 100
 101 static int
 102 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 103 {
 104     int i;
 105     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 106     i = *(int*)value;
 107     if(i < 0 || i > 1000000)
 108         return -3;
 109     maxDiskEntries = i;
 110     while(numDiskEntries > maxDiskEntries)
 111         destroyDiskEntry(diskEntriesLast->object, 0);
 112     return 1;
 113 }
 114
 115 static int
 116 atomSetterFlush(ConfigVariablePtr var, void *value)
 117 {
 118     discardObjects(1, 0);
 119     return configAtomSetter(var, value);
 120 }
 121
 122 static int
 123 checkRoot(AtomPtr root)
 124 {
 125     struct stat ss;
 126     int rc;
 127
 128     if(!root || root->length == 0)
 129         return 0;
 130
 131     if(root->string[0] != '/') {
 132         return -2;
 133     }
 134
 135     rc = stat(root->string, &ss);
 136     if(rc < 0)
 137         return -1;
 138     else if(!S_ISDIR(ss.st_mode)) {
 139         errno = ENOTDIR;
 140         return -1;
 141     }
 142     return 1;
 143 }
 144
 145 static AtomPtr
 146 maybeAddSlash(AtomPtr atom)
 147 {
 148     AtomPtr newAtom = NULL;
 149     if(!atom) return NULL;
 150     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 151         newAtom = atomCat(atom, "/");
 152         releaseAtom(atom);
 153         return newAtom;
 154     }
 155     return atom;
 156 }
 157
 158 void
 159 initDiskcache()
 160 {
 161     int rc;
 162
 163     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 164     rc = checkRoot(diskCacheRoot);
 165     if(rc <= 0) {
 166         switch(rc) {
 167         case 0: break;
 168         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 169         case -2:
 170             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 171                    diskCacheRoot->string);
 172             break;
 173         default: abort();
 174         }
 175         releaseAtom(diskCacheRoot);
 176         diskCacheRoot = NULL;
 177     }
 178
 179     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 180     rc = checkRoot(localDocumentRoot);
 181     if(rc <= 0) {
 182         switch(rc) {
 183         case 0: break;
 184         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 185         case -2:
 186             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 187             break;
 188         default: abort();
 189         }
 190         releaseAtom(localDocumentRoot);
 191         localDocumentRoot = NULL;
 192     }
 193 }
 194
 195 #ifdef DEBUG_DISK_CACHE
 196 #define CHECK_ENTRY(entry) check_entry((entry))
 197 static void
 198 check_entry(DiskCacheEntryPtr entry)
 199 {
 200     if(entry && entry->fd < 0)
 201         assert(entry == &negativeEntry);
 202     if(entry && entry->fd >= 0) {
 203         assert((!entry->previous) == (entry == diskEntries));
 204         assert((!entry->next) == (entry == diskEntriesLast));
 205         if(entry->size >= 0)
 206             assert(entry->size + entry->body_offset >= entry->offset);
 207         assert(entry->body_offset >= 0);
 208         if(entry->offset >= 0) {
 209             off_t offset;
 210             offset = lseek(entry->fd, 0, SEEK_CUR);
 211             assert(offset == entry->offset);
 212         }
 213         if(entry->size >= 0) {
 214             int rc;
 215             struct stat ss;
 216             rc = fstat(entry->fd, &ss);
 217             assert(rc >= 0);
 218             assert(ss.st_size == entry->size + entry->body_offset);
 219         }
 220     }
 221 }
 222 #else
 223 #define CHECK_ENTRY(entry) do {} while(0)
 224 #endif
 225
 226 int
 227 diskEntrySize(ObjectPtr object)
 228 {
 229     struct stat buf;
 230     int rc;
 231     DiskCacheEntryPtr entry = object->disk_entry;
 232
 233     if(!entry || entry == &negativeEntry)
 234         return -1;
 235
 236     if(entry->size >= 0)
 237         return entry->size;
 238
 239     rc = fstat(entry->fd, &buf);
 240     if(rc < 0) {
 241         do_log_error(L_ERROR, errno, "Couldn't stat");
 242         return -1;
 243     }
 244
 245     if(buf.st_size <= entry->body_offset)
 246         entry->size = 0;
 247     else
 248         entry->size =  buf.st_size - entry->body_offset;
 249     CHECK_ENTRY(entry);
 250     if(object->length >= 0 && entry->size == object->length)
 251         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 252     return entry->size;
 253 }
 254
 255 static int
 256 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 257 {
 258     off_t rc;
 259
 260     CHECK_ENTRY(entry);
 261     assert(entry != &negativeEntry);
 262     if(entry->offset == offset)
 263         return 1;
 264     if(offset > entry->body_offset) {
 265         /* Avoid extending the file by mistake */
 266         if(entry->size < 0)
 267             diskEntrySize(entry->object);
 268         if(entry->size < 0)
 269             return -1;
 270         if(entry->size + entry->body_offset < offset)
 271             return -1;
 272     }
 273     rc = lseek(entry->fd, offset, SEEK_SET);
 274     if(rc < 0) {
 275         do_log_error(L_ERROR, errno, "Couldn't seek");
 276         entry->offset = -1;
 277         return -1;
 278     }
 279     entry->offset = offset;
 280     return 1;
 281 }
 282
 283 /* Given a local URL, constructs the filename where it can be found. */
 284
 285 int
 286 localFilename(char *buf, int n, char *key, int len)
 287 {
 288     int i, j;
 289     if(len <= 0 || key[0] != '/') return -1;
 290
 291     if(urlIsSpecial(key, len)) return -1;
 292
 293     if(localDocumentRoot == NULL ||
 294        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 295         return -1;
 296
 297     if(n <= localDocumentRoot->length)
 298         return -1;
 299
 300     i = 0;
 301     if(key[i] != '/')
 302         return -1;
 303
 304     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 305     j = localDocumentRoot->length;
 306     if(buf[j - 1] == '/')
 307         j--;
 308
 309     while(i < len) {
 310         if(j >= n - 1)
 311             return -1;
 312         if(key[i] == '/' && i < len - 2)
 313             if(key[i + 1] == '.' &&
 314                (key[i + 2] == '.' || key[i + 2] == '/'))
 315                 return -1;
 316         buf[j++] = key[i++];
 317     }
 318
 319     if(buf[j - 1] == '/') {
 320         if(j >= n - 11)
 321             return -1;
 322         memcpy(buf + j, "index.html", 10);
 323         j += 10;
 324     }
 325
 326     buf[j] = '\0';
 327     return j;
 328 }
 329
 330 static void
 331 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 332 {
 333     static MD5_CTX ctx;
 334     MD5Init(&ctx);
 335     MD5Update(&ctx, key, len);
 336     MD5Final(&ctx);
 337     memcpy(dst, ctx.digest, 16);
 338 }
 339
 340 /* Check whether a character can be stored in a filename.  This is
 341    needed since we want to support deficient file systems. */
 342 static int
 343 fssafe(char c)
 344 {
 345     if(c <= 31 || c >= 127)
 346         return 0;
 347     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 348        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 349         return 1;
 350     return 0;
 351 }
 352
 353 /* Given a URL, returns the directory name within which all files
 354    starting with this URL can be found. */
 355 static int
 356 urlDirname(char *buf, int n, const char *url, int len)
 357 {
 358     int i, j;
 359     if(len < 8)
 360         return -1;
 361     if(memcmp(url, "http://", 7) != 0)
 362         return -1;
 363
 364     if(diskCacheRoot == NULL ||
 365        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 366         return -1;
 367
 368     if(n <= diskCacheRoot->length)
 369         return -1;
 370
 371     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 372     j = diskCacheRoot->length;
 373
 374     if(buf[j - 1] != '/')
 375         buf[j++] = '/';
 376
 377     for(i = 7; i < len; i++) {
 378         if(i >= len || url[i] == '/')
 379             break;
 380         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 381             return -1;
 382         if(url[i] == '%' || !fssafe(url[i])) {
 383             if(j + 3 >= n) return -1;
 384             buf[j++] = '%';
 385             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 386             buf[j++] = i2h(url[i] & 0x0F);
 387         } else {
 388             buf[j++] = url[i]; if(j >= n) return -1;
 389         }
 390     }
 391     buf[j++] = '/'; if(j >= n) return -1;
 392     buf[j] = '\0';
 393     return j;
 394 }
 395
 396 /* Given a URL, returns the filename where the cached data can be
 397    found. */
 398 static int
 399 urlFilename(char *restrict buf, int n, const char *url, int len)
 400 {
 401     int j;
 402     unsigned char md5buf[18];
 403     j = urlDirname(buf, n, url, len);
 404     if(j < 0 || j + 24 >= n)
 405         return -1;
 406     md5((unsigned char*)url, len, md5buf);
 407     b64cpy(buf + j, (char*)md5buf, 16, 1);
 408     buf[j + 24] = '\0';
 409     return j + 24;
 410 }
 411
 412 static char *
 413 dirnameUrl(char *url, int n, char *name, int len)
 414 {
 415     int i, j, k, c1, c2;
 416     k = diskCacheRoot->length;
 417     if(len < k)
 418         return NULL;
 419     if(memcmp(name, diskCacheRoot->string, k) != 0)
 420         return NULL;
 421     if(n < 8)
 422         return NULL;
 423     memcpy(url, "http://", 7);
 424     if(name[len - 1] == '/')
 425         len --;
 426     j = 7;
 427     for(i = k; i < len; i++) {
 428         if(name[i] == '%') {
 429             if(i >= len - 2)
 430                 return NULL;
 431             c1 = h2i(name[i + 1]);
 432             c2 = h2i(name[i + 2]);
 433             if(c1 < 0 || c2 < 0)
 434                 return NULL;
 435             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 436             i += 2;             /* skip extra digits */
 437         } else if(i < len - 1 &&
 438                   name[i] == '.' && name[i + 1] == '/') {
 439                 return NULL;
 440         } else if(i == len - 1 && name[i] == '.') {
 441             return NULL;
 442         } else {
 443             url[j++] = name[i]; if(j >= n) goto fail;
 444         }
 445     }
 446     url[j++] = '/'; if(j >= n) goto fail;
 447     url[j] = '\0';
 448     return url;
 449
 450  fail:
 451     return NULL;
 452 }
 453
 454 /* Create a file and all intermediate directories. */
 455 static int
 456 createFile(const char *name, int path_start)
 457 {
 458     int fd;
 459     char buf[1024];
 460     int n;
 461     int rc;
 462
 463     if(name[path_start] == '/')
 464         path_start++;
 465
 466     if(path_start < 2 || name[path_start - 1] != '/' ) {
 467         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 468         return -1;
 469     }
 470
 471     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 472               diskCacheFilePermissions);
 473     if(fd >= 0)
 474         return fd;
 475     if(errno != ENOENT) {
 476         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 477         return -1;
 478     }
 479
 480     n = path_start;
 481     while(name[n] != '\0' && n < 1024) {
 482         while(name[n] != '/' && name[n] != '\0' && n < 512)
 483             n++;
 484         if(name[n] != '/' || n >= 1024)
 485             break;
 486         memcpy(buf, name, n + 1);
 487         buf[n + 1] = '\0';
 488         rc = mkdir(buf, diskCacheDirectoryPermissions);
 489         if(rc < 0 && errno != EEXIST) {
 490             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 491             return -1;
 492         }
 493         n++;
 494     }
 495     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 496               diskCacheFilePermissions);
 497     if(fd < 0) {
 498         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 499         return -1;
 500     }
 501
 502     return fd;
 503 }
 504
 505 static int
 506 chooseBodyOffset(int n, ObjectPtr object)
 507 {
 508     int length = MAX(object->size, object->length);
 509     int body_offset;
 510
 511     if(object->length >= 0 && object->length + n < 4096 - 4)
 512         return -1;              /* no gap for small objects */
 513
 514     if(n <= 128)
 515         body_offset = 256;
 516     else if(n <= 192)
 517         body_offset = 384;
 518     else if(n <= 256)
 519         body_offset = 512;
 520     else if(n <= 384)
 521         body_offset = 768;
 522     else if(n <= 512)
 523         body_offset = 1024;
 524     else if(n <= 1024)
 525         body_offset = 2048;
 526     else if(n < 2048)
 527         body_offset = 4096;
 528     else
 529         body_offset = ((n + 32 + 4095) / 4096 + 1) * 4096;
 530
 531     /* Tweak the gap so that we don't use up a full disk block for
 532        a small tail */
 533     if(object->length >= 0 && object->length < 64 * 1024) {
 534         int last = (body_offset + object->length) % 4096;
 535         int gap = body_offset - n - 32;
 536         if(last < gap / 2)
 537             body_offset -= last;
 538     }
 539
 540     /* Rewriting large objects is expensive -- don't use small gaps.
 541        This has the additional benefit of block-aligning large bodies. */
 542     if(length >= 64 * 1024) {
 543         int min_gap, min_offset;
 544         if(length >= 512 * 1024)
 545             min_gap = 4096;
 546         else if(length >= 256 * 1024)
 547             min_gap = 2048;
 548         else
 549             min_gap = 1024;
 550
 551         min_offset = ((n + 32 + min_gap - 1) / min_gap + 1) * min_gap;
 552         body_offset = MAX(body_offset, min_offset);
 553     }
 554
 555     return body_offset;
 556 }
 557
 558 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 559    otherwise the offset at which the file descriptor is left. */
 560 /* If chunk is not null, it should be the first chunk of the object,
 561    and will be written out in the same operation if possible. */
 562 static int
 563 writeHeaders(int fd, int *body_offset_return,
 564              ObjectPtr object, char *chunk, int chunk_len)
 565 {
 566     int n, rc, error = -1;
 567     int body_offset = *body_offset_return;
 568     char *buf = NULL;
 569     int buf_is_chunk = 0;
 570     int bufsize = 0;
 571
 572     if(object->flags & OBJECT_LOCAL)
 573         return -1;
 574
 575     if(body_offset > CHUNK_SIZE)
 576         goto overflow;
 577
 578     /* get_chunk might trigger object expiry */
 579     bufsize = CHUNK_SIZE;
 580     buf_is_chunk = 1;
 581     buf = maybe_get_chunk();
 582     if(!buf) {
 583         bufsize = 2048;
 584         buf_is_chunk = 0;
 585         buf = malloc(2048);
 586         if(buf == NULL) {
 587             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 588             return -1;
 589         }
 590     }
 591
 592  format_again:
 593     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 594                   object->code, object->message->string);
 595
 596     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 597     if(n < 0)
 598         goto overflow;
 599
 600     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 601     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 602
 603     if(object->age >= 0 && object->age != object->date) {
 604         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 605         n = format_time(buf, n, bufsize, object->age);
 606     }
 607
 608     if(object->atime >= 0) {
 609         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 610         n = format_time(buf, n, bufsize, object->atime);
 611     }
 612
 613     if(n < 0)
 614         goto overflow;
 615
 616     if(body_offset < 0)
 617         body_offset = chooseBodyOffset(n, object);
 618
 619     if(body_offset > bufsize)
 620         goto overflow;
 621
 622     if(body_offset > 0 && body_offset != n + 4)
 623         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 624                       body_offset);
 625
 626     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 627     if(n < 0)
 628         goto overflow;
 629
 630     if(body_offset < 0)
 631         body_offset = n;
 632     if(n > body_offset) {
 633         error = -2;
 634         goto fail;
 635     }
 636
 637     if(n < body_offset)
 638         memset(buf + n, 0, body_offset - n);
 639
 640  again:
 641 #ifdef HAVE_READV_WRITEV
 642     if(chunk_len > 0) {
 643         struct iovec iov[2];
 644         iov[0].iov_base = buf;
 645         iov[0].iov_len = body_offset;
 646         iov[1].iov_base = chunk;
 647         iov[1].iov_len = chunk_len;
 648         rc = writev(fd, iov, 2);
 649     } else
 650 #endif
 651         rc = write(fd, buf, body_offset);
 652
 653     if(rc < 0 && errno == EINTR)
 654         goto again;
 655
 656     if(rc < body_offset)
 657         goto fail;
 658     if(object->length >= 0 &&
 659        rc - body_offset >= object->length)
 660         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 661
 662     *body_offset_return = body_offset;
 663     if(buf_is_chunk)
 664         dispose_chunk(buf);
 665     else
 666         free(buf);
 667     return rc;
 668
 669  overflow:
 670     if(bufsize < bigBufferSize) {
 671         char *oldbuf = buf;
 672         buf = malloc(bigBufferSize);
 673         if(!buf) {
 674             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 675             goto fail;
 676         }
 677         bufsize = bigBufferSize;
 678         if(oldbuf) {
 679             if(buf_is_chunk)
 680                 dispose_chunk(oldbuf);
 681             else
 682                 free(oldbuf);
 683         }
 684         buf_is_chunk = 0;
 685         goto format_again;
 686     }
 687     /* fall through */
 688
 689  fail:
 690     if(buf_is_chunk)
 691         dispose_chunk(buf);
 692     else
 693         free(buf);
 694     return error;
 695 }
 696
 697 typedef struct _MimeEntry {
 698     char *extension;
 699     char *mime;
 700 } MimeEntryRec;
 701
 702 static const MimeEntryRec mimeEntries[] = {
 703     { "html", "text/html" },
 704     { "htm", "text/html" },
 705     { "text", "text/plain" },
 706     { "txt", "text/plain" },
 707     { "png", "image/png" },
 708     { "gif", "image/gif" },
 709     { "jpeg", "image/jpeg" },
 710     { "jpg", "image/jpeg" },
 711     { "ico", "image/x-icon" },
 712     { "pdf", "application/pdf" },
 713     { "ps", "application/postscript" },
 714     { "tar", "application/x-tar" },
 715     { "pac", "application/x-ns-proxy-autoconfig" },
 716     { "css", "text/css" },
 717     { "js",  "application/x-javascript" },
 718     { "xml", "text/xml" },
 719     { "swf", "application/x-shockwave-flash" },
 720 };
 721
 722 static char*
 723 localObjectMimeType(ObjectPtr object, char **encoding_return)
 724 {
 725     char *name = object->key;
 726     int nlen = object->key_size;
 727     int i;
 728
 729     assert(nlen >= 1);
 730
 731     if(name[nlen - 1] == '/') {
 732         *encoding_return = NULL;
 733         return "text/html";
 734     }
 735
 736     if(nlen < 3) {
 737         *encoding_return = NULL;
 738         return "application/octet-stream";
 739     }
 740
 741     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 742         *encoding_return = "x-gzip";
 743         nlen -= 3;
 744     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 745         *encoding_return = "x-compress";
 746         nlen -= 2;
 747     } else {
 748         *encoding_return = NULL;
 749     }
 750
 751     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 752         int len = strlen(mimeEntries[i].extension);
 753         if(nlen > len &&
 754            name[nlen - len - 1] == '.' &&
 755            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 756             return mimeEntries[i].mime;
 757     }
 758
 759     return "application/octet-stream";
 760 }
 761
 762 /* Same interface as validateEntry -- see below */
 763 int
 764 validateLocalEntry(ObjectPtr object, int fd,
 765                    int *body_offset_return, off_t *offset_return)
 766 {
 767     struct stat ss;
 768     char buf[512];
 769     int n, rc;
 770     char *encoding;
 771
 772     rc = fstat(fd, &ss);
 773     if(rc < 0) {
 774         do_log_error(L_ERROR, errno, "Couldn't stat");
 775         return -1;
 776     }
 777
 778     if(S_ISREG(ss.st_mode)) {
 779         if(!(ss.st_mode & S_IROTH) ||
 780            (object->length >= 0 && object->length != ss.st_size) ||
 781            (object->last_modified >= 0 &&
 782             object->last_modified != ss.st_mtime))
 783             return -1;
 784     } else {
 785         notifyObject(object);
 786         return -1;
 787     }
 788
 789     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 790                   (unsigned long)ss.st_ino,
 791                   (unsigned long)ss.st_size,
 792                   (unsigned long)ss.st_mtime);
 793     if(n >= 512)
 794         n = -1;
 795
 796     if(n > 0 && object->etag) {
 797         if(strlen(object->etag) != n ||
 798            memcmp(object->etag, buf, n) != 0)
 799             return -1;
 800     }
 801
 802     if(!(object->flags & OBJECT_INITIAL)) {
 803         if(!object->last_modified && !object->etag)
 804             return -1;
 805     }
 806
 807     if(object->flags & OBJECT_INITIAL) {
 808         object->length = ss.st_size;
 809         object->last_modified = ss.st_mtime;
 810         object->date = current_time.tv_sec;
 811         object->age = current_time.tv_sec;
 812         object->code = 200;
 813         if(n > 0)
 814             object->etag = strdup(buf); /* okay if fails */
 815         object->message = internAtom("Okay");
 816         n = snnprintf(buf, 0, 512,
 817                       "\r\nServer: Polipo"
 818                       "\r\nContent-Type: %s",
 819                       localObjectMimeType(object, &encoding));
 820         if(encoding != NULL)
 821             n = snnprintf(buf, n, 512,
 822                           "\r\nContent-Encoding: %s", encoding);
 823         if(n < 0)
 824             return -1;
 825         object->headers = internAtomN(buf, n);
 826         if(object->headers == NULL)
 827             return -1;
 828         object->flags &= ~OBJECT_INITIAL;
 829     }
 830
 831     if(body_offset_return)
 832         *body_offset_return = 0;
 833     if(offset_return)
 834         *offset_return = 0;
 835     return 0;
 836 }
 837
 838 /* Assumes fd is at offset 0.
 839    Returns -1 if not valid, 1 if metadata should be written out, 0
 840    otherwise. */
 841 int
 842 validateEntry(ObjectPtr object, int fd,
 843               int *body_offset_return, off_t *offset_return)
 844 {
 845     char *buf;
 846     int buf_is_chunk, bufsize;
 847     int rc, n;
 848     int dummy;
 849     int code;
 850     AtomPtr headers;
 851     time_t date, last_modified, expires, polipo_age, polipo_access;
 852     int length;
 853     off_t offset = -1;
 854     int body_offset;
 855     char *etag;
 856     AtomPtr via;
 857     CacheControlRec cache_control;
 858     char *location;
 859     AtomPtr message;
 860     int dirty = 0;
 861
 862     if(object->flags & OBJECT_LOCAL)
 863         return validateLocalEntry(object, fd,
 864                                   body_offset_return, offset_return);
 865
 866     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 867         return 0;
 868
 869     /* get_chunk might trigger object expiry */
 870     bufsize = CHUNK_SIZE;
 871     buf_is_chunk = 1;
 872     buf = maybe_get_chunk();
 873     if(!buf) {
 874         bufsize = 2048;
 875         buf_is_chunk = 0;
 876         buf = malloc(2048);
 877         if(buf == NULL) {
 878             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 879             return -1;
 880         }
 881     }
 882
 883  again:
 884     rc = read(fd, buf, bufsize);
 885     if(rc < 0) {
 886         if(errno == EINTR)
 887             goto again;
 888         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 889         goto fail;
 890     }
 891     offset = rc;
 892
 893  parse_again:
 894     n = findEndOfHeaders(buf, 0, rc, &dummy);
 895     if(n < 0) {
 896         char *oldbuf = buf;
 897         if(bufsize < bigBufferSize) {
 898             buf = malloc(bigBufferSize);
 899             if(!buf) {
 900                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 901                 goto fail;
 902             }
 903             bufsize = bigBufferSize;
 904             memcpy(buf, oldbuf, offset);
 905             if(buf_is_chunk)
 906                 dispose_chunk(oldbuf);
 907             else
 908                 free(oldbuf);
 909             buf_is_chunk = 0;
 910         again2:
 911             rc = read(fd, buf + offset, bufsize - offset);
 912             if(rc < 0) {
 913                 if(errno == EINTR)
 914                     goto again2;
 915                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 916                 goto fail;
 917             }
 918             offset += rc;
 919             goto parse_again;
 920         }
 921         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 922         goto fail;
 923     }
 924
 925     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 926     if(rc < 0) {
 927         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 928         goto fail;
 929     }
 930
 931     if(object->code != 0 && object->code != code) {
 932         releaseAtom(message);
 933         goto fail;
 934     }
 935
 936     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 937                           &headers, &length, &cache_control, NULL, NULL,
 938                           &date, &last_modified, &expires, &polipo_age,
 939                           &polipo_access, &body_offset,
 940                           NULL, &etag, NULL,
 941                           NULL, NULL, &location, &via, NULL);
 942     if(rc < 0) {
 943         releaseAtom(message);
 944         goto fail;
 945     }
 946     if(body_offset < 0)
 947         body_offset = n;
 948
 949     if(!location || strlen(location) != object->key_size ||
 950        memcmp(location, object->key, object->key_size) != 0) {
 951         do_log(L_ERROR, "Inconsistent cache file for %s.\n", scrub(location));
 952         goto invalid;
 953     }
 954
 955     if(polipo_age < 0)
 956         polipo_age = date;
 957
 958     if(polipo_age < 0) {
 959         do_log(L_ERROR, "Undated disk entry for %s.\n", scrub(location));
 960         goto invalid;
 961     }
 962
 963     if(!(object->flags & OBJECT_INITIAL)) {
 964         if((last_modified >= 0) != (object->last_modified >= 0))
 965             goto invalid;
 966
 967         if((object->cache_control & CACHE_MISMATCH) ||
 968            (cache_control.flags & CACHE_MISMATCH))
 969             goto invalid;
 970
 971         if(last_modified >= 0 && object->last_modified >= 0 &&
 972            last_modified != object->last_modified)
 973             goto invalid;
 974
 975         if(length >= 0 && object->length >= 0)
 976             if(length != object->length)
 977                 goto invalid;
 978
 979         if(!!etag != !!object->etag)
 980             goto invalid;
 981
 982         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 983             goto invalid;
 984
 985         /* If we don't have a usable ETag, and either CACHE_VARY or we
 986            don't have a last-modified date, we validate disk entries by
 987            using their date. */
 988         if(!(etag && object->etag) &&
 989            (!(last_modified >= 0 && object->last_modified >= 0) ||
 990             ((cache_control.flags & CACHE_VARY) ||
 991              (object->cache_control & CACHE_VARY)))) {
 992             if(date >= 0 && date != object->date)
 993                 goto invalid;
 994             if(polipo_age >= 0 && polipo_age != object->age)
 995                 goto invalid;
 996         }
 997         if((object->cache_control & CACHE_VARY) && dontTrustVaryETag >= 1) {
 998             /* Check content-type to work around mod_gzip bugs */
 999             if(!httpHeaderMatch(atomContentType, object->headers, headers) ||
1000                !httpHeaderMatch(atomContentEncoding, object->headers, headers))
1001                 goto invalid;
1002         }
1003     }
1004
1005     if(location)
1006         free(location);
1007
1008     if(headers) {
1009         if(!object->headers)
1010             object->headers = headers;
1011         else
1012             releaseAtom(headers);
1013     }
1014
1015     if(object->code == 0) {
1016         object->code = code;
1017         object->message = retainAtom(message);
1018     }
1019     if(object->date <= date)
1020         object->date = date;
1021     else
1022         dirty = 1;
1023     if(object->last_modified < 0)
1024         object->last_modified = last_modified;
1025     if(object->expires < 0)
1026         object->expires = expires;
1027     else if(object->expires > expires)
1028         dirty = 1;
1029     if(object->age < 0)
1030         object->age = polipo_age;
1031     else if(object->age > polipo_age)
1032         dirty = 1;
1033     if(object->atime <= polipo_access)
1034         object->atime = polipo_access;
1035     else
1036         dirty = 1;
1037
1038     object->cache_control |= cache_control.flags;
1039
1040     if(object->age < 0) object->age = object->date;
1041     if(object->age < 0) object->age = 0; /* a long time ago */
1042     if(object->length < 0) object->length = length;
1043     if(!object->etag)
1044         object->etag = etag;
1045     else {
1046         if(etag)
1047             free(etag);
1048     }
1049     releaseAtom(message);
1050
1051     if(object->flags & OBJECT_INITIAL) object->via = via;
1052     object->flags &= ~OBJECT_INITIAL;
1053     if(offset > body_offset) {
1054         /* We need to make sure we don't invoke object expiry recursively */
1055         objectSetChunks(object, 1);
1056         if(object->numchunks >= 1) {
1057             if(object->chunks[0].data == NULL)
1058                 object->chunks[0].data = maybe_get_chunk();
1059             if(object->chunks[0].data)
1060                 objectAddData(object, buf + body_offset,
1061                               0, MIN(offset - body_offset, CHUNK_SIZE));
1062         }
1063     }
1064
1065     httpTweakCachability(object);
1066
1067     if(buf_is_chunk)
1068         dispose_chunk(buf);
1069     else
1070         free(buf);
1071     if(body_offset_return) *body_offset_return = body_offset;
1072     if(offset_return) *offset_return = offset;
1073     return dirty;
1074
1075  invalid:
1076     releaseAtom(message);
1077     if(etag) free(etag);
1078     if(location) free(location);
1079     if(via) releaseAtom(via);
1080     /* fall through */
1081
1082  fail:
1083     if(buf_is_chunk)
1084         dispose_chunk(buf);
1085     else
1086         free(buf);
1087     return -1;
1088 }
1089
1090 void
1091 dirtyDiskEntry(ObjectPtr object)
1092 {
1093     DiskCacheEntryPtr entry = object->disk_entry;
1094     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1095 }
1096
1097 int
1098 revalidateDiskEntry(ObjectPtr object)
1099 {
1100     DiskCacheEntryPtr entry = object->disk_entry;
1101     int rc;
1102     int body_offset;
1103
1104     if(!entry || entry == &negativeEntry)
1105         return 1;
1106
1107     CHECK_ENTRY(entry);
1108     rc = entrySeek(entry, 0);
1109     if(rc < 0) return 0;
1110
1111     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1112     if(rc < 0) {
1113         destroyDiskEntry(object, 0);
1114         return 0;
1115     }
1116     if(body_offset != entry->body_offset) {
1117         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1118                body_offset, entry->body_offset);
1119         destroyDiskEntry(object, 0);
1120         return 0;
1121     }
1122
1123     entry->metadataDirty |= !!rc;
1124     CHECK_ENTRY(entry);
1125     return 1;
1126 }
1127
1128 static inline int
1129 objectHasDiskEntry(ObjectPtr object)
1130 {
1131     return object->disk_entry && object->disk_entry != &negativeEntry;
1132 }
1133
1134 static DiskCacheEntryPtr
1135 makeDiskEntry(ObjectPtr object, int writeable, int create)
1136 {
1137     DiskCacheEntryPtr entry = NULL;
1138     char buf[1024];
1139     int fd = -1;
1140     int negative = 0, isWriteable = 0, size = -1, name_len = -1;
1141     char *name = NULL;
1142     off_t offset = -1;
1143     int body_offset = -1;
1144     int rc;
1145     int local = (object->flags & OBJECT_LOCAL) != 0;
1146     int dirty = 0;
1147
1148     if(local && (writeable || create))
1149         return NULL;
1150
1151     if(!local && !(object->flags & OBJECT_PUBLIC))
1152         return NULL;
1153
1154     if(maxDiskCacheEntrySize >= 0) {
1155         if(object->length > 0) {
1156             if(object->length > maxDiskCacheEntrySize)
1157                 return NULL;
1158         } else {
1159             if(object->size > maxDiskCacheEntrySize)
1160                 return NULL;
1161         }
1162     }
1163
1164     if(object->disk_entry) {
1165         entry = object->disk_entry;
1166         CHECK_ENTRY(entry);
1167         if(entry != &negativeEntry && (!writeable || entry->writeable)) {
1168             /* We'll keep the entry -- put it at the front. */
1169             if(entry != diskEntries && entry != &negativeEntry) {
1170                 entry->previous->next = entry->next;
1171                 if(entry->next)
1172                     entry->next->previous = entry->previous;
1173                 else
1174                     diskEntriesLast = entry->previous;
1175                 entry->next = diskEntries;
1176                 diskEntries->previous = entry;
1177                 entry->previous = NULL;
1178                 diskEntries = entry;
1179             }
1180             return entry;
1181         } else {
1182             if(entry == &negativeEntry) {
1183                 negative = 1;
1184                 if(!create) return NULL;
1185                 object->disk_entry = NULL;
1186             }
1187             entry = NULL;
1188             destroyDiskEntry(object, 0);
1189         }
1190     }
1191
1192     if(numDiskEntries > maxDiskEntries)
1193         destroyDiskEntry(diskEntriesLast->object, 0);
1194
1195     if(!local) {
1196         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1197             return NULL;
1198         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1199         if(name_len < 0) return NULL;
1200         if(!negative) {
1201             isWriteable = 1;
1202             fd = open(buf, O_RDWR | O_BINARY);
1203             if(fd < 0 && !writeable && errno == EACCES) {
1204                 isWriteable = 0;
1205                 fd = open(buf, O_RDONLY | O_BINARY);
1206             }
1207         }
1208         if(fd >= 0) {
1209             rc = validateEntry(object, fd, &body_offset, &offset);
1210             if(rc >= 0) {
1211                 dirty = rc;
1212             } else {
1213                 close(fd);
1214                 fd = -1;
1215                 rc = unlink(buf);
1216                 if(rc < 0 && errno != ENOENT) {
1217                     do_log_error(L_WARN,  errno,
1218                                  "Couldn't unlink stale disk entry %s",
1219                                  scrub(buf));
1220                     /* But continue -- it's okay to have stale entries. */
1221                 }
1222             }
1223         }
1224
1225         if(fd < 0 && create && name_len > 0 &&
1226            !(object->flags & OBJECT_INITIAL)) {
1227             isWriteable = 1;
1228             fd = createFile(buf, diskCacheRoot->length);
1229             if(fd < 0)
1230                 return NULL;
1231
1232             if(fd >= 0) {
1233                 char *data = NULL;
1234                 int dsize = 0;
1235                 if(object->numchunks > 0) {
1236                     data = object->chunks[0].data;
1237                     dsize = object->chunks[0].size;
1238                 }
1239                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1240                 if(rc < 0) {
1241                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1242                     rc = unlink(buf);
1243                     if(rc < 0 && errno != ENOENT)
1244                         do_log_error(L_ERROR, errno,
1245                                      "Couldn't unlink truncated entry %s",
1246                                      scrub(buf));
1247                     close(fd);
1248                     return NULL;
1249                 }
1250                 assert(rc >= body_offset);
1251                 size = rc - body_offset;
1252                 offset = rc;
1253                 dirty = 0;
1254             }
1255         }
1256     } else {
1257         /* local */
1258         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1259             return NULL;
1260
1261         name_len =
1262             localFilename(buf, 1024, object->key, object->key_size);
1263         if(name_len < 0)
1264             return NULL;
1265         isWriteable = 0;
1266         fd = open(buf, O_RDONLY | O_BINARY);
1267         if(fd >= 0) {
1268             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1269                 close(fd);
1270                 fd = -1;
1271             }
1272         }
1273         offset = 0;
1274     }
1275
1276     if(fd < 0) {
1277         object->disk_entry = &negativeEntry;
1278         return NULL;
1279     }
1280     assert(body_offset >= 0);
1281
1282     name = strdup_n(buf, name_len);
1283     if(name == NULL) {
1284         do_log(L_ERROR, "Couldn't allocate name.\n");
1285         close(fd);
1286         return NULL;
1287     }
1288
1289     entry = malloc(sizeof(DiskCacheEntryRec));
1290     if(entry == NULL) {
1291         do_log(L_ERROR, "Couldn't allocate entry.\n");
1292         free(name);
1293         close(fd);
1294         return NULL;
1295     }
1296
1297     entry->filename = name;
1298     entry->object = object;
1299     entry->fd = fd;
1300     entry->body_offset = body_offset;
1301     entry->local = local;
1302     entry->offset = offset;
1303     entry->size = size;
1304     entry->metadataDirty = dirty;
1305     entry->writeable = isWriteable;
1306
1307     entry->next = diskEntries;
1308     if(diskEntries)
1309         diskEntries->previous = entry;
1310     diskEntries = entry;
1311     if(diskEntriesLast == NULL)
1312         diskEntriesLast = entry;
1313     entry->previous = NULL;
1314     numDiskEntries++;
1315
1316     object->disk_entry = entry;
1317
1318     CHECK_ENTRY(entry);
1319     return entry;
1320 }
1321
1322 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1323 static int
1324 rewriteEntry(ObjectPtr object)
1325 {
1326     int old_body_offset = object->disk_entry->body_offset;
1327     int fd, rc, n;
1328     DiskCacheEntryPtr entry;
1329     char* buf;
1330     int buf_is_chunk, bufsize;
1331     int offset;
1332
1333     fd = dup(object->disk_entry->fd);
1334     if(fd < 0) {
1335         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1336         return -1;
1337     }
1338
1339     rc = destroyDiskEntry(object, 1);
1340     if(rc < 0) {
1341         close(fd);
1342         return -1;
1343     }
1344     entry = makeDiskEntry(object, 1, 1);
1345     if(!entry) {
1346         close(fd);
1347         return -1;
1348     }
1349
1350     offset = diskEntrySize(object);
1351     if(offset < 0) {
1352         close(fd);
1353         return -1;
1354     }
1355
1356     bufsize = CHUNK_SIZE;
1357     buf_is_chunk = 1;
1358     buf = maybe_get_chunk();
1359     if(!buf) {
1360         bufsize = 2048;
1361         buf_is_chunk = 0;
1362         buf = malloc(2048);
1363         if(buf == NULL) {
1364             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1365             close(fd);
1366             return -1;
1367         }
1368     }
1369
1370     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1371     if(rc < 0)
1372         goto done;
1373
1374     while(1) {
1375         CHECK_ENTRY(entry);
1376         n = read(fd, buf, bufsize);
1377         if(n < 0 && errno == EINTR)
1378             continue;
1379         if(n <= 0)
1380             goto done;
1381         rc = entrySeek(entry, entry->body_offset + offset);
1382         if(rc < 0)
1383             goto done;
1384     write_again:
1385         rc = write(entry->fd, buf, n);
1386         if(rc >= 0) {
1387             entry->offset += rc;
1388             entry->size += rc;
1389         } else if(errno == EINTR) {
1390             goto write_again;
1391         }
1392         if(rc < n)
1393             goto done;
1394     }
1395
1396  done:
1397     CHECK_ENTRY(entry);
1398     if(object->length >= 0 && entry->size == object->length)
1399         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1400     close(fd);
1401     if(buf_is_chunk)
1402         dispose_chunk(buf);
1403     else
1404         free(buf);
1405     return 1;
1406 }
1407
1408 int
1409 destroyDiskEntry(ObjectPtr object, int d)
1410 {
1411     DiskCacheEntryPtr entry = object->disk_entry;
1412     int rc, urc = 1;
1413
1414     assert(!entry || !entry->local || !d);
1415
1416     if(d && !entry)
1417         entry = makeDiskEntry(object, 1, 0);
1418
1419     CHECK_ENTRY(entry);
1420
1421     if(!entry || entry == &negativeEntry) {
1422         return 1;
1423     }
1424
1425     assert(entry->object == object);
1426
1427     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1428         /* See writeoutToDisk */
1429         d = 1;
1430     }
1431
1432     if(d) {
1433         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1434         if(entry->filename) {
1435             urc = unlink(entry->filename);
1436             if(urc < 0)
1437                 do_log_error(L_WARN, errno,
1438                              "Couldn't unlink %s", scrub(entry->filename));
1439         }
1440     } else {
1441         if(entry && entry->metadataDirty)
1442             writeoutMetadata(object);
1443         makeDiskEntry(object, 1, 0);
1444         /* rewriteDiskEntry may change the disk entry */
1445         entry = object->disk_entry;
1446         if(entry == NULL || entry == &negativeEntry)
1447             return 0;
1448         if(entry->writeable && diskCacheWriteoutOnClose > 0)
1449             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1450     }
1451  again:
1452     rc = close(entry->fd);
1453     if(rc < 0 && errno == EINTR)
1454         goto again;
1455
1456     entry->fd = -1;
1457
1458     if(entry->filename)
1459         free(entry->filename);
1460     entry->filename = NULL;
1461
1462     if(entry->previous)
1463         entry->previous->next = entry->next;
1464     else
1465         diskEntries = entry->next;
1466     if(entry->next)
1467         entry->next->previous = entry->previous;
1468     else
1469         diskEntriesLast = entry->previous;
1470
1471     numDiskEntries--;
1472     assert(numDiskEntries >= 0);
1473
1474     free(entry);
1475     object->disk_entry = NULL;
1476     if(urc < 0)
1477         return -1;
1478     else
1479         return 1;
1480 }
1481
1482 ObjectPtr
1483 objectGetFromDisk(ObjectPtr object)
1484 {
1485     DiskCacheEntryPtr entry = makeDiskEntry(object, 0, 0);
1486     if(!entry) return NULL;
1487     return object;
1488 }
1489
1490
1491 int
1492 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1493 {
1494     DiskCacheEntryPtr entry;
1495     int rc, result;
1496     int i, j, k;
1497     int complete;
1498
1499     if(object->type != OBJECT_HTTP)
1500         return 0;
1501
1502     if(object->flags & OBJECT_LINEAR)
1503         return 0;
1504
1505     if(object->length >= 0) {
1506         chunks = MIN(chunks,
1507                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1508     }
1509
1510     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1511     if(rc < 0)
1512         return 0;
1513
1514     complete = 1;
1515     if(object->flags & OBJECT_INITIAL) {
1516         complete = 0;
1517     } else if((object->length < 0 || object->size < object->length) &&
1518               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1519         complete = 0;
1520     } else {
1521         for(k = 0; k < chunks; k++) {
1522             int s;
1523             i = offset / CHUNK_SIZE + k;
1524             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1525             if(object->chunks[i].size < s) {
1526                 complete = 0;
1527                 break;
1528             }
1529         }
1530     }
1531
1532     if(complete)
1533         return 1;
1534
1535     /* This has the side-effect of revalidating the entry, which is
1536        what makes HEAD requests work. */
1537     entry = makeDiskEntry(object, 0, 0);
1538     if(!entry)
1539         return 0;
1540
1541     for(k = 0; k < chunks; k++) {
1542         i = offset / CHUNK_SIZE + k;
1543         if(!object->chunks[i].data)
1544             object->chunks[i].data = get_chunk();
1545         if(!object->chunks[i].data) {
1546             chunks = k;
1547             break;
1548         }
1549         lockChunk(object, i);
1550     }
1551
1552     result = 0;
1553
1554     for(k = 0; k < chunks; k++) {
1555         int o;
1556         i = offset / CHUNK_SIZE + k;
1557         j = object->chunks[i].size;
1558         o = i * CHUNK_SIZE + j;
1559
1560         if(object->chunks[i].size == CHUNK_SIZE)
1561             continue;
1562
1563         if(entry->size >= 0 && entry->size <= o)
1564             break;
1565
1566         if(entry->offset != entry->body_offset + o) {
1567             rc = entrySeek(entry, entry->body_offset + o);
1568             if(rc < 0) {
1569                 result = 0;
1570                 break;
1571             }
1572         }
1573
1574         CHECK_ENTRY(entry);
1575         again:
1576         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1577         if(rc < 0) {
1578             if(errno == EINTR)
1579                 goto again;
1580             entry->offset = -1;
1581             do_log_error(L_ERROR, errno, "Couldn't read");
1582             break;
1583         }
1584
1585         entry->offset += rc;
1586         object->chunks[i].size += rc;
1587         if(object->size < o + rc)
1588             object->size = o + rc;
1589
1590         if(entry->object->length >= 0 && entry->size < 0 &&
1591            entry->offset - entry->body_offset == entry->object->length)
1592             entry->size = entry->object->length;
1593
1594         if(rc < CHUNK_SIZE - j) {
1595             /* Paranoia: the read may have been interrupted half-way. */
1596             if(entry->size < 0) {
1597                 if(rc == 0 ||
1598                    (entry->object->length >= 0 &&
1599                     entry->object->length ==
1600                     entry->offset - entry->body_offset))
1601                     entry->size = entry->offset - entry->body_offset;
1602                 break;
1603             } else if(entry->size != entry->offset - entry->body_offset) {
1604                 if(rc == 0 ||
1605                    entry->size < entry->offset - entry->body_offset) {
1606                     do_log(L_WARN,
1607                            "Disk entry size changed behind our back: "
1608                            "%ld -> %ld (%d).\n",
1609                            (long)entry->size,
1610                            (long)entry->offset - entry->body_offset,
1611                            object->size);
1612                     entry->size = -1;
1613                 }
1614             }
1615             break;
1616         }
1617
1618         CHECK_ENTRY(entry);
1619         result = 1;
1620     }
1621
1622     CHECK_ENTRY(object->disk_entry);
1623     for(k = 0; k < chunks; k++) {
1624         i = offset / CHUNK_SIZE + k;
1625         unlockChunk(object, i);
1626     }
1627
1628     if(result > 0) {
1629         notifyObject(object);
1630         return 1;
1631     } else {
1632         return 0;
1633     }
1634 }
1635
1636 int
1637 writeoutToDisk(ObjectPtr object, int upto, int max)
1638 {
1639     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1640         /* An object was created with an unknown length, and then grew
1641            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1642         destroyDiskEntry(object, 1);
1643         return 0;
1644     }
1645
1646     return reallyWriteoutToDisk(object, upto, max);
1647 }
1648
1649 static int
1650 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1651 {
1652     DiskCacheEntryPtr entry;
1653     int rc;
1654     int i, j;
1655     int offset;
1656     int bytes = 0;
1657
1658     if(upto < 0)
1659         upto = object->size;
1660
1661     if((object->cache_control & CACHE_NO_STORE) ||
1662        (object->flags & OBJECT_LOCAL))
1663         return 0;
1664
1665     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1666         return 0;
1667
1668     entry = makeDiskEntry(object, 1, 1);
1669     if(!entry) return 0;
1670
1671     assert(!entry->local);
1672
1673     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1674         goto done;
1675
1676     diskEntrySize(object);
1677     if(entry->size < 0)
1678         return 0;
1679
1680     if(object->length >= 0 && entry->size >= object->length) {
1681         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1682         goto done;
1683     }
1684
1685     if(entry->size >= upto)
1686         goto done;
1687
1688     if(!entry->writeable) {
1689         entry = makeDiskEntry(object, 1, 1);
1690         if(!entry)
1691             return 0;
1692         if(!entry->writeable)
1693             return 0;
1694         diskEntrySize(object);
1695         if(entry->size < 0)
1696             return 0;
1697     }
1698
1699     offset = entry->size;
1700
1701     /* Avoid a seek in case we start writing at the beginning */
1702     if(offset == 0 && entry->metadataDirty) {
1703         writeoutMetadata(object);
1704         /* rewriteDiskEntry may change the entry */
1705         entry = makeDiskEntry(object, 1, 0);
1706         if(entry == NULL || !entry->writeable)
1707             return 0;
1708     }
1709
1710     rc = entrySeek(entry, offset + entry->body_offset);
1711     if(rc < 0) return 0;
1712
1713     do {
1714         if(max >= 0 && bytes >= max)
1715             break;
1716         CHECK_ENTRY(entry);
1717         assert(entry->offset == offset + entry->body_offset);
1718         i = offset / CHUNK_SIZE;
1719         j = offset % CHUNK_SIZE;
1720         if(i >= object->numchunks)
1721             break;
1722         if(object->chunks[i].size <= j)
1723             break;
1724     again:
1725         rc = write(entry->fd, object->chunks[i].data + j,
1726                    object->chunks[i].size - j);
1727         if(rc < 0) {
1728             if(errno == EINTR)
1729                 goto again;
1730             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1731             break;
1732         }
1733         entry->offset += rc;
1734         offset += rc;
1735         bytes += rc;
1736         if(entry->size < offset)
1737             entry->size = offset;
1738     } while(j + rc >= CHUNK_SIZE);
1739
1740  done:
1741     CHECK_ENTRY(entry);
1742     if(entry->metadataDirty)
1743         writeoutMetadata(object);
1744
1745     return bytes;
1746 }
1747
1748 int
1749 writeoutMetadata(ObjectPtr object)
1750 {
1751     DiskCacheEntryPtr entry;
1752     int rc;
1753
1754     if((object->cache_control & CACHE_NO_STORE) ||
1755        (object->flags & OBJECT_LOCAL))
1756         return 0;
1757
1758     entry = makeDiskEntry(object, 1, 0);
1759     if(entry == NULL || entry == &negativeEntry)
1760         goto fail;
1761
1762     assert(!entry->local);
1763
1764     rc = entrySeek(entry, 0);
1765     if(rc < 0) goto fail;
1766
1767     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1768     if(rc == -2) {
1769         rc = rewriteEntry(object);
1770         if(rc < 0) return 0;
1771         return 1;
1772     }
1773     if(rc < 0) goto fail;
1774     entry->offset = rc;
1775     entry->metadataDirty = 0;
1776     return 1;
1777
1778  fail:
1779     /* We need this in order to avoid trying to write this entry out
1780        multiple times. */
1781     if(entry && entry != &negativeEntry)
1782         entry->metadataDirty = 0;
1783     return 0;
1784 }
1785
1786 static void
1787 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1788 {
1789     if(dst->filename == NULL) {
1790         dst->filename = src->filename;
1791         dst->body_offset = src->body_offset;
1792     } else
1793         free(src->filename);
1794     free(src->location);
1795     if(dst->length < 0)
1796         dst->length = src->length;
1797     if(dst->size < 0)
1798         dst->size = src->size;
1799     if(dst->age < 0)
1800         dst->age = src->age;
1801     if(dst->date < 0)
1802         dst->date = src->date;
1803     if(dst->last_modified < 0)
1804         dst->last_modified = src->last_modified;
1805     free(src);
1806 }
1807
1808 DiskObjectPtr
1809 readDiskObject(char *filename, struct stat *sb)
1810 {
1811     int fd, rc, n, dummy, code;
1812     int length, size;
1813     time_t date, last_modified, age, atime, expires;
1814     char *location = NULL, *fn = NULL;
1815     DiskObjectPtr dobject;
1816     char *buf;
1817     int buf_is_chunk, bufsize;
1818     int body_offset;
1819     struct stat ss;
1820
1821     fd = -1;
1822
1823     if(sb == NULL) {
1824         rc = stat(filename, &ss);
1825         if(rc < 0) {
1826             do_log_error(L_WARN, errno, "Couldn't stat %s", scrub(filename));
1827             return NULL;
1828         }
1829         sb = &ss;
1830     }
1831
1832     buf_is_chunk = 1;
1833     bufsize = CHUNK_SIZE;
1834     buf = get_chunk();
1835     if(buf == NULL) {
1836         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1837         return NULL;
1838     }
1839
1840     if(S_ISREG(sb->st_mode)) {
1841         fd = open(filename, O_RDONLY | O_BINARY);
1842         if(fd < 0)
1843             goto fail;
1844     again:
1845         rc = read(fd, buf, bufsize);
1846         if(rc < 0)
1847             goto fail;
1848
1849         n = findEndOfHeaders(buf, 0, rc, &dummy);
1850         if(n < 0) {
1851             long lrc;
1852             if(buf_is_chunk) {
1853                 dispose_chunk(buf);
1854                 buf_is_chunk = 0;
1855                 bufsize = bigBufferSize;
1856                 buf = malloc(bigBufferSize);
1857                 if(buf == NULL)
1858                     goto fail2;
1859                 lrc = lseek(fd, 0, SEEK_SET);
1860                 if(lrc < 0)
1861                     goto fail;
1862                 goto again;
1863             }
1864             goto fail;
1865         }
1866
1867         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1868         if(rc < 0)
1869             goto fail;
1870
1871         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1872                               NULL, &length, NULL, NULL, NULL,
1873                               &date, &last_modified, &expires, &age,
1874                               &atime, &body_offset, NULL,
1875                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1876         if(rc < 0 || location == NULL)
1877             goto fail;
1878         if(body_offset < 0)
1879             body_offset = n;
1880
1881         size = sb->st_size - body_offset;
1882         if(size < 0)
1883             size = 0;
1884     } else if(S_ISDIR(sb->st_mode)) {
1885         char *n;
1886         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1887         if(n == NULL)
1888             goto fail;
1889         location = strdup(n);
1890         if(location == NULL)
1891             goto fail;
1892         length = -1;
1893         size = -1;
1894         body_offset = -1;
1895         age = -1;
1896         atime = -1;
1897         date = -1;
1898         last_modified = -1;
1899         expires = -1;
1900     } else {
1901         goto fail;
1902     }
1903
1904     dobject = malloc(sizeof(DiskObjectRec));
1905     if(!dobject)
1906         goto fail;
1907
1908     fn = strdup(filename);
1909     if(!fn)
1910         goto fail;
1911
1912     if(buf_is_chunk)
1913         dispose_chunk(buf);
1914     else
1915         free(buf);
1916
1917     dobject->location = location;
1918     dobject->filename = fn;
1919     dobject->length = length;
1920     dobject->body_offset = body_offset;
1921     dobject->size = size;
1922     dobject->age = age;
1923     dobject->access = atime;
1924     dobject->date = date;
1925     dobject->last_modified = last_modified;
1926     dobject->expires = expires;
1927     if(fd >= 0) close(fd);
1928     return dobject;
1929
1930  fail:
1931     if(buf_is_chunk)
1932         dispose_chunk(buf);
1933     else
1934         free(buf);
1935  fail2:
1936     if(fd >= 0) close(fd);
1937     if(location) free(location);
1938     return NULL;
1939 }
1940
1941
1942 DiskObjectPtr
1943 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1944 {
1945     DiskObjectPtr dobject = NULL;
1946     int c = 0;
1947
1948     dobject = readDiskObject((char*)filename, sb);
1949     if(dobject == NULL)
1950         return 0;
1951
1952     if(!dobjects ||
1953        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1954         if(dobjects && c == 0) {
1955             mergeDobjects(dobjects, dobject);
1956         } else {
1957             dobject->next = dobjects;
1958             dobjects = dobject;
1959         }
1960     } else {
1961         DiskObjectPtr other = dobjects;
1962         while(other->next) {
1963             c = strcmp(dobject->location, other->next->location);
1964             if(c < 0)
1965                 break;
1966             other = other->next;
1967         }
1968         if(strcmp(dobject->location, other->location) == 0) {
1969             mergeDobjects(other, dobject);
1970         } else {
1971             dobject->next = other->next;
1972             other->next = dobject;
1973         }
1974     }
1975     return dobjects;
1976 }
1977
1978 /* Determine whether p is below root */
1979 static int
1980 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1981 {
1982     char *cp;
1983     int m = strlen(p->location);
1984     if(m < n)
1985         return 0;
1986     if(memcmp(root, p->location, n) != 0)
1987         return 0;
1988     if(recursive)
1989         return 1;
1990     if(m == 0 || p->location[m - 1] == '/')
1991         return 1;
1992     cp = strchr(p->location + n, '/');
1993     if(cp && cp - p->location != m - 1)
1994         return 0;
1995     return 1;
1996 }
1997
1998 /* Filter out all disk objects that are not under root */
1999 DiskObjectPtr
2000 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
2001 {
2002     int n = strlen(root);
2003     DiskObjectPtr p, q;
2004
2005     while(from && !filter(from, root, n, recursive)) {
2006         p = from;
2007         from = p->next;
2008         free(p->location);
2009         free(p);
2010     }
2011
2012     p = from;
2013     while(p && p->next) {
2014         if(!filter(p->next, root, n, recursive)) {
2015             q = p->next;
2016             p->next = q->next;
2017             free(q->location);
2018             free(q);
2019         } else {
2020             p = p->next;
2021         }
2022     }
2023     return from;
2024 }
2025
2026 DiskObjectPtr
2027 insertRoot(DiskObjectPtr from, const char *root)
2028 {
2029     DiskObjectPtr p;
2030
2031     p = from;
2032     while(p) {
2033         if(strcmp(root, p->location) == 0)
2034             return from;
2035         p = p->next;
2036     }
2037
2038     p = malloc(sizeof(DiskObjectRec));
2039     if(!p) return from;
2040     p->location = strdup(root);
2041     if(p->location == NULL) {
2042         free(p);
2043         return from;
2044     }
2045     p->filename = NULL;
2046     p->length = -1;
2047     p->size = -1;
2048     p->age = -1;
2049     p->access = -1;
2050     p->last_modified = -1;
2051     p->expires = -1;
2052     p->next = from;
2053     return p;
2054 }
2055
2056 /* Insert all missing directories in a sorted list of dobjects */
2057 DiskObjectPtr
2058 insertDirs(DiskObjectPtr from)
2059 {
2060     DiskObjectPtr p, q, new;
2061     int n, m;
2062     char *cp;
2063
2064     p = NULL; q = from;
2065     while(q) {
2066         n = strlen(q->location);
2067         if(n > 0 && q->location[n - 1] != '/') {
2068             cp = strrchr(q->location, '/');
2069             m = cp - q->location + 1;
2070             if(cp && (!p || strlen(p->location) < m ||
2071                       memcmp(p->location, q->location, m) != 0)) {
2072                 new = malloc(sizeof(DiskObjectRec));
2073                 if(!new) break;
2074                 new->location = strdup_n(q->location, m);
2075                 if(new->location == NULL) {
2076                     free(new);
2077                     break;
2078                 }
2079                 new->filename = NULL;
2080                 new->length = -1;
2081                 new->size = -1;
2082                 new->age = -1;
2083                 new->access = -1;
2084                 new->last_modified = -1;
2085                 new->expires = -1;
2086                 new->next = q;
2087                 if(p)
2088                     p->next = new;
2089                 else
2090                     from = new;
2091             }
2092         }
2093         p = q;
2094         q = q->next;
2095     }
2096     return from;
2097 }
2098
2099 void
2100 indexDiskObjects(FILE *out, const char *root, int recursive)
2101 {
2102     int n, i, isdir;
2103     DIR *dir;
2104     struct dirent *dirent;
2105     char buf[1024];
2106     char *fts_argv[2];
2107     FTS *fts;
2108     FTSENT *fe;
2109     DiskObjectPtr dobjects = NULL;
2110     char *of = root[0] == '\0' ? "" : " of ";
2111
2112     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2113             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2114             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2115             "<html><head>\n"
2116             "<title>%s%s%s</title>\n"
2117             "</head><body>\n"
2118             "<h1>%s%s%s</h1>\n",
2119             recursive ? "Recursive index" : "Index", of, root,
2120             recursive ? "Recursive index" : "Index", of, root);
2121
2122     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2123         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2124         goto trailer;
2125     }
2126
2127     if(diskCacheRoot->length >= 1024) {
2128         fprintf(out,
2129                 "<p>The value of <tt>diskCacheRoot</tt> is "
2130                 "too long (%d).</p>\n",
2131                 diskCacheRoot->length);
2132         goto trailer;
2133     }
2134
2135     if(strlen(root) < 8) {
2136         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2137         buf[diskCacheRoot->length] = '\0';
2138         n = diskCacheRoot->length;
2139     } else {
2140         n = urlDirname(buf, 1024, root, strlen(root));
2141     }
2142     if(n > 0) {
2143         if(recursive) {
2144             dir = NULL;
2145             fts_argv[0] = buf;
2146             fts_argv[1] = NULL;
2147             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2148             if(fts) {
2149                 while(1) {
2150                     fe = fts_read(fts);
2151                     if(!fe) break;
2152                     if(fe->fts_info != FTS_DP)
2153                         dobjects =
2154                             processObject(dobjects,
2155                                           fe->fts_path,
2156                                           fe->fts_info == FTS_NS ||
2157                                           fe->fts_info == FTS_NSOK ?
2158                                           fe->fts_statp : NULL);
2159                 }
2160                 fts_close(fts);
2161             }
2162         } else {
2163             dir = opendir(buf);
2164             if(dir) {
2165                 while(1) {
2166                     dirent = readdir(dir);
2167                     if(!dirent) break;
2168                     if(n + strlen(dirent->d_name) < 1024) {
2169                         strcpy(buf + n, dirent->d_name);
2170                     } else {
2171                         continue;
2172                     }
2173                     dobjects = processObject(dobjects, buf, NULL);
2174                 }
2175                 closedir(dir);
2176             } else {
2177                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2178                         strerror(errno), errno);
2179                 goto trailer;
2180             }
2181         }
2182     }
2183
2184     if(dobjects) {
2185         DiskObjectPtr dobject;
2186         int entryno;
2187         dobjects = insertRoot(dobjects, root);
2188         dobjects = insertDirs(dobjects);
2189         dobjects = filterDiskObjects(dobjects, root, recursive);
2190         buf[0] = '\0';
2191         alternatingHttpStyle(out, "diskcachelist");
2192         fprintf(out, "<table id=diskcachelist>\n");
2193         fprintf(out, "<tbody>\n");
2194         entryno = 0;
2195         while(dobjects) {
2196             dobject = dobjects;
2197             i = strlen(dobject->location);
2198             isdir = (i == 0 || dobject->location[i - 1] == '/');
2199             if(entryno % 2)
2200                 fprintf(out, "<tr class=odd>");
2201             else
2202                 fprintf(out, "<tr class=even>");
2203             if(dobject->size >= 0) {
2204                 fprintf(out, "<td><a href=\"%s\"><tt>",
2205                         dobject->location);
2206                 htmlPrint(out,
2207                           dobject->location, strlen(dobject->location));
2208                 fprintf(out, "</tt></a></td> ");
2209                 if(dobject->length >= 0) {
2210                     if(dobject->size == dobject->length)
2211                         fprintf(out, "<td>%d</td> ", dobject->length);
2212                     else
2213                         fprintf(out, "<td>%d/%d</td> ",
2214                                dobject->size, dobject->length);
2215                 } else {
2216                     /* Avoid a trigraph. */
2217                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2218                 }
2219                 if(dobject->last_modified >= 0) {
2220                     struct tm *tm = gmtime(&dobject->last_modified);
2221                     if(tm == NULL)
2222                         n = -1;
2223                     else
2224                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2225                 } else
2226                     n = -1;
2227                 if(n > 0) {
2228                     buf[n] = '\0';
2229                     fprintf(out, "<td>%s</td> ", buf);
2230                 } else {
2231                     fprintf(out, "<td></td>");
2232                 }
2233
2234                 if(dobject->date >= 0) {
2235                     struct tm *tm = gmtime(&dobject->date);
2236                     if(tm == NULL)
2237                         n = -1;
2238                     else
2239                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2240                 } else
2241                     n = -1;
2242                 if(n > 0) {
2243                     buf[n] = '\0';
2244                     fprintf(out, "<td>%s</td>", buf);
2245                 } else {
2246                     fprintf(out, "<td></td>");
2247                 }
2248             } else {
2249                 fprintf(out, "<td><tt>");
2250                 htmlPrint(out, dobject->location,
2251                           strlen(dobject->location));
2252                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2253             }
2254             if(isdir) {
2255                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2256                         "<td><a href=\"/polipo/recursive-index?%s\">"
2257                         "recursive</a></td>",
2258                         dobject->location, dobject->location);
2259             }
2260             fprintf(out, "</tr>\n");
2261             entryno++;
2262             dobjects = dobject->next;
2263             free(dobject->location);
2264             free(dobject->filename);
2265             free(dobject);
2266         }
2267         fprintf(out, "</tbody>\n");
2268         fprintf(out, "</table>\n");
2269     }
2270
2271  trailer:
2272     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2273     fprintf(out, "</body></html>\n");
2274     return;
2275 }
2276
2277 static int
2278 checkForZeroes(char *buf, int n)
2279 {
2280     int i, j;
2281     unsigned long *lbuf = (unsigned long *)buf;
2282     assert(n % sizeof(unsigned long) == 0);
2283
2284     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2285         if(lbuf[i] != 0L)
2286             return i * sizeof(unsigned long);
2287     }
2288     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2289         if(buf[i * sizeof(unsigned long) + j] != 0)
2290             break;
2291     }
2292
2293     return i * sizeof(unsigned long) + j;
2294 }
2295
2296 static int
2297 copyFile(int from, char *filename, int n)
2298 {
2299     char *buf;
2300     int to, offset, nread, nzeroes, rc;
2301
2302     buf = malloc(CHUNK_SIZE);
2303     if(buf == NULL)
2304         return -1;
2305
2306     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2307               diskCacheFilePermissions);
2308     if(to < 0) {
2309         free(buf);
2310         return -1;
2311     }
2312
2313     offset = 0;
2314     while(offset < n) {
2315         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2316         if(nread <= 0)
2317             break;
2318         nzeroes = checkForZeroes(buf, nread & -8);
2319         if(nzeroes > 0) {
2320             /* I like holes */
2321             rc = lseek(to, nzeroes, SEEK_CUR);
2322             if(rc != offset + nzeroes) {
2323                 if(rc < 0)
2324                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2325                 else
2326                     do_log(L_ERROR,
2327                            "Couldn't extend file: "
2328                            "unexpected offset %d != %d + %d.\n",
2329                            rc, offset, nread);
2330                 break;
2331             }
2332         }
2333         if(nread > nzeroes) {
2334             rc = write(to, buf + nzeroes, nread - nzeroes);
2335             if(rc != nread - nzeroes) {
2336                 if(rc < 0)
2337                     do_log_error(L_ERROR, errno, "Couldn't write");
2338                 else
2339                     do_log(L_ERROR, "Short write.\n");
2340                 break;
2341             }
2342         }
2343         offset += nread;
2344     }
2345     free(buf);
2346     close(to);
2347     if(offset <= 0)
2348         unlink(filename);       /* something went wrong straight away */
2349     return 1;
2350 }
2351
2352 static long int
2353 expireFile(char *filename, struct stat *sb,
2354            int *considered, int *unlinked, int *truncated)
2355 {
2356     DiskObjectPtr dobject = NULL;
2357     time_t t;
2358     int fd, rc;
2359     long int ret = sb->st_size;
2360
2361     if(!preciseExpiry) {
2362         t = sb->st_mtime;
2363         if(t > current_time.tv_sec + 1) {
2364             do_log(L_WARN, "File %s has access time in the future.\n",
2365                    filename);
2366             t = current_time.tv_sec;
2367         }
2368
2369         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2370            (sb->st_size < diskCacheTruncateSize ||
2371             t > current_time.tv_sec - diskCacheTruncateTime))
2372             return ret;
2373     }
2374
2375     (*considered)++;
2376
2377     dobject = readDiskObject(filename, sb);
2378     if(!dobject) {
2379         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n",
2380                scrub(filename));
2381         rc = unlink(filename);
2382         if(rc < 0) {
2383             do_log_error(L_ERROR, errno,
2384                          "Couldn't unlink %s", scrub(filename));
2385             return ret;
2386         } else {
2387             (*unlinked)++;
2388             return 0;
2389         }
2390     }
2391
2392     t = dobject->access;
2393     if(t < 0) t = dobject->age;
2394     if(t < 0) t = dobject->date;
2395
2396     if(t > current_time.tv_sec)
2397         do_log(L_WARN,
2398                "Disk entry %s (%s) has access time in the future.\n",
2399                scrub(dobject->location), scrub(dobject->filename));
2400
2401     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2402         rc = unlink(dobject->filename);
2403         if(rc < 0) {
2404             do_log_error(L_ERROR, errno, "Couldn't unlink %s",
2405                          scrub(filename));
2406         } else {
2407             (*unlinked)++;
2408             ret = 0;
2409         }
2410     } else if(dobject->size >
2411               diskCacheTruncateSize + 4 * dobject->body_offset &&
2412               t < current_time.tv_sec - diskCacheTruncateTime) {
2413         /* We need to copy rather than simply truncate in place: the
2414            latter would confuse a running polipo. */
2415         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2416         rc = unlink(dobject->filename);
2417         if(rc < 0) {
2418             do_log_error(L_ERROR, errno, "Couldn't unlink %s",
2419                          scrub(filename));
2420             close(fd);
2421             fd = -1;
2422         } else {
2423             (*unlinked)++;
2424             copyFile(fd, dobject->filename,
2425                      dobject->body_offset + diskCacheTruncateSize);
2426             close(fd);
2427             (*unlinked)--;
2428             (*truncated)++;
2429             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2430         }
2431     }
2432     free(dobject->location);
2433     free(dobject->filename);
2434     free(dobject);
2435     return ret;
2436 }
2437
2438 void
2439 expireDiskObjects()
2440 {
2441     int rc;
2442     char *fts_argv[2];
2443     FTS *fts;
2444     FTSENT *fe;
2445     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2446     int dirs = 0, rmdirs = 0;
2447     long left = 0, total = 0;
2448
2449     if(diskCacheRoot == NULL ||
2450        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2451         return;
2452
2453     fts_argv[0] = diskCacheRoot->string;
2454     fts_argv[1] = NULL;
2455     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2456     if(fts == NULL) {
2457         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2458     } else {
2459         while(1) {
2460             gettimeofday(&current_time, NULL);
2461
2462             fe = fts_read(fts);
2463             if(!fe) break;
2464
2465             if(fe->fts_info == FTS_D)
2466                 continue;
2467
2468             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2469                fe->fts_info == FTS_DNR) {
2470                 if(fe->fts_accpath[0] == '/' &&
2471                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2472                     continue;
2473                 dirs++;
2474                 rc = rmdir(fe->fts_accpath);
2475                 if(rc >= 0)
2476                     rmdirs++;
2477                 else if(errno != ENOTEMPTY && errno != EEXIST)
2478                     do_log_error(L_ERROR, errno,
2479                                  "Couldn't remove directory %s",
2480                                  scrub(fe->fts_accpath));
2481                 continue;
2482             } else if(fe->fts_info == FTS_NS) {
2483                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2484                              scrub(fe->fts_accpath));
2485                 continue;
2486             } else if(fe->fts_info == FTS_ERR) {
2487                 do_log_error(L_ERROR, fe->fts_errno,
2488                              "Couldn't fts_read disk cache");
2489                 break;
2490             }
2491
2492             if(!S_ISREG(fe->fts_statp->st_mode)) {
2493                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2494                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2495                 continue;
2496             }
2497
2498             files++;
2499             left += expireFile(fe->fts_accpath, fe->fts_statp,
2500                                &considered, &unlinked, &truncated);
2501             total += fe->fts_statp->st_size;
2502         }
2503         fts_close(fts);
2504     }
2505
2506     printf("Disk cache purged.\n");
2507     printf("%d files, %d considered, %d removed, %d truncated "
2508            "(%ldkB -> %ldkB).\n",
2509            files, considered, unlinked, truncated, total/1024, left/1024);
2510     printf("%d directories, %d removed.\n", dirs, rmdirs);
2511     return;
2512 }
2513
2514 #else
2515
2516 void
2517 preinitDiskcache()
2518 {
2519     return;
2520 }
2521
2522 void
2523 initDiskcache()
2524 {
2525     return;
2526 }
2527
2528 int
2529 writeoutToDisk(ObjectPtr object, int upto, int max)
2530 {
2531     return 0;
2532 }
2533
2534 int
2535 destroyDiskEntry(ObjectPtr object, int d)
2536 {
2537     return 0;
2538 }
2539
2540 ObjectPtr
2541 objectGetFromDisk(ObjectPtr object)
2542 {
2543     return NULL;
2544 }
2545
2546 int
2547 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2548 {
2549     return 0;
2550 }
2551
2552 int
2553 revalidateDiskEntry(ObjectPtr object)
2554 {
2555     return 0;
2556 }
2557
2558 void
2559 dirtyDiskEntry(ObjectPtr object)
2560 {
2561     return;
2562 }
2563
2564 void
2565 expireDiskObjects()
2566 {
2567     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2568 }
2569
2570 int
2571 diskEntrySize(ObjectPtr object)
2572 {
2573     return -1;
2574 }
2575 #endif