diskcache.c

   1 /*
   2 Copyright (c) 2003-2006 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheNoAtime = 1;
  42 int diskCacheWriteoutOnClose = (32 * 1024);
  43
  44 int maxDiskCacheEntrySize = -1;
  45
  46 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  47 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  48 int diskCacheTruncateSize =  1024 * 1024;
  49 int preciseExpiry = 0;
  50
  51 static DiskCacheEntryRec negativeEntry = {
  52     NULL, NULL,
  53     -1, -1, -1, -1, 0, 0, 0, NULL, NULL
  54 };
  55
  56 #ifndef LOCAL_ROOT
  57 #define LOCAL_ROOT "/usr/share/polipo/www/"
  58 #endif
  59
  60 #ifndef DISK_CACHE_ROOT
  61 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  62 #endif
  63
  64 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  65 static int atomSetterFlush(ConfigVariablePtr, void*);
  66 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  67
  68 void
  69 preinitDiskcache()
  70 {
  71     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  72     localDocumentRoot = internAtom(LOCAL_ROOT);
  73
  74     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  75                              configIntSetter,
  76                              "Access rights for new directories.");
  77     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  78                              configIntSetter,
  79                              "Access rights for new cache files.");
  80     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  81                              configIntSetter,
  82                              "Number of bytes to write out eagerly.");
  83     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  84                              "Root of the disk cache.");
  85     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  86                              "Root of the local tree.");
  87     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  88                     "File descriptors used by the on-disk cache.");
  89     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  90                     "Time after which on-disk objects are removed.");
  91     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  92                     "Time after which on-disk objects are truncated.");
  93     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  94                     "Size to which on-disk objects are truncated.");
  95     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  96                     "Whether to consider all files for purging.");
  97     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  98                              configIntSetter,
  99                              "Maximum size of objects cached on disk.");
 100     CONFIG_VARIABLE_SETTABLE(diskCacheNoAtime, CONFIG_BOOLEAN,
 101                              configIntSetter,
 102                              "Try to set noatime on the disk cache.");
 103 }
 104
 105 static int
 106 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 107 {
 108     int i;
 109     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 110     i = *(int*)value;
 111     if(i < 0 || i > 1000000)
 112         return -3;
 113     maxDiskEntries = i;
 114     while(numDiskEntries > maxDiskEntries)
 115         destroyDiskEntry(diskEntriesLast->object, 0);
 116     return 1;
 117 }
 118
 119 static int
 120 atomSetterFlush(ConfigVariablePtr var, void *value)
 121 {
 122     discardObjects(1, 0);
 123     return configAtomSetter(var, value);
 124 }
 125
 126 static int
 127 checkRoot(AtomPtr root)
 128 {
 129     struct stat ss;
 130     int rc;
 131
 132     if(!root || root->length == 0)
 133         return 0;
 134
 135     if(root->string[0] != '/') {
 136         return -2;
 137     }
 138
 139     rc = stat(root->string, &ss);
 140     if(rc < 0)
 141         return -1;
 142     else if(!S_ISDIR(ss.st_mode)) {
 143         errno = ENOTDIR;
 144         return -1;
 145     }
 146     return 1;
 147 }
 148
 149 static AtomPtr
 150 maybeAddSlash(AtomPtr atom)
 151 {
 152     AtomPtr newAtom = NULL;
 153     if(!atom) return NULL;
 154     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 155         newAtom = atomCat(atom, "/");
 156         releaseAtom(atom);
 157         return newAtom;
 158     }
 159     return atom;
 160 }
 161
 162 void
 163 initDiskcache()
 164 {
 165     int rc;
 166
 167     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 168     rc = checkRoot(diskCacheRoot);
 169     if(rc <= 0) {
 170         switch(rc) {
 171         case 0: break;
 172         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 173         case -2:
 174             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 175                    diskCacheRoot->string);
 176             break;
 177         default: abort();
 178         }
 179         releaseAtom(diskCacheRoot);
 180         diskCacheRoot = NULL;
 181     }
 182
 183     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 184     rc = checkRoot(localDocumentRoot);
 185     if(rc <= 0) {
 186         switch(rc) {
 187         case 0: break;
 188         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 189         case -2:
 190             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 191             break;
 192         default: abort();
 193         }
 194         releaseAtom(localDocumentRoot);
 195         localDocumentRoot = NULL;
 196     }
 197 }
 198
 199 #ifdef DEBUG_DISK_CACHE
 200 #define CHECK_ENTRY(entry) check_entry((entry))
 201 static void
 202 check_entry(DiskCacheEntryPtr entry)
 203 {
 204     if(entry && entry->fd < 0)
 205         assert(entry == &negativeEntry);
 206     if(entry && entry->fd >= 0) {
 207         assert((!entry->previous) == (entry == diskEntries));
 208         assert((!entry->next) == (entry == diskEntriesLast));
 209         if(entry->size >= 0)
 210             assert(entry->size + entry->body_offset >= entry->offset);
 211         assert(entry->body_offset >= 0);
 212         if(entry->offset >= 0) {
 213             off_t offset;
 214             offset = lseek(entry->fd, 0, SEEK_CUR);
 215             assert(offset == entry->offset);
 216         }
 217         if(entry->size >= 0) {
 218             int rc;
 219             struct stat ss;
 220             rc = fstat(entry->fd, &ss);
 221             assert(rc >= 0);
 222             assert(ss.st_size == entry->size + entry->body_offset);
 223         }
 224     }
 225 }
 226 #else
 227 #define CHECK_ENTRY(entry) do {} while(0)
 228 #endif
 229
 230 int
 231 diskEntrySize(ObjectPtr object)
 232 {
 233     struct stat buf;
 234     int rc;
 235     DiskCacheEntryPtr entry = object->disk_entry;
 236
 237     if(!entry || entry == &negativeEntry)
 238         return -1;
 239
 240     if(entry->size >= 0)
 241         return entry->size;
 242
 243     rc = fstat(entry->fd, &buf);
 244     if(rc < 0) {
 245         do_log_error(L_ERROR, errno, "Couldn't stat");
 246         return -1;
 247     }
 248
 249     if(buf.st_size <= entry->body_offset)
 250         entry->size = 0;
 251     else
 252         entry->size =  buf.st_size - entry->body_offset;
 253     CHECK_ENTRY(entry);
 254     if(object->length >= 0 && entry->size == object->length)
 255         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 256     return entry->size;
 257 }
 258
 259 static int
 260 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 261 {
 262     off_t rc;
 263
 264     CHECK_ENTRY(entry);
 265     assert(entry != &negativeEntry);
 266     if(entry->offset == offset)
 267         return 1;
 268     if(offset > entry->body_offset) {
 269         /* Avoid extending the file by mistake */
 270         if(entry->size < 0)
 271             diskEntrySize(entry->object);
 272         if(entry->size < 0)
 273             return -1;
 274         if(entry->size + entry->body_offset < offset)
 275             return -1;
 276     }
 277     rc = lseek(entry->fd, offset, SEEK_SET);
 278     if(rc < 0) {
 279         do_log_error(L_ERROR, errno, "Couldn't seek");
 280         entry->offset = -1;
 281         return -1;
 282     }
 283     entry->offset = offset;
 284     return 1;
 285 }
 286
 287 /* Given a local URL, constructs the filename where it can be found. */
 288
 289 int
 290 localFilename(char *buf, int n, char *key, int len)
 291 {
 292     int i, j;
 293     if(len <= 0 || key[0] != '/') return -1;
 294
 295     if(urlIsSpecial(key, len)) return -1;
 296
 297     if(localDocumentRoot == NULL ||
 298        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 299         return -1;
 300
 301     if(n <= localDocumentRoot->length)
 302         return -1;
 303
 304     i = 0;
 305     if(key[i] != '/')
 306         return -1;
 307
 308     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 309     j = localDocumentRoot->length;
 310     if(buf[j - 1] == '/')
 311         j--;
 312
 313     while(i < len) {
 314         if(j >= n - 1)
 315             return -1;
 316         if(key[i] == '/' && i < len - 2)
 317             if(key[i + 1] == '.' &&
 318                (key[i + 2] == '.' || key[i + 2] == '/'))
 319                 return -1;
 320         buf[j++] = key[i++];
 321     }
 322
 323     if(buf[j - 1] == '/') {
 324         if(j >= n - 11)
 325             return -1;
 326         memcpy(buf + j, "index.html", 10);
 327         j += 10;
 328     }
 329
 330     buf[j] = '\0';
 331     return j;
 332 }
 333
 334 static void
 335 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 336 {
 337     static MD5_CTX ctx;
 338     MD5Init(&ctx);
 339     MD5Update(&ctx, key, len);
 340     MD5Final(&ctx);
 341     memcpy(dst, ctx.digest, 16);
 342 }
 343
 344 /* Check whether a character can be stored in a filename.  This is
 345    needed since we want to support deficient file systems. */
 346 static int
 347 fssafe(char c)
 348 {
 349     if(c <= 31 || c >= 127)
 350         return 0;
 351     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 352        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 353         return 1;
 354     return 0;
 355 }
 356
 357 /* Given a URL, returns the directory name within which all files
 358    starting with this URL can be found. */
 359 static int
 360 urlDirname(char *buf, int n, const char *url, int len)
 361 {
 362     int i, j;
 363     if(len < 8)
 364         return -1;
 365     if(memcmp(url, "http://", 7) != 0)
 366         return -1;
 367
 368     if(diskCacheRoot == NULL ||
 369        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 370         return -1;
 371
 372     if(n <= diskCacheRoot->length)
 373         return -1;
 374
 375     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 376     j = diskCacheRoot->length;
 377
 378     if(buf[j - 1] != '/')
 379         buf[j++] = '/';
 380
 381     for(i = 7; i < len; i++) {
 382         if(i >= len || url[i] == '/')
 383             break;
 384         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 385             return -1;
 386         if(url[i] == '%' || !fssafe(url[i])) {
 387             if(j + 3 >= n) return -1;
 388             buf[j++] = '%';
 389             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 390             buf[j++] = i2h(url[i] & 0x0F);
 391         } else {
 392             buf[j++] = url[i]; if(j >= n) return -1;
 393         }
 394     }
 395     buf[j++] = '/'; if(j >= n) return -1;
 396     buf[j] = '\0';
 397     return j;
 398 }
 399
 400 /* Given a URL, returns the filename where the cached data can be
 401    found. */
 402 static int
 403 urlFilename(char *restrict buf, int n, const char *url, int len)
 404 {
 405     int j;
 406     unsigned char md5buf[18];
 407     j = urlDirname(buf, n, url, len);
 408     if(j < 0 || j + 24 >= n)
 409         return -1;
 410     md5((unsigned char*)url, len, md5buf);
 411     b64cpy(buf + j, (char*)md5buf, 16, 1);
 412     buf[j + 24] = '\0';
 413     return j + 24;
 414 }
 415
 416 static char *
 417 dirnameUrl(char *url, int n, char *name, int len)
 418 {
 419     int i, j, k, c1, c2;
 420     k = diskCacheRoot->length;
 421     if(len < k)
 422         return NULL;
 423     if(memcmp(name, diskCacheRoot->string, k) != 0)
 424         return NULL;
 425     if(n < 8)
 426         return NULL;
 427     memcpy(url, "http://", 7);
 428     if(name[len - 1] == '/')
 429         len --;
 430     j = 7;
 431     for(i = k; i < len; i++) {
 432         if(name[i] == '%') {
 433             if(i >= len - 2)
 434                 return NULL;
 435             c1 = h2i(name[i + 1]);
 436             c2 = h2i(name[i + 2]);
 437             if(c1 < 0 || c2 < 0)
 438                 return NULL;
 439             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 440             i += 2;             /* skip extra digits */
 441         } else if(i < len - 1 &&
 442                   name[i] == '.' && name[i + 1] == '/') {
 443                 return NULL;
 444         } else if(i == len - 1 && name[i] == '.') {
 445             return NULL;
 446         } else {
 447             url[j++] = name[i]; if(j >= n) goto fail;
 448         }
 449     }
 450     url[j++] = '/'; if(j >= n) goto fail;
 451     url[j] = '\0';
 452     return url;
 453
 454  fail:
 455     return NULL;
 456 }
 457
 458 /* Create a file and all intermediate directories. */
 459 static int
 460 createFile(const char *name, int path_start)
 461 {
 462     int fd;
 463     char buf[1024];
 464     int n;
 465     int rc;
 466     int noatime = diskCacheNoAtime ? O_NOATIME : 0;
 467
 468     if(name[path_start] == '/')
 469         path_start++;
 470
 471     if(path_start < 2 || name[path_start - 1] != '/' ) {
 472         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 473         return -1;
 474     }
 475
 476     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY | noatime,
 477               diskCacheFilePermissions);
 478     if(fd >= 0)
 479         return fd;
 480     if(errno != ENOENT) {
 481         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 482         return -1;
 483     }
 484
 485     n = path_start;
 486     while(name[n] != '\0' && n < 1024) {
 487         while(name[n] != '/' && name[n] != '\0' && n < 512)
 488             n++;
 489         if(name[n] != '/' || n >= 1024)
 490             break;
 491         memcpy(buf, name, n + 1);
 492         buf[n + 1] = '\0';
 493         rc = mkdir(buf, diskCacheDirectoryPermissions);
 494         if(rc < 0 && errno != EEXIST) {
 495             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 496             return -1;
 497         }
 498         n++;
 499     }
 500     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY | noatime,
 501               diskCacheFilePermissions);
 502     if(fd < 0) {
 503         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 504         return -1;
 505     }
 506
 507     return fd;
 508 }
 509
 510 static int
 511 chooseBodyOffset(int n, ObjectPtr object)
 512 {
 513     int length = MAX(object->size, object->length);
 514     int body_offset;
 515
 516     if(object->length >= 0 && object->length + n < 4096 - 4)
 517         return -1;              /* no gap for small objects */
 518
 519     if(n <= 128)
 520         body_offset = 256;
 521     else if(n <= 192)
 522         body_offset = 384;
 523     else if(n <= 256)
 524         body_offset = 512;
 525     else if(n <= 384)
 526         body_offset = 768;
 527     else if(n <= 512)
 528         body_offset = 1024;
 529     else if(n <= 1024)
 530         body_offset = 2048;
 531     else if(n < 2048)
 532         body_offset = 4096;
 533     else
 534         body_offset = ((n + 32 + 4095) / 4096 + 1) * 4096;
 535
 536     /* Tweak the gap so that we don't use up a full disk block for
 537        a small tail */
 538     if(object->length >= 0 && object->length < 64 * 1024) {
 539         int last = (body_offset + object->length) % 4096;
 540         int gap = body_offset - n - 32;
 541         if(last < gap / 2)
 542             body_offset -= last;
 543     }
 544
 545     /* Rewriting large objects is expensive -- don't use small gaps.
 546        This has the additional benefit of block-aligning large bodies. */
 547     if(length >= 64 * 1024) {
 548         int min_gap, min_offset;
 549         if(length >= 512 * 1024)
 550             min_gap = 4096;
 551         else if(length >= 256 * 1024)
 552             min_gap = 2048;
 553         else
 554             min_gap = 1024;
 555
 556         min_offset = ((n + 32 + min_gap - 1) / min_gap + 1) * min_gap;
 557         body_offset = MAX(body_offset, min_offset);
 558     }
 559
 560     return body_offset;
 561 }
 562
 563 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 564    otherwise the offset at which the file descriptor is left. */
 565 /* If chunk is not null, it should be the first chunk of the object,
 566    and will be written out in the same operation if possible. */
 567 static int
 568 writeHeaders(int fd, int *body_offset_return,
 569              ObjectPtr object, char *chunk, int chunk_len)
 570 {
 571     int n;
 572     int rc;
 573     int body_offset = *body_offset_return;
 574     char *buf = NULL;
 575     int buf_is_chunk = 0;
 576     int bufsize = 0;
 577
 578     if(object->flags & OBJECT_LOCAL)
 579         return -1;
 580
 581     if(body_offset > CHUNK_SIZE)
 582         goto overflow;
 583
 584     /* get_chunk might trigger object expiry */
 585     bufsize = CHUNK_SIZE;
 586     buf_is_chunk = 1;
 587     buf = maybe_get_chunk();
 588     if(!buf) {
 589         bufsize = 2048;
 590         buf_is_chunk = 0;
 591         buf = malloc(2048);
 592         if(buf == NULL) {
 593             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 594             return -1;
 595         }
 596     }
 597
 598  format_again:
 599     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 600                   object->code, object->message->string);
 601
 602     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 603     if(n < 0)
 604         goto overflow;
 605
 606     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 607     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 608
 609     if(object->age >= 0 && object->age != object->date) {
 610         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 611         n = format_time(buf, n, bufsize, object->age);
 612     }
 613
 614     if(object->atime >= 0) {
 615         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 616         n = format_time(buf, n, bufsize, object->atime);
 617     }
 618
 619     if(n < 0)
 620         goto overflow;
 621
 622     if(body_offset < 0)
 623         body_offset = chooseBodyOffset(n, object);
 624
 625     if(body_offset > bufsize)
 626         goto overflow;
 627
 628     if(body_offset > 0 && body_offset != n + 4)
 629         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 630                       body_offset);
 631
 632     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 633     if(n < 0)
 634         goto overflow;
 635
 636     if(body_offset < 0)
 637         body_offset = n;
 638     if(n > body_offset)
 639         goto fail;
 640
 641     if(n < body_offset)
 642         memset(buf + n, 0, body_offset - n);
 643
 644  again:
 645 #ifdef HAVE_READV_WRITEV
 646     if(chunk_len > 0) {
 647         struct iovec iov[2];
 648         iov[0].iov_base = buf;
 649         iov[0].iov_len = body_offset;
 650         iov[1].iov_base = chunk;
 651         iov[1].iov_len = chunk_len;
 652         rc = writev(fd, iov, 2);
 653     } else
 654 #endif
 655         rc = write(fd, buf, body_offset);
 656
 657     if(rc < 0 && errno == EINTR)
 658         goto again;
 659
 660     if(rc < body_offset)
 661         goto fail;
 662     if(object->length >= 0 &&
 663        rc - body_offset >= object->length)
 664         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 665
 666     *body_offset_return = body_offset;
 667     if(buf_is_chunk)
 668         dispose_chunk(buf);
 669     else
 670         free(buf);
 671     return rc;
 672
 673  overflow:
 674     if(bufsize < bigBufferSize) {
 675         char *oldbuf = buf;
 676         buf = malloc(bigBufferSize);
 677         if(!buf) {
 678             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 679             goto fail;
 680         }
 681         bufsize = bigBufferSize;
 682         if(oldbuf) {
 683             if(buf_is_chunk)
 684                 dispose_chunk(oldbuf);
 685             else
 686                 free(oldbuf);
 687         }
 688         buf_is_chunk = 0;
 689         goto format_again;
 690     }
 691     /* fall through */
 692
 693  fail:
 694     if(buf_is_chunk)
 695         dispose_chunk(buf);
 696     else
 697         free(buf);
 698     return -1;
 699 }
 700
 701 typedef struct _MimeEntry {
 702     char *extension;
 703     char *mime;
 704 } MimeEntryRec;
 705
 706 static const MimeEntryRec mimeEntries[] = {
 707     { "html", "text/html" },
 708     { "htm", "text/html" },
 709     { "text", "text/plain" },
 710     { "txt", "text/plain" },
 711     { "png", "image/png" },
 712     { "gif", "image/gif" },
 713     { "jpeg", "image/jpeg" },
 714     { "jpg", "image/jpeg" },
 715     { "ico", "image/x-icon" },
 716     { "pdf", "application/pdf" },
 717     { "ps", "application/postscript" },
 718     { "tar", "application/x-tar" },
 719     { "pac", "application/x-ns-proxy-autoconfig" },
 720     { "css", "text/css" },
 721     { "js",  "application/x-javascript" },
 722     { "xml", "text/xml" },
 723     { "swf", "application/x-shockwave-flash" },
 724 };
 725
 726 static char*
 727 localObjectMimeType(ObjectPtr object, char **encoding_return)
 728 {
 729     char *name = object->key;
 730     int nlen = object->key_size;
 731     int i;
 732
 733     assert(nlen >= 1);
 734
 735     if(name[nlen - 1] == '/') {
 736         *encoding_return = NULL;
 737         return "text/html";
 738     }
 739
 740     if(nlen < 3) {
 741         *encoding_return = NULL;
 742         return "application/octet-stream";
 743     }
 744
 745     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 746         *encoding_return = "x-gzip";
 747         nlen -= 3;
 748     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 749         *encoding_return = "x-compress";
 750         nlen -= 2;
 751     } else {
 752         *encoding_return = NULL;
 753     }
 754
 755     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 756         int len = strlen(mimeEntries[i].extension);
 757         if(nlen > len &&
 758            name[nlen - len - 1] == '.' &&
 759            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 760             return mimeEntries[i].mime;
 761     }
 762
 763     return "application/octet-stream";
 764 }
 765
 766 /* Same interface as validateEntry -- see below */
 767 int
 768 validateLocalEntry(ObjectPtr object, int fd,
 769                    int *body_offset_return, off_t *offset_return)
 770 {
 771     struct stat ss;
 772     char buf[512];
 773     int n, rc;
 774     char *encoding;
 775
 776     rc = fstat(fd, &ss);
 777     if(rc < 0) {
 778         do_log_error(L_ERROR, errno, "Couldn't stat");
 779         return -1;
 780     }
 781
 782     if(S_ISREG(ss.st_mode)) {
 783         if(!(ss.st_mode & S_IROTH) ||
 784            (object->length >= 0 && object->length != ss.st_size) ||
 785            (object->last_modified >= 0 &&
 786             object->last_modified != ss.st_mtime))
 787             return -1;
 788     } else {
 789         notifyObject(object);
 790         return -1;
 791     }
 792
 793     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 794                   (unsigned long)ss.st_ino,
 795                   (unsigned long)ss.st_size,
 796                   (unsigned long)ss.st_mtime);
 797     if(n >= 512)
 798         n = -1;
 799
 800     if(n > 0 && object->etag) {
 801         if(strlen(object->etag) != n ||
 802            memcmp(object->etag, buf, n) != 0)
 803             return -1;
 804     }
 805
 806     if(!(object->flags & OBJECT_INITIAL)) {
 807         if(!object->last_modified && !object->etag)
 808             return -1;
 809     }
 810
 811     if(object->flags & OBJECT_INITIAL) {
 812         object->length = ss.st_size;
 813         object->last_modified = ss.st_mtime;
 814         object->date = current_time.tv_sec;
 815         object->age = current_time.tv_sec;
 816         object->code = 200;
 817         if(n > 0)
 818             object->etag = strdup(buf); /* okay if fails */
 819         object->message = internAtom("Okay");
 820         n = snnprintf(buf, 0, 512,
 821                       "\r\nServer: Polipo"
 822                       "\r\nContent-Type: %s",
 823                       localObjectMimeType(object, &encoding));
 824         if(encoding != NULL)
 825             n = snnprintf(buf, n, 512,
 826                           "\r\nContent-Encoding: %s", encoding);
 827         if(n < 0)
 828             return -1;
 829         object->headers = internAtomN(buf, n);
 830         if(object->headers == NULL)
 831             return -1;
 832         object->flags &= ~OBJECT_INITIAL;
 833     }
 834
 835     if(body_offset_return)
 836         *body_offset_return = 0;
 837     if(offset_return)
 838         *offset_return = 0;
 839     return 0;
 840 }
 841
 842 /* Assumes fd is at offset 0.
 843    Returns -1 if not valid, 1 if metadata should be written out, 0
 844    otherwise. */
 845 int
 846 validateEntry(ObjectPtr object, int fd,
 847               int *body_offset_return, off_t *offset_return)
 848 {
 849     char *buf;
 850     int buf_is_chunk, bufsize;
 851     int rc, n;
 852     int dummy;
 853     int code;
 854     AtomPtr headers;
 855     time_t date, last_modified, expires, polipo_age, polipo_access;
 856     int length;
 857     off_t offset = -1;
 858     int body_offset;
 859     char *etag;
 860     AtomPtr via;
 861     CacheControlRec cache_control;
 862     char *location;
 863     AtomPtr message;
 864     int dirty = 0;
 865
 866     if(object->flags & OBJECT_LOCAL)
 867         return validateLocalEntry(object, fd,
 868                                   body_offset_return, offset_return);
 869
 870     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 871         return 0;
 872
 873     /* get_chunk might trigger object expiry */
 874     bufsize = CHUNK_SIZE;
 875     buf_is_chunk = 1;
 876     buf = maybe_get_chunk();
 877     if(!buf) {
 878         bufsize = 2048;
 879         buf_is_chunk = 0;
 880         buf = malloc(2048);
 881         if(buf == NULL) {
 882             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 883             return -1;
 884         }
 885     }
 886
 887  again:
 888     rc = read(fd, buf, bufsize);
 889     if(rc < 0) {
 890         if(errno == EINTR)
 891             goto again;
 892         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 893         goto fail;
 894     }
 895     offset = rc;
 896
 897  parse_again:
 898     n = findEndOfHeaders(buf, 0, rc, &dummy);
 899     if(n < 0) {
 900         char *oldbuf = buf;
 901         if(bufsize < bigBufferSize) {
 902             buf = malloc(bigBufferSize);
 903             if(!buf) {
 904                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 905                 goto fail;
 906             }
 907             bufsize = bigBufferSize;
 908             memcpy(buf, oldbuf, offset);
 909             if(buf_is_chunk)
 910                 dispose_chunk(oldbuf);
 911             else
 912                 free(oldbuf);
 913             buf_is_chunk = 0;
 914         again2:
 915             rc = read(fd, buf + offset, bufsize - offset);
 916             if(rc < 0) {
 917                 if(errno == EINTR)
 918                     goto again2;
 919                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 920                 goto fail;
 921             }
 922             offset += rc;
 923             goto parse_again;
 924         }
 925         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 926         goto fail;
 927     }
 928
 929     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 930     if(rc < 0) {
 931         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 932         goto fail;
 933     }
 934
 935     if(object->code != 0 && object->code != code) {
 936         releaseAtom(message);
 937         goto fail;
 938     }
 939
 940     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 941                           &headers, &length, &cache_control, NULL, NULL,
 942                           &date, &last_modified, &expires, &polipo_age,
 943                           &polipo_access, &body_offset,
 944                           NULL, &etag, NULL,
 945                           NULL, NULL, &location, &via, NULL);
 946     if(rc < 0) {
 947         releaseAtom(message);
 948         goto fail;
 949     }
 950     if(body_offset < 0)
 951         body_offset = n;
 952
 953     if(!location || strlen(location) != object->key_size ||
 954        memcmp(location, object->key, object->key_size) != 0) {
 955         do_log(L_ERROR, "Inconsistent cache file for %s.\n", location);
 956         goto invalid;
 957     }
 958
 959     if(polipo_age < 0)
 960         polipo_age = date;
 961
 962     if(polipo_age < 0) {
 963         do_log(L_ERROR, "Undated disk entry for %s.\n", location);
 964         goto invalid;
 965     }
 966
 967     if(!(object->flags & OBJECT_INITIAL)) {
 968         if((last_modified >= 0) != (object->last_modified >= 0))
 969             goto invalid;
 970
 971         if((object->cache_control & CACHE_MISMATCH) ||
 972            (cache_control.flags & CACHE_MISMATCH))
 973             goto invalid;
 974
 975         if(last_modified >= 0 && object->last_modified >= 0 &&
 976            last_modified != object->last_modified)
 977             goto invalid;
 978
 979         if(length >= 0 && object->length >= 0)
 980             if(length != object->length)
 981                 goto invalid;
 982
 983         if(!!etag != !!object->etag)
 984             goto invalid;
 985
 986         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 987             goto invalid;
 988
 989         /* If we don't have a usable ETag, and either CACHE_VARY or we
 990            don't have a last-modified date, we validate disk entries by
 991            using their date. */
 992         if(!(etag && object->etag) &&
 993            (!(last_modified >= 0 && object->last_modified >= 0) ||
 994             ((cache_control.flags & CACHE_VARY) ||
 995              (object->cache_control & CACHE_VARY)))) {
 996             if(date >= 0 && date != object->date)
 997                 goto invalid;
 998             if(polipo_age >= 0 && polipo_age != object->age)
 999                 goto invalid;
1000         }
1001         if((object->cache_control & CACHE_VARY) && dontTrustVaryETag >= 1) {
1002             /* Check content-type to work around mod_gzip bugs */
1003             if(!httpHeaderMatch(atomContentType, object->headers, headers) ||
1004                !httpHeaderMatch(atomContentEncoding, object->headers, headers))
1005                 goto invalid;
1006         }
1007     }
1008
1009     if(location)
1010         free(location);
1011
1012     if(headers) {
1013         if(!object->headers)
1014             object->headers = headers;
1015         else
1016             releaseAtom(headers);
1017     }
1018
1019     if(object->code == 0) {
1020         object->code = code;
1021         object->message = retainAtom(message);
1022     }
1023     if(object->date <= date)
1024         object->date = date;
1025     else
1026         dirty = 1;
1027     if(object->last_modified < 0)
1028         object->last_modified = last_modified;
1029     if(object->expires < 0)
1030         object->expires = expires;
1031     else if(object->expires > expires)
1032         dirty = 1;
1033     if(object->age < 0)
1034         object->age = polipo_age;
1035     else if(object->age > polipo_age)
1036         dirty = 1;
1037     if(object->atime <= polipo_access)
1038         object->atime = polipo_access;
1039     else
1040         dirty = 1;
1041
1042     object->cache_control |= cache_control.flags;
1043
1044     if(object->age < 0) object->age = object->date;
1045     if(object->age < 0) object->age = 0; /* a long time ago */
1046     if(object->length < 0) object->length = length;
1047     if(!object->etag)
1048         object->etag = etag;
1049     else {
1050         if(etag)
1051             free(etag);
1052     }
1053     releaseAtom(message);
1054
1055     if(object->flags & OBJECT_INITIAL) object->via = via;
1056     object->flags &= ~OBJECT_INITIAL;
1057     if(offset > body_offset) {
1058         /* We need to make sure we don't invoke object expiry recursively */
1059         objectSetChunks(object, 1);
1060         if(object->numchunks >= 1) {
1061             if(object->chunks[0].data == NULL)
1062                 object->chunks[0].data = maybe_get_chunk();
1063             if(object->chunks[0].data)
1064                 objectAddData(object, buf + body_offset,
1065                               0, MIN(offset - body_offset, CHUNK_SIZE));
1066         }
1067     }
1068
1069     httpTweakCachability(object);
1070
1071     if(buf_is_chunk)
1072         dispose_chunk(buf);
1073     else
1074         free(buf);
1075     if(body_offset_return) *body_offset_return = body_offset;
1076     if(offset_return) *offset_return = offset;
1077     return dirty;
1078
1079  invalid:
1080     releaseAtom(message);
1081     if(etag) free(etag);
1082     if(location) free(location);
1083     if(via) releaseAtom(via);
1084     /* fall through */
1085
1086  fail:
1087     if(buf_is_chunk)
1088         dispose_chunk(buf);
1089     else
1090         free(buf);
1091     return -1;
1092 }
1093
1094 void
1095 dirtyDiskEntry(ObjectPtr object)
1096 {
1097     DiskCacheEntryPtr entry = object->disk_entry;
1098     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1099 }
1100
1101 int
1102 revalidateDiskEntry(ObjectPtr object)
1103 {
1104     DiskCacheEntryPtr entry = object->disk_entry;
1105     int rc;
1106     int body_offset;
1107
1108     if(!entry || entry == &negativeEntry)
1109         return 1;
1110
1111     CHECK_ENTRY(entry);
1112     rc = entrySeek(entry, 0);
1113     if(rc < 0) return 0;
1114
1115     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1116     if(rc < 0) {
1117         destroyDiskEntry(object, 0);
1118         return 0;
1119     }
1120     if(body_offset != entry->body_offset) {
1121         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1122                body_offset, entry->body_offset);
1123         destroyDiskEntry(object, 0);
1124         return 0;
1125     }
1126
1127     entry->metadataDirty |= !!rc;
1128     CHECK_ENTRY(entry);
1129     return 1;
1130 }
1131
1132 static inline int
1133 objectHasDiskEntry(ObjectPtr object)
1134 {
1135     return object->disk_entry && object->disk_entry != &negativeEntry;
1136 }
1137
1138 static DiskCacheEntryPtr
1139 makeDiskEntry(ObjectPtr object, int writeable, int create)
1140 {
1141     DiskCacheEntryPtr entry = NULL;
1142     char buf[1024];
1143     int fd = -1;
1144     int negative = 0, isWriteable = 0, size = -1, name_len = -1;
1145     char *name = NULL;
1146     off_t offset = -1;
1147     int body_offset = -1;
1148     int rc;
1149     int local = (object->flags & OBJECT_LOCAL) != 0;
1150     int dirty = 0;
1151     int noatime = diskCacheNoAtime ? O_NOATIME : 0;
1152
1153     if(local && (writeable || create))
1154         return NULL;
1155
1156     if(!local && !(object->flags & OBJECT_PUBLIC))
1157         return NULL;
1158
1159     if(maxDiskCacheEntrySize >= 0) {
1160         if(object->length > 0) {
1161             if(object->length > maxDiskCacheEntrySize)
1162                 return NULL;
1163         } else {
1164             if(object->size > maxDiskCacheEntrySize)
1165                 return NULL;
1166         }
1167     }
1168
1169     if(object->disk_entry) {
1170         entry = object->disk_entry;
1171         CHECK_ENTRY(entry);
1172         if(entry != &negativeEntry && (!writeable || entry->writeable)) {
1173             /* We'll keep the entry -- put it at the front. */
1174             if(entry != diskEntries && entry != &negativeEntry) {
1175                 entry->previous->next = entry->next;
1176                 if(entry->next)
1177                     entry->next->previous = entry->previous;
1178                 else
1179                     diskEntriesLast = entry->previous;
1180                 entry->next = diskEntries;
1181                 diskEntries->previous = entry;
1182                 entry->previous = NULL;
1183                 diskEntries = entry;
1184             }
1185             return entry;
1186         } else {
1187             if(entry == &negativeEntry) {
1188                 negative = 1;
1189                 if(!create) return NULL;
1190                 object->disk_entry = NULL;
1191             }
1192             entry = NULL;
1193             destroyDiskEntry(object, 0);
1194         }
1195     }
1196
1197     if(numDiskEntries > maxDiskEntries)
1198         destroyDiskEntry(diskEntriesLast->object, 0);
1199
1200     if(!local) {
1201         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1202             return NULL;
1203         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1204         if(name_len < 0) return NULL;
1205         if(!negative) {
1206             isWriteable = 1;
1207             fd = open(buf, O_RDWR | O_BINARY | noatime);
1208             if(fd < 0 && !writeable && errno == EACCES) {
1209                 writeable = 0;
1210                 fd = open(buf, O_RDONLY | O_BINARY | noatime);
1211             }
1212         }
1213         if(fd >= 0) {
1214             rc = validateEntry(object, fd, &body_offset, &offset);
1215             if(rc >= 0) {
1216                 dirty = rc;
1217             } else {
1218                 close(fd);
1219                 fd = -1;
1220                 rc = unlink(buf);
1221                 if(rc < 0 && errno != ENOENT) {
1222                     do_log_error(L_WARN,  errno,
1223                                  "Couldn't unlink stale disk entry %s",
1224                                  buf);
1225                     /* But continue -- it's okay to have stale entries. */
1226                 }
1227             }
1228         }
1229
1230         if(fd < 0 && create && name_len > 0 &&
1231            !(object->flags & OBJECT_INITIAL)) {
1232             isWriteable = 1;
1233             fd = createFile(buf, diskCacheRoot->length);
1234             if(fd < 0)
1235                 return NULL;
1236
1237             if(fd >= 0) {
1238                 char *data = NULL;
1239                 int dsize = 0;
1240                 if(object->numchunks > 0) {
1241                     data = object->chunks[0].data;
1242                     dsize = object->chunks[0].size;
1243                 }
1244                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1245                 if(rc < 0) {
1246                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1247                     rc = unlink(buf);
1248                     if(rc < 0 && errno != ENOENT)
1249                         do_log_error(L_ERROR, errno,
1250                                      "Couldn't unlink truncated entry %s",
1251                                      buf);
1252                     close(fd);
1253                     return NULL;
1254                 }
1255                 assert(rc >= body_offset);
1256                 size = rc - body_offset;
1257                 offset = rc;
1258                 dirty = 0;
1259             }
1260         }
1261     } else {
1262         /* local */
1263         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1264             return NULL;
1265
1266         name_len =
1267             localFilename(buf, 1024, object->key, object->key_size);
1268         if(name_len < 0)
1269             return NULL;
1270         isWriteable = 0;
1271         fd = open(buf, O_RDONLY | O_BINARY | noatime);
1272         if(fd >= 0) {
1273             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1274                 close(fd);
1275                 fd = -1;
1276             }
1277         }
1278         offset = 0;
1279     }
1280
1281     if(fd < 0) {
1282         object->disk_entry = &negativeEntry;
1283         return NULL;
1284     }
1285     assert(body_offset >= 0);
1286
1287     name = strdup_n(buf, name_len);
1288     if(name == NULL) {
1289         do_log(L_ERROR, "Couldn't allocate name.\n");
1290         close(fd);
1291         fd = -1;
1292         return NULL;
1293     }
1294
1295     entry = malloc(sizeof(DiskCacheEntryRec));
1296     if(entry == NULL) {
1297         do_log(L_ERROR, "Couldn't allocate entry.\n");
1298         free(name);
1299         close(fd);
1300         return NULL;
1301     }
1302
1303     entry->filename = name;
1304     entry->object = object;
1305     entry->fd = fd;
1306     entry->body_offset = body_offset;
1307     entry->local = local;
1308     entry->offset = offset;
1309     entry->size = size;
1310     entry->metadataDirty = dirty;
1311     entry->writeable = isWriteable;
1312
1313     entry->next = diskEntries;
1314     if(diskEntries)
1315         diskEntries->previous = entry;
1316     diskEntries = entry;
1317     if(diskEntriesLast == NULL)
1318         diskEntriesLast = entry;
1319     entry->previous = NULL;
1320     numDiskEntries++;
1321
1322     object->disk_entry = entry;
1323
1324     CHECK_ENTRY(entry);
1325     return entry;
1326 }
1327
1328 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1329 static int
1330 rewriteEntry(ObjectPtr object)
1331 {
1332     int old_body_offset = object->disk_entry->body_offset;
1333     int fd, rc, n;
1334     DiskCacheEntryPtr entry;
1335     char* buf;
1336     int buf_is_chunk, bufsize;
1337     int offset;
1338
1339     fd = dup(object->disk_entry->fd);
1340     if(fd < 0) {
1341         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1342         return -1;
1343     }
1344
1345     rc = destroyDiskEntry(object, 1);
1346     if(rc < 0) {
1347         close(fd);
1348         return -1;
1349     }
1350     entry = makeDiskEntry(object, 1, 1);
1351     if(!entry) {
1352         close(fd);
1353         return -1;
1354     }
1355
1356     offset = diskEntrySize(object);
1357     if(offset < 0) {
1358         close(fd);
1359         return -1;
1360     }
1361
1362     bufsize = CHUNK_SIZE;
1363     buf_is_chunk = 1;
1364     buf = maybe_get_chunk();
1365     if(!buf) {
1366         bufsize = 2048;
1367         buf_is_chunk = 0;
1368         buf = malloc(2048);
1369         if(buf == NULL) {
1370             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1371             close(fd);
1372             return -1;
1373         }
1374     }
1375
1376     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1377     if(rc < 0)
1378         goto done;
1379
1380     while(1) {
1381         CHECK_ENTRY(entry);
1382         n = read(fd, buf, bufsize);
1383         if(n <= 0)
1384             goto done;
1385         rc = entrySeek(entry, entry->body_offset + offset);
1386         if(rc < 0)
1387             goto done;
1388         rc = write(entry->fd, buf, n);
1389         if(rc >= 0) {
1390             entry->offset += rc;
1391             entry->size += rc;
1392         }
1393         if(rc < n)
1394             goto done;
1395     }
1396
1397  done:
1398     CHECK_ENTRY(entry);
1399     if(object->length >= 0 && entry->size == object->length)
1400         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1401     close(fd);
1402     if(buf_is_chunk)
1403         dispose_chunk(buf);
1404     else
1405         free(buf);
1406     return 1;
1407 }
1408
1409 int
1410 destroyDiskEntry(ObjectPtr object, int d)
1411 {
1412     DiskCacheEntryPtr entry = object->disk_entry;
1413     int rc, urc = 1;
1414
1415     assert(!entry || !entry->local || !d);
1416
1417     if(d && !entry)
1418         entry = makeDiskEntry(object, 1, 0);
1419
1420     CHECK_ENTRY(entry);
1421
1422     if(!entry || entry == &negativeEntry) {
1423         return 1;
1424     }
1425
1426     assert(entry->object == object);
1427
1428     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1429         /* See writeoutToDisk */
1430         d = 1;
1431     }
1432
1433     if(d) {
1434         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1435         if(entry->filename) {
1436             urc = unlink(entry->filename);
1437             if(urc < 0)
1438                 do_log_error(L_WARN, errno,
1439                              "Couldn't unlink %s", entry->filename);
1440         }
1441     } else {
1442         if(entry && entry->metadataDirty)
1443             writeoutMetadata(object);
1444         makeDiskEntry(object, 1, 0);
1445         /* rewriteDiskEntry may change the disk entry */
1446         entry = object->disk_entry;
1447         if(entry == NULL || entry == &negativeEntry)
1448             return 0;
1449         if(entry->writeable && diskCacheWriteoutOnClose > 0)
1450             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1451     }
1452  again:
1453     rc = close(entry->fd);
1454     if(rc < 0 && errno == EINTR)
1455         goto again;
1456
1457     entry->fd = -1;
1458
1459     if(entry->filename)
1460         free(entry->filename);
1461     entry->filename = NULL;
1462
1463     if(entry->previous)
1464         entry->previous->next = entry->next;
1465     else
1466         diskEntries = entry->next;
1467     if(entry->next)
1468         entry->next->previous = entry->previous;
1469     else
1470         diskEntriesLast = entry->previous;
1471
1472     numDiskEntries--;
1473     assert(numDiskEntries >= 0);
1474
1475     free(entry);
1476     object->disk_entry = NULL;
1477     if(urc < 0)
1478         return -1;
1479     else
1480         return 1;
1481 }
1482
1483 ObjectPtr
1484 objectGetFromDisk(ObjectPtr object)
1485 {
1486     DiskCacheEntryPtr entry = makeDiskEntry(object, 0, 0);
1487     if(!entry) return NULL;
1488     return object;
1489 }
1490
1491
1492 int
1493 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1494 {
1495     DiskCacheEntryPtr entry;
1496     int rc, result;
1497     int i, j, k;
1498     int complete;
1499
1500     if(object->type != OBJECT_HTTP)
1501         return 0;
1502
1503     if(object->flags & OBJECT_LINEAR)
1504         return 0;
1505
1506     if(object->length >= 0) {
1507         chunks = MIN(chunks,
1508                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1509     }
1510
1511     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1512     if(rc < 0)
1513         return 0;
1514
1515     complete = 1;
1516     if(object->flags & OBJECT_INITIAL) {
1517         complete = 0;
1518     } else if((object->length < 0 || object->size < object->length) &&
1519               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1520         complete = 0;
1521     } else {
1522         for(k = 0; k < chunks; k++) {
1523             int s;
1524             i = offset / CHUNK_SIZE + k;
1525             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1526             if(object->chunks[i].size < s) {
1527                 complete = 0;
1528                 break;
1529             }
1530         }
1531     }
1532
1533     if(complete)
1534         return 1;
1535
1536     /* This has the side-effect of revalidating the entry, which is
1537        what makes HEAD requests work. */
1538     entry = makeDiskEntry(object, 0, 0);
1539     if(!entry)
1540         return 0;
1541
1542     for(k = 0; k < chunks; k++) {
1543         i = offset / CHUNK_SIZE + k;
1544         if(!object->chunks[i].data)
1545             object->chunks[i].data = get_chunk();
1546         if(!object->chunks[i].data) {
1547             chunks = k;
1548             break;
1549         }
1550         lockChunk(object, i);
1551     }
1552
1553     result = 0;
1554
1555     for(k = 0; k < chunks; k++) {
1556         int o;
1557         i = offset / CHUNK_SIZE + k;
1558         j = object->chunks[i].size;
1559         o = i * CHUNK_SIZE + j;
1560
1561         if(object->chunks[i].size == CHUNK_SIZE)
1562             continue;
1563
1564         if(entry->size >= 0 && entry->size <= o)
1565             break;
1566
1567         if(entry->offset != entry->body_offset + o) {
1568             rc = entrySeek(entry, entry->body_offset + o);
1569             if(rc < 0) {
1570                 result = 0;
1571                 break;
1572             }
1573         }
1574
1575         CHECK_ENTRY(entry);
1576         again:
1577         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1578         if(rc < 0) {
1579             if(errno == EINTR)
1580                 goto again;
1581             entry->offset = -1;
1582             do_log_error(L_ERROR, errno, "Couldn't read");
1583             break;
1584         }
1585
1586         entry->offset += rc;
1587         object->chunks[i].size += rc;
1588         if(object->size < o + rc)
1589             object->size = o + rc;
1590
1591         if(entry->object->length >= 0 && entry->size < 0 &&
1592            entry->offset - entry->body_offset == entry->object->length)
1593             entry->size = entry->object->length;
1594
1595         if(rc < CHUNK_SIZE - j) {
1596             /* Paranoia: the read may have been interrupted half-way. */
1597             if(entry->size < 0) {
1598                 if(rc == 0 ||
1599                    (entry->object->length >= 0 &&
1600                     entry->object->length ==
1601                     entry->offset - entry->body_offset))
1602                     entry->size = entry->offset - entry->body_offset;
1603                 break;
1604             } else if(entry->size != entry->offset - entry->body_offset) {
1605                 if(rc == 0 ||
1606                    entry->size < entry->offset - entry->body_offset) {
1607                     do_log(L_WARN,
1608                            "Disk entry size changed behind our back: "
1609                            "%ld -> %ld (%d).\n",
1610                            (long)entry->size,
1611                            (long)entry->offset - entry->body_offset,
1612                            object->size);
1613                     entry->size = -1;
1614                 }
1615             }
1616             break;
1617         }
1618
1619         CHECK_ENTRY(entry);
1620         result = 1;
1621     }
1622
1623     CHECK_ENTRY(object->disk_entry);
1624     for(k = 0; k < chunks; k++) {
1625         i = offset / CHUNK_SIZE + k;
1626         unlockChunk(object, i);
1627     }
1628
1629     if(result > 0) {
1630         notifyObject(object);
1631         return 1;
1632     } else {
1633         return 0;
1634     }
1635 }
1636
1637 int
1638 writeoutToDisk(ObjectPtr object, int upto, int max)
1639 {
1640     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1641         /* An object was created with an unknown length, and then grew
1642            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1643         destroyDiskEntry(object, 1);
1644         return 0;
1645     }
1646
1647     return reallyWriteoutToDisk(object, upto, max);
1648 }
1649
1650 static int
1651 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1652 {
1653     DiskCacheEntryPtr entry;
1654     int rc;
1655     int i, j;
1656     int offset;
1657     int bytes = 0;
1658
1659     if(upto < 0)
1660         upto = object->size;
1661
1662     if((object->cache_control & CACHE_NO_STORE) ||
1663        (object->flags & OBJECT_LOCAL))
1664         return 0;
1665
1666     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1667         return 0;
1668
1669     entry = makeDiskEntry(object, 1, 1);
1670     if(!entry) return 0;
1671
1672     assert(!entry->local);
1673
1674     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1675         goto done;
1676
1677     diskEntrySize(object);
1678     if(entry->size < 0)
1679         return 0;
1680
1681     if(object->length >= 0 && entry->size >= object->length) {
1682         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1683         goto done;
1684     }
1685
1686     if(entry->size >= upto)
1687         goto done;
1688
1689     if(!entry->writeable) {
1690         entry = makeDiskEntry(object, 1, 1);
1691         if(!entry)
1692             return 0;
1693         if(!entry->writeable)
1694             return 0;
1695         diskEntrySize(object);
1696         if(entry->size < 0)
1697             return 0;
1698     }
1699
1700     offset = entry->size;
1701
1702     /* Avoid a seek in case we start writing at the beginning */
1703     if(offset == 0 && entry->metadataDirty) {
1704         writeoutMetadata(object);
1705         /* rewriteDiskEntry may change the entry */
1706         entry = makeDiskEntry(object, 1, 0);
1707         if(entry == NULL || !entry->writeable)
1708             return 0;
1709     }
1710
1711     rc = entrySeek(entry, offset + entry->body_offset);
1712     if(rc < 0) return 0;
1713
1714     do {
1715         if(max >= 0 && bytes >= max)
1716             break;
1717         CHECK_ENTRY(entry);
1718         assert(entry->offset == offset + entry->body_offset);
1719         i = offset / CHUNK_SIZE;
1720         j = offset % CHUNK_SIZE;
1721         if(i >= object->numchunks)
1722             break;
1723         if(object->chunks[i].size <= j)
1724             break;
1725     again:
1726         rc = write(entry->fd, object->chunks[i].data + j,
1727                    object->chunks[i].size - j);
1728         if(rc < 0) {
1729             if(errno == EINTR)
1730                 goto again;
1731             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1732             break;
1733         }
1734         entry->offset += rc;
1735         offset += rc;
1736         bytes += rc;
1737         if(entry->size < offset)
1738             entry->size = offset;
1739     } while(j + rc >= CHUNK_SIZE);
1740
1741  done:
1742     CHECK_ENTRY(entry);
1743     if(entry->metadataDirty)
1744         writeoutMetadata(object);
1745
1746     return bytes;
1747 }
1748
1749 int
1750 writeoutMetadata(ObjectPtr object)
1751 {
1752     DiskCacheEntryPtr entry;
1753     int rc;
1754
1755     if((object->cache_control & CACHE_NO_STORE) ||
1756        (object->flags & OBJECT_LOCAL))
1757         return 0;
1758
1759     entry = makeDiskEntry(object, 1, 0);
1760     if(entry == NULL || entry == &negativeEntry)
1761         goto fail;
1762
1763     assert(!entry->local);
1764
1765     rc = entrySeek(entry, 0);
1766     if(rc < 0) goto fail;
1767
1768     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1769     if(rc == -2) {
1770         rc = rewriteEntry(object);
1771         if(rc < 0) return 0;
1772         return 1;
1773     }
1774     if(rc < 0) goto fail;
1775     entry->offset = rc;
1776     entry->metadataDirty = 0;
1777     return 1;
1778
1779  fail:
1780     /* We need this in order to avoid trying to write this entry out
1781        multiple times. */
1782     if(entry && entry != &negativeEntry)
1783         entry->metadataDirty = 0;
1784     return 0;
1785 }
1786
1787 static void
1788 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1789 {
1790     if(dst->filename == NULL) {
1791         dst->filename = src->filename;
1792         dst->body_offset = src->body_offset;
1793     } else
1794         free(src->filename);
1795     free(src->location);
1796     if(dst->length < 0)
1797         dst->length = src->length;
1798     if(dst->size < 0)
1799         dst->size = src->size;
1800     if(dst->age < 0)
1801         dst->age = src->age;
1802     if(dst->date < 0)
1803         dst->date = src->date;
1804     if(dst->last_modified < 0)
1805         dst->last_modified = src->last_modified;
1806     free(src);
1807 }
1808
1809 DiskObjectPtr
1810 readDiskObject(char *filename, struct stat *sb)
1811 {
1812     int fd, rc, n, dummy, code;
1813     int length, size;
1814     time_t date, last_modified, age, atime, expires;
1815     char *location = NULL, *fn = NULL;
1816     DiskObjectPtr dobject;
1817     char *buf;
1818     int buf_is_chunk, bufsize;
1819     int body_offset;
1820     struct stat ss;
1821     int noatime = diskCacheNoAtime ? O_NOATIME : 0;
1822
1823     fd = -1;
1824
1825     if(sb == NULL) {
1826         rc = stat(filename, &ss);
1827         if(rc < 0) {
1828             do_log_error(L_WARN, errno, "Couldn't stat %s", filename);
1829             return NULL;
1830         }
1831         sb = &ss;
1832     }
1833
1834     buf_is_chunk = 1;
1835     bufsize = CHUNK_SIZE;
1836     buf = get_chunk();
1837     if(buf == NULL) {
1838         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1839         return NULL;
1840     }
1841
1842     if(S_ISREG(sb->st_mode)) {
1843         fd = open(filename, O_RDONLY | O_BINARY | noatime);
1844         if(fd < 0)
1845             goto fail;
1846     again:
1847         rc = read(fd, buf, bufsize);
1848         if(rc < 0)
1849             goto fail;
1850
1851         n = findEndOfHeaders(buf, 0, rc, &dummy);
1852         if(n < 0) {
1853             long lrc;
1854             if(buf_is_chunk) {
1855                 dispose_chunk(buf);
1856                 buf_is_chunk = 0;
1857                 bufsize = bigBufferSize;
1858                 buf = malloc(bigBufferSize);
1859                 if(buf == NULL)
1860                     goto fail2;
1861                 lrc = lseek(fd, 0, SEEK_SET);
1862                 if(lrc < 0)
1863                     goto fail;
1864                 goto again;
1865             }
1866             goto fail;
1867         }
1868
1869         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1870         if(rc < 0)
1871             goto fail;
1872
1873         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1874                               NULL, &length, NULL, NULL, NULL,
1875                               &date, &last_modified, &expires, &age,
1876                               &atime, &body_offset, NULL,
1877                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1878         if(rc < 0 || location == NULL)
1879             goto fail;
1880         if(body_offset < 0)
1881             body_offset = n;
1882
1883         size = sb->st_size - body_offset;
1884         if(size < 0)
1885             size = 0;
1886     } else if(S_ISDIR(sb->st_mode)) {
1887         char *n;
1888         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1889         if(n == NULL)
1890             goto fail;
1891         location = strdup(n);
1892         if(location == NULL)
1893             goto fail;
1894         length = -1;
1895         size = -1;
1896         body_offset = -1;
1897         age = -1;
1898         atime = -1;
1899         date = -1;
1900         last_modified = -1;
1901     } else {
1902         goto fail;
1903     }
1904
1905     dobject = malloc(sizeof(DiskObjectRec));
1906     if(!dobject)
1907         goto fail;
1908
1909     fn = strdup(filename);
1910     if(!fn)
1911         goto fail;
1912
1913     if(buf_is_chunk)
1914         dispose_chunk(buf);
1915     else
1916         free(buf);
1917
1918     dobject->location = location;
1919     dobject->filename = fn;
1920     dobject->length = length;
1921     dobject->body_offset = body_offset;
1922     dobject->size = size;
1923     dobject->age = age;
1924     dobject->access = atime;
1925     dobject->date = date;
1926     dobject->last_modified = last_modified;
1927     dobject->expires = expires;
1928     if(fd >= 0) close(fd);
1929     return dobject;
1930
1931  fail:
1932     if(buf_is_chunk)
1933         dispose_chunk(buf);
1934     else
1935         free(buf);
1936  fail2:
1937     if(fd >= 0) close(fd);
1938     if(location) free(location);
1939     return NULL;
1940 }
1941
1942
1943 DiskObjectPtr
1944 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1945 {
1946     DiskObjectPtr dobject = NULL;
1947     int c = 0;
1948
1949     dobject = readDiskObject((char*)filename, sb);
1950     if(dobject == NULL)
1951         return 0;
1952
1953     if(!dobjects ||
1954        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1955         if(dobjects && c == 0) {
1956             mergeDobjects(dobjects, dobject);
1957         } else {
1958             dobject->next = dobjects;
1959             dobjects = dobject;
1960         }
1961     } else {
1962         DiskObjectPtr other = dobjects;
1963         while(other->next) {
1964             c = strcmp(dobject->location, other->next->location);
1965             if(c < 0)
1966                 break;
1967             other = other->next;
1968         }
1969         if(strcmp(dobject->location, other->location) == 0) {
1970             mergeDobjects(other, dobject);
1971         } else {
1972             dobject->next = other->next;
1973             other->next = dobject;
1974         }
1975     }
1976     return dobjects;
1977 }
1978
1979 /* Determine whether p is below root */
1980 static int
1981 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1982 {
1983     char *cp;
1984     int m = strlen(p->location);
1985     if(m < n)
1986         return 0;
1987     if(memcmp(root, p->location, n) != 0)
1988         return 0;
1989     if(recursive)
1990         return 1;
1991     if(m == 0 || p->location[m - 1] == '/')
1992         return 1;
1993     cp = strchr(p->location + n, '/');
1994     if(cp && cp - p->location != m - 1)
1995         return 0;
1996     return 1;
1997 }
1998
1999 /* Filter out all disk objects that are not under root */
2000 DiskObjectPtr
2001 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
2002 {
2003     int n = strlen(root);
2004     DiskObjectPtr p, q;
2005
2006     while(from && !filter(from, root, n, recursive)) {
2007         p = from;
2008         from = p->next;
2009         free(p->location);
2010         free(p);
2011     }
2012
2013     p = from;
2014     while(p && p->next) {
2015         if(!filter(p->next, root, n, recursive)) {
2016             q = p->next;
2017             p->next = q->next;
2018             free(q->location);
2019             free(q);
2020         } else {
2021             p = p->next;
2022         }
2023     }
2024     return from;
2025 }
2026
2027 DiskObjectPtr
2028 insertRoot(DiskObjectPtr from, const char *root)
2029 {
2030     DiskObjectPtr p;
2031
2032     p = from;
2033     while(p) {
2034         if(strcmp(root, p->location) == 0)
2035             return from;
2036         p = p->next;
2037     }
2038
2039     p = malloc(sizeof(DiskObjectRec));
2040     if(!p) return from;
2041     p->location = strdup(root);
2042     if(p->location == NULL) {
2043         free(p);
2044         return from;
2045     }
2046     p->filename = NULL;
2047     p->length = -1;
2048     p->size = -1;
2049     p->age = -1;
2050     p->access = -1;
2051     p->last_modified = -1;
2052     p->expires = -1;
2053     p->next = from;
2054     return p;
2055 }
2056
2057 /* Insert all missing directories in a sorted list of dobjects */
2058 DiskObjectPtr
2059 insertDirs(DiskObjectPtr from)
2060 {
2061     DiskObjectPtr p, q, new;
2062     int n, m;
2063     char *cp;
2064
2065     p = NULL; q = from;
2066     while(q) {
2067         n = strlen(q->location);
2068         if(n > 0 && q->location[n - 1] != '/') {
2069             cp = strrchr(q->location, '/');
2070             m = cp - q->location + 1;
2071             if(cp && (!p || strlen(p->location) < m ||
2072                       memcmp(p->location, q->location, m) != 0)) {
2073                 new = malloc(sizeof(DiskObjectRec));
2074                 if(!new) break;
2075                 new->location = strdup_n(q->location, m);
2076                 if(new->location == NULL) {
2077                     free(new);
2078                     break;
2079                 }
2080                 new->filename = NULL;
2081                 new->length = -1;
2082                 new->size = -1;
2083                 new->age = -1;
2084                 new->access = -1;
2085                 new->last_modified = -1;
2086                 new->expires = -1;
2087                 new->next = q;
2088                 if(p)
2089                     p->next = new;
2090                 else
2091                     from = new;
2092             }
2093         }
2094         p = q;
2095         q = q->next;
2096     }
2097     return from;
2098 }
2099
2100 void
2101 indexDiskObjects(FILE *out, const char *root, int recursive)
2102 {
2103     int n, i, isdir;
2104     DIR *dir;
2105     struct dirent *dirent;
2106     char buf[1024];
2107     char *fts_argv[2];
2108     FTS *fts;
2109     FTSENT *fe;
2110     DiskObjectPtr dobjects = NULL;
2111     char *of = root[0] == '\0' ? "" : " of ";
2112
2113     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2114             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2115             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2116             "<html><head>\n"
2117             "<title>%s%s%s</title>\n"
2118             "</head><body>\n"
2119             "<h1>%s%s%s</h1>\n",
2120             recursive ? "Recursive index" : "Index", of, root,
2121             recursive ? "Recursive index" : "Index", of, root);
2122
2123     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2124         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2125         goto trailer;
2126     }
2127
2128     if(diskCacheRoot->length >= 1024) {
2129         fprintf(out,
2130                 "<p>The value of <tt>diskCacheRoot</tt> is "
2131                 "too long (%d).</p>\n",
2132                 diskCacheRoot->length);
2133         goto trailer;
2134     }
2135
2136     if(strlen(root) < 8) {
2137         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2138         buf[diskCacheRoot->length] = '\0';
2139         n = diskCacheRoot->length;
2140     } else {
2141         n = urlDirname(buf, 1024, root, strlen(root));
2142     }
2143     if(n > 0) {
2144         if(recursive) {
2145             dir = NULL;
2146             fts_argv[0] = buf;
2147             fts_argv[1] = NULL;
2148             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2149             if(fts) {
2150                 while(1) {
2151                     fe = fts_read(fts);
2152                     if(!fe) break;
2153                     if(fe->fts_info != FTS_DP)
2154                         dobjects =
2155                             processObject(dobjects,
2156                                           fe->fts_path,
2157                                           fe->fts_info == FTS_NS ||
2158                                           fe->fts_info == FTS_NSOK ?
2159                                           fe->fts_statp : NULL);
2160                 }
2161                 fts_close(fts);
2162             }
2163         } else {
2164             dir = opendir(buf);
2165             if(dir) {
2166                 while(1) {
2167                     dirent = readdir(dir);
2168                     if(!dirent) break;
2169                     if(n + strlen(dirent->d_name) < 1024) {
2170                         strcpy(buf + n, dirent->d_name);
2171                     } else {
2172                         continue;
2173                     }
2174                     dobjects = processObject(dobjects, buf, NULL);
2175                 }
2176                 closedir(dir);
2177             } else {
2178                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2179                         strerror(errno), errno);
2180                 goto trailer;
2181             }
2182         }
2183     }
2184
2185     if(dobjects) {
2186         DiskObjectPtr dobject;
2187         int entryno;
2188         dobjects = insertRoot(dobjects, root);
2189         dobjects = insertDirs(dobjects);
2190         dobjects = filterDiskObjects(dobjects, root, recursive);
2191         dobject = dobjects;
2192         buf[0] = '\0';
2193         alternatingHttpStyle(out, "diskcachelist");
2194         fprintf(out, "<table id=diskcachelist>\n");
2195         fprintf(out, "<tbody>\n");
2196         entryno = 0;
2197         while(dobjects) {
2198             dobject = dobjects;
2199             i = strlen(dobject->location);
2200             isdir = (i == 0 || dobject->location[i - 1] == '/');
2201             if(entryno % 2)
2202                 fprintf(out, "<tr class=odd>");
2203             else
2204                 fprintf(out, "<tr class=even>");
2205             if(dobject->size >= 0) {
2206                 fprintf(out, "<td><a href=\"%s\"><tt>",
2207                         dobject->location);
2208                 htmlPrint(out,
2209                           dobject->location, strlen(dobject->location));
2210                 fprintf(out, "</tt></a></td> ");
2211                 if(dobject->length >= 0) {
2212                     if(dobject->size == dobject->length)
2213                         fprintf(out, "<td>%d</td> ", dobject->length);
2214                     else
2215                         fprintf(out, "<td>%d/%d</td> ",
2216                                dobject->size, dobject->length);
2217                 } else {
2218                     /* Avoid a trigraph. */
2219                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2220                 }
2221                 if(dobject->last_modified >= 0) {
2222                     struct tm *tm = gmtime(&dobject->last_modified);
2223                     if(tm == NULL)
2224                         n = -1;
2225                     else
2226                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2227                 } else
2228                     n = -1;
2229                 if(n > 0) {
2230                     buf[n] = '\0';
2231                     fprintf(out, "<td>%s</td> ", buf);
2232                 } else {
2233                     fprintf(out, "<td></td>");
2234                 }
2235
2236                 if(dobject->date >= 0) {
2237                     struct tm *tm = gmtime(&dobject->date);
2238                     if(tm == NULL)
2239                         n = -1;
2240                     else
2241                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2242                 } else
2243                     n = -1;
2244                 if(n > 0) {
2245                     buf[n] = '\0';
2246                     fprintf(out, "<td>%s</td>", buf);
2247                 } else {
2248                     fprintf(out, "<td></td>");
2249                 }
2250             } else {
2251                 fprintf(out, "<td><tt>");
2252                 htmlPrint(out, dobject->location,
2253                           strlen(dobject->location));
2254                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2255             }
2256             if(isdir) {
2257                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2258                         "<td><a href=\"/polipo/recursive-index?%s\">"
2259                         "recursive</a></td>",
2260                         dobject->location, dobject->location);
2261             }
2262             fprintf(out, "</tr>\n");
2263             entryno++;
2264             dobjects = dobject->next;
2265             free(dobject->location);
2266             free(dobject->filename);
2267             free(dobject);
2268         }
2269         fprintf(out, "</tbody>\n");
2270         fprintf(out, "</table>\n");
2271     }
2272
2273  trailer:
2274     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2275     fprintf(out, "</body></html>\n");
2276     return;
2277 }
2278
2279 static int
2280 checkForZeroes(char *buf, int n)
2281 {
2282     int i, j;
2283     unsigned long *lbuf = (unsigned long *)buf;
2284     assert(n % sizeof(unsigned long) == 0);
2285
2286     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2287         if(lbuf[i] != 0L)
2288             return i * sizeof(unsigned long);
2289     }
2290     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2291         if(buf[i * sizeof(unsigned long) + j] != 0)
2292             break;
2293     }
2294
2295     return i * sizeof(unsigned long) + j;
2296 }
2297
2298 static int
2299 copyFile(int from, char *filename, int n)
2300 {
2301     char *buf;
2302     int to, offset, nread, nzeroes, rc;
2303
2304     buf = malloc(CHUNK_SIZE);
2305     if(buf == NULL)
2306         return -1;
2307
2308     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2309               diskCacheFilePermissions);
2310     if(to < 0) {
2311         free(buf);
2312         return -1;
2313     }
2314
2315     offset = 0;
2316     while(offset < n) {
2317         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2318         if(nread <= 0)
2319             break;
2320         nzeroes = checkForZeroes(buf, nread & -8);
2321         if(nzeroes > 0) {
2322             /* I like holes */
2323             rc = lseek(to, nzeroes, SEEK_CUR);
2324             if(rc != offset + nzeroes) {
2325                 if(rc < 0)
2326                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2327                 else
2328                     do_log(L_ERROR,
2329                            "Couldn't extend file: "
2330                            "unexpected offset %d != %d + %d.\n",
2331                            rc, offset, nread);
2332                 break;
2333             }
2334         }
2335         if(nread > nzeroes) {
2336             rc = write(to, buf + nzeroes, nread - nzeroes);
2337             if(rc != nread - nzeroes) {
2338                 if(rc < 0)
2339                     do_log_error(L_ERROR, errno, "Couldn't write");
2340                 else
2341                     do_log(L_ERROR, "Short write.\n");
2342                 break;
2343             }
2344         }
2345         offset += nread;
2346     }
2347     free(buf);
2348     close(to);
2349     if(offset <= 0)
2350         unlink(filename);       /* something went wrong straight away */
2351     return 1;
2352 }
2353
2354 static long int
2355 expireFile(char *filename, struct stat *sb,
2356            int *considered, int *unlinked, int *truncated)
2357 {
2358     DiskObjectPtr dobject = NULL;
2359     time_t t;
2360     int fd, rc;
2361     long int ret = sb->st_size;
2362
2363     if(!preciseExpiry) {
2364         t = sb->st_mtime;
2365         if(t > current_time.tv_sec + 1) {
2366             do_log(L_WARN, "File %s has access time in the future.\n",
2367                    filename);
2368             t = current_time.tv_sec;
2369         }
2370
2371         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2372            (sb->st_size < diskCacheTruncateSize ||
2373             t > current_time.tv_sec - diskCacheTruncateTime))
2374             return ret;
2375     }
2376
2377     (*considered)++;
2378
2379     dobject = readDiskObject(filename, sb);
2380     if(!dobject) {
2381         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n", filename);
2382         rc = unlink(filename);
2383         if(rc < 0) {
2384             do_log_error(L_ERROR, errno,
2385                          "Couldn't unlink %s", filename);
2386             return ret;
2387         } else {
2388             (*unlinked)++;
2389             return 0;
2390         }
2391     }
2392
2393     t = dobject->access;
2394     if(t < 0) t = dobject->age;
2395     if(t < 0) t = dobject->date;
2396
2397     if(t > current_time.tv_sec)
2398         do_log(L_WARN,
2399                "Disk entry %s (%s) has access time in the future.\n",
2400                dobject->location, dobject->filename);
2401
2402     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2403         rc = unlink(dobject->filename);
2404         if(rc < 0) {
2405             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2406         } else {
2407             (*unlinked)++;
2408             ret = 0;
2409         }
2410     } else if(dobject->size >
2411               diskCacheTruncateSize + 4 * dobject->body_offset &&
2412               t < current_time.tv_sec - diskCacheTruncateTime) {
2413         /* We need to copy rather than simply truncate in place: the
2414            latter would confuse a running polipo. */
2415         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2416         rc = unlink(dobject->filename);
2417         if(rc < 0) {
2418             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2419             close(fd);
2420             fd = -1;
2421         } else {
2422             (*unlinked)++;
2423             copyFile(fd, dobject->filename,
2424                      dobject->body_offset + diskCacheTruncateSize);
2425             close(fd);
2426             (*unlinked)--;
2427             (*truncated)++;
2428             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2429         }
2430     }
2431     free(dobject->location);
2432     free(dobject->filename);
2433     free(dobject);
2434     return ret;
2435 }
2436
2437 void
2438 expireDiskObjects()
2439 {
2440     int rc;
2441     char *fts_argv[2];
2442     FTS *fts;
2443     FTSENT *fe;
2444     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2445     int dirs = 0, rmdirs = 0;
2446     long left = 0, total = 0;
2447
2448     if(diskCacheRoot == NULL ||
2449        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2450         return;
2451
2452     fts_argv[0] = diskCacheRoot->string;
2453     fts_argv[1] = NULL;
2454     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2455     if(fts == NULL) {
2456         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2457     } else {
2458         while(1) {
2459             gettimeofday(&current_time, NULL);
2460
2461             fe = fts_read(fts);
2462             if(!fe) break;
2463
2464             if(fe->fts_info == FTS_D)
2465                 continue;
2466
2467             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2468                fe->fts_info == FTS_DNR) {
2469                 if(fe->fts_accpath[0] == '/' &&
2470                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2471                     continue;
2472                 dirs++;
2473                 rc = rmdir(fe->fts_accpath);
2474                 if(rc >= 0)
2475                     rmdirs++;
2476                 else if(errno != ENOTEMPTY && errno != EEXIST)
2477                     do_log_error(L_ERROR, errno,
2478                                  "Couldn't remove directory %s",
2479                                  fe->fts_accpath);
2480                 continue;
2481             } else if(fe->fts_info == FTS_NS) {
2482                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2483                              fe->fts_accpath);
2484                 continue;
2485             } else if(fe->fts_info == FTS_ERR) {
2486                 do_log_error(L_ERROR, fe->fts_errno,
2487                              "Couldn't fts_read disk cache");
2488                 break;
2489             }
2490
2491             if(!S_ISREG(fe->fts_statp->st_mode)) {
2492                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2493                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2494                 continue;
2495             }
2496
2497             files++;
2498             left += expireFile(fe->fts_accpath, fe->fts_statp,
2499                                &considered, &unlinked, &truncated);
2500             total += fe->fts_statp->st_size;
2501         }
2502         fts_close(fts);
2503     }
2504
2505     printf("Disk cache purged.\n");
2506     printf("%d files, %d considered, %d removed, %d truncated "
2507            "(%ldkB -> %ldkB).\n",
2508            files, considered, unlinked, truncated, total/1024, left/1024);
2509     printf("%d directories, %d removed.\n", dirs, rmdirs);
2510     return;
2511 }
2512
2513 #else
2514
2515 void
2516 preinitDiskcache()
2517 {
2518     return;
2519 }
2520
2521 void
2522 initDiskcache()
2523 {
2524     return;
2525 }
2526
2527 int
2528 writeoutToDisk(ObjectPtr object, int upto, int max)
2529 {
2530     return 0;
2531 }
2532
2533 int
2534 destroyDiskEntry(ObjectPtr object, int d)
2535 {
2536     return 0;
2537 }
2538
2539 ObjectPtr
2540 objectGetFromDisk(ObjectPtr object)
2541 {
2542     return NULL;
2543 }
2544
2545 int
2546 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2547 {
2548     return 0;
2549 }
2550
2551 int
2552 revalidateDiskEntry(ObjectPtr object)
2553 {
2554     return 0;
2555 }
2556
2557 void
2558 dirtyDiskEntry(ObjectPtr object)
2559 {
2560     return;
2561 }
2562
2563 void
2564 expireDiskObjects()
2565 {
2566     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2567 }
2568
2569 int
2570 diskEntrySize(ObjectPtr object)
2571 {
2572     return -1;
2573 }
2574 #endif