diskcache.c

   1 /*
   2 Copyright (c) 2003-2006 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheWriteoutOnClose = (32 * 1024);
  42
  43 int maxDiskCacheEntrySize = -1;
  44
  45 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  46 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  47 int diskCacheTruncateSize =  1024 * 1024;
  48 int preciseExpiry = 0;
  49
  50 static DiskCacheEntryRec negativeEntry = {
  51     NULL, NULL,
  52     -1, -1, -1, -1, 0, 0, 0, NULL, NULL
  53 };
  54
  55 #ifndef LOCAL_ROOT
  56 #define LOCAL_ROOT "/usr/share/polipo/www/"
  57 #endif
  58
  59 #ifndef DISK_CACHE_ROOT
  60 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  61 #endif
  62
  63 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  64 static int atomSetterFlush(ConfigVariablePtr, void*);
  65 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  66
  67 void
  68 preinitDiskcache()
  69 {
  70     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  71     localDocumentRoot = internAtom(LOCAL_ROOT);
  72
  73     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  74                              configIntSetter,
  75                              "Access rights for new directories.");
  76     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  77                              configIntSetter,
  78                              "Access rights for new cache files.");
  79     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  80                              configIntSetter,
  81                              "Number of bytes to write out eagerly.");
  82     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  83                              "Root of the disk cache.");
  84     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  85                              "Root of the local tree.");
  86     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  87                     "File descriptors used by the on-disk cache.");
  88     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  89                     "Time after which on-disk objects are removed.");
  90     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  91                     "Time after which on-disk objects are truncated.");
  92     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  93                     "Size to which on-disk objects are truncated.");
  94     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  95                     "Whether to consider all files for purging.");
  96     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  97                              configIntSetter,
  98                              "Maximum size of objects cached on disk.");
  99 }
 100
 101 static int
 102 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 103 {
 104     int i;
 105     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 106     i = *(int*)value;
 107     if(i < 0 || i > 1000000)
 108         return -3;
 109     maxDiskEntries = i;
 110     while(numDiskEntries > maxDiskEntries)
 111         destroyDiskEntry(diskEntriesLast->object, 0);
 112     return 1;
 113 }
 114
 115 static int
 116 atomSetterFlush(ConfigVariablePtr var, void *value)
 117 {
 118     discardObjects(1, 0);
 119     return configAtomSetter(var, value);
 120 }
 121
 122 static int
 123 checkRoot(AtomPtr root)
 124 {
 125     struct stat ss;
 126     int rc;
 127
 128     if(!root || root->length == 0)
 129         return 0;
 130
 131     if(root->string[0] != '/') {
 132         return -2;
 133     }
 134
 135     rc = stat(root->string, &ss);
 136     if(rc < 0)
 137         return -1;
 138     else if(!S_ISDIR(ss.st_mode)) {
 139         errno = ENOTDIR;
 140         return -1;
 141     }
 142     return 1;
 143 }
 144
 145 static AtomPtr
 146 maybeAddSlash(AtomPtr atom)
 147 {
 148     AtomPtr newAtom = NULL;
 149     if(!atom) return NULL;
 150     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 151         newAtom = atomCat(atom, "/");
 152         releaseAtom(atom);
 153         return newAtom;
 154     }
 155     return atom;
 156 }
 157
 158 void
 159 initDiskcache()
 160 {
 161     int rc;
 162
 163     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 164     rc = checkRoot(diskCacheRoot);
 165     if(rc <= 0) {
 166         switch(rc) {
 167         case 0: break;
 168         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 169         case -2:
 170             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 171                    diskCacheRoot->string);
 172             break;
 173         default: abort();
 174         }
 175         releaseAtom(diskCacheRoot);
 176         diskCacheRoot = NULL;
 177     }
 178
 179     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 180     rc = checkRoot(localDocumentRoot);
 181     if(rc <= 0) {
 182         switch(rc) {
 183         case 0: break;
 184         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 185         case -2:
 186             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 187             break;
 188         default: abort();
 189         }
 190         releaseAtom(localDocumentRoot);
 191         localDocumentRoot = NULL;
 192     }
 193 }
 194
 195 #ifdef DEBUG_DISK_CACHE
 196 #define CHECK_ENTRY(entry) check_entry((entry))
 197 static void
 198 check_entry(DiskCacheEntryPtr entry)
 199 {
 200     if(entry && entry->fd < 0)
 201         assert(entry == &negativeEntry);
 202     if(entry && entry->fd >= 0) {
 203         assert((!entry->previous) == (entry == diskEntries));
 204         assert((!entry->next) == (entry == diskEntriesLast));
 205         if(entry->size >= 0)
 206             assert(entry->size + entry->body_offset >= entry->offset);
 207         assert(entry->body_offset >= 0);
 208         if(entry->offset >= 0) {
 209             off_t offset;
 210             offset = lseek(entry->fd, 0, SEEK_CUR);
 211             assert(offset == entry->offset);
 212         }
 213         if(entry->size >= 0) {
 214             int rc;
 215             struct stat ss;
 216             rc = fstat(entry->fd, &ss);
 217             assert(rc >= 0);
 218             assert(ss.st_size == entry->size + entry->body_offset);
 219         }
 220     }
 221 }
 222 #else
 223 #define CHECK_ENTRY(entry) do {} while(0)
 224 #endif
 225
 226 int
 227 diskEntrySize(ObjectPtr object)
 228 {
 229     struct stat buf;
 230     int rc;
 231     DiskCacheEntryPtr entry = object->disk_entry;
 232
 233     if(!entry || entry == &negativeEntry)
 234         return -1;
 235
 236     if(entry->size >= 0)
 237         return entry->size;
 238
 239     rc = fstat(entry->fd, &buf);
 240     if(rc < 0) {
 241         do_log_error(L_ERROR, errno, "Couldn't stat");
 242         return -1;
 243     }
 244
 245     if(buf.st_size <= entry->body_offset)
 246         entry->size = 0;
 247     else
 248         entry->size =  buf.st_size - entry->body_offset;
 249     CHECK_ENTRY(entry);
 250     if(object->length >= 0 && entry->size == object->length)
 251         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 252     return entry->size;
 253 }
 254
 255 static int
 256 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 257 {
 258     off_t rc;
 259
 260     CHECK_ENTRY(entry);
 261     assert(entry != &negativeEntry);
 262     if(entry->offset == offset)
 263         return 1;
 264     if(offset > entry->body_offset) {
 265         /* Avoid extending the file by mistake */
 266         if(entry->size < 0)
 267             diskEntrySize(entry->object);
 268         if(entry->size < 0)
 269             return -1;
 270         if(entry->size + entry->body_offset < offset)
 271             return -1;
 272     }
 273     rc = lseek(entry->fd, offset, SEEK_SET);
 274     if(rc < 0) {
 275         do_log_error(L_ERROR, errno, "Couldn't seek");
 276         entry->offset = -1;
 277         return -1;
 278     }
 279     entry->offset = offset;
 280     return 1;
 281 }
 282
 283 /* Given a local URL, constructs the filename where it can be found. */
 284
 285 int
 286 localFilename(char *buf, int n, char *key, int len)
 287 {
 288     int i, j;
 289     if(len <= 0 || key[0] != '/') return -1;
 290
 291     if(urlIsSpecial(key, len)) return -1;
 292
 293     if(localDocumentRoot == NULL ||
 294        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 295         return -1;
 296
 297     if(n <= localDocumentRoot->length)
 298         return -1;
 299
 300     i = 0;
 301     if(key[i] != '/')
 302         return -1;
 303
 304     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 305     j = localDocumentRoot->length;
 306     if(buf[j - 1] == '/')
 307         j--;
 308
 309     while(i < len) {
 310         if(j >= n - 1)
 311             return -1;
 312         if(key[i] == '/' && i < len - 2)
 313             if(key[i + 1] == '.' &&
 314                (key[i + 2] == '.' || key[i + 2] == '/'))
 315                 return -1;
 316         buf[j++] = key[i++];
 317     }
 318
 319     if(buf[j - 1] == '/') {
 320         if(j >= n - 11)
 321             return -1;
 322         memcpy(buf + j, "index.html", 10);
 323         j += 10;
 324     }
 325
 326     buf[j] = '\0';
 327     return j;
 328 }
 329
 330 static void
 331 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 332 {
 333     static MD5_CTX ctx;
 334     MD5Init(&ctx);
 335     MD5Update(&ctx, key, len);
 336     MD5Final(&ctx);
 337     memcpy(dst, ctx.digest, 16);
 338 }
 339
 340 /* Check whether a character can be stored in a filename.  This is
 341    needed since we want to support deficient file systems. */
 342 static int
 343 fssafe(char c)
 344 {
 345     if(c <= 31 || c >= 127)
 346         return 0;
 347     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 348        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 349         return 1;
 350     return 0;
 351 }
 352
 353 /* Given a URL, returns the directory name within which all files
 354    starting with this URL can be found. */
 355 static int
 356 urlDirname(char *buf, int n, const char *url, int len)
 357 {
 358     int i, j;
 359     if(len < 8)
 360         return -1;
 361     if(memcmp(url, "http://", 7) != 0)
 362         return -1;
 363
 364     if(diskCacheRoot == NULL ||
 365        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 366         return -1;
 367
 368     if(n <= diskCacheRoot->length)
 369         return -1;
 370
 371     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 372     j = diskCacheRoot->length;
 373
 374     if(buf[j - 1] != '/')
 375         buf[j++] = '/';
 376
 377     for(i = 7; i < len; i++) {
 378         if(i >= len || url[i] == '/')
 379             break;
 380         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 381             return -1;
 382         if(url[i] == '%' || !fssafe(url[i])) {
 383             if(j >= n + 3) return -1;
 384             buf[j++] = '%';
 385             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 386             buf[j++] = i2h(url[i] & 0x0F);
 387         } else {
 388             buf[j++] = url[i]; if(j >= n) return -1;
 389         }
 390     }
 391     buf[j++] = '/'; if(j >= n) return -1;
 392     buf[j] = '\0';
 393     return j;
 394 }
 395
 396 /* Given a URL, returns the filename where the cached data can be
 397    found. */
 398 static int
 399 urlFilename(char *restrict buf, int n, const char *url, int len)
 400 {
 401     int j;
 402     unsigned char md5buf[18];
 403     j = urlDirname(buf, n, url, len);
 404     if(j < 0 || j + 24 >= n)
 405         return -1;
 406     md5((unsigned char*)url, len, md5buf);
 407     b64cpy(buf + j, (char*)md5buf, 16, 1);
 408     buf[j + 24] = '\0';
 409     return j + 24;
 410 }
 411
 412 static char *
 413 dirnameUrl(char *url, int n, char *name, int len)
 414 {
 415     int i, j, k, c1, c2;
 416     k = diskCacheRoot->length;
 417     if(len < k)
 418         return NULL;
 419     if(memcmp(name, diskCacheRoot->string, k) != 0)
 420         return NULL;
 421     if(n < 8)
 422         return NULL;
 423     memcpy(url, "http://", 7);
 424     if(name[len - 1] == '/')
 425         len --;
 426     j = 7;
 427     for(i = k; i < len; i++) {
 428         if(name[i] == '%') {
 429             if(i >= len - 2)
 430                 return NULL;
 431             c1 = h2i(name[i + 1]);
 432             c2 = h2i(name[i + 2]);
 433             if(c1 < 0 || c2 < 0)
 434                 return NULL;
 435             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 436         } else if(i < len - 1 &&
 437                   name[i] == '.' && name[i + 1] == '/') {
 438                 return NULL;
 439         } else if(i == len - 1 && name[i] == '.') {
 440             return NULL;
 441         } else {
 442             url[j++] = name[i]; if(j >= n) goto fail;
 443         }
 444     }
 445     url[j++] = '/'; if(j >= n) goto fail;
 446     url[j] = '\0';
 447     return url;
 448
 449  fail:
 450     return NULL;
 451 }
 452
 453 /* Create a file and all intermediate directories. */
 454 static int
 455 createFile(const char *name, int path_start)
 456 {
 457     int fd;
 458     char buf[1024];
 459     int n;
 460     int rc;
 461
 462     if(name[path_start] == '/')
 463         path_start++;
 464
 465     if(path_start < 2 || name[path_start - 1] != '/' ) {
 466         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 467         return -1;
 468     }
 469
 470     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 471               diskCacheFilePermissions);
 472     if(fd >= 0)
 473         return fd;
 474     if(errno != ENOENT) {
 475         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 476         return -1;
 477     }
 478
 479     n = path_start;
 480     while(name[n] != '\0' && n < 1024) {
 481         while(name[n] != '/' && name[n] != '\0' && n < 512)
 482             n++;
 483         if(name[n] != '/' || n >= 1024)
 484             break;
 485         memcpy(buf, name, n + 1);
 486         buf[n + 1] = '\0';
 487         rc = mkdir(buf, diskCacheDirectoryPermissions);
 488         if(rc < 0 && errno != EEXIST) {
 489             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 490             return -1;
 491         }
 492         n++;
 493     }
 494     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 495               diskCacheFilePermissions);
 496     if(fd < 0) {
 497         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 498         return -1;
 499     }
 500
 501     return fd;
 502 }
 503
 504 static int
 505 chooseBodyOffset(int n, ObjectPtr object)
 506 {
 507     int length = MAX(object->size, object->length);
 508     int body_offset;
 509
 510     if(object->length >= 0 && object->length + n < 4096 - 50)
 511         return -1;              /* no gap for small objects */
 512
 513     if(n <= 128)
 514         body_offset = 256;
 515     else if(n <= 192)
 516         body_offset = 384;
 517     else if(n <= 256)
 518         body_offset = 512;
 519     else if(n <= 384)
 520         body_offset = 768;
 521     else if(n <= 512)
 522         body_offset = 1024;
 523     else if(n <= 1024)
 524         body_offset = 2048;
 525     else if(n < 2048)
 526         body_offset = 4096;
 527     else body_offset = ((n + 4095) / 4096 + 1) * 4096;
 528
 529     /* Tweak the gap so that we don't use up a full disk block for
 530        a small tail */
 531     if(object->length >= 0 && object->length < 64 * 1024) {
 532         int last = (body_offset + object->length) % 4096;
 533         if(last < body_offset / 4 && n < body_offset - last - 50)
 534             body_offset -= last;
 535     }
 536
 537     /* Rewriting large objects is expensive -- don't use small gaps */
 538     if(length >= 64 * 1024)
 539         body_offset = MAX(body_offset, 1024);
 540     if(length >= 256 * 1024)
 541         body_offset = MAX(body_offset, 2048);
 542     if(length >= 512 * 1024)
 543         body_offset = MAX(body_offset, 4096);
 544     return body_offset;
 545 }
 546
 547 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 548    -2 on overflow, otherwise the offset at which the file descriptor is
 549    left. */
 550 static int
 551 writeHeaders(int fd, int *body_offset_return,
 552              ObjectPtr object, char *chunk, int chunk_len)
 553 {
 554     int n;
 555     int rc;
 556     int body_offset = *body_offset_return;
 557     char *buf;
 558     int buf_is_chunk;
 559     int bufsize;
 560
 561     if(object->flags & OBJECT_LOCAL)
 562         return -1;
 563
 564     /* get_chunk might trigger object expiry */
 565     bufsize = CHUNK_SIZE;
 566     buf_is_chunk = 1;
 567     buf = maybe_get_chunk();
 568     if(!buf) {
 569         bufsize = 2048;
 570         buf_is_chunk = 0;
 571         buf = malloc(2048);
 572         if(buf == NULL) {
 573             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 574             return -1;
 575         }
 576     }
 577
 578  format_again:
 579     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 580                   object->code, object->message->string);
 581
 582     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 583     if(n < 0)
 584         goto overflow;
 585
 586     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 587     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 588
 589     if(object->age >= 0 && object->age != object->date) {
 590         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 591         n = format_time(buf, n, bufsize, object->age);
 592     }
 593
 594     if(object->atime >= 0) {
 595         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 596         n = format_time(buf, n, bufsize, object->atime);
 597     }
 598
 599     if(n < 0)
 600         goto overflow;
 601
 602     if(body_offset < 0)
 603         body_offset = chooseBodyOffset(n, object);
 604
 605     if(body_offset > bufsize)
 606         goto overflow;
 607
 608     if(body_offset > 0 && body_offset != n + 4)
 609         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 610                       body_offset);
 611
 612     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 613
 614     if(body_offset < 0)
 615         body_offset = n;
 616     if(n > body_offset)
 617         goto fail2;
 618
 619     if(n < body_offset)
 620         memset(buf + n, 0, body_offset - n);
 621
 622  again:
 623 #ifdef HAVE_READV_WRITEV
 624     if(chunk_len > 0) {
 625         struct iovec iov[2];
 626         iov[0].iov_base = buf;
 627         iov[0].iov_len = body_offset;
 628         iov[1].iov_base = chunk;
 629         iov[1].iov_len = chunk_len;
 630         rc = writev(fd, iov, 2);
 631     } else
 632 #endif
 633         rc = write(fd, buf, body_offset);
 634
 635     if(rc < 0 && errno == EINTR)
 636         goto again;
 637
 638     if(rc < body_offset)
 639         goto fail;
 640     if(object->length >= 0 &&
 641        rc - body_offset >= object->length)
 642         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 643
 644     *body_offset_return = body_offset;
 645     if(buf_is_chunk)
 646         dispose_chunk(buf);
 647     else
 648         free(buf);
 649     return rc;
 650
 651  overflow:
 652     if(bufsize < bigBufferSize) {
 653         char *oldbuf = buf;
 654         buf = malloc(bigBufferSize);
 655         if(!buf) {
 656             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 657             goto fail;
 658         }
 659         bufsize = bigBufferSize;
 660         if(buf_is_chunk)
 661             dispose_chunk(oldbuf);
 662         else
 663             free(oldbuf);
 664         buf_is_chunk = 0;
 665         goto format_again;
 666     }
 667     /* fall through */
 668
 669  fail:
 670     if(buf_is_chunk)
 671         dispose_chunk(buf);
 672     else
 673         free(buf);
 674     return -1;
 675
 676  fail2:
 677     if(buf_is_chunk)
 678         dispose_chunk(buf);
 679     else
 680         free(buf);
 681     return -1;
 682 }
 683
 684 typedef struct _MimeEntry {
 685     char *extension;
 686     char *mime;
 687 } MimeEntryRec;
 688
 689 static const MimeEntryRec mimeEntries[] = {
 690     { "html", "text/html" },
 691     { "htm", "text/html" },
 692     { "text", "text/plain" },
 693     { "txt", "text/plain" },
 694     { "png", "image/png" },
 695     { "gif", "image/gif" },
 696     { "jpeg", "image/jpeg" },
 697     { "jpg", "image/jpeg" },
 698     { "ico", "image/x-icon" },
 699     { "pdf", "application/pdf" },
 700     { "ps", "application/postscript" },
 701     { "tar", "application/x-tar" },
 702     { "pac", "application/x-ns-proxy-autoconfig" },
 703     { "css", "text/css" },
 704     { "js",  "application/x-javascript" },
 705     { "xml", "text/xml" },
 706     { "swf", "application/x-shockwave-flash" },
 707 };
 708
 709 static char*
 710 localObjectMimeType(ObjectPtr object, char **encoding_return)
 711 {
 712     char *name = object->key;
 713     int nlen = object->key_size;
 714     int i;
 715
 716     assert(nlen >= 1);
 717
 718     if(name[nlen - 1] == '/') {
 719         *encoding_return = NULL;
 720         return "text/html";
 721     }
 722
 723     if(nlen < 3) {
 724         *encoding_return = NULL;
 725         return "application/octet-stream";
 726     }
 727
 728     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 729         *encoding_return = "x-gzip";
 730         nlen -= 3;
 731     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 732         *encoding_return = "x-compress";
 733         nlen -= 2;
 734     } else {
 735         *encoding_return = NULL;
 736     }
 737
 738     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 739         int len = strlen(mimeEntries[i].extension);
 740         if(nlen > len &&
 741            name[nlen - len - 1] == '.' &&
 742            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 743             return mimeEntries[i].mime;
 744     }
 745
 746     return "application/octet-stream";
 747 }
 748
 749 /* Same interface as validateEntry -- see below */
 750 int
 751 validateLocalEntry(ObjectPtr object, int fd,
 752                    int *body_offset_return, off_t *offset_return)
 753 {
 754     struct stat ss;
 755     char buf[512];
 756     int n, rc;
 757     char *encoding;
 758
 759     rc = fstat(fd, &ss);
 760     if(rc < 0) {
 761         do_log_error(L_ERROR, errno, "Couldn't stat");
 762         return -1;
 763     }
 764
 765     if(S_ISREG(ss.st_mode)) {
 766         if(!(ss.st_mode & S_IROTH) ||
 767            (object->length >= 0 && object->length != ss.st_size) ||
 768            (object->last_modified >= 0 &&
 769             object->last_modified != ss.st_mtime))
 770             return -1;
 771     } else {
 772         notifyObject(object);
 773         return -1;
 774     }
 775
 776     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 777                   (unsigned long)ss.st_ino,
 778                   (unsigned long)ss.st_size,
 779                   (unsigned long)ss.st_mtime);
 780     if(n >= 512)
 781         n = -1;
 782
 783     if(n > 0 && object->etag) {
 784         if(strlen(object->etag) != n ||
 785            memcmp(object->etag, buf, n) != 0)
 786             return -1;
 787     }
 788
 789     if(!(object->flags & OBJECT_INITIAL)) {
 790         if(!object->last_modified && !object->etag)
 791             return -1;
 792     }
 793
 794     if(object->flags & OBJECT_INITIAL) {
 795         object->length = ss.st_size;
 796         object->last_modified = ss.st_mtime;
 797         object->date = current_time.tv_sec;
 798         object->age = current_time.tv_sec;
 799         object->code = 200;
 800         if(n > 0)
 801             object->etag = strdup(buf); /* okay if fails */
 802         object->message = internAtom("Okay");
 803         n = snnprintf(buf, 0, 512,
 804                       "\r\nServer: Polipo"
 805                       "\r\nContent-Type: %s",
 806                       localObjectMimeType(object, &encoding));
 807         if(encoding != NULL)
 808             n = snnprintf(buf, n, 512,
 809                           "\r\nContent-Encoding: %s", encoding);
 810         if(n < 0)
 811             return -1;
 812         object->headers = internAtomN(buf, n);
 813         if(object->headers == NULL)
 814             return -1;
 815         object->flags &= ~OBJECT_INITIAL;
 816     }
 817
 818     if(body_offset_return)
 819         *body_offset_return = 0;
 820     if(offset_return)
 821         *offset_return = 0;
 822     return 0;
 823 }
 824
 825 /* Assumes fd is at offset 0.
 826    Returns -1 if not valid, 1 if metadata should be written out, 0
 827    otherwise. */
 828 int
 829 validateEntry(ObjectPtr object, int fd,
 830               int *body_offset_return, off_t *offset_return)
 831 {
 832     char *buf;
 833     int buf_is_chunk, bufsize;
 834     int rc, n;
 835     int dummy;
 836     int code;
 837     AtomPtr headers;
 838     time_t date, last_modified, expires, polipo_age, polipo_access;
 839     int length;
 840     off_t offset = -1;
 841     int body_offset;
 842     char *etag;
 843     AtomPtr via;
 844     CacheControlRec cache_control;
 845     char *location;
 846     AtomPtr message;
 847     int dirty = 0;
 848
 849     if(object->flags & OBJECT_LOCAL)
 850         return validateLocalEntry(object, fd,
 851                                   body_offset_return, offset_return);
 852
 853     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 854         return 0;
 855
 856     /* get_chunk might trigger object expiry */
 857     bufsize = CHUNK_SIZE;
 858     buf_is_chunk = 1;
 859     buf = maybe_get_chunk();
 860     if(!buf) {
 861         bufsize = 2048;
 862         buf_is_chunk = 0;
 863         buf = malloc(2048);
 864         if(buf == NULL) {
 865             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 866             return -1;
 867         }
 868     }
 869
 870  again:
 871     rc = read(fd, buf, bufsize);
 872     if(rc < 0) {
 873         if(errno == EINTR)
 874             goto again;
 875         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 876         goto fail;
 877     }
 878     offset = rc;
 879
 880  parse_again:
 881     n = findEndOfHeaders(buf, 0, rc, &dummy);
 882     if(n < 0) {
 883         char *oldbuf = buf;
 884         if(bufsize < bigBufferSize) {
 885             buf = malloc(bigBufferSize);
 886             if(!buf) {
 887                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 888                 goto fail;
 889             }
 890             bufsize = bigBufferSize;
 891             memcpy(buf, oldbuf, offset);
 892             if(buf_is_chunk)
 893                 dispose_chunk(oldbuf);
 894             else
 895                 free(oldbuf);
 896             buf_is_chunk = 0;
 897         again2:
 898             rc = read(fd, buf + offset, bufsize - offset);
 899             if(rc < 0) {
 900                 if(errno == EINTR)
 901                     goto again2;
 902                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 903                 goto fail;
 904             }
 905             offset += rc;
 906             goto parse_again;
 907         }
 908         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 909         goto fail;
 910     }
 911
 912     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 913     if(rc < 0) {
 914         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 915         goto fail;
 916     }
 917
 918     if(object->code != 0 && object->code != code) {
 919         releaseAtom(message);
 920         goto fail;
 921     }
 922
 923     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 924                           &headers, &length, &cache_control, NULL, NULL,
 925                           &date, &last_modified, &expires, &polipo_age,
 926                           &polipo_access, &body_offset,
 927                           NULL, &etag, NULL,
 928                           NULL, NULL, &location, &via, NULL);
 929     if(rc < 0) {
 930         releaseAtom(message);
 931         goto fail;
 932     }
 933     if(body_offset < 0)
 934         body_offset = n;
 935
 936     if(!location || strlen(location) != object->key_size ||
 937        memcmp(location, object->key, object->key_size) != 0) {
 938         do_log(L_ERROR, "Inconsistent cache file for %s.\n", location);
 939         goto invalid;
 940     }
 941
 942     if(polipo_age < 0)
 943         polipo_age = date;
 944
 945     if(polipo_age < 0) {
 946         do_log(L_ERROR, "Undated disk entry for %s.\n", location);
 947         goto invalid;
 948     }
 949
 950     if(!(object->flags & OBJECT_INITIAL)) {
 951         if((last_modified >=0) != (object->last_modified >= 0))
 952             goto invalid;
 953
 954         if(last_modified >= 0 && object->last_modified >= 0 &&
 955            last_modified != object->last_modified)
 956             goto invalid;
 957
 958         if(length >= 0 && object->length >= 0)
 959             if(length != object->length)
 960                 goto invalid;
 961
 962         if(!!etag != !!object->etag)
 963             goto invalid;
 964
 965         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 966             goto invalid;
 967
 968         /* If we don't have a usable ETag, and either CACHE_VARY or we
 969            don't have a last-modified date, we validate disk entries by
 970            using their date. */
 971         if(!(etag && object->etag) &&
 972            (!(last_modified >= 0 && object->last_modified >= 0) ||
 973             ((cache_control.flags & CACHE_VARY) ||
 974              (object->cache_control & CACHE_VARY)))) {
 975             if(date >= 0 && date != object->date)
 976                 goto invalid;
 977             if(polipo_age >= 0 && polipo_age != object->age)
 978                 goto invalid;
 979         }
 980     }
 981
 982     if(location)
 983         free(location);
 984
 985     if(headers) {
 986         if(!object->headers)
 987             object->headers = headers;
 988         else
 989             releaseAtom(headers);
 990     }
 991
 992     if(object->code == 0) {
 993         object->code = code;
 994         object->message = retainAtom(message);
 995     }
 996     if(object->date <= date)
 997         object->date = date;
 998     else
 999         dirty = 1;
1000     if(object->last_modified < 0)
1001         object->last_modified = last_modified;
1002     if(object->expires < 0)
1003         object->expires = expires;
1004     else if(object->expires > expires)
1005         dirty = 1;
1006     if(object->age < 0)
1007         object->age = polipo_age;
1008     else if(object->age > polipo_age)
1009         dirty = 1;
1010     if(object->atime <= polipo_access)
1011         object->atime = polipo_access;
1012     else
1013         dirty = 1;
1014
1015     object->cache_control |= cache_control.flags;
1016
1017     if(object->age < 0) object->age = object->date;
1018     if(object->age < 0) object->age = 0; /* a long time ago */
1019     if(object->length < 0) object->length = length;
1020     if(!object->etag)
1021         object->etag = etag;
1022     else {
1023         if(etag)
1024             free(etag);
1025     }
1026     releaseAtom(message);
1027
1028     if(object->flags & OBJECT_INITIAL) object->via = via;
1029     object->flags &= ~OBJECT_INITIAL;
1030     if(offset > body_offset) {
1031         /* We need to make sure we don't invoke object expiry recursively */
1032         objectSetChunks(object, 1);
1033         if(object->numchunks >= 1) {
1034             if(object->chunks[0].data == NULL)
1035                 object->chunks[0].data = maybe_get_chunk();
1036             if(object->chunks[0].data)
1037                 objectAddData(object, buf + body_offset,
1038                               0, MIN(offset - body_offset, CHUNK_SIZE));
1039         }
1040     }
1041
1042     if(buf_is_chunk)
1043         dispose_chunk(buf);
1044     else
1045         free(buf);
1046     if(body_offset_return) *body_offset_return = body_offset;
1047     if(offset_return) *offset_return = offset;
1048     return dirty;
1049
1050  invalid:
1051     releaseAtom(message);
1052     if(etag) free(etag);
1053     if(location) free(location);
1054     if(via) releaseAtom(via);
1055     /* fall through */
1056
1057  fail:
1058     if(buf_is_chunk)
1059         dispose_chunk(buf);
1060     else
1061         free(buf);
1062     return -1;
1063 }
1064
1065 void
1066 dirtyDiskEntry(ObjectPtr object)
1067 {
1068     DiskCacheEntryPtr entry = object->disk_entry;
1069     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1070 }
1071
1072 int
1073 revalidateDiskEntry(ObjectPtr object)
1074 {
1075     DiskCacheEntryPtr entry = object->disk_entry;
1076     int rc;
1077     int body_offset;
1078
1079     if(!entry || entry == &negativeEntry)
1080         return 1;
1081
1082     CHECK_ENTRY(entry);
1083     rc = entrySeek(entry, 0);
1084     if(rc < 0) return 0;
1085
1086     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1087     if(rc < 0) {
1088         destroyDiskEntry(object, 0);
1089         return 0;
1090     }
1091     if(body_offset != entry->body_offset) {
1092         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1093                body_offset, entry->body_offset);
1094         destroyDiskEntry(object, 0);
1095         return 0;
1096     }
1097
1098     entry->metadataDirty |= !!rc;
1099     CHECK_ENTRY(entry);
1100     return 1;
1101 }
1102
1103 static inline int
1104 objectHasDiskEntry(ObjectPtr object)
1105 {
1106     return object->disk_entry && object->disk_entry != &negativeEntry;
1107 }
1108
1109 static DiskCacheEntryPtr
1110 makeDiskEntry(ObjectPtr object, int writeable, int create)
1111 {
1112     DiskCacheEntryPtr entry = NULL;
1113     char buf[1024];
1114     int fd = -1;
1115     int negative = 0, isWriteable = 0, size = -1, name_len = -1;
1116     char *name = NULL;
1117     off_t offset = -1;
1118     int body_offset = -1;
1119     int rc;
1120     int local = (object->flags & OBJECT_LOCAL) != 0;
1121     int dirty = 0;
1122
1123     if(local && (writeable || create))
1124         return NULL;
1125
1126     if(!local && !(object->flags & OBJECT_PUBLIC))
1127         return NULL;
1128
1129     if(maxDiskCacheEntrySize >= 0) {
1130         if(object->length > 0) {
1131             if(object->length > maxDiskCacheEntrySize)
1132                 return NULL;
1133         } else {
1134             if(object->size > maxDiskCacheEntrySize)
1135                 return NULL;
1136         }
1137     }
1138
1139     if(object->disk_entry) {
1140         entry = object->disk_entry;
1141         CHECK_ENTRY(entry);
1142         if(entry != &negativeEntry && (!writeable || entry->writeable)) {
1143             /* We'll keep the entry -- put it at the front. */
1144             if(entry != diskEntries && entry != &negativeEntry) {
1145                 entry->previous->next = entry->next;
1146                 if(entry->next)
1147                     entry->next->previous = entry->previous;
1148                 else
1149                     diskEntriesLast = entry->previous;
1150                 entry->next = diskEntries;
1151                 diskEntries->previous = entry;
1152                 entry->previous = NULL;
1153                 diskEntries = entry;
1154             }
1155             return entry;
1156         } else {
1157             if(entry == &negativeEntry) {
1158                 negative = 1;
1159                 if(!create) return NULL;
1160                 object->disk_entry = NULL;
1161             }
1162             entry = NULL;
1163             destroyDiskEntry(object, 0);
1164         }
1165     }
1166
1167     if(numDiskEntries > maxDiskEntries)
1168         destroyDiskEntry(diskEntriesLast->object, 0);
1169
1170     if(!local) {
1171         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1172             return NULL;
1173         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1174         if(name_len < 0) return NULL;
1175         if(!negative) {
1176             isWriteable = 1;
1177             fd = open(buf, O_RDWR | O_BINARY);
1178             if(fd < 0 && !writeable && errno == EACCES) {
1179                 writeable = 0;
1180                 fd = open(buf, O_RDONLY | O_BINARY);
1181             }
1182         }
1183         if(fd >= 0) {
1184             rc = validateEntry(object, fd, &body_offset, &offset);
1185             if(rc >= 0) {
1186                 dirty = rc;
1187             } else {
1188                 close(fd);
1189                 fd = -1;
1190                 rc = unlink(buf);
1191                 if(rc < 0 && errno != ENOENT) {
1192                     do_log_error(L_WARN,  errno,
1193                                  "Couldn't unlink stale disk entry %s",
1194                                  buf);
1195                     /* But continue -- it's okay to have stale entries. */
1196                 }
1197             }
1198         }
1199
1200         if(fd < 0 && create && name_len > 0 &&
1201            !(object->flags & OBJECT_INITIAL)) {
1202             isWriteable = 1;
1203             fd = createFile(buf, diskCacheRoot->length);
1204             if(fd < 0)
1205                 return NULL;
1206
1207             if(fd >= 0) {
1208                 char *data = NULL;
1209                 int dsize = 0;
1210                 if(object->numchunks > 0) {
1211                     data = object->chunks[0].data;
1212                     dsize = object->chunks[0].size;
1213                 }
1214                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1215                 if(rc < 0) {
1216                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1217                     rc = unlink(buf);
1218                     if(rc < 0 && errno != ENOENT)
1219                         do_log_error(L_ERROR, errno,
1220                                      "Couldn't unlink truncated entry %s",
1221                                      buf);
1222                     close(fd);
1223                     return NULL;
1224                 }
1225                 assert(rc >= body_offset);
1226                 size = rc - body_offset;
1227                 offset = rc;
1228                 dirty = 0;
1229             }
1230         }
1231     } else {
1232         /* local */
1233         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1234             return NULL;
1235
1236         name_len =
1237             localFilename(buf, 1024, object->key, object->key_size);
1238         if(name_len < 0)
1239             return NULL;
1240         isWriteable = 0;
1241         fd = open(buf, O_RDONLY | O_BINARY);
1242         if(fd >= 0) {
1243             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1244                 close(fd);
1245                 fd = -1;
1246             }
1247         }
1248         offset = 0;
1249     }
1250
1251     if(fd < 0) {
1252         object->disk_entry = &negativeEntry;
1253         return NULL;
1254     }
1255     assert(body_offset >= 0);
1256
1257     name = strdup_n(buf, name_len);
1258     if(name == NULL) {
1259         do_log(L_ERROR, "Couldn't allocate name.\n");
1260         close(fd);
1261         fd = -1;
1262         return NULL;
1263     }
1264
1265     entry = malloc(sizeof(DiskCacheEntryRec));
1266     if(entry == NULL) {
1267         do_log(L_ERROR, "Couldn't allocate entry.\n");
1268         free(name);
1269         close(fd);
1270         return NULL;
1271     }
1272
1273     entry->filename = name;
1274     entry->object = object;
1275     entry->fd = fd;
1276     entry->body_offset = body_offset;
1277     entry->local = local;
1278     entry->offset = offset;
1279     entry->size = size;
1280     entry->metadataDirty = dirty;
1281     entry->writeable = isWriteable;
1282
1283     entry->next = diskEntries;
1284     if(diskEntries)
1285         diskEntries->previous = entry;
1286     diskEntries = entry;
1287     if(diskEntriesLast == NULL)
1288         diskEntriesLast = entry;
1289     entry->previous = NULL;
1290     numDiskEntries++;
1291
1292     object->disk_entry = entry;
1293
1294     CHECK_ENTRY(entry);
1295     return entry;
1296 }
1297
1298 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1299 static int
1300 rewriteEntry(ObjectPtr object)
1301 {
1302     int old_body_offset = object->disk_entry->body_offset;
1303     int fd, rc, n;
1304     DiskCacheEntryPtr entry;
1305     char* buf;
1306     int buf_is_chunk, bufsize;
1307     int offset;
1308
1309     fd = dup(object->disk_entry->fd);
1310     if(fd < 0) {
1311         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1312         return -1;
1313     }
1314
1315     rc = destroyDiskEntry(object, 1);
1316     if(rc < 0) {
1317         close(fd);
1318         return -1;
1319     }
1320     entry = makeDiskEntry(object, 1, 1);
1321     if(!entry) {
1322         close(fd);
1323         return -1;
1324     }
1325
1326     offset = diskEntrySize(object);
1327     if(offset < 0) {
1328         close(fd);
1329         return -1;
1330     }
1331
1332     bufsize = CHUNK_SIZE;
1333     buf_is_chunk = 1;
1334     buf = maybe_get_chunk();
1335     if(!buf) {
1336         bufsize = 2048;
1337         buf_is_chunk = 0;
1338         buf = malloc(2048);
1339         if(buf == NULL) {
1340             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1341             close(fd);
1342             return -1;
1343         }
1344     }
1345
1346     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1347     if(rc < 0)
1348         goto done;
1349
1350     while(1) {
1351         CHECK_ENTRY(entry);
1352         n = read(fd, buf, bufsize);
1353         if(n <= 0)
1354             goto done;
1355         rc = entrySeek(entry, entry->body_offset + offset);
1356         if(rc < 0)
1357             goto done;
1358         rc = write(entry->fd, buf, n);
1359         if(rc >= 0) {
1360             entry->offset += rc;
1361             entry->size += rc;
1362         }
1363         if(rc < n)
1364             goto done;
1365     }
1366
1367  done:
1368     CHECK_ENTRY(entry);
1369     if(object->length >= 0 && entry->size == object->length)
1370         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1371     close(fd);
1372     if(buf_is_chunk)
1373         dispose_chunk(buf);
1374     else
1375         free(buf);
1376     return 1;
1377 }
1378
1379 int
1380 destroyDiskEntry(ObjectPtr object, int d)
1381 {
1382     DiskCacheEntryPtr entry = object->disk_entry;
1383     int rc, urc = 1;
1384
1385     assert(!entry || !entry->local || !d);
1386
1387     if(d && !entry)
1388         entry = makeDiskEntry(object, 1, 0);
1389
1390     CHECK_ENTRY(entry);
1391
1392     if(!entry || entry == &negativeEntry) {
1393         return 1;
1394     }
1395
1396     assert(entry->object == object);
1397
1398     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1399         /* See writeoutToDisk */
1400         d = 1;
1401     }
1402
1403     if(d) {
1404         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1405         if(entry->filename) {
1406             urc = unlink(entry->filename);
1407             if(urc < 0)
1408                 do_log_error(L_WARN, errno,
1409                              "Couldn't unlink %s", entry->filename);
1410         }
1411     } else {
1412         if(entry && entry->metadataDirty)
1413             writeoutMetadata(object);
1414         makeDiskEntry(object, 1, 0);
1415         /* rewriteDiskEntry may change the disk entry */
1416         entry = object->disk_entry;
1417         if(entry == NULL || entry == &negativeEntry)
1418             return 0;
1419         if(entry->writeable && diskCacheWriteoutOnClose > 0)
1420             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1421     }
1422  again:
1423     rc = close(entry->fd);
1424     if(rc < 0 && errno == EINTR)
1425         goto again;
1426
1427     entry->fd = -1;
1428
1429     if(entry->filename)
1430         free(entry->filename);
1431     entry->filename = NULL;
1432
1433     if(entry->previous)
1434         entry->previous->next = entry->next;
1435     else
1436         diskEntries = entry->next;
1437     if(entry->next)
1438         entry->next->previous = entry->previous;
1439     else
1440         diskEntriesLast = entry->previous;
1441
1442     numDiskEntries--;
1443     assert(numDiskEntries >= 0);
1444
1445     free(entry);
1446     object->disk_entry = NULL;
1447     if(urc < 0)
1448         return -1;
1449     else
1450         return 1;
1451 }
1452
1453 ObjectPtr
1454 objectGetFromDisk(ObjectPtr object)
1455 {
1456     DiskCacheEntryPtr entry = makeDiskEntry(object, 0, 0);
1457     if(!entry) return NULL;
1458     return object;
1459 }
1460
1461
1462 int
1463 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1464 {
1465     DiskCacheEntryPtr entry;
1466     int rc, result;
1467     int i, j, k;
1468     int complete;
1469
1470     if(object->type != OBJECT_HTTP)
1471         return 0;
1472
1473     if(object->flags & OBJECT_LINEAR)
1474         return 0;
1475
1476     if(object->length >= 0) {
1477         chunks = MIN(chunks,
1478                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1479     }
1480
1481     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1482     if(rc < 0)
1483         return 0;
1484
1485     complete = 1;
1486     if(object->flags & OBJECT_INITIAL) {
1487         complete = 0;
1488     } else if((object->length < 0 || object->size < object->length) &&
1489               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1490         complete = 0;
1491     } else {
1492         for(k = 0; k < chunks; k++) {
1493             int s;
1494             i = offset / CHUNK_SIZE + k;
1495             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1496             if(object->chunks[i].size < s) {
1497                 complete = 0;
1498                 break;
1499             }
1500         }
1501     }
1502
1503     if(complete)
1504         return 1;
1505
1506     /* This has the side-effect of revalidating the entry, which is
1507        what makes HEAD requests work. */
1508     entry = makeDiskEntry(object, 0, 0);
1509     if(!entry)
1510         return 0;
1511
1512     for(k = 0; k < chunks; k++) {
1513         i = offset / CHUNK_SIZE + k;
1514         if(!object->chunks[i].data)
1515             object->chunks[i].data = get_chunk();
1516         if(!object->chunks[i].data) {
1517             chunks = k;
1518             break;
1519         }
1520         lockChunk(object, i);
1521     }
1522
1523     result = 0;
1524
1525     for(k = 0; k < chunks; k++) {
1526         int o;
1527         i = offset / CHUNK_SIZE + k;
1528         j = object->chunks[i].size;
1529         o = i * CHUNK_SIZE + j;
1530
1531         if(object->chunks[i].size == CHUNK_SIZE)
1532             continue;
1533
1534         if(entry->size >= 0 && entry->size <= o)
1535             break;
1536
1537         if(entry->offset != entry->body_offset + o) {
1538             rc = entrySeek(entry, entry->body_offset + o);
1539             if(rc < 0) {
1540                 result = 0;
1541                 break;
1542             }
1543         }
1544
1545         CHECK_ENTRY(entry);
1546         again:
1547         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1548         if(rc < 0) {
1549             if(errno == EINTR)
1550                 goto again;
1551             entry->offset = -1;
1552             do_log_error(L_ERROR, errno, "Couldn't read");
1553             break;
1554         }
1555
1556         entry->offset += rc;
1557         object->chunks[i].size += rc;
1558         if(object->size < o + rc)
1559             object->size = o + rc;
1560
1561         if(entry->object->length >= 0 && entry->size < 0 &&
1562            entry->offset - entry->body_offset == entry->object->length)
1563             entry->size = entry->object->length;
1564
1565         if(rc < CHUNK_SIZE - j) {
1566             /* Paranoia: the read may have been interrupted half-way. */
1567             if(entry->size < 0) {
1568                 if(rc == 0 ||
1569                    (entry->object->length >= 0 &&
1570                     entry->object->length ==
1571                     entry->offset - entry->body_offset))
1572                     entry->size = entry->offset - entry->body_offset;
1573                 break;
1574             } else if(entry->size != entry->offset - entry->body_offset) {
1575                 if(rc == 0 ||
1576                    entry->size < entry->offset - entry->body_offset) {
1577                     do_log(L_WARN,
1578                            "Disk entry size changed behind our back: "
1579                            "%ld -> %ld (%d).\n",
1580                            (long)entry->size,
1581                            (long)entry->offset - entry->body_offset,
1582                            object->size);
1583                     entry->size = -1;
1584                 }
1585             }
1586             break;
1587         }
1588
1589         CHECK_ENTRY(entry);
1590         result = 1;
1591     }
1592
1593     CHECK_ENTRY(object->disk_entry);
1594     for(k = 0; k < chunks; k++) {
1595         i = offset / CHUNK_SIZE + k;
1596         unlockChunk(object, i);
1597     }
1598
1599     if(result > 0) {
1600         notifyObject(object);
1601         return 1;
1602     } else {
1603         return 0;
1604     }
1605 }
1606
1607 int
1608 writeoutToDisk(ObjectPtr object, int upto, int max)
1609 {
1610     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1611         /* An object was created with an unknown length, and then grew
1612            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1613         destroyDiskEntry(object, 1);
1614         return 0;
1615     }
1616
1617     return reallyWriteoutToDisk(object, upto, max);
1618 }
1619
1620 static int
1621 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1622 {
1623     DiskCacheEntryPtr entry;
1624     int rc;
1625     int i, j;
1626     int offset;
1627     int bytes = 0;
1628
1629     if(upto < 0)
1630         upto = object->size;
1631
1632     if((object->cache_control & CACHE_NO_STORE) ||
1633        (object->flags & OBJECT_LOCAL))
1634         return 0;
1635
1636     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1637         return 0;
1638
1639     entry = makeDiskEntry(object, 1, 1);
1640     if(!entry) return 0;
1641
1642     assert(!entry->local);
1643
1644     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1645         goto done;
1646
1647     diskEntrySize(object);
1648     if(entry->size < 0)
1649         return 0;
1650
1651     if(object->length >= 0 && entry->size >= object->length) {
1652         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1653         goto done;
1654     }
1655
1656     if(entry->size >= upto)
1657         goto done;
1658
1659     if(!entry->writeable) {
1660         entry = makeDiskEntry(object, 1, 1);
1661         if(!entry)
1662             return 0;
1663         if(!entry->writeable)
1664             return 0;
1665         diskEntrySize(object);
1666         if(entry->size < 0)
1667             return 0;
1668     }
1669
1670     offset = entry->size;
1671
1672     /* Avoid a seek in case we start writing at the beginning */
1673     if(offset == 0 && entry->metadataDirty) {
1674         writeoutMetadata(object);
1675         /* rewriteDiskEntry may change the entry */
1676         entry = makeDiskEntry(object, 1, 0);
1677         if(entry == NULL || !entry->writeable)
1678             return 0;
1679     }
1680
1681     rc = entrySeek(entry, offset + entry->body_offset);
1682     if(rc < 0) return 0;
1683
1684     do {
1685         if(max >= 0 && bytes >= max)
1686             break;
1687         CHECK_ENTRY(entry);
1688         assert(entry->offset == offset + entry->body_offset);
1689         i = offset / CHUNK_SIZE;
1690         j = offset % CHUNK_SIZE;
1691         if(i >= object->numchunks)
1692             break;
1693         if(object->chunks[i].size <= j)
1694             break;
1695     again:
1696         rc = write(entry->fd, object->chunks[i].data + j,
1697                    object->chunks[i].size - j);
1698         if(rc < 0) {
1699             if(errno == EINTR)
1700                 goto again;
1701             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1702             break;
1703         }
1704         entry->offset += rc;
1705         offset += rc;
1706         bytes += rc;
1707         if(entry->size < offset)
1708             entry->size = offset;
1709     } while(j + rc >= CHUNK_SIZE);
1710
1711  done:
1712     CHECK_ENTRY(entry);
1713     if(entry->metadataDirty)
1714         writeoutMetadata(object);
1715
1716     return bytes;
1717 }
1718
1719 int
1720 writeoutMetadata(ObjectPtr object)
1721 {
1722     DiskCacheEntryPtr entry;
1723     int rc;
1724
1725     if((object->cache_control & CACHE_NO_STORE) ||
1726        (object->flags & OBJECT_LOCAL))
1727         return 0;
1728
1729     entry = makeDiskEntry(object, 1, 0);
1730     if(entry == NULL || entry == &negativeEntry)
1731         goto fail;
1732
1733     assert(!entry->local);
1734
1735     rc = entrySeek(entry, 0);
1736     if(rc < 0) goto fail;
1737
1738     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1739     if(rc == -2) {
1740         rc = rewriteEntry(object);
1741         if(rc < 0) return 0;
1742         return 1;
1743     }
1744     if(rc < 0) goto fail;
1745     entry->offset = rc;
1746     entry->metadataDirty = 0;
1747     return 1;
1748
1749  fail:
1750     /* We need this in order to avoid trying to write this entry out
1751        multiple times. */
1752     if(entry && entry != &negativeEntry)
1753         entry->metadataDirty = 0;
1754     return 0;
1755 }
1756
1757 static void
1758 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1759 {
1760     if(dst->filename == NULL) {
1761         dst->filename = src->filename;
1762         dst->body_offset = src->body_offset;
1763     } else
1764         free(src->filename);
1765     free(src->location);
1766     if(dst->length < 0)
1767         dst->length = src->length;
1768     if(dst->size < 0)
1769         dst->size = src->size;
1770     if(dst->age < 0)
1771         dst->age = src->age;
1772     if(dst->date < 0)
1773         dst->date = src->date;
1774     if(dst->last_modified < 0)
1775         dst->last_modified = src->last_modified;
1776     free(src);
1777 }
1778
1779 DiskObjectPtr
1780 readDiskObject(char *filename, struct stat *sb)
1781 {
1782     int fd, rc, n, dummy, code;
1783     int length, size;
1784     time_t date, last_modified, age, atime, expires;
1785     char *location = NULL, *fn = NULL;
1786     DiskObjectPtr dobject;
1787     char *buf;
1788     int buf_is_chunk, bufsize;
1789     int body_offset;
1790     struct stat ss;
1791
1792     fd = -1;
1793
1794     if(sb == NULL) {
1795         rc = stat(filename, &ss);
1796         if(rc < 0) {
1797             do_log_error(L_WARN, errno, "Couldn't stat %s", filename);
1798             return NULL;
1799         }
1800         sb = &ss;
1801     }
1802
1803     buf_is_chunk = 1;
1804     bufsize = CHUNK_SIZE;
1805     buf = get_chunk();
1806     if(buf == NULL) {
1807         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1808         return NULL;
1809     }
1810
1811     if(S_ISREG(sb->st_mode)) {
1812         fd = open(filename, O_RDONLY | O_BINARY);
1813         if(fd < 0)
1814             goto fail;
1815     again:
1816         rc = read(fd, buf, bufsize);
1817         if(rc < 0)
1818             goto fail;
1819
1820         n = findEndOfHeaders(buf, 0, rc, &dummy);
1821         if(n < 0) {
1822             long lrc;
1823             if(buf_is_chunk) {
1824                 dispose_chunk(buf);
1825                 buf_is_chunk = 0;
1826                 bufsize = bigBufferSize;
1827                 buf = malloc(bigBufferSize);
1828                 if(buf == NULL)
1829                     goto fail2;
1830                 lrc = lseek(fd, 0, SEEK_SET);
1831                 if(lrc < 0)
1832                     goto fail;
1833                 goto again;
1834             }
1835             goto fail;
1836         }
1837
1838         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1839         if(rc < 0)
1840             goto fail;
1841
1842         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1843                               NULL, &length, NULL, NULL, NULL,
1844                               &date, &last_modified, &expires, &age,
1845                               &atime, &body_offset, NULL,
1846                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1847         if(rc < 0 || location == NULL)
1848             goto fail;
1849         if(body_offset < 0)
1850             body_offset = n;
1851
1852         size = sb->st_size - body_offset;
1853         if(size < 0)
1854             size = 0;
1855     } else if(S_ISDIR(sb->st_mode)) {
1856         char *n;
1857         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1858         if(n == NULL)
1859             goto fail;
1860         location = strdup(n);
1861         if(location == NULL)
1862             goto fail;
1863         length = -1;
1864         size = -1;
1865         body_offset = -1;
1866         age = -1;
1867         atime = -1;
1868         date = -1;
1869         last_modified = -1;
1870     } else {
1871         goto fail;
1872     }
1873
1874     dobject = malloc(sizeof(DiskObjectRec));
1875     if(!dobject)
1876         goto fail;
1877
1878     fn = strdup(filename);
1879     if(!fn)
1880         goto fail;
1881
1882     if(buf_is_chunk)
1883         dispose_chunk(buf);
1884     else
1885         free(buf);
1886
1887     dobject->location = location;
1888     dobject->filename = fn;
1889     dobject->length = length;
1890     dobject->body_offset = body_offset;
1891     dobject->size = size;
1892     dobject->age = age;
1893     dobject->access = atime;
1894     dobject->date = date;
1895     dobject->last_modified = last_modified;
1896     dobject->expires = expires;
1897     if(fd >= 0) close(fd);
1898     return dobject;
1899
1900  fail:
1901     if(buf_is_chunk)
1902         dispose_chunk(buf);
1903     else
1904         free(buf);
1905  fail2:
1906     if(fd >= 0) close(fd);
1907     if(location) free(location);
1908     return NULL;
1909 }
1910
1911
1912 DiskObjectPtr
1913 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1914 {
1915     DiskObjectPtr dobject = NULL;
1916     int c = 0;
1917
1918     dobject = readDiskObject((char*)filename, sb);
1919     if(dobject == NULL)
1920         return 0;
1921
1922     if(!dobjects ||
1923        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1924         if(dobjects && c == 0) {
1925             mergeDobjects(dobjects, dobject);
1926         } else {
1927             dobject->next = dobjects;
1928             dobjects = dobject;
1929         }
1930     } else {
1931         DiskObjectPtr other = dobjects;
1932         while(other->next) {
1933             c = strcmp(dobject->location, other->next->location);
1934             if(c < 0)
1935                 break;
1936             other = other->next;
1937         }
1938         if(strcmp(dobject->location, other->location) == 0) {
1939             mergeDobjects(other, dobject);
1940         } else {
1941             dobject->next = other->next;
1942             other->next = dobject;
1943         }
1944     }
1945     return dobjects;
1946 }
1947
1948 /* Determine whether p is below root */
1949 static int
1950 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1951 {
1952     char *cp;
1953     int m = strlen(p->location);
1954     if(m < n)
1955         return 0;
1956     if(memcmp(root, p->location, n) != 0)
1957         return 0;
1958     if(recursive)
1959         return 1;
1960     if(m == 0 || p->location[m - 1] == '/')
1961         return 1;
1962     cp = strchr(p->location + n, '/');
1963     if(cp && cp - p->location != m - 1)
1964         return 0;
1965     return 1;
1966 }
1967
1968 /* Filter out all disk objects that are not under root */
1969 DiskObjectPtr
1970 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
1971 {
1972     int n = strlen(root);
1973     DiskObjectPtr p, q;
1974
1975     while(from && !filter(from, root, n, recursive)) {
1976         p = from;
1977         from = p->next;
1978         free(p->location);
1979         free(p);
1980     }
1981
1982     p = from;
1983     while(p && p->next) {
1984         if(!filter(p->next, root, n, recursive)) {
1985             q = p->next;
1986             p->next = q->next;
1987             free(q->location);
1988             free(q);
1989         } else {
1990             p = p->next;
1991         }
1992     }
1993     return from;
1994 }
1995
1996 DiskObjectPtr
1997 insertRoot(DiskObjectPtr from, const char *root)
1998 {
1999     DiskObjectPtr p;
2000
2001     p = from;
2002     while(p) {
2003         if(strcmp(root, p->location) == 0)
2004             return from;
2005         p = p->next;
2006     }
2007
2008     p = malloc(sizeof(DiskObjectRec));
2009     if(!p) return from;
2010     p->location = strdup(root);
2011     if(p->location == NULL) {
2012         free(p);
2013         return from;
2014     }
2015     p->filename = NULL;
2016     p->length = -1;
2017     p->size = -1;
2018     p->age = -1;
2019     p->access = -1;
2020     p->last_modified = -1;
2021     p->expires = -1;
2022     p->next = from;
2023     return p;
2024 }
2025
2026 /* Insert all missing directories in a sorted list of dobjects */
2027 DiskObjectPtr
2028 insertDirs(DiskObjectPtr from)
2029 {
2030     DiskObjectPtr p, q, new;
2031     int n, m;
2032     char *cp;
2033
2034     p = NULL; q = from;
2035     while(q) {
2036         n = strlen(q->location);
2037         if(n > 0 && q->location[n - 1] != '/') {
2038             cp = strrchr(q->location, '/');
2039             m = cp - q->location + 1;
2040             if(cp && (!p || strlen(p->location) < m ||
2041                       memcmp(p->location, q->location, m) != 0)) {
2042                 new = malloc(sizeof(DiskObjectRec));
2043                 if(!new) break;
2044                 new->location = strdup_n(q->location, m);
2045                 if(new->location == NULL) {
2046                     free(new);
2047                     break;
2048                 }
2049                 new->filename = NULL;
2050                 new->length = -1;
2051                 new->size = -1;
2052                 new->age = -1;
2053                 new->access = -1;
2054                 new->last_modified = -1;
2055                 new->expires = -1;
2056                 new->next = q;
2057                 if(p)
2058                     p->next = new;
2059                 else
2060                     from = new;
2061             }
2062         }
2063         p = q;
2064         q = q->next;
2065     }
2066     return from;
2067 }
2068
2069 void
2070 indexDiskObjects(FILE *out, const char *root, int recursive)
2071 {
2072     int n, i, isdir;
2073     DIR *dir;
2074     struct dirent *dirent;
2075     char buf[1024];
2076     char *fts_argv[2];
2077     FTS *fts;
2078     FTSENT *fe;
2079     DiskObjectPtr dobjects = NULL;
2080     char *of = root[0] == '\0' ? "" : " of ";
2081
2082     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2083             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2084             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2085             "<html><head>\n"
2086             "<title>%s%s%s</title>\n"
2087             "</head><body>\n"
2088             "<h1>%s%s%s</h1>\n",
2089             recursive ? "Recursive index" : "Index", of, root,
2090             recursive ? "Recursive index" : "Index", of, root);
2091
2092     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2093         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2094         goto trailer;
2095     }
2096
2097     if(diskCacheRoot->length >= 1024) {
2098         fprintf(out,
2099                 "<p>The value of <tt>diskCacheRoot</tt> is "
2100                 "too long (%d).</p>\n",
2101                 diskCacheRoot->length);
2102         goto trailer;
2103     }
2104
2105     if(strlen(root) < 8) {
2106         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2107         buf[diskCacheRoot->length] = '\0';
2108         n = diskCacheRoot->length;
2109     } else {
2110         n = urlDirname(buf, 1024, root, strlen(root));
2111     }
2112     if(n > 0) {
2113         if(recursive) {
2114             dir = NULL;
2115             fts_argv[0] = buf;
2116             fts_argv[1] = NULL;
2117             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2118             if(fts) {
2119                 while(1) {
2120                     fe = fts_read(fts);
2121                     if(!fe) break;
2122                     if(fe->fts_info != FTS_DP)
2123                         dobjects =
2124                             processObject(dobjects,
2125                                           fe->fts_path,
2126                                           fe->fts_info == FTS_NS ||
2127                                           fe->fts_info == FTS_NSOK ?
2128                                           fe->fts_statp : NULL);
2129                 }
2130                 fts_close(fts);
2131             }
2132         } else {
2133             dir = opendir(buf);
2134             if(dir) {
2135                 while(1) {
2136                     dirent = readdir(dir);
2137                     if(!dirent) break;
2138                     if(n + strlen(dirent->d_name) < 1024) {
2139                         strcpy(buf + n, dirent->d_name);
2140                     } else {
2141                         continue;
2142                     }
2143                     dobjects = processObject(dobjects, buf, NULL);
2144                 }
2145                 closedir(dir);
2146             } else {
2147                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2148                         strerror(errno), errno);
2149                 goto trailer;
2150             }
2151         }
2152     }
2153
2154     if(dobjects) {
2155         DiskObjectPtr dobject;
2156         int entryno;
2157         dobjects = insertRoot(dobjects, root);
2158         dobjects = insertDirs(dobjects);
2159         dobjects = filterDiskObjects(dobjects, root, recursive);
2160         dobject = dobjects;
2161         buf[0] = '\0';
2162         alternatingHttpStyle(out, "diskcachelist");
2163         fprintf(out, "<table id=diskcachelist>\n");
2164         fprintf(out, "<tbody>\n");
2165         entryno = 0;
2166         while(dobjects) {
2167             dobject = dobjects;
2168             i = strlen(dobject->location);
2169             isdir = (i == 0 || dobject->location[i - 1] == '/');
2170             if(entryno % 2)
2171                 fprintf(out, "<tr class=odd>");
2172             else
2173                 fprintf(out, "<tr class=even>");
2174             if(dobject->size >= 0) {
2175                 fprintf(out, "<td><a href=\"%s\"><tt>",
2176                         dobject->location);
2177                 htmlPrint(out,
2178                           dobject->location, strlen(dobject->location));
2179                 fprintf(out, "</tt></a></td> ");
2180                 if(dobject->length >= 0) {
2181                     if(dobject->size == dobject->length)
2182                         fprintf(out, "<td>%d</td> ", dobject->length);
2183                     else
2184                         fprintf(out, "<td>%d/%d</td> ",
2185                                dobject->size, dobject->length);
2186                 } else {
2187                     /* Avoid a trigraph. */
2188                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2189                 }
2190                 if(dobject->last_modified >= 0) {
2191                     struct tm *tm = gmtime(&dobject->last_modified);
2192                     if(tm == NULL)
2193                         n = -1;
2194                     else
2195                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2196                 } else
2197                     n = -1;
2198                 if(n > 0) {
2199                     buf[n] = '\0';
2200                     fprintf(out, "<td>%s</td> ", buf);
2201                 } else {
2202                     fprintf(out, "<td></td>");
2203                 }
2204
2205                 if(dobject->date >= 0) {
2206                     struct tm *tm = gmtime(&dobject->date);
2207                     if(tm == NULL)
2208                         n = -1;
2209                     else
2210                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2211                 } else
2212                     n = -1;
2213                 if(n > 0) {
2214                     buf[n] = '\0';
2215                     fprintf(out, "<td>%s</td>", buf);
2216                 } else {
2217                     fprintf(out, "<td></td>");
2218                 }
2219             } else {
2220                 fprintf(out, "<td><tt>");
2221                 htmlPrint(out, dobject->location,
2222                           strlen(dobject->location));
2223                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2224             }
2225             if(isdir) {
2226                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2227                         "<td><a href=\"/polipo/recursive-index?%s\">"
2228                         "recursive</a></td>",
2229                         dobject->location, dobject->location);
2230             }
2231             fprintf(out, "</tr>\n");
2232             entryno++;
2233             dobjects = dobject->next;
2234             free(dobject->location);
2235             free(dobject->filename);
2236             free(dobject);
2237         }
2238         fprintf(out, "</tbody>\n");
2239         fprintf(out, "</table>\n");
2240     }
2241
2242  trailer:
2243     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2244     fprintf(out, "</body></html>\n");
2245     return;
2246 }
2247
2248 static int
2249 checkForZeroes(char *buf, int n)
2250 {
2251     int i, j;
2252     unsigned long *lbuf = (unsigned long *)buf;
2253     assert(n % sizeof(unsigned long) == 0);
2254
2255     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2256         if(lbuf[i] != 0L)
2257             return i * sizeof(unsigned long);
2258     }
2259     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2260         if(buf[i * sizeof(unsigned long) + j] != 0)
2261             break;
2262     }
2263
2264     return i * sizeof(unsigned long) + j;
2265 }
2266
2267 static int
2268 copyFile(int from, char *filename, int n)
2269 {
2270     char *buf;
2271     int to, offset, nread, nzeroes, rc;
2272
2273     buf = malloc(CHUNK_SIZE);
2274     if(buf == NULL)
2275         return -1;
2276
2277     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2278               diskCacheFilePermissions);
2279     if(to < 0) {
2280         free(buf);
2281         return -1;
2282     }
2283
2284     offset = 0;
2285     while(offset < n) {
2286         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2287         if(nread <= 0)
2288             break;
2289         nzeroes = checkForZeroes(buf, nread & -8);
2290         if(nzeroes > 0) {
2291             /* I like holes */
2292             rc = lseek(to, nzeroes, SEEK_CUR);
2293             if(rc != offset + nzeroes) {
2294                 if(rc < 0)
2295                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2296                 else
2297                     do_log(L_ERROR,
2298                            "Couldn't extend file: "
2299                            "unexpected offset %d != %d + %d.\n",
2300                            rc, offset, nread);
2301                 break;
2302             }
2303         }
2304         if(nread > nzeroes) {
2305             rc = write(to, buf + nzeroes, nread - nzeroes);
2306             if(rc != nread - nzeroes) {
2307                 if(rc < 0)
2308                     do_log_error(L_ERROR, errno, "Couldn't write");
2309                 else
2310                     do_log(L_ERROR, "Short write.\n");
2311                 break;
2312             }
2313         }
2314         offset += nread;
2315     }
2316     free(buf);
2317     close(to);
2318     if(offset <= 0)
2319         unlink(filename);       /* something went wrong straight away */
2320     return 1;
2321 }
2322
2323 static long int
2324 expireFile(char *filename, struct stat *sb,
2325            int *considered, int *unlinked, int *truncated)
2326 {
2327     DiskObjectPtr dobject = NULL;
2328     time_t t;
2329     int fd, rc;
2330     long int ret = sb->st_size;
2331
2332     if(!preciseExpiry) {
2333         t = sb->st_mtime;
2334         if(t > current_time.tv_sec + 1) {
2335             do_log(L_WARN, "File %s has access time in the future.\n",
2336                    filename);
2337             t = current_time.tv_sec;
2338         }
2339
2340         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2341            (sb->st_size < diskCacheTruncateSize ||
2342             t > current_time.tv_sec - diskCacheTruncateTime))
2343             return ret;
2344     }
2345
2346     (*considered)++;
2347
2348     dobject = readDiskObject(filename, sb);
2349     if(!dobject) {
2350         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n", filename);
2351         rc = unlink(filename);
2352         if(rc < 0) {
2353             do_log_error(L_ERROR, errno,
2354                          "Couldn't unlink %s", filename);
2355             return ret;
2356         } else {
2357             (*unlinked)++;
2358             return 0;
2359         }
2360     }
2361
2362     t = dobject->access;
2363     if(t < 0) t = dobject->age;
2364     if(t < 0) t = dobject->date;
2365
2366     if(t > current_time.tv_sec)
2367         do_log(L_WARN,
2368                "Disk entry %s (%s) has access time in the future.\n",
2369                dobject->location, dobject->filename);
2370
2371     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2372         rc = unlink(dobject->filename);
2373         if(rc < 0) {
2374             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2375         } else {
2376             (*unlinked)++;
2377             ret = 0;
2378         }
2379     } else if(dobject->size >
2380               diskCacheTruncateSize + 4 * dobject->body_offset &&
2381               t < current_time.tv_sec - diskCacheTruncateTime) {
2382         /* We need to copy rather than simply truncate in place: the
2383            latter would confuse a running polipo. */
2384         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2385         rc = unlink(dobject->filename);
2386         if(rc < 0) {
2387             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2388             close(fd);
2389             fd = -1;
2390         } else {
2391             (*unlinked)++;
2392             copyFile(fd, dobject->filename,
2393                      dobject->body_offset + diskCacheTruncateSize);
2394             close(fd);
2395             (*unlinked)--;
2396             (*truncated)++;
2397             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2398         }
2399     }
2400     free(dobject->location);
2401     free(dobject->filename);
2402     free(dobject);
2403     return ret;
2404 }
2405
2406 void
2407 expireDiskObjects()
2408 {
2409     int rc;
2410     char *fts_argv[2];
2411     FTS *fts;
2412     FTSENT *fe;
2413     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2414     int dirs = 0, rmdirs = 0;
2415     long left = 0, total = 0;
2416
2417     if(diskCacheRoot == NULL ||
2418        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2419         return;
2420
2421     fts_argv[0] = diskCacheRoot->string;
2422     fts_argv[1] = NULL;
2423     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2424     if(fts == NULL) {
2425         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2426     } else {
2427         while(1) {
2428             gettimeofday(&current_time, NULL);
2429
2430             fe = fts_read(fts);
2431             if(!fe) break;
2432
2433             if(fe->fts_info == FTS_D)
2434                 continue;
2435
2436             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2437                fe->fts_info == FTS_DNR) {
2438                 if(fe->fts_accpath[0] == '/' &&
2439                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2440                     continue;
2441                 dirs++;
2442                 rc = rmdir(fe->fts_accpath);
2443                 if(rc >= 0)
2444                     rmdirs++;
2445                 else if(errno != ENOTEMPTY && errno != EEXIST)
2446                     do_log_error(L_ERROR, errno,
2447                                  "Couldn't remove directory %s",
2448                                  fe->fts_accpath);
2449                 continue;
2450             } else if(fe->fts_info == FTS_NS) {
2451                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2452                              fe->fts_accpath);
2453                 continue;
2454             } else if(fe->fts_info == FTS_ERR) {
2455                 do_log_error(L_ERROR, fe->fts_errno,
2456                              "Couldn't fts_read disk cache");
2457                 break;
2458             }
2459
2460             if(!S_ISREG(fe->fts_statp->st_mode)) {
2461                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2462                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2463                 continue;
2464             }
2465
2466             files++;
2467             left += expireFile(fe->fts_accpath, fe->fts_statp,
2468                                &considered, &unlinked, &truncated);
2469             total += fe->fts_statp->st_size;
2470         }
2471         fts_close(fts);
2472     }
2473
2474     printf("Disk cache purged.\n");
2475     printf("%d files, %d considered, %d removed, %d truncated "
2476            "(%ldkB -> %ldkB).\n",
2477            files, considered, unlinked, truncated, total/1024, left/1024);
2478     printf("%d directories, %d removed.\n", dirs, rmdirs);
2479     return;
2480 }
2481
2482 #else
2483
2484 void
2485 preinitDiskcache()
2486 {
2487     return;
2488 }
2489
2490 void
2491 initDiskcache()
2492 {
2493     return;
2494 }
2495
2496 int
2497 writeoutToDisk(ObjectPtr object, int upto, int max)
2498 {
2499     return 0;
2500 }
2501
2502 int
2503 destroyDiskEntry(ObjectPtr object, int d)
2504 {
2505     return 0;
2506 }
2507
2508 ObjectPtr
2509 objectGetFromDisk(ObjectPtr object)
2510 {
2511     return NULL;
2512 }
2513
2514 int
2515 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2516 {
2517     return 0;
2518 }
2519
2520 int
2521 revalidateDiskEntry(ObjectPtr object)
2522 {
2523     return 0;
2524 }
2525
2526 void
2527 dirtyDiskEntry(ObjectPtr object)
2528 {
2529     return;
2530 }
2531
2532 void
2533 expireDiskObjects()
2534 {
2535     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2536 }
2537
2538 int
2539 diskEntrySize(ObjectPtr object)
2540 {
2541     return -1;
2542 }
2543 #endif