diskcache.c

   1 /*
   2 Copyright (c) 2003-2006 by Juliusz Chroboczek
   3
   4 Permission is hereby granted, free of charge, to any person obtaining a copy
   5 of this software and associated documentation files (the "Software"), to deal
   6 in the Software without restriction, including without limitation the rights
   7 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 copies of the Software, and to permit persons to whom the Software is
   9 furnished to do so, subject to the following conditions:
  10
  11 The above copyright notice and this permission notice shall be included in
  12 all copies or substantial portions of the Software.
  13
  14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
  17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  20 THE SOFTWARE.
  21 */
  22
  23 #include "polipo.h"
  24
  25 #ifndef NO_DISK_CACHE
  26
  27 #include "md5import.h"
  28
  29 int maxDiskEntries = 32;
  30
  31 /* Because the functions in this file can be called during object
  32    expiry, we cannot use get_chunk. */
  33
  34 AtomPtr diskCacheRoot;
  35 AtomPtr localDocumentRoot;
  36
  37 DiskCacheEntryPtr diskEntries = NULL, diskEntriesLast = NULL;
  38 int numDiskEntries = 0;
  39 int diskCacheDirectoryPermissions = 0700;
  40 int diskCacheFilePermissions = 0600;
  41 int diskCacheWriteoutOnClose = (32 * 1024);
  42
  43 int maxDiskCacheEntrySize = -1;
  44
  45 int diskCacheUnlinkTime = 32 * 24 * 60 * 60;
  46 int diskCacheTruncateTime = 4 * 24 * 60 * 60 + 12 * 60 * 60;
  47 int diskCacheTruncateSize =  1024 * 1024;
  48 int preciseExpiry = 0;
  49
  50 static DiskCacheEntryRec negativeEntry = {
  51     NULL, NULL,
  52     -1, -1, -1, -1, 0, 0, 0, NULL, NULL
  53 };
  54
  55 #ifndef LOCAL_ROOT
  56 #define LOCAL_ROOT "/usr/share/polipo/www/"
  57 #endif
  58
  59 #ifndef DISK_CACHE_ROOT
  60 #define DISK_CACHE_ROOT "/var/cache/polipo/"
  61 #endif
  62
  63 static int maxDiskEntriesSetter(ConfigVariablePtr, void*);
  64 static int atomSetterFlush(ConfigVariablePtr, void*);
  65 static int reallyWriteoutToDisk(ObjectPtr object, int upto, int max);
  66
  67 void
  68 preinitDiskcache()
  69 {
  70     diskCacheRoot = internAtom(DISK_CACHE_ROOT);
  71     localDocumentRoot = internAtom(LOCAL_ROOT);
  72
  73     CONFIG_VARIABLE_SETTABLE(diskCacheDirectoryPermissions, CONFIG_OCTAL,
  74                              configIntSetter,
  75                              "Access rights for new directories.");
  76     CONFIG_VARIABLE_SETTABLE(diskCacheFilePermissions, CONFIG_OCTAL,
  77                              configIntSetter,
  78                              "Access rights for new cache files.");
  79     CONFIG_VARIABLE_SETTABLE(diskCacheWriteoutOnClose, CONFIG_INT,
  80                              configIntSetter,
  81                              "Number of bytes to write out eagerly.");
  82     CONFIG_VARIABLE_SETTABLE(diskCacheRoot, CONFIG_ATOM, atomSetterFlush,
  83                              "Root of the disk cache.");
  84     CONFIG_VARIABLE_SETTABLE(localDocumentRoot, CONFIG_ATOM, atomSetterFlush,
  85                              "Root of the local tree.");
  86     CONFIG_VARIABLE_SETTABLE(maxDiskEntries, CONFIG_INT, maxDiskEntriesSetter,
  87                     "File descriptors used by the on-disk cache.");
  88     CONFIG_VARIABLE(diskCacheUnlinkTime, CONFIG_TIME,
  89                     "Time after which on-disk objects are removed.");
  90     CONFIG_VARIABLE(diskCacheTruncateTime, CONFIG_TIME,
  91                     "Time after which on-disk objects are truncated.");
  92     CONFIG_VARIABLE(diskCacheTruncateSize, CONFIG_INT,
  93                     "Size to which on-disk objects are truncated.");
  94     CONFIG_VARIABLE(preciseExpiry, CONFIG_BOOLEAN,
  95                     "Whether to consider all files for purging.");
  96     CONFIG_VARIABLE_SETTABLE(maxDiskCacheEntrySize, CONFIG_INT,
  97                              configIntSetter,
  98                              "Maximum size of objects cached on disk.");
  99 }
 100
 101 static int
 102 maxDiskEntriesSetter(ConfigVariablePtr var, void *value)
 103 {
 104     int i;
 105     assert(var->type == CONFIG_INT && var->value.i == &maxDiskEntries);
 106     i = *(int*)value;
 107     if(i < 0 || i > 1000000)
 108         return -3;
 109     maxDiskEntries = i;
 110     while(numDiskEntries > maxDiskEntries)
 111         destroyDiskEntry(diskEntriesLast->object, 0);
 112     return 1;
 113 }
 114
 115 static int
 116 atomSetterFlush(ConfigVariablePtr var, void *value)
 117 {
 118     discardObjects(1, 0);
 119     return configAtomSetter(var, value);
 120 }
 121
 122 static int
 123 checkRoot(AtomPtr root)
 124 {
 125     struct stat ss;
 126     int rc;
 127
 128     if(!root || root->length == 0)
 129         return 0;
 130
 131     if(root->string[0] != '/') {
 132         return -2;
 133     }
 134
 135     rc = stat(root->string, &ss);
 136     if(rc < 0)
 137         return -1;
 138     else if(!S_ISDIR(ss.st_mode)) {
 139         errno = ENOTDIR;
 140         return -1;
 141     }
 142     return 1;
 143 }
 144
 145 static AtomPtr
 146 maybeAddSlash(AtomPtr atom)
 147 {
 148     AtomPtr newAtom = NULL;
 149     if(!atom) return NULL;
 150     if(atom->length > 0 && atom->string[atom->length - 1] != '/') {
 151         newAtom = atomCat(atom, "/");
 152         releaseAtom(atom);
 153         return newAtom;
 154     }
 155     return atom;
 156 }
 157
 158 void
 159 initDiskcache()
 160 {
 161     int rc;
 162
 163     diskCacheRoot = expandTilde(maybeAddSlash(diskCacheRoot));
 164     rc = checkRoot(diskCacheRoot);
 165     if(rc <= 0) {
 166         switch(rc) {
 167         case 0: break;
 168         case -1: do_log_error(L_WARN, errno, "Disabling disk cache"); break;
 169         case -2:
 170             do_log(L_WARN, "Disabling disk cache: path %s is not absolute.\n",
 171                    diskCacheRoot->string);
 172             break;
 173         default: abort();
 174         }
 175         releaseAtom(diskCacheRoot);
 176         diskCacheRoot = NULL;
 177     }
 178
 179     localDocumentRoot = expandTilde(maybeAddSlash(localDocumentRoot));
 180     rc = checkRoot(localDocumentRoot);
 181     if(rc <= 0) {
 182         switch(rc) {
 183         case 0: break;
 184         case -1: do_log_error(L_WARN, errno, "Disabling local tree"); break;
 185         case -2:
 186             do_log(L_WARN, "Disabling local tree: path is not absolute.\n");
 187             break;
 188         default: abort();
 189         }
 190         releaseAtom(localDocumentRoot);
 191         localDocumentRoot = NULL;
 192     }
 193 }
 194
 195 #ifdef DEBUG_DISK_CACHE
 196 #define CHECK_ENTRY(entry) check_entry((entry))
 197 static void
 198 check_entry(DiskCacheEntryPtr entry)
 199 {
 200     if(entry && entry->fd < 0)
 201         assert(entry == &negativeEntry);
 202     if(entry && entry->fd >= 0) {
 203         assert((!entry->previous) == (entry == diskEntries));
 204         assert((!entry->next) == (entry == diskEntriesLast));
 205         if(entry->size >= 0)
 206             assert(entry->size + entry->body_offset >= entry->offset);
 207         assert(entry->body_offset >= 0);
 208         if(entry->offset >= 0) {
 209             off_t offset;
 210             offset = lseek(entry->fd, 0, SEEK_CUR);
 211             assert(offset == entry->offset);
 212         }
 213         if(entry->size >= 0) {
 214             int rc;
 215             struct stat ss;
 216             rc = fstat(entry->fd, &ss);
 217             assert(rc >= 0);
 218             assert(ss.st_size == entry->size + entry->body_offset);
 219         }
 220     }
 221 }
 222 #else
 223 #define CHECK_ENTRY(entry) do {} while(0)
 224 #endif
 225
 226 int
 227 diskEntrySize(ObjectPtr object)
 228 {
 229     struct stat buf;
 230     int rc;
 231     DiskCacheEntryPtr entry = object->disk_entry;
 232
 233     if(!entry || entry == &negativeEntry)
 234         return -1;
 235
 236     if(entry->size >= 0)
 237         return entry->size;
 238
 239     rc = fstat(entry->fd, &buf);
 240     if(rc < 0) {
 241         do_log_error(L_ERROR, errno, "Couldn't stat");
 242         return -1;
 243     }
 244
 245     if(buf.st_size <= entry->body_offset)
 246         entry->size = 0;
 247     else
 248         entry->size =  buf.st_size - entry->body_offset;
 249     CHECK_ENTRY(entry);
 250     if(object->length >= 0 && entry->size == object->length)
 251         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 252     return entry->size;
 253 }
 254
 255 static int
 256 entrySeek(DiskCacheEntryPtr entry, off_t offset)
 257 {
 258     off_t rc;
 259
 260     CHECK_ENTRY(entry);
 261     assert(entry != &negativeEntry);
 262     if(entry->offset == offset)
 263         return 1;
 264     if(offset > entry->body_offset) {
 265         /* Avoid extending the file by mistake */
 266         if(entry->size < 0)
 267             diskEntrySize(entry->object);
 268         if(entry->size < 0)
 269             return -1;
 270         if(entry->size + entry->body_offset < offset)
 271             return -1;
 272     }
 273     rc = lseek(entry->fd, offset, SEEK_SET);
 274     if(rc < 0) {
 275         do_log_error(L_ERROR, errno, "Couldn't seek");
 276         entry->offset = -1;
 277         return -1;
 278     }
 279     entry->offset = offset;
 280     return 1;
 281 }
 282
 283 /* Given a local URL, constructs the filename where it can be found. */
 284
 285 int
 286 localFilename(char *buf, int n, char *key, int len)
 287 {
 288     int i, j;
 289     if(len <= 0 || key[0] != '/') return -1;
 290
 291     if(urlIsSpecial(key, len)) return -1;
 292
 293     if(localDocumentRoot == NULL ||
 294        localDocumentRoot->length <= 0 || localDocumentRoot->string[0] != '/')
 295         return -1;
 296
 297     if(n <= localDocumentRoot->length)
 298         return -1;
 299
 300     i = 0;
 301     if(key[i] != '/')
 302         return -1;
 303
 304     memcpy(buf, localDocumentRoot->string, localDocumentRoot->length);
 305     j = localDocumentRoot->length;
 306     if(buf[j - 1] == '/')
 307         j--;
 308
 309     while(i < len) {
 310         if(j >= n - 1)
 311             return -1;
 312         if(key[i] == '/' && i < len - 2)
 313             if(key[i + 1] == '.' &&
 314                (key[i + 2] == '.' || key[i + 2] == '/'))
 315                 return -1;
 316         buf[j++] = key[i++];
 317     }
 318
 319     if(buf[j - 1] == '/') {
 320         if(j >= n - 11)
 321             return -1;
 322         memcpy(buf + j, "index.html", 10);
 323         j += 10;
 324     }
 325
 326     buf[j] = '\0';
 327     return j;
 328 }
 329
 330 static void
 331 md5(unsigned char *restrict key, int len, unsigned char *restrict dst)
 332 {
 333     static MD5_CTX ctx;
 334     MD5Init(&ctx);
 335     MD5Update(&ctx, key, len);
 336     MD5Final(&ctx);
 337     memcpy(dst, ctx.digest, 16);
 338 }
 339
 340 /* Check whether a character can be stored in a filename.  This is
 341    needed since we want to support deficient file systems. */
 342 static int
 343 fssafe(char c)
 344 {
 345     if(c <= 31 || c >= 127)
 346         return 0;
 347     if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
 348        (c >= '0' && c <= '9') ||  c == '.' || c == '-' || c == '_')
 349         return 1;
 350     return 0;
 351 }
 352
 353 /* Given a URL, returns the directory name within which all files
 354    starting with this URL can be found. */
 355 static int
 356 urlDirname(char *buf, int n, const char *url, int len)
 357 {
 358     int i, j;
 359     if(len < 8)
 360         return -1;
 361     if(memcmp(url, "http://", 7) != 0)
 362         return -1;
 363
 364     if(diskCacheRoot == NULL ||
 365        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
 366         return -1;
 367
 368     if(n <= diskCacheRoot->length)
 369         return -1;
 370
 371     memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
 372     j = diskCacheRoot->length;
 373
 374     if(buf[j - 1] != '/')
 375         buf[j++] = '/';
 376
 377     for(i = 7; i < len; i++) {
 378         if(i >= len || url[i] == '/')
 379             break;
 380         if(url[i] == '.' && i != len - 1 && url[i + 1] == '.')
 381             return -1;
 382         if(url[i] == '%' || !fssafe(url[i])) {
 383             if(j >= n + 3) return -1;
 384             buf[j++] = '%';
 385             buf[j++] = i2h((url[i] & 0xF0) >> 4);
 386             buf[j++] = i2h(url[i] & 0x0F);
 387         } else {
 388             buf[j++] = url[i]; if(j >= n) return -1;
 389         }
 390     }
 391     buf[j++] = '/'; if(j >= n) return -1;
 392     buf[j] = '\0';
 393     return j;
 394 }
 395
 396 /* Given a URL, returns the filename where the cached data can be
 397    found. */
 398 static int
 399 urlFilename(char *restrict buf, int n, const char *url, int len)
 400 {
 401     int j;
 402     unsigned char md5buf[18];
 403     j = urlDirname(buf, n, url, len);
 404     if(j < 0 || j + 24 >= n)
 405         return -1;
 406     md5((unsigned char*)url, len, md5buf);
 407     b64cpy(buf + j, (char*)md5buf, 16, 1);
 408     buf[j + 24] = '\0';
 409     return j + 24;
 410 }
 411
 412 static char *
 413 dirnameUrl(char *url, int n, char *name, int len)
 414 {
 415     int i, j, k, c1, c2;
 416     k = diskCacheRoot->length;
 417     if(len < k)
 418         return NULL;
 419     if(memcmp(name, diskCacheRoot->string, k) != 0)
 420         return NULL;
 421     if(n < 8)
 422         return NULL;
 423     memcpy(url, "http://", 7);
 424     if(name[len - 1] == '/')
 425         len --;
 426     j = 7;
 427     for(i = k; i < len; i++) {
 428         if(name[i] == '%') {
 429             if(i >= len - 2)
 430                 return NULL;
 431             c1 = h2i(name[i + 1]);
 432             c2 = h2i(name[i + 2]);
 433             if(c1 < 0 || c2 < 0)
 434                 return NULL;
 435             url[j++] = c1 * 16 + c2; if(j >= n) goto fail;
 436         } else if(i < len - 1 &&
 437                   name[i] == '.' && name[i + 1] == '/') {
 438                 return NULL;
 439         } else if(i == len - 1 && name[i] == '.') {
 440             return NULL;
 441         } else {
 442             url[j++] = name[i]; if(j >= n) goto fail;
 443         }
 444     }
 445     url[j++] = '/'; if(j >= n) goto fail;
 446     url[j] = '\0';
 447     return url;
 448
 449  fail:
 450     return NULL;
 451 }
 452
 453 /* Create a file and all intermediate directories. */
 454 static int
 455 createFile(const char *name, int path_start)
 456 {
 457     int fd;
 458     char buf[1024];
 459     int n;
 460     int rc;
 461
 462     if(name[path_start] == '/')
 463         path_start++;
 464
 465     if(path_start < 2 || name[path_start - 1] != '/' ) {
 466         do_log(L_ERROR, "Incorrect name %s (%d).\n", name, path_start);
 467         return -1;
 468     }
 469
 470     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 471               diskCacheFilePermissions);
 472     if(fd >= 0)
 473         return fd;
 474     if(errno != ENOENT) {
 475         do_log_error(L_ERROR, errno, "Couldn't create disk file %s", name);
 476         return -1;
 477     }
 478
 479     n = path_start;
 480     while(name[n] != '\0' && n < 1024) {
 481         while(name[n] != '/' && name[n] != '\0' && n < 512)
 482             n++;
 483         if(name[n] != '/' || n >= 1024)
 484             break;
 485         memcpy(buf, name, n + 1);
 486         buf[n + 1] = '\0';
 487         rc = mkdir(buf, diskCacheDirectoryPermissions);
 488         if(rc < 0 && errno != EEXIST) {
 489             do_log_error(L_ERROR, errno, "Couldn't create directory %s", buf);
 490             return -1;
 491         }
 492         n++;
 493     }
 494     fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
 495               diskCacheFilePermissions);
 496     if(fd < 0) {
 497         do_log_error(L_ERROR, errno, "Couldn't create file %s", name);
 498         return -1;
 499     }
 500
 501     return fd;
 502 }
 503
 504 static int
 505 chooseBodyOffset(int n, ObjectPtr object)
 506 {
 507     int length = MAX(object->size, object->length);
 508     int body_offset;
 509
 510     if(object->length >= 0 && object->length + n < 4096 - 4)
 511         return -1;              /* no gap for small objects */
 512
 513     if(n <= 128)
 514         body_offset = 256;
 515     else if(n <= 192)
 516         body_offset = 384;
 517     else if(n <= 256)
 518         body_offset = 512;
 519     else if(n <= 384)
 520         body_offset = 768;
 521     else if(n <= 512)
 522         body_offset = 1024;
 523     else if(n <= 1024)
 524         body_offset = 2048;
 525     else if(n < 2048)
 526         body_offset = 4096;
 527     else
 528         body_offset = ((n + 32 + 4095) / 4096 + 1) * 4096;
 529
 530     /* Tweak the gap so that we don't use up a full disk block for
 531        a small tail */
 532     if(object->length >= 0 && object->length < 64 * 1024) {
 533         int last = (body_offset + object->length) % 4096;
 534         int gap = body_offset - n - 32;
 535         if(last < gap / 2)
 536             body_offset -= last;
 537     }
 538
 539     /* Rewriting large objects is expensive -- don't use small gaps.
 540        This has the additional benefit of block-aligning large bodies. */
 541     if(length >= 64 * 1024) {
 542         int min_gap, min_offset;
 543         if(length >= 512 * 1024)
 544             min_gap = 4096;
 545         else if(length >= 256 * 1024)
 546             min_gap = 2048;
 547         else
 548             min_gap = 1024;
 549
 550         min_offset = ((n + 32 + min_gap - 1) / min_gap + 1) * min_gap;
 551         body_offset = MAX(body_offset, min_offset);
 552     }
 553
 554     return body_offset;
 555 }
 556
 557 /* Assumes the file descriptor is at offset 0.  Returns -1 on failure,
 558    otherwise the offset at which the file descriptor is left. */
 559 /* If chunk is not null, it should be the first chunk of the object,
 560    and will be written out in the same operation if possible. */
 561 static int
 562 writeHeaders(int fd, int *body_offset_return,
 563              ObjectPtr object, char *chunk, int chunk_len)
 564 {
 565     int n;
 566     int rc;
 567     int body_offset = *body_offset_return;
 568     char *buf = NULL;
 569     int buf_is_chunk = 0;
 570     int bufsize = 0;
 571
 572     if(object->flags & OBJECT_LOCAL)
 573         return -1;
 574
 575     if(body_offset > CHUNK_SIZE)
 576         goto overflow;
 577
 578     /* get_chunk might trigger object expiry */
 579     bufsize = CHUNK_SIZE;
 580     buf_is_chunk = 1;
 581     buf = maybe_get_chunk();
 582     if(!buf) {
 583         bufsize = 2048;
 584         buf_is_chunk = 0;
 585         buf = malloc(2048);
 586         if(buf == NULL) {
 587             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 588             return -1;
 589         }
 590     }
 591
 592  format_again:
 593     n = snnprintf(buf, 0, bufsize, "HTTP/1.1 %3d %s",
 594                   object->code, object->message->string);
 595
 596     n = httpWriteObjectHeaders(buf, n, bufsize, object, 0, -1);
 597     if(n < 0)
 598         goto overflow;
 599
 600     n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Location: ");
 601     n = snnprint_n(buf, n, bufsize, object->key, object->key_size);
 602
 603     if(object->age >= 0 && object->age != object->date) {
 604         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Date: ");
 605         n = format_time(buf, n, bufsize, object->age);
 606     }
 607
 608     if(object->atime >= 0) {
 609         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Access: ");
 610         n = format_time(buf, n, bufsize, object->atime);
 611     }
 612
 613     if(n < 0)
 614         goto overflow;
 615
 616     if(body_offset < 0)
 617         body_offset = chooseBodyOffset(n, object);
 618
 619     if(body_offset > bufsize)
 620         goto overflow;
 621
 622     if(body_offset > 0 && body_offset != n + 4)
 623         n = snnprintf(buf, n, bufsize, "\r\nX-Polipo-Body-Offset: %d",
 624                       body_offset);
 625
 626     n = snnprintf(buf, n, bufsize, "\r\n\r\n");
 627     if(n < 0)
 628         goto overflow;
 629
 630     if(body_offset < 0)
 631         body_offset = n;
 632     if(n > body_offset)
 633         goto fail;
 634
 635     if(n < body_offset)
 636         memset(buf + n, 0, body_offset - n);
 637
 638  again:
 639 #ifdef HAVE_READV_WRITEV
 640     if(chunk_len > 0) {
 641         struct iovec iov[2];
 642         iov[0].iov_base = buf;
 643         iov[0].iov_len = body_offset;
 644         iov[1].iov_base = chunk;
 645         iov[1].iov_len = chunk_len;
 646         rc = writev(fd, iov, 2);
 647     } else
 648 #endif
 649         rc = write(fd, buf, body_offset);
 650
 651     if(rc < 0 && errno == EINTR)
 652         goto again;
 653
 654     if(rc < body_offset)
 655         goto fail;
 656     if(object->length >= 0 &&
 657        rc - body_offset >= object->length)
 658         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
 659
 660     *body_offset_return = body_offset;
 661     if(buf_is_chunk)
 662         dispose_chunk(buf);
 663     else
 664         free(buf);
 665     return rc;
 666
 667  overflow:
 668     if(bufsize < bigBufferSize) {
 669         char *oldbuf = buf;
 670         buf = malloc(bigBufferSize);
 671         if(!buf) {
 672             do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 673             goto fail;
 674         }
 675         bufsize = bigBufferSize;
 676         if(oldbuf) {
 677             if(buf_is_chunk)
 678                 dispose_chunk(oldbuf);
 679             else
 680                 free(oldbuf);
 681         }
 682         buf_is_chunk = 0;
 683         goto format_again;
 684     }
 685     /* fall through */
 686
 687  fail:
 688     if(buf_is_chunk)
 689         dispose_chunk(buf);
 690     else
 691         free(buf);
 692     return -1;
 693 }
 694
 695 typedef struct _MimeEntry {
 696     char *extension;
 697     char *mime;
 698 } MimeEntryRec;
 699
 700 static const MimeEntryRec mimeEntries[] = {
 701     { "html", "text/html" },
 702     { "htm", "text/html" },
 703     { "text", "text/plain" },
 704     { "txt", "text/plain" },
 705     { "png", "image/png" },
 706     { "gif", "image/gif" },
 707     { "jpeg", "image/jpeg" },
 708     { "jpg", "image/jpeg" },
 709     { "ico", "image/x-icon" },
 710     { "pdf", "application/pdf" },
 711     { "ps", "application/postscript" },
 712     { "tar", "application/x-tar" },
 713     { "pac", "application/x-ns-proxy-autoconfig" },
 714     { "css", "text/css" },
 715     { "js",  "application/x-javascript" },
 716     { "xml", "text/xml" },
 717     { "swf", "application/x-shockwave-flash" },
 718 };
 719
 720 static char*
 721 localObjectMimeType(ObjectPtr object, char **encoding_return)
 722 {
 723     char *name = object->key;
 724     int nlen = object->key_size;
 725     int i;
 726
 727     assert(nlen >= 1);
 728
 729     if(name[nlen - 1] == '/') {
 730         *encoding_return = NULL;
 731         return "text/html";
 732     }
 733
 734     if(nlen < 3) {
 735         *encoding_return = NULL;
 736         return "application/octet-stream";
 737     }
 738
 739     if(memcmp(name + nlen - 3, ".gz", 3) == 0) {
 740         *encoding_return = "x-gzip";
 741         nlen -= 3;
 742     } else if(memcmp(name + nlen - 2, ".Z", 2) == 0) {
 743         *encoding_return = "x-compress";
 744         nlen -= 2;
 745     } else {
 746         *encoding_return = NULL;
 747     }
 748
 749     for(i = 0; i < sizeof(mimeEntries) / sizeof(mimeEntries[0]); i++) {
 750         int len = strlen(mimeEntries[i].extension);
 751         if(nlen > len &&
 752            name[nlen - len - 1] == '.' &&
 753            memcmp(name + nlen - len, mimeEntries[i].extension, len) == 0)
 754             return mimeEntries[i].mime;
 755     }
 756
 757     return "application/octet-stream";
 758 }
 759
 760 /* Same interface as validateEntry -- see below */
 761 int
 762 validateLocalEntry(ObjectPtr object, int fd,
 763                    int *body_offset_return, off_t *offset_return)
 764 {
 765     struct stat ss;
 766     char buf[512];
 767     int n, rc;
 768     char *encoding;
 769
 770     rc = fstat(fd, &ss);
 771     if(rc < 0) {
 772         do_log_error(L_ERROR, errno, "Couldn't stat");
 773         return -1;
 774     }
 775
 776     if(S_ISREG(ss.st_mode)) {
 777         if(!(ss.st_mode & S_IROTH) ||
 778            (object->length >= 0 && object->length != ss.st_size) ||
 779            (object->last_modified >= 0 &&
 780             object->last_modified != ss.st_mtime))
 781             return -1;
 782     } else {
 783         notifyObject(object);
 784         return -1;
 785     }
 786
 787     n = snnprintf(buf, 0, 512, "%lx-%lx-%lx",
 788                   (unsigned long)ss.st_ino,
 789                   (unsigned long)ss.st_size,
 790                   (unsigned long)ss.st_mtime);
 791     if(n >= 512)
 792         n = -1;
 793
 794     if(n > 0 && object->etag) {
 795         if(strlen(object->etag) != n ||
 796            memcmp(object->etag, buf, n) != 0)
 797             return -1;
 798     }
 799
 800     if(!(object->flags & OBJECT_INITIAL)) {
 801         if(!object->last_modified && !object->etag)
 802             return -1;
 803     }
 804
 805     if(object->flags & OBJECT_INITIAL) {
 806         object->length = ss.st_size;
 807         object->last_modified = ss.st_mtime;
 808         object->date = current_time.tv_sec;
 809         object->age = current_time.tv_sec;
 810         object->code = 200;
 811         if(n > 0)
 812             object->etag = strdup(buf); /* okay if fails */
 813         object->message = internAtom("Okay");
 814         n = snnprintf(buf, 0, 512,
 815                       "\r\nServer: Polipo"
 816                       "\r\nContent-Type: %s",
 817                       localObjectMimeType(object, &encoding));
 818         if(encoding != NULL)
 819             n = snnprintf(buf, n, 512,
 820                           "\r\nContent-Encoding: %s", encoding);
 821         if(n < 0)
 822             return -1;
 823         object->headers = internAtomN(buf, n);
 824         if(object->headers == NULL)
 825             return -1;
 826         object->flags &= ~OBJECT_INITIAL;
 827     }
 828
 829     if(body_offset_return)
 830         *body_offset_return = 0;
 831     if(offset_return)
 832         *offset_return = 0;
 833     return 0;
 834 }
 835
 836 /* Assumes fd is at offset 0.
 837    Returns -1 if not valid, 1 if metadata should be written out, 0
 838    otherwise. */
 839 int
 840 validateEntry(ObjectPtr object, int fd,
 841               int *body_offset_return, off_t *offset_return)
 842 {
 843     char *buf;
 844     int buf_is_chunk, bufsize;
 845     int rc, n;
 846     int dummy;
 847     int code;
 848     AtomPtr headers;
 849     time_t date, last_modified, expires, polipo_age, polipo_access;
 850     int length;
 851     off_t offset = -1;
 852     int body_offset;
 853     char *etag;
 854     AtomPtr via;
 855     CacheControlRec cache_control;
 856     char *location;
 857     AtomPtr message;
 858     int dirty = 0;
 859
 860     if(object->flags & OBJECT_LOCAL)
 861         return validateLocalEntry(object, fd,
 862                                   body_offset_return, offset_return);
 863
 864     if(!(object->flags & OBJECT_PUBLIC) && (object->flags & OBJECT_INITIAL))
 865         return 0;
 866
 867     /* get_chunk might trigger object expiry */
 868     bufsize = CHUNK_SIZE;
 869     buf_is_chunk = 1;
 870     buf = maybe_get_chunk();
 871     if(!buf) {
 872         bufsize = 2048;
 873         buf_is_chunk = 0;
 874         buf = malloc(2048);
 875         if(buf == NULL) {
 876             do_log(L_ERROR, "Couldn't allocate buffer.\n");
 877             return -1;
 878         }
 879     }
 880
 881  again:
 882     rc = read(fd, buf, bufsize);
 883     if(rc < 0) {
 884         if(errno == EINTR)
 885             goto again;
 886         do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 887         goto fail;
 888     }
 889     offset = rc;
 890
 891  parse_again:
 892     n = findEndOfHeaders(buf, 0, rc, &dummy);
 893     if(n < 0) {
 894         char *oldbuf = buf;
 895         if(bufsize < bigBufferSize) {
 896             buf = malloc(bigBufferSize);
 897             if(!buf) {
 898                 do_log(L_ERROR, "Couldn't allocate big buffer.\n");
 899                 goto fail;
 900             }
 901             bufsize = bigBufferSize;
 902             memcpy(buf, oldbuf, offset);
 903             if(buf_is_chunk)
 904                 dispose_chunk(oldbuf);
 905             else
 906                 free(oldbuf);
 907             buf_is_chunk = 0;
 908         again2:
 909             rc = read(fd, buf + offset, bufsize - offset);
 910             if(rc < 0) {
 911                 if(errno == EINTR)
 912                     goto again2;
 913                 do_log_error(L_ERROR, errno, "Couldn't read disk entry");
 914                 goto fail;
 915             }
 916             offset += rc;
 917             goto parse_again;
 918         }
 919         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 920         goto fail;
 921     }
 922
 923     rc = httpParseServerFirstLine(buf, &code, &dummy, &message);
 924     if(rc < 0) {
 925         do_log(L_ERROR, "Couldn't parse disk entry.\n");
 926         goto fail;
 927     }
 928
 929     if(object->code != 0 && object->code != code) {
 930         releaseAtom(message);
 931         goto fail;
 932     }
 933
 934     rc = httpParseHeaders(0, NULL, buf, rc, NULL,
 935                           &headers, &length, &cache_control, NULL, NULL,
 936                           &date, &last_modified, &expires, &polipo_age,
 937                           &polipo_access, &body_offset,
 938                           NULL, &etag, NULL,
 939                           NULL, NULL, &location, &via, NULL);
 940     if(rc < 0) {
 941         releaseAtom(message);
 942         goto fail;
 943     }
 944     if(body_offset < 0)
 945         body_offset = n;
 946
 947     if(!location || strlen(location) != object->key_size ||
 948        memcmp(location, object->key, object->key_size) != 0) {
 949         do_log(L_ERROR, "Inconsistent cache file for %s.\n", location);
 950         goto invalid;
 951     }
 952
 953     if(polipo_age < 0)
 954         polipo_age = date;
 955
 956     if(polipo_age < 0) {
 957         do_log(L_ERROR, "Undated disk entry for %s.\n", location);
 958         goto invalid;
 959     }
 960
 961     if(!(object->flags & OBJECT_INITIAL)) {
 962         if((last_modified >=0) != (object->last_modified >= 0))
 963             goto invalid;
 964
 965         if((object->cache_control & CACHE_MISMATCH) ||
 966            (cache_control.flags & CACHE_MISMATCH))
 967             goto invalid;
 968
 969         if(last_modified >= 0 && object->last_modified >= 0 &&
 970            last_modified != object->last_modified)
 971             goto invalid;
 972
 973         if(length >= 0 && object->length >= 0)
 974             if(length != object->length)
 975                 goto invalid;
 976
 977         if(!!etag != !!object->etag)
 978             goto invalid;
 979
 980         if(etag && object->etag && strcmp(etag, object->etag) != 0)
 981             goto invalid;
 982
 983         /* If we don't have a usable ETag, and either CACHE_VARY or we
 984            don't have a last-modified date, we validate disk entries by
 985            using their date. */
 986         if(!(etag && object->etag) &&
 987            (!(last_modified >= 0 && object->last_modified >= 0) ||
 988             ((cache_control.flags & CACHE_VARY) ||
 989              (object->cache_control & CACHE_VARY)))) {
 990             if(date >= 0 && date != object->date)
 991                 goto invalid;
 992             if(polipo_age >= 0 && polipo_age != object->age)
 993                 goto invalid;
 994         }
 995     }
 996
 997     if(location)
 998         free(location);
 999
1000     if(headers) {
1001         if(!object->headers)
1002             object->headers = headers;
1003         else
1004             releaseAtom(headers);
1005     }
1006
1007     if(object->code == 0) {
1008         object->code = code;
1009         object->message = retainAtom(message);
1010     }
1011     if(object->date <= date)
1012         object->date = date;
1013     else
1014         dirty = 1;
1015     if(object->last_modified < 0)
1016         object->last_modified = last_modified;
1017     if(object->expires < 0)
1018         object->expires = expires;
1019     else if(object->expires > expires)
1020         dirty = 1;
1021     if(object->age < 0)
1022         object->age = polipo_age;
1023     else if(object->age > polipo_age)
1024         dirty = 1;
1025     if(object->atime <= polipo_access)
1026         object->atime = polipo_access;
1027     else
1028         dirty = 1;
1029
1030     object->cache_control |= cache_control.flags;
1031
1032     if(object->age < 0) object->age = object->date;
1033     if(object->age < 0) object->age = 0; /* a long time ago */
1034     if(object->length < 0) object->length = length;
1035     if(!object->etag)
1036         object->etag = etag;
1037     else {
1038         if(etag)
1039             free(etag);
1040     }
1041     releaseAtom(message);
1042
1043     if(object->flags & OBJECT_INITIAL) object->via = via;
1044     object->flags &= ~OBJECT_INITIAL;
1045     if(offset > body_offset) {
1046         /* We need to make sure we don't invoke object expiry recursively */
1047         objectSetChunks(object, 1);
1048         if(object->numchunks >= 1) {
1049             if(object->chunks[0].data == NULL)
1050                 object->chunks[0].data = maybe_get_chunk();
1051             if(object->chunks[0].data)
1052                 objectAddData(object, buf + body_offset,
1053                               0, MIN(offset - body_offset, CHUNK_SIZE));
1054         }
1055     }
1056
1057     httpTweakCachability(object);
1058
1059     if(buf_is_chunk)
1060         dispose_chunk(buf);
1061     else
1062         free(buf);
1063     if(body_offset_return) *body_offset_return = body_offset;
1064     if(offset_return) *offset_return = offset;
1065     return dirty;
1066
1067  invalid:
1068     releaseAtom(message);
1069     if(etag) free(etag);
1070     if(location) free(location);
1071     if(via) releaseAtom(via);
1072     /* fall through */
1073
1074  fail:
1075     if(buf_is_chunk)
1076         dispose_chunk(buf);
1077     else
1078         free(buf);
1079     return -1;
1080 }
1081
1082 void
1083 dirtyDiskEntry(ObjectPtr object)
1084 {
1085     DiskCacheEntryPtr entry = object->disk_entry;
1086     if(entry && entry != &negativeEntry) entry->metadataDirty = 1;
1087 }
1088
1089 int
1090 revalidateDiskEntry(ObjectPtr object)
1091 {
1092     DiskCacheEntryPtr entry = object->disk_entry;
1093     int rc;
1094     int body_offset;
1095
1096     if(!entry || entry == &negativeEntry)
1097         return 1;
1098
1099     CHECK_ENTRY(entry);
1100     rc = entrySeek(entry, 0);
1101     if(rc < 0) return 0;
1102
1103     rc = validateEntry(object, entry->fd, &body_offset, &entry->offset);
1104     if(rc < 0) {
1105         destroyDiskEntry(object, 0);
1106         return 0;
1107     }
1108     if(body_offset != entry->body_offset) {
1109         do_log(L_WARN, "Inconsistent body offset (%d != %d).\n",
1110                body_offset, entry->body_offset);
1111         destroyDiskEntry(object, 0);
1112         return 0;
1113     }
1114
1115     entry->metadataDirty |= !!rc;
1116     CHECK_ENTRY(entry);
1117     return 1;
1118 }
1119
1120 static inline int
1121 objectHasDiskEntry(ObjectPtr object)
1122 {
1123     return object->disk_entry && object->disk_entry != &negativeEntry;
1124 }
1125
1126 static DiskCacheEntryPtr
1127 makeDiskEntry(ObjectPtr object, int writeable, int create)
1128 {
1129     DiskCacheEntryPtr entry = NULL;
1130     char buf[1024];
1131     int fd = -1;
1132     int negative = 0, isWriteable = 0, size = -1, name_len = -1;
1133     char *name = NULL;
1134     off_t offset = -1;
1135     int body_offset = -1;
1136     int rc;
1137     int local = (object->flags & OBJECT_LOCAL) != 0;
1138     int dirty = 0;
1139
1140     if(local && (writeable || create))
1141         return NULL;
1142
1143     if(!local && !(object->flags & OBJECT_PUBLIC))
1144         return NULL;
1145
1146     if(maxDiskCacheEntrySize >= 0) {
1147         if(object->length > 0) {
1148             if(object->length > maxDiskCacheEntrySize)
1149                 return NULL;
1150         } else {
1151             if(object->size > maxDiskCacheEntrySize)
1152                 return NULL;
1153         }
1154     }
1155
1156     if(object->disk_entry) {
1157         entry = object->disk_entry;
1158         CHECK_ENTRY(entry);
1159         if(entry != &negativeEntry && (!writeable || entry->writeable)) {
1160             /* We'll keep the entry -- put it at the front. */
1161             if(entry != diskEntries && entry != &negativeEntry) {
1162                 entry->previous->next = entry->next;
1163                 if(entry->next)
1164                     entry->next->previous = entry->previous;
1165                 else
1166                     diskEntriesLast = entry->previous;
1167                 entry->next = diskEntries;
1168                 diskEntries->previous = entry;
1169                 entry->previous = NULL;
1170                 diskEntries = entry;
1171             }
1172             return entry;
1173         } else {
1174             if(entry == &negativeEntry) {
1175                 negative = 1;
1176                 if(!create) return NULL;
1177                 object->disk_entry = NULL;
1178             }
1179             entry = NULL;
1180             destroyDiskEntry(object, 0);
1181         }
1182     }
1183
1184     if(numDiskEntries > maxDiskEntries)
1185         destroyDiskEntry(diskEntriesLast->object, 0);
1186
1187     if(!local) {
1188         if(diskCacheRoot == NULL || diskCacheRoot->length <= 0)
1189             return NULL;
1190         name_len = urlFilename(buf, 1024, object->key, object->key_size);
1191         if(name_len < 0) return NULL;
1192         if(!negative) {
1193             isWriteable = 1;
1194             fd = open(buf, O_RDWR | O_BINARY);
1195             if(fd < 0 && !writeable && errno == EACCES) {
1196                 writeable = 0;
1197                 fd = open(buf, O_RDONLY | O_BINARY);
1198             }
1199         }
1200         if(fd >= 0) {
1201             rc = validateEntry(object, fd, &body_offset, &offset);
1202             if(rc >= 0) {
1203                 dirty = rc;
1204             } else {
1205                 close(fd);
1206                 fd = -1;
1207                 rc = unlink(buf);
1208                 if(rc < 0 && errno != ENOENT) {
1209                     do_log_error(L_WARN,  errno,
1210                                  "Couldn't unlink stale disk entry %s",
1211                                  buf);
1212                     /* But continue -- it's okay to have stale entries. */
1213                 }
1214             }
1215         }
1216
1217         if(fd < 0 && create && name_len > 0 &&
1218            !(object->flags & OBJECT_INITIAL)) {
1219             isWriteable = 1;
1220             fd = createFile(buf, diskCacheRoot->length);
1221             if(fd < 0)
1222                 return NULL;
1223
1224             if(fd >= 0) {
1225                 char *data = NULL;
1226                 int dsize = 0;
1227                 if(object->numchunks > 0) {
1228                     data = object->chunks[0].data;
1229                     dsize = object->chunks[0].size;
1230                 }
1231                 rc = writeHeaders(fd, &body_offset, object, data, dsize);
1232                 if(rc < 0) {
1233                     do_log_error(L_ERROR, errno, "Couldn't write headers");
1234                     rc = unlink(buf);
1235                     if(rc < 0 && errno != ENOENT)
1236                         do_log_error(L_ERROR, errno,
1237                                      "Couldn't unlink truncated entry %s",
1238                                      buf);
1239                     close(fd);
1240                     return NULL;
1241                 }
1242                 assert(rc >= body_offset);
1243                 size = rc - body_offset;
1244                 offset = rc;
1245                 dirty = 0;
1246             }
1247         }
1248     } else {
1249         /* local */
1250         if(localDocumentRoot == NULL || localDocumentRoot->length == 0)
1251             return NULL;
1252
1253         name_len =
1254             localFilename(buf, 1024, object->key, object->key_size);
1255         if(name_len < 0)
1256             return NULL;
1257         isWriteable = 0;
1258         fd = open(buf, O_RDONLY | O_BINARY);
1259         if(fd >= 0) {
1260             if(validateEntry(object, fd, &body_offset, NULL) < 0) {
1261                 close(fd);
1262                 fd = -1;
1263             }
1264         }
1265         offset = 0;
1266     }
1267
1268     if(fd < 0) {
1269         object->disk_entry = &negativeEntry;
1270         return NULL;
1271     }
1272     assert(body_offset >= 0);
1273
1274     name = strdup_n(buf, name_len);
1275     if(name == NULL) {
1276         do_log(L_ERROR, "Couldn't allocate name.\n");
1277         close(fd);
1278         fd = -1;
1279         return NULL;
1280     }
1281
1282     entry = malloc(sizeof(DiskCacheEntryRec));
1283     if(entry == NULL) {
1284         do_log(L_ERROR, "Couldn't allocate entry.\n");
1285         free(name);
1286         close(fd);
1287         return NULL;
1288     }
1289
1290     entry->filename = name;
1291     entry->object = object;
1292     entry->fd = fd;
1293     entry->body_offset = body_offset;
1294     entry->local = local;
1295     entry->offset = offset;
1296     entry->size = size;
1297     entry->metadataDirty = dirty;
1298     entry->writeable = isWriteable;
1299
1300     entry->next = diskEntries;
1301     if(diskEntries)
1302         diskEntries->previous = entry;
1303     diskEntries = entry;
1304     if(diskEntriesLast == NULL)
1305         diskEntriesLast = entry;
1306     entry->previous = NULL;
1307     numDiskEntries++;
1308
1309     object->disk_entry = entry;
1310
1311     CHECK_ENTRY(entry);
1312     return entry;
1313 }
1314
1315 /* Rewrite a disk cache entry, used when the body offset needs to change. */
1316 static int
1317 rewriteEntry(ObjectPtr object)
1318 {
1319     int old_body_offset = object->disk_entry->body_offset;
1320     int fd, rc, n;
1321     DiskCacheEntryPtr entry;
1322     char* buf;
1323     int buf_is_chunk, bufsize;
1324     int offset;
1325
1326     fd = dup(object->disk_entry->fd);
1327     if(fd < 0) {
1328         do_log_error(L_ERROR, errno, "Couldn't duplicate file descriptor");
1329         return -1;
1330     }
1331
1332     rc = destroyDiskEntry(object, 1);
1333     if(rc < 0) {
1334         close(fd);
1335         return -1;
1336     }
1337     entry = makeDiskEntry(object, 1, 1);
1338     if(!entry) {
1339         close(fd);
1340         return -1;
1341     }
1342
1343     offset = diskEntrySize(object);
1344     if(offset < 0) {
1345         close(fd);
1346         return -1;
1347     }
1348
1349     bufsize = CHUNK_SIZE;
1350     buf_is_chunk = 1;
1351     buf = maybe_get_chunk();
1352     if(!buf) {
1353         bufsize = 2048;
1354         buf_is_chunk = 0;
1355         buf = malloc(2048);
1356         if(buf == NULL) {
1357             do_log(L_ERROR, "Couldn't allocate buffer.\n");
1358             close(fd);
1359             return -1;
1360         }
1361     }
1362
1363     rc = lseek(fd, old_body_offset + offset, SEEK_SET);
1364     if(rc < 0)
1365         goto done;
1366
1367     while(1) {
1368         CHECK_ENTRY(entry);
1369         n = read(fd, buf, bufsize);
1370         if(n <= 0)
1371             goto done;
1372         rc = entrySeek(entry, entry->body_offset + offset);
1373         if(rc < 0)
1374             goto done;
1375         rc = write(entry->fd, buf, n);
1376         if(rc >= 0) {
1377             entry->offset += rc;
1378             entry->size += rc;
1379         }
1380         if(rc < n)
1381             goto done;
1382     }
1383
1384  done:
1385     CHECK_ENTRY(entry);
1386     if(object->length >= 0 && entry->size == object->length)
1387         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1388     close(fd);
1389     if(buf_is_chunk)
1390         dispose_chunk(buf);
1391     else
1392         free(buf);
1393     return 1;
1394 }
1395
1396 int
1397 destroyDiskEntry(ObjectPtr object, int d)
1398 {
1399     DiskCacheEntryPtr entry = object->disk_entry;
1400     int rc, urc = 1;
1401
1402     assert(!entry || !entry->local || !d);
1403
1404     if(d && !entry)
1405         entry = makeDiskEntry(object, 1, 0);
1406
1407     CHECK_ENTRY(entry);
1408
1409     if(!entry || entry == &negativeEntry) {
1410         return 1;
1411     }
1412
1413     assert(entry->object == object);
1414
1415     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1416         /* See writeoutToDisk */
1417         d = 1;
1418     }
1419
1420     if(d) {
1421         entry->object->flags &= ~OBJECT_DISK_ENTRY_COMPLETE;
1422         if(entry->filename) {
1423             urc = unlink(entry->filename);
1424             if(urc < 0)
1425                 do_log_error(L_WARN, errno,
1426                              "Couldn't unlink %s", entry->filename);
1427         }
1428     } else {
1429         if(entry && entry->metadataDirty)
1430             writeoutMetadata(object);
1431         makeDiskEntry(object, 1, 0);
1432         /* rewriteDiskEntry may change the disk entry */
1433         entry = object->disk_entry;
1434         if(entry == NULL || entry == &negativeEntry)
1435             return 0;
1436         if(entry->writeable && diskCacheWriteoutOnClose > 0)
1437             reallyWriteoutToDisk(object, -1, diskCacheWriteoutOnClose);
1438     }
1439  again:
1440     rc = close(entry->fd);
1441     if(rc < 0 && errno == EINTR)
1442         goto again;
1443
1444     entry->fd = -1;
1445
1446     if(entry->filename)
1447         free(entry->filename);
1448     entry->filename = NULL;
1449
1450     if(entry->previous)
1451         entry->previous->next = entry->next;
1452     else
1453         diskEntries = entry->next;
1454     if(entry->next)
1455         entry->next->previous = entry->previous;
1456     else
1457         diskEntriesLast = entry->previous;
1458
1459     numDiskEntries--;
1460     assert(numDiskEntries >= 0);
1461
1462     free(entry);
1463     object->disk_entry = NULL;
1464     if(urc < 0)
1465         return -1;
1466     else
1467         return 1;
1468 }
1469
1470 ObjectPtr
1471 objectGetFromDisk(ObjectPtr object)
1472 {
1473     DiskCacheEntryPtr entry = makeDiskEntry(object, 0, 0);
1474     if(!entry) return NULL;
1475     return object;
1476 }
1477
1478
1479 int
1480 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
1481 {
1482     DiskCacheEntryPtr entry;
1483     int rc, result;
1484     int i, j, k;
1485     int complete;
1486
1487     if(object->type != OBJECT_HTTP)
1488         return 0;
1489
1490     if(object->flags & OBJECT_LINEAR)
1491         return 0;
1492
1493     if(object->length >= 0) {
1494         chunks = MIN(chunks,
1495                      (object->length - offset + CHUNK_SIZE - 1) / CHUNK_SIZE);
1496     }
1497
1498     rc = objectSetChunks(object, offset / CHUNK_SIZE + chunks);
1499     if(rc < 0)
1500         return 0;
1501
1502     complete = 1;
1503     if(object->flags & OBJECT_INITIAL) {
1504         complete = 0;
1505     } else if((object->length < 0 || object->size < object->length) &&
1506               object->size < (offset / CHUNK_SIZE + chunks) * CHUNK_SIZE) {
1507         complete = 0;
1508     } else {
1509         for(k = 0; k < chunks; k++) {
1510             int s;
1511             i = offset / CHUNK_SIZE + k;
1512             s = MIN(CHUNK_SIZE, object->size - i * CHUNK_SIZE);
1513             if(object->chunks[i].size < s) {
1514                 complete = 0;
1515                 break;
1516             }
1517         }
1518     }
1519
1520     if(complete)
1521         return 1;
1522
1523     /* This has the side-effect of revalidating the entry, which is
1524        what makes HEAD requests work. */
1525     entry = makeDiskEntry(object, 0, 0);
1526     if(!entry)
1527         return 0;
1528
1529     for(k = 0; k < chunks; k++) {
1530         i = offset / CHUNK_SIZE + k;
1531         if(!object->chunks[i].data)
1532             object->chunks[i].data = get_chunk();
1533         if(!object->chunks[i].data) {
1534             chunks = k;
1535             break;
1536         }
1537         lockChunk(object, i);
1538     }
1539
1540     result = 0;
1541
1542     for(k = 0; k < chunks; k++) {
1543         int o;
1544         i = offset / CHUNK_SIZE + k;
1545         j = object->chunks[i].size;
1546         o = i * CHUNK_SIZE + j;
1547
1548         if(object->chunks[i].size == CHUNK_SIZE)
1549             continue;
1550
1551         if(entry->size >= 0 && entry->size <= o)
1552             break;
1553
1554         if(entry->offset != entry->body_offset + o) {
1555             rc = entrySeek(entry, entry->body_offset + o);
1556             if(rc < 0) {
1557                 result = 0;
1558                 break;
1559             }
1560         }
1561
1562         CHECK_ENTRY(entry);
1563         again:
1564         rc = read(entry->fd, object->chunks[i].data + j, CHUNK_SIZE - j);
1565         if(rc < 0) {
1566             if(errno == EINTR)
1567                 goto again;
1568             entry->offset = -1;
1569             do_log_error(L_ERROR, errno, "Couldn't read");
1570             break;
1571         }
1572
1573         entry->offset += rc;
1574         object->chunks[i].size += rc;
1575         if(object->size < o + rc)
1576             object->size = o + rc;
1577
1578         if(entry->object->length >= 0 && entry->size < 0 &&
1579            entry->offset - entry->body_offset == entry->object->length)
1580             entry->size = entry->object->length;
1581
1582         if(rc < CHUNK_SIZE - j) {
1583             /* Paranoia: the read may have been interrupted half-way. */
1584             if(entry->size < 0) {
1585                 if(rc == 0 ||
1586                    (entry->object->length >= 0 &&
1587                     entry->object->length ==
1588                     entry->offset - entry->body_offset))
1589                     entry->size = entry->offset - entry->body_offset;
1590                 break;
1591             } else if(entry->size != entry->offset - entry->body_offset) {
1592                 if(rc == 0 ||
1593                    entry->size < entry->offset - entry->body_offset) {
1594                     do_log(L_WARN,
1595                            "Disk entry size changed behind our back: "
1596                            "%ld -> %ld (%d).\n",
1597                            (long)entry->size,
1598                            (long)entry->offset - entry->body_offset,
1599                            object->size);
1600                     entry->size = -1;
1601                 }
1602             }
1603             break;
1604         }
1605
1606         CHECK_ENTRY(entry);
1607         result = 1;
1608     }
1609
1610     CHECK_ENTRY(object->disk_entry);
1611     for(k = 0; k < chunks; k++) {
1612         i = offset / CHUNK_SIZE + k;
1613         unlockChunk(object, i);
1614     }
1615
1616     if(result > 0) {
1617         notifyObject(object);
1618         return 1;
1619     } else {
1620         return 0;
1621     }
1622 }
1623
1624 int
1625 writeoutToDisk(ObjectPtr object, int upto, int max)
1626 {
1627     if(maxDiskCacheEntrySize >= 0 && object->size > maxDiskCacheEntrySize) {
1628         /* An object was created with an unknown length, and then grew
1629            beyond maxDiskCacheEntrySize.  Destroy the disk entry. */
1630         destroyDiskEntry(object, 1);
1631         return 0;
1632     }
1633
1634     return reallyWriteoutToDisk(object, upto, max);
1635 }
1636
1637 static int
1638 reallyWriteoutToDisk(ObjectPtr object, int upto, int max)
1639 {
1640     DiskCacheEntryPtr entry;
1641     int rc;
1642     int i, j;
1643     int offset;
1644     int bytes = 0;
1645
1646     if(upto < 0)
1647         upto = object->size;
1648
1649     if((object->cache_control & CACHE_NO_STORE) ||
1650        (object->flags & OBJECT_LOCAL))
1651         return 0;
1652
1653     if((object->flags & OBJECT_DISK_ENTRY_COMPLETE) && !object->disk_entry)
1654         return 0;
1655
1656     entry = makeDiskEntry(object, 1, 1);
1657     if(!entry) return 0;
1658
1659     assert(!entry->local);
1660
1661     if(object->flags & OBJECT_DISK_ENTRY_COMPLETE)
1662         goto done;
1663
1664     diskEntrySize(object);
1665     if(entry->size < 0)
1666         return 0;
1667
1668     if(object->length >= 0 && entry->size >= object->length) {
1669         object->flags |= OBJECT_DISK_ENTRY_COMPLETE;
1670         goto done;
1671     }
1672
1673     if(entry->size >= upto)
1674         goto done;
1675
1676     if(!entry->writeable) {
1677         entry = makeDiskEntry(object, 1, 1);
1678         if(!entry)
1679             return 0;
1680         if(!entry->writeable)
1681             return 0;
1682         diskEntrySize(object);
1683         if(entry->size < 0)
1684             return 0;
1685     }
1686
1687     offset = entry->size;
1688
1689     /* Avoid a seek in case we start writing at the beginning */
1690     if(offset == 0 && entry->metadataDirty) {
1691         writeoutMetadata(object);
1692         /* rewriteDiskEntry may change the entry */
1693         entry = makeDiskEntry(object, 1, 0);
1694         if(entry == NULL || !entry->writeable)
1695             return 0;
1696     }
1697
1698     rc = entrySeek(entry, offset + entry->body_offset);
1699     if(rc < 0) return 0;
1700
1701     do {
1702         if(max >= 0 && bytes >= max)
1703             break;
1704         CHECK_ENTRY(entry);
1705         assert(entry->offset == offset + entry->body_offset);
1706         i = offset / CHUNK_SIZE;
1707         j = offset % CHUNK_SIZE;
1708         if(i >= object->numchunks)
1709             break;
1710         if(object->chunks[i].size <= j)
1711             break;
1712     again:
1713         rc = write(entry->fd, object->chunks[i].data + j,
1714                    object->chunks[i].size - j);
1715         if(rc < 0) {
1716             if(errno == EINTR)
1717                 goto again;
1718             do_log_error(L_ERROR, errno, "Couldn't write disk entry");
1719             break;
1720         }
1721         entry->offset += rc;
1722         offset += rc;
1723         bytes += rc;
1724         if(entry->size < offset)
1725             entry->size = offset;
1726     } while(j + rc >= CHUNK_SIZE);
1727
1728  done:
1729     CHECK_ENTRY(entry);
1730     if(entry->metadataDirty)
1731         writeoutMetadata(object);
1732
1733     return bytes;
1734 }
1735
1736 int
1737 writeoutMetadata(ObjectPtr object)
1738 {
1739     DiskCacheEntryPtr entry;
1740     int rc;
1741
1742     if((object->cache_control & CACHE_NO_STORE) ||
1743        (object->flags & OBJECT_LOCAL))
1744         return 0;
1745
1746     entry = makeDiskEntry(object, 1, 0);
1747     if(entry == NULL || entry == &negativeEntry)
1748         goto fail;
1749
1750     assert(!entry->local);
1751
1752     rc = entrySeek(entry, 0);
1753     if(rc < 0) goto fail;
1754
1755     rc = writeHeaders(entry->fd, &entry->body_offset, object, NULL, 0);
1756     if(rc == -2) {
1757         rc = rewriteEntry(object);
1758         if(rc < 0) return 0;
1759         return 1;
1760     }
1761     if(rc < 0) goto fail;
1762     entry->offset = rc;
1763     entry->metadataDirty = 0;
1764     return 1;
1765
1766  fail:
1767     /* We need this in order to avoid trying to write this entry out
1768        multiple times. */
1769     if(entry && entry != &negativeEntry)
1770         entry->metadataDirty = 0;
1771     return 0;
1772 }
1773
1774 static void
1775 mergeDobjects(DiskObjectPtr dst, DiskObjectPtr src)
1776 {
1777     if(dst->filename == NULL) {
1778         dst->filename = src->filename;
1779         dst->body_offset = src->body_offset;
1780     } else
1781         free(src->filename);
1782     free(src->location);
1783     if(dst->length < 0)
1784         dst->length = src->length;
1785     if(dst->size < 0)
1786         dst->size = src->size;
1787     if(dst->age < 0)
1788         dst->age = src->age;
1789     if(dst->date < 0)
1790         dst->date = src->date;
1791     if(dst->last_modified < 0)
1792         dst->last_modified = src->last_modified;
1793     free(src);
1794 }
1795
1796 DiskObjectPtr
1797 readDiskObject(char *filename, struct stat *sb)
1798 {
1799     int fd, rc, n, dummy, code;
1800     int length, size;
1801     time_t date, last_modified, age, atime, expires;
1802     char *location = NULL, *fn = NULL;
1803     DiskObjectPtr dobject;
1804     char *buf;
1805     int buf_is_chunk, bufsize;
1806     int body_offset;
1807     struct stat ss;
1808
1809     fd = -1;
1810
1811     if(sb == NULL) {
1812         rc = stat(filename, &ss);
1813         if(rc < 0) {
1814             do_log_error(L_WARN, errno, "Couldn't stat %s", filename);
1815             return NULL;
1816         }
1817         sb = &ss;
1818     }
1819
1820     buf_is_chunk = 1;
1821     bufsize = CHUNK_SIZE;
1822     buf = get_chunk();
1823     if(buf == NULL) {
1824         do_log(L_ERROR, "Couldn't allocate buffer.\n");
1825         return NULL;
1826     }
1827
1828     if(S_ISREG(sb->st_mode)) {
1829         fd = open(filename, O_RDONLY | O_BINARY);
1830         if(fd < 0)
1831             goto fail;
1832     again:
1833         rc = read(fd, buf, bufsize);
1834         if(rc < 0)
1835             goto fail;
1836
1837         n = findEndOfHeaders(buf, 0, rc, &dummy);
1838         if(n < 0) {
1839             long lrc;
1840             if(buf_is_chunk) {
1841                 dispose_chunk(buf);
1842                 buf_is_chunk = 0;
1843                 bufsize = bigBufferSize;
1844                 buf = malloc(bigBufferSize);
1845                 if(buf == NULL)
1846                     goto fail2;
1847                 lrc = lseek(fd, 0, SEEK_SET);
1848                 if(lrc < 0)
1849                     goto fail;
1850                 goto again;
1851             }
1852             goto fail;
1853         }
1854
1855         rc = httpParseServerFirstLine(buf, &code, &dummy, NULL);
1856         if(rc < 0)
1857             goto fail;
1858
1859         rc = httpParseHeaders(0, NULL, buf, rc, NULL,
1860                               NULL, &length, NULL, NULL, NULL,
1861                               &date, &last_modified, &expires, &age,
1862                               &atime, &body_offset, NULL,
1863                               NULL, NULL, NULL, NULL, &location, NULL, NULL);
1864         if(rc < 0 || location == NULL)
1865             goto fail;
1866         if(body_offset < 0)
1867             body_offset = n;
1868
1869         size = sb->st_size - body_offset;
1870         if(size < 0)
1871             size = 0;
1872     } else if(S_ISDIR(sb->st_mode)) {
1873         char *n;
1874         n = dirnameUrl(buf, 512, (char*)filename, strlen(filename));
1875         if(n == NULL)
1876             goto fail;
1877         location = strdup(n);
1878         if(location == NULL)
1879             goto fail;
1880         length = -1;
1881         size = -1;
1882         body_offset = -1;
1883         age = -1;
1884         atime = -1;
1885         date = -1;
1886         last_modified = -1;
1887     } else {
1888         goto fail;
1889     }
1890
1891     dobject = malloc(sizeof(DiskObjectRec));
1892     if(!dobject)
1893         goto fail;
1894
1895     fn = strdup(filename);
1896     if(!fn)
1897         goto fail;
1898
1899     if(buf_is_chunk)
1900         dispose_chunk(buf);
1901     else
1902         free(buf);
1903
1904     dobject->location = location;
1905     dobject->filename = fn;
1906     dobject->length = length;
1907     dobject->body_offset = body_offset;
1908     dobject->size = size;
1909     dobject->age = age;
1910     dobject->access = atime;
1911     dobject->date = date;
1912     dobject->last_modified = last_modified;
1913     dobject->expires = expires;
1914     if(fd >= 0) close(fd);
1915     return dobject;
1916
1917  fail:
1918     if(buf_is_chunk)
1919         dispose_chunk(buf);
1920     else
1921         free(buf);
1922  fail2:
1923     if(fd >= 0) close(fd);
1924     if(location) free(location);
1925     return NULL;
1926 }
1927
1928
1929 DiskObjectPtr
1930 processObject(DiskObjectPtr dobjects, char *filename, struct stat *sb)
1931 {
1932     DiskObjectPtr dobject = NULL;
1933     int c = 0;
1934
1935     dobject = readDiskObject((char*)filename, sb);
1936     if(dobject == NULL)
1937         return 0;
1938
1939     if(!dobjects ||
1940        (c = strcmp(dobject->location, dobjects->location)) <= 0) {
1941         if(dobjects && c == 0) {
1942             mergeDobjects(dobjects, dobject);
1943         } else {
1944             dobject->next = dobjects;
1945             dobjects = dobject;
1946         }
1947     } else {
1948         DiskObjectPtr other = dobjects;
1949         while(other->next) {
1950             c = strcmp(dobject->location, other->next->location);
1951             if(c < 0)
1952                 break;
1953             other = other->next;
1954         }
1955         if(strcmp(dobject->location, other->location) == 0) {
1956             mergeDobjects(other, dobject);
1957         } else {
1958             dobject->next = other->next;
1959             other->next = dobject;
1960         }
1961     }
1962     return dobjects;
1963 }
1964
1965 /* Determine whether p is below root */
1966 static int
1967 filter(DiskObjectPtr p, const char *root, int n, int recursive)
1968 {
1969     char *cp;
1970     int m = strlen(p->location);
1971     if(m < n)
1972         return 0;
1973     if(memcmp(root, p->location, n) != 0)
1974         return 0;
1975     if(recursive)
1976         return 1;
1977     if(m == 0 || p->location[m - 1] == '/')
1978         return 1;
1979     cp = strchr(p->location + n, '/');
1980     if(cp && cp - p->location != m - 1)
1981         return 0;
1982     return 1;
1983 }
1984
1985 /* Filter out all disk objects that are not under root */
1986 DiskObjectPtr
1987 filterDiskObjects(DiskObjectPtr from, const char *root, int recursive)
1988 {
1989     int n = strlen(root);
1990     DiskObjectPtr p, q;
1991
1992     while(from && !filter(from, root, n, recursive)) {
1993         p = from;
1994         from = p->next;
1995         free(p->location);
1996         free(p);
1997     }
1998
1999     p = from;
2000     while(p && p->next) {
2001         if(!filter(p->next, root, n, recursive)) {
2002             q = p->next;
2003             p->next = q->next;
2004             free(q->location);
2005             free(q);
2006         } else {
2007             p = p->next;
2008         }
2009     }
2010     return from;
2011 }
2012
2013 DiskObjectPtr
2014 insertRoot(DiskObjectPtr from, const char *root)
2015 {
2016     DiskObjectPtr p;
2017
2018     p = from;
2019     while(p) {
2020         if(strcmp(root, p->location) == 0)
2021             return from;
2022         p = p->next;
2023     }
2024
2025     p = malloc(sizeof(DiskObjectRec));
2026     if(!p) return from;
2027     p->location = strdup(root);
2028     if(p->location == NULL) {
2029         free(p);
2030         return from;
2031     }
2032     p->filename = NULL;
2033     p->length = -1;
2034     p->size = -1;
2035     p->age = -1;
2036     p->access = -1;
2037     p->last_modified = -1;
2038     p->expires = -1;
2039     p->next = from;
2040     return p;
2041 }
2042
2043 /* Insert all missing directories in a sorted list of dobjects */
2044 DiskObjectPtr
2045 insertDirs(DiskObjectPtr from)
2046 {
2047     DiskObjectPtr p, q, new;
2048     int n, m;
2049     char *cp;
2050
2051     p = NULL; q = from;
2052     while(q) {
2053         n = strlen(q->location);
2054         if(n > 0 && q->location[n - 1] != '/') {
2055             cp = strrchr(q->location, '/');
2056             m = cp - q->location + 1;
2057             if(cp && (!p || strlen(p->location) < m ||
2058                       memcmp(p->location, q->location, m) != 0)) {
2059                 new = malloc(sizeof(DiskObjectRec));
2060                 if(!new) break;
2061                 new->location = strdup_n(q->location, m);
2062                 if(new->location == NULL) {
2063                     free(new);
2064                     break;
2065                 }
2066                 new->filename = NULL;
2067                 new->length = -1;
2068                 new->size = -1;
2069                 new->age = -1;
2070                 new->access = -1;
2071                 new->last_modified = -1;
2072                 new->expires = -1;
2073                 new->next = q;
2074                 if(p)
2075                     p->next = new;
2076                 else
2077                     from = new;
2078             }
2079         }
2080         p = q;
2081         q = q->next;
2082     }
2083     return from;
2084 }
2085
2086 void
2087 indexDiskObjects(FILE *out, const char *root, int recursive)
2088 {
2089     int n, i, isdir;
2090     DIR *dir;
2091     struct dirent *dirent;
2092     char buf[1024];
2093     char *fts_argv[2];
2094     FTS *fts;
2095     FTSENT *fe;
2096     DiskObjectPtr dobjects = NULL;
2097     char *of = root[0] == '\0' ? "" : " of ";
2098
2099     fprintf(out, "<!DOCTYPE HTML PUBLIC "
2100             "\"-//W3C//DTD HTML 4.01 Transitional//EN\" "
2101             "\"http://www.w3.org/TR/html4/loose.dtd\">\n"
2102             "<html><head>\n"
2103             "<title>%s%s%s</title>\n"
2104             "</head><body>\n"
2105             "<h1>%s%s%s</h1>\n",
2106             recursive ? "Recursive index" : "Index", of, root,
2107             recursive ? "Recursive index" : "Index", of, root);
2108
2109     if(diskCacheRoot == NULL || diskCacheRoot->length <= 0) {
2110         fprintf(out, "<p>No <tt>diskCacheRoot</tt>.</p>\n");
2111         goto trailer;
2112     }
2113
2114     if(diskCacheRoot->length >= 1024) {
2115         fprintf(out,
2116                 "<p>The value of <tt>diskCacheRoot</tt> is "
2117                 "too long (%d).</p>\n",
2118                 diskCacheRoot->length);
2119         goto trailer;
2120     }
2121
2122     if(strlen(root) < 8) {
2123         memcpy(buf, diskCacheRoot->string, diskCacheRoot->length);
2124         buf[diskCacheRoot->length] = '\0';
2125         n = diskCacheRoot->length;
2126     } else {
2127         n = urlDirname(buf, 1024, root, strlen(root));
2128     }
2129     if(n > 0) {
2130         if(recursive) {
2131             dir = NULL;
2132             fts_argv[0] = buf;
2133             fts_argv[1] = NULL;
2134             fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2135             if(fts) {
2136                 while(1) {
2137                     fe = fts_read(fts);
2138                     if(!fe) break;
2139                     if(fe->fts_info != FTS_DP)
2140                         dobjects =
2141                             processObject(dobjects,
2142                                           fe->fts_path,
2143                                           fe->fts_info == FTS_NS ||
2144                                           fe->fts_info == FTS_NSOK ?
2145                                           fe->fts_statp : NULL);
2146                 }
2147                 fts_close(fts);
2148             }
2149         } else {
2150             dir = opendir(buf);
2151             if(dir) {
2152                 while(1) {
2153                     dirent = readdir(dir);
2154                     if(!dirent) break;
2155                     if(n + strlen(dirent->d_name) < 1024) {
2156                         strcpy(buf + n, dirent->d_name);
2157                     } else {
2158                         continue;
2159                     }
2160                     dobjects = processObject(dobjects, buf, NULL);
2161                 }
2162                 closedir(dir);
2163             } else {
2164                 fprintf(out, "<p>Couldn't open directory: %s (%d).</p>\n",
2165                         strerror(errno), errno);
2166                 goto trailer;
2167             }
2168         }
2169     }
2170
2171     if(dobjects) {
2172         DiskObjectPtr dobject;
2173         int entryno;
2174         dobjects = insertRoot(dobjects, root);
2175         dobjects = insertDirs(dobjects);
2176         dobjects = filterDiskObjects(dobjects, root, recursive);
2177         dobject = dobjects;
2178         buf[0] = '\0';
2179         alternatingHttpStyle(out, "diskcachelist");
2180         fprintf(out, "<table id=diskcachelist>\n");
2181         fprintf(out, "<tbody>\n");
2182         entryno = 0;
2183         while(dobjects) {
2184             dobject = dobjects;
2185             i = strlen(dobject->location);
2186             isdir = (i == 0 || dobject->location[i - 1] == '/');
2187             if(entryno % 2)
2188                 fprintf(out, "<tr class=odd>");
2189             else
2190                 fprintf(out, "<tr class=even>");
2191             if(dobject->size >= 0) {
2192                 fprintf(out, "<td><a href=\"%s\"><tt>",
2193                         dobject->location);
2194                 htmlPrint(out,
2195                           dobject->location, strlen(dobject->location));
2196                 fprintf(out, "</tt></a></td> ");
2197                 if(dobject->length >= 0) {
2198                     if(dobject->size == dobject->length)
2199                         fprintf(out, "<td>%d</td> ", dobject->length);
2200                     else
2201                         fprintf(out, "<td>%d/%d</td> ",
2202                                dobject->size, dobject->length);
2203                 } else {
2204                     /* Avoid a trigraph. */
2205                     fprintf(out, "<td>%d/<em>??" "?</em></td> ", dobject->size);
2206                 }
2207                 if(dobject->last_modified >= 0) {
2208                     struct tm *tm = gmtime(&dobject->last_modified);
2209                     if(tm == NULL)
2210                         n = -1;
2211                     else
2212                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2213                 } else
2214                     n = -1;
2215                 if(n > 0) {
2216                     buf[n] = '\0';
2217                     fprintf(out, "<td>%s</td> ", buf);
2218                 } else {
2219                     fprintf(out, "<td></td>");
2220                 }
2221
2222                 if(dobject->date >= 0) {
2223                     struct tm *tm = gmtime(&dobject->date);
2224                     if(tm == NULL)
2225                         n = -1;
2226                     else
2227                         n = strftime(buf, 1024, "%d.%m.%Y", tm);
2228                 } else
2229                     n = -1;
2230                 if(n > 0) {
2231                     buf[n] = '\0';
2232                     fprintf(out, "<td>%s</td>", buf);
2233                 } else {
2234                     fprintf(out, "<td></td>");
2235                 }
2236             } else {
2237                 fprintf(out, "<td><tt>");
2238                 htmlPrint(out, dobject->location,
2239                           strlen(dobject->location));
2240                 fprintf(out, "</tt></td><td></td><td></td><td></td>");
2241             }
2242             if(isdir) {
2243                 fprintf(out, "<td><a href=\"/polipo/index?%s\">plain</a></td>"
2244                         "<td><a href=\"/polipo/recursive-index?%s\">"
2245                         "recursive</a></td>",
2246                         dobject->location, dobject->location);
2247             }
2248             fprintf(out, "</tr>\n");
2249             entryno++;
2250             dobjects = dobject->next;
2251             free(dobject->location);
2252             free(dobject->filename);
2253             free(dobject);
2254         }
2255         fprintf(out, "</tbody>\n");
2256         fprintf(out, "</table>\n");
2257     }
2258
2259  trailer:
2260     fprintf(out, "<p><a href=\"/polipo/\">back</a></p>\n");
2261     fprintf(out, "</body></html>\n");
2262     return;
2263 }
2264
2265 static int
2266 checkForZeroes(char *buf, int n)
2267 {
2268     int i, j;
2269     unsigned long *lbuf = (unsigned long *)buf;
2270     assert(n % sizeof(unsigned long) == 0);
2271
2272     for(i = 0; i * sizeof(unsigned long) < n; i++) {
2273         if(lbuf[i] != 0L)
2274             return i * sizeof(unsigned long);
2275     }
2276     for(j = 0; i * sizeof(unsigned long) + j < n; j++) {
2277         if(buf[i * sizeof(unsigned long) + j] != 0)
2278             break;
2279     }
2280
2281     return i * sizeof(unsigned long) + j;
2282 }
2283
2284 static int
2285 copyFile(int from, char *filename, int n)
2286 {
2287     char *buf;
2288     int to, offset, nread, nzeroes, rc;
2289
2290     buf = malloc(CHUNK_SIZE);
2291     if(buf == NULL)
2292         return -1;
2293
2294     to = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY,
2295               diskCacheFilePermissions);
2296     if(to < 0) {
2297         free(buf);
2298         return -1;
2299     }
2300
2301     offset = 0;
2302     while(offset < n) {
2303         nread = read(from, buf, MIN(CHUNK_SIZE, n - offset));
2304         if(nread <= 0)
2305             break;
2306         nzeroes = checkForZeroes(buf, nread & -8);
2307         if(nzeroes > 0) {
2308             /* I like holes */
2309             rc = lseek(to, nzeroes, SEEK_CUR);
2310             if(rc != offset + nzeroes) {
2311                 if(rc < 0)
2312                     do_log_error(L_ERROR, errno, "Couldn't extend file");
2313                 else
2314                     do_log(L_ERROR,
2315                            "Couldn't extend file: "
2316                            "unexpected offset %d != %d + %d.\n",
2317                            rc, offset, nread);
2318                 break;
2319             }
2320         }
2321         if(nread > nzeroes) {
2322             rc = write(to, buf + nzeroes, nread - nzeroes);
2323             if(rc != nread - nzeroes) {
2324                 if(rc < 0)
2325                     do_log_error(L_ERROR, errno, "Couldn't write");
2326                 else
2327                     do_log(L_ERROR, "Short write.\n");
2328                 break;
2329             }
2330         }
2331         offset += nread;
2332     }
2333     free(buf);
2334     close(to);
2335     if(offset <= 0)
2336         unlink(filename);       /* something went wrong straight away */
2337     return 1;
2338 }
2339
2340 static long int
2341 expireFile(char *filename, struct stat *sb,
2342            int *considered, int *unlinked, int *truncated)
2343 {
2344     DiskObjectPtr dobject = NULL;
2345     time_t t;
2346     int fd, rc;
2347     long int ret = sb->st_size;
2348
2349     if(!preciseExpiry) {
2350         t = sb->st_mtime;
2351         if(t > current_time.tv_sec + 1) {
2352             do_log(L_WARN, "File %s has access time in the future.\n",
2353                    filename);
2354             t = current_time.tv_sec;
2355         }
2356
2357         if(t > current_time.tv_sec - diskCacheUnlinkTime &&
2358            (sb->st_size < diskCacheTruncateSize ||
2359             t > current_time.tv_sec - diskCacheTruncateTime))
2360             return ret;
2361     }
2362
2363     (*considered)++;
2364
2365     dobject = readDiskObject(filename, sb);
2366     if(!dobject) {
2367         do_log(L_ERROR, "Incorrect disk entry %s -- removing.\n", filename);
2368         rc = unlink(filename);
2369         if(rc < 0) {
2370             do_log_error(L_ERROR, errno,
2371                          "Couldn't unlink %s", filename);
2372             return ret;
2373         } else {
2374             (*unlinked)++;
2375             return 0;
2376         }
2377     }
2378
2379     t = dobject->access;
2380     if(t < 0) t = dobject->age;
2381     if(t < 0) t = dobject->date;
2382
2383     if(t > current_time.tv_sec)
2384         do_log(L_WARN,
2385                "Disk entry %s (%s) has access time in the future.\n",
2386                dobject->location, dobject->filename);
2387
2388     if(t < current_time.tv_sec - diskCacheUnlinkTime) {
2389         rc = unlink(dobject->filename);
2390         if(rc < 0) {
2391             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2392         } else {
2393             (*unlinked)++;
2394             ret = 0;
2395         }
2396     } else if(dobject->size >
2397               diskCacheTruncateSize + 4 * dobject->body_offset &&
2398               t < current_time.tv_sec - diskCacheTruncateTime) {
2399         /* We need to copy rather than simply truncate in place: the
2400            latter would confuse a running polipo. */
2401         fd = open(dobject->filename, O_RDONLY | O_BINARY, 0);
2402         rc = unlink(dobject->filename);
2403         if(rc < 0) {
2404             do_log_error(L_ERROR, errno, "Couldn't unlink %s", filename);
2405             close(fd);
2406             fd = -1;
2407         } else {
2408             (*unlinked)++;
2409             copyFile(fd, dobject->filename,
2410                      dobject->body_offset + diskCacheTruncateSize);
2411             close(fd);
2412             (*unlinked)--;
2413             (*truncated)++;
2414             ret = sb->st_size - dobject->body_offset + diskCacheTruncateSize;
2415         }
2416     }
2417     free(dobject->location);
2418     free(dobject->filename);
2419     free(dobject);
2420     return ret;
2421 }
2422
2423 void
2424 expireDiskObjects()
2425 {
2426     int rc;
2427     char *fts_argv[2];
2428     FTS *fts;
2429     FTSENT *fe;
2430     int files = 0, considered = 0, unlinked = 0, truncated = 0;
2431     int dirs = 0, rmdirs = 0;
2432     long left = 0, total = 0;
2433
2434     if(diskCacheRoot == NULL ||
2435        diskCacheRoot->length <= 0 || diskCacheRoot->string[0] != '/')
2436         return;
2437
2438     fts_argv[0] = diskCacheRoot->string;
2439     fts_argv[1] = NULL;
2440     fts = fts_open(fts_argv, FTS_LOGICAL, NULL);
2441     if(fts == NULL) {
2442         do_log_error(L_ERROR, errno, "Couldn't fts_open disk cache");
2443     } else {
2444         while(1) {
2445             gettimeofday(&current_time, NULL);
2446
2447             fe = fts_read(fts);
2448             if(!fe) break;
2449
2450             if(fe->fts_info == FTS_D)
2451                 continue;
2452
2453             if(fe->fts_info == FTS_DP || fe->fts_info == FTS_DC ||
2454                fe->fts_info == FTS_DNR) {
2455                 if(fe->fts_accpath[0] == '/' &&
2456                    strlen(fe->fts_accpath) <= diskCacheRoot->length)
2457                     continue;
2458                 dirs++;
2459                 rc = rmdir(fe->fts_accpath);
2460                 if(rc >= 0)
2461                     rmdirs++;
2462                 else if(errno != ENOTEMPTY && errno != EEXIST)
2463                     do_log_error(L_ERROR, errno,
2464                                  "Couldn't remove directory %s",
2465                                  fe->fts_accpath);
2466                 continue;
2467             } else if(fe->fts_info == FTS_NS) {
2468                 do_log_error(L_ERROR, fe->fts_errno, "Couldn't stat file %s",
2469                              fe->fts_accpath);
2470                 continue;
2471             } else if(fe->fts_info == FTS_ERR) {
2472                 do_log_error(L_ERROR, fe->fts_errno,
2473                              "Couldn't fts_read disk cache");
2474                 break;
2475             }
2476
2477             if(!S_ISREG(fe->fts_statp->st_mode)) {
2478                 do_log(L_ERROR, "Unexpected file %s type 0%o.\n",
2479                        fe->fts_accpath, (unsigned int)fe->fts_statp->st_mode);
2480                 continue;
2481             }
2482
2483             files++;
2484             left += expireFile(fe->fts_accpath, fe->fts_statp,
2485                                &considered, &unlinked, &truncated);
2486             total += fe->fts_statp->st_size;
2487         }
2488         fts_close(fts);
2489     }
2490
2491     printf("Disk cache purged.\n");
2492     printf("%d files, %d considered, %d removed, %d truncated "
2493            "(%ldkB -> %ldkB).\n",
2494            files, considered, unlinked, truncated, total/1024, left/1024);
2495     printf("%d directories, %d removed.\n", dirs, rmdirs);
2496     return;
2497 }
2498
2499 #else
2500
2501 void
2502 preinitDiskcache()
2503 {
2504     return;
2505 }
2506
2507 void
2508 initDiskcache()
2509 {
2510     return;
2511 }
2512
2513 int
2514 writeoutToDisk(ObjectPtr object, int upto, int max)
2515 {
2516     return 0;
2517 }
2518
2519 int
2520 destroyDiskEntry(ObjectPtr object, int d)
2521 {
2522     return 0;
2523 }
2524
2525 ObjectPtr
2526 objectGetFromDisk(ObjectPtr object)
2527 {
2528     return NULL;
2529 }
2530
2531 int
2532 objectFillFromDisk(ObjectPtr object, int offset, int chunks)
2533 {
2534     return 0;
2535 }
2536
2537 int
2538 revalidateDiskEntry(ObjectPtr object)
2539 {
2540     return 0;
2541 }
2542
2543 void
2544 dirtyDiskEntry(ObjectPtr object)
2545 {
2546     return;
2547 }
2548
2549 void
2550 expireDiskObjects()
2551 {
2552     do_log(L_ERROR, "Disk cache not supported in this version.\n");
2553 }
2554
2555 int
2556 diskEntrySize(ObjectPtr object)
2557 {
2558     return -1;
2559 }
2560 #endif