cache.c

   1 /* cache.c: cache management
   2  *
   3  * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com>
   4  *
   5  * Licensed under GNU General Public License v2
   6  *   (see COPYING for full license text)
   7  *
   8  *
   9  * The cache is just a directory structure where each file is a cache slot,
  10  * and each filename is based on the hash of some key (e.g. the cgit url).
  11  * Each file contains the full key followed by the cached content for that
  12  * key.
  13  *
  14  */
  15
  16 #include "cgit.h"
  17 #include "cache.h"
  18 #include "html.h"
  19 #ifdef HAVE_LINUX_SENDFILE
  20 #include <sys/sendfile.h>
  21 #endif
  22
  23 #define CACHE_BUFSIZE (1024 * 4)
  24
  25 struct cache_slot {
  26         const char *key;
  27         size_t keylen;
  28         int ttl;
  29         cache_fill_fn fn;
  30         int cache_fd;
  31         int lock_fd;
  32         int stdout_fd;
  33         const char *cache_name;
  34         const char *lock_name;
  35         int match;
  36         struct stat cache_st;
  37         int bufsize;
  38         char buf[CACHE_BUFSIZE];
  39 };
  40
  41 /* Open an existing cache slot and fill the cache buffer with
  42  * (part of) the content of the cache file. Return 0 on success
  43  * and errno otherwise.
  44  */
  45 static int open_slot(struct cache_slot *slot)
  46 {
  47         char *bufz;
  48         ssize_t bufkeylen = -1;
  49
  50         slot->cache_fd = open(slot->cache_name, O_RDONLY);
  51         if (slot->cache_fd == -1)
  52                 return errno;
  53
  54         if (fstat(slot->cache_fd, &slot->cache_st))
  55                 return errno;
  56
  57         slot->bufsize = xread(slot->cache_fd, slot->buf, sizeof(slot->buf));
  58         if (slot->bufsize < 0)
  59                 return errno;
  60
  61         bufz = memchr(slot->buf, 0, slot->bufsize);
  62         if (bufz)
  63                 bufkeylen = bufz - slot->buf;
  64
  65         if (slot->key)
  66                 slot->match = bufkeylen == slot->keylen &&
  67                     !memcmp(slot->key, slot->buf, bufkeylen + 1);
  68
  69         return 0;
  70 }
  71
  72 /* Close the active cache slot */
  73 static int close_slot(struct cache_slot *slot)
  74 {
  75         int err = 0;
  76         if (slot->cache_fd > 0) {
  77                 if (close(slot->cache_fd))
  78                         err = errno;
  79                 else
  80                         slot->cache_fd = -1;
  81         }
  82         return err;
  83 }
  84
  85 /* Print the content of the active cache slot (but skip the key). */
  86 static int print_slot(struct cache_slot *slot)
  87 {
  88         off_t off;
  89 #ifdef HAVE_LINUX_SENDFILE
  90         off_t size;
  91 #endif
  92
  93         off = slot->keylen + 1;
  94
  95 #ifdef HAVE_LINUX_SENDFILE
  96         size = slot->cache_st.st_size;
  97
  98         do {
  99                 ssize_t ret;
 100                 ret = sendfile(STDOUT_FILENO, slot->cache_fd, &off, size - off);
 101                 if (ret < 0) {
 102                         if (errno == EAGAIN || errno == EINTR)
 103                                 continue;
 104                         /* Fall back to read/write on EINVAL or ENOSYS */
 105                         if (errno == EINVAL || errno == ENOSYS)
 106                                 break;
 107                         return errno;
 108                 }
 109                 if (off == size)
 110                         return 0;
 111         } while (1);
 112 #endif
 113
 114         if (lseek(slot->cache_fd, off, SEEK_SET) != off)
 115                 return errno;
 116
 117         do {
 118                 ssize_t ret;
 119                 ret = xread(slot->cache_fd, slot->buf, sizeof(slot->buf));
 120                 if (ret < 0)
 121                         return errno;
 122                 if (ret == 0)
 123                         return 0;
 124                 if (write_in_full(STDOUT_FILENO, slot->buf, ret) < 0)
 125                         return errno;
 126         } while (1);
 127 }
 128
 129 /* Check if the slot has expired */
 130 static int is_expired(struct cache_slot *slot)
 131 {
 132         if (slot->ttl < 0)
 133                 return 0;
 134         else
 135                 return slot->cache_st.st_mtime + slot->ttl * 60 < time(NULL);
 136 }
 137
 138 /* Check if the slot has been modified since we opened it.
 139  * NB: If stat() fails, we pretend the file is modified.
 140  */
 141 static int is_modified(struct cache_slot *slot)
 142 {
 143         struct stat st;
 144
 145         if (stat(slot->cache_name, &st))
 146                 return 1;
 147         return (st.st_ino != slot->cache_st.st_ino ||
 148                 st.st_mtime != slot->cache_st.st_mtime ||
 149                 st.st_size != slot->cache_st.st_size);
 150 }
 151
 152 /* Close an open lockfile */
 153 static int close_lock(struct cache_slot *slot)
 154 {
 155         int err = 0;
 156         if (slot->lock_fd > 0) {
 157                 if (close(slot->lock_fd))
 158                         err = errno;
 159                 else
 160                         slot->lock_fd = -1;
 161         }
 162         return err;
 163 }
 164
 165 /* Create a lockfile used to store the generated content for a cache
 166  * slot, and write the slot key + \0 into it.
 167  * Returns 0 on success and errno otherwise.
 168  */
 169 static int lock_slot(struct cache_slot *slot)
 170 {
 171         struct flock lock = {
 172                 .l_type = F_WRLCK,
 173                 .l_whence = SEEK_SET,
 174                 .l_start = 0,
 175                 .l_len = 0,
 176         };
 177
 178         slot->lock_fd = open(slot->lock_name, O_RDWR | O_CREAT,
 179                              S_IRUSR | S_IWUSR);
 180         if (slot->lock_fd == -1)
 181                 return errno;
 182         if (fcntl(slot->lock_fd, F_SETLK, &lock) < 0) {
 183                 int saved_errno = errno;
 184                 close(slot->lock_fd);
 185                 slot->lock_fd = -1;
 186                 return saved_errno;
 187         }
 188         if (xwrite(slot->lock_fd, slot->key, slot->keylen + 1) < 0)
 189                 return errno;
 190         return 0;
 191 }
 192
 193 /* Release the current lockfile. If `replace_old_slot` is set the
 194  * lockfile replaces the old cache slot, otherwise the lockfile is
 195  * just deleted.
 196  */
 197 static int unlock_slot(struct cache_slot *slot, int replace_old_slot)
 198 {
 199         int err;
 200
 201         if (replace_old_slot)
 202                 err = rename(slot->lock_name, slot->cache_name);
 203         else
 204                 err = unlink(slot->lock_name);
 205
 206         /* Restore stdout and close the temporary FD. */
 207         if (slot->stdout_fd >= 0) {
 208                 dup2(slot->stdout_fd, STDOUT_FILENO);
 209                 close(slot->stdout_fd);
 210                 slot->stdout_fd = -1;
 211         }
 212
 213         if (err)
 214                 return errno;
 215
 216         return 0;
 217 }
 218
 219 /* Generate the content for the current cache slot by redirecting
 220  * stdout to the lock-fd and invoking the callback function
 221  */
 222 static int fill_slot(struct cache_slot *slot)
 223 {
 224         /* Preserve stdout */
 225         slot->stdout_fd = dup(STDOUT_FILENO);
 226         if (slot->stdout_fd == -1)
 227                 return errno;
 228
 229         /* Redirect stdout to lockfile */
 230         if (dup2(slot->lock_fd, STDOUT_FILENO) == -1)
 231                 return errno;
 232
 233         /* Generate cache content */
 234         slot->fn();
 235
 236         /* Make sure any buffered data is flushed to the file */
 237         if (fflush(stdout))
 238                 return errno;
 239
 240         /* update stat info */
 241         if (fstat(slot->lock_fd, &slot->cache_st))
 242                 return errno;
 243
 244         return 0;
 245 }
 246
 247 /* Crude implementation of 32-bit FNV-1 hash algorithm,
 248  * see http://www.isthe.com/chongo/tech/comp/fnv/ for details
 249  * about the magic numbers.
 250  */
 251 #define FNV_OFFSET 0x811c9dc5
 252 #define FNV_PRIME  0x01000193
 253
 254 unsigned long hash_str(const char *str)
 255 {
 256         unsigned long h = FNV_OFFSET;
 257         unsigned char *s = (unsigned char *)str;
 258
 259         if (!s)
 260                 return h;
 261
 262         while (*s) {
 263                 h *= FNV_PRIME;
 264                 h ^= *s++;
 265         }
 266         return h;
 267 }
 268
 269 static int process_slot(struct cache_slot *slot)
 270 {
 271         int err;
 272
 273         err = open_slot(slot);
 274         if (!err && slot->match) {
 275                 if (is_expired(slot)) {
 276                         if (!lock_slot(slot)) {
 277                                 /* If the cachefile has been replaced between
 278                                  * `open_slot` and `lock_slot`, we'll just
 279                                  * serve the stale content from the original
 280                                  * cachefile. This way we avoid pruning the
 281                                  * newly generated slot. The same code-path
 282                                  * is chosen if fill_slot() fails for some
 283                                  * reason.
 284                                  *
 285                                  * TODO? check if the new slot contains the
 286                                  * same key as the old one, since we would
 287                                  * prefer to serve the newest content.
 288                                  * This will require us to open yet another
 289                                  * file-descriptor and read and compare the
 290                                  * key from the new file, so for now we're
 291                                  * lazy and just ignore the new file.
 292                                  */
 293                                 if (is_modified(slot) || fill_slot(slot)) {
 294                                         unlock_slot(slot, 0);
 295                                         close_lock(slot);
 296                                 } else {
 297                                         close_slot(slot);
 298                                         unlock_slot(slot, 1);
 299                                         slot->cache_fd = slot->lock_fd;
 300                                 }
 301                         }
 302                 }
 303                 if ((err = print_slot(slot)) != 0) {
 304                         cache_log("[cgit] error printing cache %s: %s (%d)\n",
 305                                   slot->cache_name,
 306                                   strerror(err),
 307                                   err);
 308                 }
 309                 close_slot(slot);
 310                 return err;
 311         }
 312
 313         /* If the cache slot does not exist (or its key doesn't match the
 314          * current key), lets try to create a new cache slot for this
 315          * request. If this fails (for whatever reason), lets just generate
 316          * the content without caching it and fool the caller to believe
 317          * everything worked out (but print a warning on stdout).
 318          */
 319
 320         close_slot(slot);
 321         if ((err = lock_slot(slot)) != 0) {
 322                 cache_log("[cgit] Unable to lock slot %s: %s (%d)\n",
 323                           slot->lock_name, strerror(err), err);
 324                 slot->fn();
 325                 return 0;
 326         }
 327
 328         if ((err = fill_slot(slot)) != 0) {
 329                 cache_log("[cgit] Unable to fill slot %s: %s (%d)\n",
 330                           slot->lock_name, strerror(err), err);
 331                 unlock_slot(slot, 0);
 332                 close_lock(slot);
 333                 slot->fn();
 334                 return 0;
 335         }
 336         // We've got a valid cache slot in the lock file, which
 337         // is about to replace the old cache slot. But if we
 338         // release the lockfile and then try to open the new cache
 339         // slot, we might get a race condition with a concurrent
 340         // writer for the same cache slot (with a different key).
 341         // Lets avoid such a race by just printing the content of
 342         // the lock file.
 343         slot->cache_fd = slot->lock_fd;
 344         unlock_slot(slot, 1);
 345         if ((err = print_slot(slot)) != 0) {
 346                 cache_log("[cgit] error printing cache %s: %s (%d)\n",
 347                           slot->cache_name,
 348                           strerror(err),
 349                           err);
 350         }
 351         close_slot(slot);
 352         return err;
 353 }
 354
 355 /* Print cached content to stdout, generate the content if necessary. */
 356 int cache_process(int size, const char *path, const char *key, int ttl,
 357                   cache_fill_fn fn)
 358 {
 359         unsigned long hash;
 360         int i;
 361         struct strbuf filename = STRBUF_INIT;
 362         struct strbuf lockname = STRBUF_INIT;
 363         struct cache_slot slot;
 364         int result;
 365
 366         /* If the cache is disabled, just generate the content */
 367         if (size <= 0 || ttl == 0) {
 368                 fn();
 369                 return 0;
 370         }
 371
 372         /* Verify input, calculate filenames */
 373         if (!path) {
 374                 cache_log("[cgit] Cache path not specified, caching is disabled\n");
 375                 fn();
 376                 return 0;
 377         }
 378         if (!key)
 379                 key = "";
 380         hash = hash_str(key) % size;
 381         strbuf_addstr(&filename, path);
 382         strbuf_ensure_end(&filename, '/');
 383         for (i = 0; i < 8; i++) {
 384                 strbuf_addf(&filename, "%x", (unsigned char)(hash & 0xf));
 385                 hash >>= 4;
 386         }
 387         strbuf_addbuf(&lockname, &filename);
 388         strbuf_addstr(&lockname, ".lock");
 389         slot.fn = fn;
 390         slot.ttl = ttl;
 391         slot.stdout_fd = -1;
 392         slot.cache_name = filename.buf;
 393         slot.lock_name = lockname.buf;
 394         slot.key = key;
 395         slot.keylen = strlen(key);
 396         result = process_slot(&slot);
 397
 398         strbuf_release(&filename);
 399         strbuf_release(&lockname);
 400         return result;
 401 }
 402
 403 /* Return a strftime formatted date/time
 404  * NB: the result from this function is to shared memory
 405  */
 406 static char *sprintftime(const char *format, time_t time)
 407 {
 408         static char buf[64];
 409         struct tm tm;
 410
 411         if (!time)
 412                 return NULL;
 413         gmtime_r(&time, &tm);
 414         strftime(buf, sizeof(buf)-1, format, &tm);
 415         return buf;
 416 }
 417
 418 int cache_ls(const char *path)
 419 {
 420         DIR *dir;
 421         struct dirent *ent;
 422         int err = 0;
 423         struct cache_slot slot = { NULL };
 424         struct strbuf fullname = STRBUF_INIT;
 425         size_t prefixlen;
 426
 427         if (!path) {
 428                 cache_log("[cgit] cache path not specified\n");
 429                 return -1;
 430         }
 431         dir = opendir(path);
 432         if (!dir) {
 433                 err = errno;
 434                 cache_log("[cgit] unable to open path %s: %s (%d)\n",
 435                           path, strerror(err), err);
 436                 return err;
 437         }
 438         strbuf_addstr(&fullname, path);
 439         strbuf_ensure_end(&fullname, '/');
 440         prefixlen = fullname.len;
 441         while ((ent = readdir(dir)) != NULL) {
 442                 if (strlen(ent->d_name) != 8)
 443                         continue;
 444                 strbuf_setlen(&fullname, prefixlen);
 445                 strbuf_addstr(&fullname, ent->d_name);
 446                 slot.cache_name = fullname.buf;
 447                 if ((err = open_slot(&slot)) != 0) {
 448                         cache_log("[cgit] unable to open path %s: %s (%d)\n",
 449                                   fullname.buf, strerror(err), err);
 450                         continue;
 451                 }
 452                 htmlf("%s %s %10"PRIuMAX" %s\n",
 453                       fullname.buf,
 454                       sprintftime("%Y-%m-%d %H:%M:%S",
 455                                   slot.cache_st.st_mtime),
 456                       (uintmax_t)slot.cache_st.st_size,
 457                       slot.buf);
 458                 close_slot(&slot);
 459         }
 460         closedir(dir);
 461         strbuf_release(&fullname);
 462         return 0;
 463 }
 464
 465 /* Print a message to stdout */
 466 void cache_log(const char *format, ...)
 467 {
 468         va_list args;
 469         va_start(args, format);
 470         vfprintf(stderr, format, args);
 471         va_end(args);
 472 }
 473