httpfs.c

   1 #define FUSE_USE_VERSION 26
   2
   3 #include <fuse.h>
   4 #include <pthread.h>
   5 #include <stdio.h>
   6 #include <string.h>
   7 #include <stdlib.h>
   8 #include <errno.h>
   9 #include <fcntl.h>
  10 #include <glib.h>
  11 #include <curl/curl.h>
  12 #include "cache.h"
  13 #include "grow.h"
  14 #include "fetch.h"
  15 #include "parser.h"
  16 #include "urlencode.h"
  17 #include "canonize.h"
  18 #include "debug.h"
  19
  20 #define INDEX_FILE "index.httpfs.html"
  21 #define IN_ORDER_THRESHOLD      (512*1024)
  22 #define OUT_ORDER_THRESHOLD     (1024*1024)
  23 #define SEEK_FORWARD_THRESHOLD  (64*1024)
  24 /*
  25  * TODO
  26  *      konfigurovatelnost
  27  *      lepsi parsery (pro apache)
  28  *      readahead
  29  *      ftp support?
  30  */
  31
  32 char *base_url;
  33 CURL *conn;
  34 int show_index = 1;
  35 pthread_mutex_t conn_lock = PTHREAD_MUTEX_INITIALIZER;
  36
  37 static inline CURL *get_conn(void)
  38 {
  39         pthread_mutex_lock(&conn_lock);
  40         return conn;
  41 }
  42
  43 static inline void put_conn(CURL *conn)
  44 {
  45         pthread_mutex_unlock(&conn_lock);
  46 }
  47
  48 static inline int is_a_index(char *s)
  49 {
  50         char *bn;
  51
  52         if (!show_index)
  53                 return 0;
  54
  55         bn = strrchr(s, '/');
  56         if (bn && !strcmp(bn+1, INDEX_FILE)) {
  57                 *(bn+1) = 0;
  58                 return 1;
  59         }
  60         return 0;
  61 }
  62
  63 void mkurl(struct grow *url, const char *path, int slashend)
  64 {
  65         grow_init(url);
  66         grow_puts(url, base_url);
  67         path_encode(url, path);
  68
  69         if (url->s[url->ptr-1] == '/') {
  70                 url->s[url->ptr-1] = 0;
  71                 url->ptr--;
  72         }
  73
  74         if (slashend)
  75                 grow_puts(url, "/");
  76         else
  77                 grow_put(url, 0);
  78 }
  79
  80 static int chunk_init(struct handle *h)
  81 {
  82         h->type = HAND_RANDOM;
  83         grow_init(&h->data);
  84         return 0;
  85 }
  86
  87 static int chunk_read(struct handle *h, char *buf, size_t size, off_t offset)
  88 {
  89         CURL *conn;
  90         int ret;
  91
  92         grow_init_static(&h->data, buf, size);
  93
  94         conn = get_conn();
  95         if (!conn)
  96                 return -EIO;
  97         ret = request(conn, h->url.s, 0, NULL, &h->data, size, offset);
  98         if (ret == -ESPIPE)
  99                 ret = 0;
 100         put_conn(conn);
 101         return ret;
 102 }
 103
 104 static void chunk_fini(struct handle *h)
 105 {
 106         return;
 107 }
 108
 109
 110 /* actually, this will copy data several times:
 111  * network stack -> curl buffer
 112  * curl buffer -> grow buffer
 113  * grow buffer -> fuse buffer
 114  * fuse buffer -> kernel buffer
 115  * kernel buffer -> user buffer
 116  *
 117  * that's FIVE TIMES!
 118  */
 119 #define min(a,b) ((a)<(b)?(a):(b))
 120
 121 static int seq_init(struct handle *h, off_t offset, size_t size)
 122 {
 123         int ret;
 124
 125         ret = request_open_async(h, offset, size);
 126         if (ret < 0)
 127                 return ret;
 128
 129         grow_init(&h->lover);
 130         h->type = HAND_HARD;
 131         return 0;
 132 }
 133
 134 static void seq_fini(struct handle *h)
 135 {
 136         if (!request_done(h))
 137                 request_close_async(h);
 138         grow_free(&h->lover);
 139 }
 140
 141 static int seq_read(struct handle *h, char *buf, size_t size, off_t offset)
 142 {
 143         int done = 0, ret = 0;
 144         struct grow *lover = &h->lover;
 145
 146         if (request_done(h)) {
 147                 if (lover->ptr)
 148                         goto fill;
 149                 return ret;
 150         }
 151
 152         if (lover->ptr < size) {
 153                 ret = request_perf_async(h, size, &done);
 154                 if (done)
 155                         request_close_async(h);
 156                 /* TODO sem prijde reconnect na timeout */
 157         }
 158
 159         if (ret < 0)
 160                 return ret;
 161
 162 fill:
 163         ret = min(lover->ptr, size);
 164         if (buf) {
 165                 memcpy(buf, lover->s, ret);
 166         }
 167         lover->ptr -= ret;
 168         memmove(lover->s, lover->s+ret, lover->ptr);
 169
 170         return ret;
 171 }
 172
 173
 174 /* TODO: mergnout to s getattrem (hint: isindex a pak zmenit prava) */
 175 static int determine_size(const char *path, off_t *size)
 176 {
 177         CURL *conn;
 178         struct grow url;
 179         struct stat stb;
 180         int ret, index;
 181
 182         memset(&stb, 0, sizeof(struct stat));
 183         stb.st_size = -1;
 184
 185         conn = get_conn();
 186         if (!conn)
 187                 return -EIO;
 188
 189         mkurl(&url, path, 0);
 190         index = is_a_index(url.s);
 191         ret = request(conn, url.s, 1, &stb, NULL, 0, 0);
 192         put_conn(conn);
 193         grow_free(&url);
 194
 195         *size = stb.st_size;
 196         return ret;
 197 }
 198
 199
 200 static int httpfs_getdir(const char *path, fuse_cache_dirh_t h, fuse_cache_dirfil_t filler)
 201 {
 202         struct stat stb;
 203         struct grow url, page = GROW_INIT;
 204         struct canon c = { .h = h, .filler = filler };
 205         int ret = 0;
 206         CURL *conn;
 207
 208         conn = get_conn();
 209         if (!conn)
 210                 return -EIO;
 211
 212         memset(&stb, 0, sizeof(stb));
 213         stb.st_size = 0;
 214         stb.st_mode = S_IFDIR | 0555;
 215
 216         filler(h, ".", &stb);
 217         filler(h, "..", &stb);
 218
 219         if (show_index) {
 220                 stb.st_mode = S_IFREG | 0444;
 221                 filler(h, INDEX_FILE, &stb);
 222         }
 223
 224         mkurl(&url, path, 1);
 225
 226         ret = request(conn, url.s, 0, NULL, &page, 0, 0);
 227         if (ret < 0)
 228                 goto err;
 229
 230         ret = 0;
 231         grow_put(&page, 0);
 232         parse(page.s, canonize, &c);
 233 err:
 234         grow_free(&url);
 235         grow_free(&page);
 236         put_conn(conn);
 237         return ret;
 238 }
 239
 240
 241 static int httpfs_getattr(const char *path, struct stat *stb)
 242 {
 243         struct grow url;
 244         CURL *conn;
 245         int ret, index;
 246
 247         memset(stb, 0, sizeof(struct stat));
 248
 249         if (!strcmp(path, "/")) {
 250                 stb->st_mode = S_IFDIR | 0555;
 251                 stb->st_size = 0;
 252                 return 0;
 253         }
 254
 255         conn = get_conn();
 256         if (!conn)
 257                 return -EIO;
 258
 259         mkurl(&url, path, 0);
 260         index = is_a_index(url.s);
 261
 262         stb->st_mode = S_IFREG | 0444;
 263         ret = request(conn, url.s, 1, stb, NULL, 0, 0);
 264         if (ret == -EMLINK) {
 265                 stb->st_mode = S_IFDIR | 0555;
 266                 stb->st_size = 0;
 267                 ret = 0;
 268         }
 269
 270         if (index)
 271                 stb->st_mode = S_IFREG | 0555;
 272
 273         put_conn(conn);
 274         grow_free(&url);
 275
 276         return ret;
 277 }
 278
 279 static int httpfs_open(const char *path, struct fuse_file_info *fi)
 280 {
 281         struct handle *h;
 282         int ret;
 283         off_t size;
 284
 285         dbg("open of '%s' with mode '%x'\n", path, fi->flags);
 286
 287         if ((fi->flags & 3) != O_RDONLY)
 288                 return -EACCES;
 289
 290         /* dry run, determine size and/or support of partial content */
 291         ret = determine_size(path, &size);
 292         if (ret < 0)
 293                 return ret;
 294         dbg("page '%s': %s\n", path, size < 0 ? "no content len" : "content len");
 295
 296
 297         h = calloc(sizeof(*h), 1);
 298         mkurl(&h->url, path, 0);
 299         is_a_index(h->url.s);
 300         pthread_mutex_init(&h->async_lock, NULL);
 301
 302         if (size >= 0) {
 303                 h->total = size;
 304                 chunk_init(h);
 305         } else {
 306                 seq_init(h, 0, 0);
 307         }
 308 /*      h->type |= HAND_HARD; */
 309
 310         fi->fh = (unsigned long) h;
 311         return 0;
 312 }
 313
 314
 315 static int httpfs_read(const char *path, char *buf, size_t size,
 316                        off_t offset, struct fuse_file_info *fi)
 317 {
 318         struct handle *h;
 319         int ret;
 320         h = (struct handle *) (uintptr_t) fi->fh;
 321
 322         pthread_mutex_lock(&h->async_lock);
 323         if (!(h->type & HAND_HARD) && offset < h->total) {
 324                 if ((h->type & HAND_RANDOM) && h->in_order > IN_ORDER_THRESHOLD) {
 325                         dbg("READ: switching to SEQUENTIAL\n");
 326                         chunk_fini(h);
 327                         ret = seq_init(h, offset, h->total - offset);
 328                         if (ret) {
 329                                 dbg("READ: cannot switch to sequential (%d), forcing SOFT HARD\n", ret);
 330                                 h->type = HAND_RANDOM | HAND_HARD;
 331                         } else {
 332                                 h->type = 0;
 333                         }
 334                 } else if (!(h->type & HAND_RANDOM) && h->out_order > OUT_ORDER_THRESHOLD) {
 335                         dbg("READ: switching to RANDOM\n");
 336                         seq_fini(h);
 337                         chunk_init(h);
 338                         h->type = HAND_RANDOM;
 339                 }
 340         }
 341
 342         dbg("read: %s %s ho:%lld io:%lld oo:%lld o:%lld s:%llu\n", (h->type&HAND_HARD)?"hard":"soft", (h->type&HAND_RANDOM)?"rand":"seq", h->offset, h->in_order, h->out_order, offset, size);
 343
 344         if (h->offset == offset) {
 345                 if (h->type & HAND_RANDOM) {
 346                         dbg("READ: reading next chunk RANDOM\n");
 347                         ret = chunk_read(h, buf, size, offset);
 348                 } else {
 349                         dbg("READ: reading next chunk SEQUENTIAL\n");
 350                         ret = seq_read(h, buf, size, offset);
 351                 }
 352                 if (ret >= 0) {
 353                         h->offset += ret;
 354                         h->in_order += ret;
 355                         h->out_order = 0;
 356                 } else {
 357                         h->in_order = 0;
 358                         h->offset = -1;
 359                 }
 360         } else {
 361                 if ((h->type & HAND_HARD) && !(h->type & HAND_RANDOM)) {
 362                         dbg("READ: reading chunk HARD SEQUENTIAL out of order\n");
 363                         ret = -ESPIPE;
 364                         if (h->offset < offset && offset - h->offset < h->total) {
 365                                 dbg("READ: small skip forwards: %d\n", offset - h->offset);
 366                                 seq_read(h, NULL, offset - h->offset, h->offset);
 367                                 ret = seq_read(h, buf, size, offset);
 368                                 if (ret >= 0) {
 369                                         h->offset = offset + ret;
 370                                 }
 371                         }
 372                 } else if (h->type & HAND_RANDOM) {
 373                         dbg("READ: reading chunk SOFT RANDOM out of order\n");
 374                         ret = chunk_read(h, buf, size, offset);
 375                         if (ret >= 0) {
 376                                 h->offset = offset+ret;
 377                                 h->in_order = size;
 378                         }
 379                 } else {
 380                         dbg("READ: reading chunk SOFT SEQUENTIAL out of order with chunk\n");
 381                         ret = chunk_read(h, buf, size, offset);
 382                         if (ret >= 0) {
 383                                 h->out_order += size;
 384                         }
 385                 }
 386         }
 387         pthread_mutex_unlock(&h->async_lock);
 388         return ret;
 389 }
 390
 391 static int httpfs_release(const char *path, struct fuse_file_info *fi)
 392 {
 393         struct handle *h;
 394         h = (struct handle *) (uintptr_t) fi->fh;
 395
 396         if (h->type & HAND_RANDOM) {
 397                 chunk_fini(h);
 398         } else {
 399                 seq_fini(h);
 400         }
 401         grow_free(&h->url);
 402         free(h);
 403         return 0;
 404 }
 405
 406 static int httpfs_opendir(const char *path, struct fuse_file_info *fi)
 407 {
 408         return 0;
 409 }
 410
 411 static int httpfs_releasedir(const char *path, struct fuse_file_info *fi)
 412 {
 413         return 0;
 414 }
 415
 416
 417
 418 static struct fuse_cache_operations httpfs_oper = {
 419         .oper = {
 420                  .getattr = httpfs_getattr,
 421                  .open = httpfs_open,
 422                  .release = httpfs_release,
 423                  .opendir = httpfs_opendir,
 424                  .releasedir = httpfs_releasedir,
 425                  .read = httpfs_read,
 426                  },
 427         .cache_getdir = httpfs_getdir,
 428 };
 429
 430
 431
 432 static int httpfs_fuse_main(struct fuse_args *args)
 433 {
 434 #if FUSE_VERSION >= 26
 435         return fuse_main(args->argc, args->argv, cache_init(&httpfs_oper), NULL);
 436 #else
 437         return fuse_main(args->argc, args->argv, cache_init(&httpfs_oper));
 438 #endif
 439 }
 440
 441
 442 int main(int argc, char *argv[])
 443 {
 444         struct fuse_args args = FUSE_ARGS_INIT(argc-1, argv+1);
 445         int res;
 446
 447         g_thread_init(NULL);
 448
 449         base_url = argv[1];
 450         if (base_url[strlen(base_url)-1] == '/')
 451                 base_url[strlen(base_url)-1] = 0;
 452
 453         conn = curl_easy_init();
 454
 455         res = cache_parse_options(&args);
 456         if (res == -1)
 457                 exit(1);
 458
 459         res = httpfs_fuse_main(&args);
 460         fuse_opt_free_args(&args);
 461
 462         return res;
 463 }