c_cpp/lib/htslib/hfile.c

   1 /*  hfile.c -- buffered low-level input/output streams.
   2
   3     Copyright (C) 2013-2016 Genome Research Ltd.
   4
   5     Author: John Marshall <jm18@sanger.ac.uk>
   6
   7 Permission is hereby granted, free of charge, to any person obtaining a copy
   8 of this software and associated documentation files (the "Software"), to deal
   9 in the Software without restriction, including without limitation the rights
  10 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11 copies of the Software, and to permit persons to whom the Software is
  12 furnished to do so, subject to the following conditions:
  13
  14 The above copyright notice and this permission notice shall be included in
  15 all copies or substantial portions of the Software.
  16
  17 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  20 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23 DEALINGS IN THE SOFTWARE.  */
  24
  25 #include <config.h>
  26
  27 #include <stdio.h>
  28 #include <stdlib.h>
  29 #include <stddef.h>
  30 #include <string.h>
  31 #include <errno.h>
  32 #include <limits.h>
  33
  34 #include <pthread.h>
  35
  36 #include "htslib/hfile.h"
  37 #include "hfile_internal.h"
  38
  39 #ifndef ENOTSUP
  40 #define ENOTSUP EINVAL
  41 #endif
  42 #ifndef EOVERFLOW
  43 #define EOVERFLOW ERANGE
  44 #endif
  45 #ifndef EPROTONOSUPPORT
  46 #define EPROTONOSUPPORT ENOSYS
  47 #endif
  48
  49 #ifndef SSIZE_MAX /* SSIZE_MAX is POSIX 1 */
  50 #define SSIZE_MAX LONG_MAX
  51 #endif
  52
  53 /* hFILE fields are used as follows:
  54
  55    char *buffer;     // Pointer to the start of the I/O buffer
  56    char *begin;      // First not-yet-read character / unused position
  57    char *end;        // First unfilled/unfillable position
  58    char *limit;      // Pointer to the first position past the buffer
  59
  60    const hFILE_backend *backend;  // Methods to refill/flush I/O buffer
  61
  62    off_t offset;     // Offset within the stream of buffer position 0
  63    unsigned at_eof:1;// For reading, whether EOF has been seen
  64    unsigned mobile:1;// Buffer is a mobile window or fixed full contents
  65    unsigned readonly:1;// Whether opened as "r" rather than "r+"/"w"/"a"
  66    int has_errno;    // Error number from the last failure on this stream
  67
  68 For reading, begin is the first unread character in the buffer and end is the
  69 first unfilled position:
  70
  71    -----------ABCDEFGHIJKLMNO---------------
  72    ^buffer    ^begin         ^end           ^limit
  73
  74 For writing, begin is the first unused position and end is unused so remains
  75 equal to buffer:
  76
  77    ABCDEFGHIJKLMNOPQRSTUVWXYZ---------------
  78    ^buffer                   ^begin         ^limit
  79    ^end
  80
  81 Thus if begin > end then there is a non-empty write buffer, if begin < end
  82 then there is a non-empty read buffer, and if begin == end then both buffers
  83 are empty.  In all cases, the stream's file position indicator corresponds
  84 to the position pointed to by begin.
  85
  86 The above is the normal scenario of a mobile window.  For in-memory streams,
  87 a fixed (immobile) buffer can be used as the full contents without any separate
  88 backend behind it.  These always have at_eof set, offset set to 0, need no
  89 read() method, and should just return EINVAL for seek():
  90
  91    abcdefghijkLMNOPQRSTUVWXYZ------
  92    ^buffer    ^begin         ^end  ^limit
  93
  94 Use hfile_init_fixed() to create one of these.  */
  95
  96 hFILE *hfile_init(size_t struct_size, const char *mode, size_t capacity)
  97 {
  98     hFILE *fp = (hFILE *) malloc(struct_size);
  99     if (fp == NULL) goto error;
 100
 101     if (capacity == 0) capacity = 32768;
 102     // FIXME For now, clamp input buffer sizes so mpileup doesn't eat memory
 103     if (strchr(mode, 'r') && capacity > 32768) capacity = 32768;
 104
 105     fp->buffer = (char *) malloc(capacity);
 106     if (fp->buffer == NULL) goto error;
 107
 108     fp->begin = fp->end = fp->buffer;
 109     fp->limit = &fp->buffer[capacity];
 110
 111     fp->offset = 0;
 112     fp->at_eof = 0;
 113     fp->mobile = 1;
 114     fp->readonly = (strchr(mode, 'r') && ! strchr(mode, '+'));
 115     fp->has_errno = 0;
 116     return fp;
 117
 118 error:
 119     hfile_destroy(fp);
 120     return NULL;
 121 }
 122
 123 hFILE *hfile_init_fixed(size_t struct_size, const char *mode,
 124                         char *buffer, size_t buf_filled, size_t buf_size)
 125 {
 126     hFILE *fp = (hFILE *) malloc(struct_size);
 127     if (fp == NULL) return NULL;
 128
 129     fp->buffer = fp->begin = buffer;
 130     fp->end = &fp->buffer[buf_filled];
 131     fp->limit = &fp->buffer[buf_size];
 132
 133     fp->offset = 0;
 134     fp->at_eof = 1;
 135     fp->mobile = 0;
 136     fp->readonly = (strchr(mode, 'r') && ! strchr(mode, '+'));
 137     fp->has_errno = 0;
 138     return fp;
 139 }
 140
 141 void hfile_destroy(hFILE *fp)
 142 {
 143     int save = errno;
 144     if (fp) free(fp->buffer);
 145     free(fp);
 146     errno = save;
 147 }
 148
 149 static inline int writebuffer_is_nonempty(hFILE *fp)
 150 {
 151     return fp->begin > fp->end;
 152 }
 153
 154 /* Refills the read buffer from the backend (once, so may only partially
 155    fill the buffer), returning the number of additional characters read
 156    (which might be 0), or negative when an error occurred.  */
 157 static ssize_t refill_buffer(hFILE *fp)
 158 {
 159     ssize_t n;
 160
 161     // Move any unread characters to the start of the buffer
 162     if (fp->mobile && fp->begin > fp->buffer) {
 163         fp->offset += fp->begin - fp->buffer;
 164         memmove(fp->buffer, fp->begin, fp->end - fp->begin);
 165         fp->end = &fp->buffer[fp->end - fp->begin];
 166         fp->begin = fp->buffer;
 167     }
 168
 169     // Read into the available buffer space at fp->[end,limit)
 170     if (fp->at_eof || fp->end == fp->limit) n = 0;
 171     else {
 172         n = fp->backend->read(fp, fp->end, fp->limit - fp->end);
 173         if (n < 0) { fp->has_errno = errno; return n; }
 174         else if (n == 0) fp->at_eof = 1;
 175     }
 176
 177     fp->end += n;
 178     return n;
 179 }
 180
 181 /*
 182  * Changes the buffer size for an hFILE.  Ideally this is done
 183  * immediately after opening.  If performed later, this function may
 184  * fail if we are reducing the buffer size and the current offset into
 185  * the buffer is beyond the new capacity.
 186  *
 187  * Returns 0 on success;
 188  *        -1 on failure.
 189  */
 190 int hfile_set_blksize(hFILE *fp, size_t bufsiz) {
 191     char *buffer;
 192     ptrdiff_t curr_used;
 193     if (!fp) return -1;
 194     curr_used = (fp->begin > fp->end ? fp->begin : fp->end) - fp->buffer;
 195     if (bufsiz == 0) bufsiz = 32768;
 196
 197     // Ensure buffer resize will not erase live data
 198     if (bufsiz < curr_used)
 199         return -1;
 200
 201     if (!(buffer = (char *) realloc(fp->buffer, bufsiz))) return -1;
 202
 203     fp->begin  = buffer + (fp->begin - fp->buffer);
 204     fp->end    = buffer + (fp->end   - fp->buffer);
 205     fp->buffer = buffer;
 206     fp->limit  = &fp->buffer[bufsiz];
 207
 208     return 0;
 209 }
 210
 211 /* Called only from hgetc(), when our buffer is empty.  */
 212 int hgetc2(hFILE *fp)
 213 {
 214     return (refill_buffer(fp) > 0)? (unsigned char) *(fp->begin++) : EOF;
 215 }
 216
 217 ssize_t hgetdelim(char *buffer, size_t size, int delim, hFILE *fp)
 218 {
 219     char *found;
 220     size_t n, copied = 0;
 221     ssize_t got;
 222
 223     if (size < 1 || size > SSIZE_MAX) {
 224         fp->has_errno = errno = EINVAL;
 225         return -1;
 226     }
 227     if (writebuffer_is_nonempty(fp)) {
 228         fp->has_errno = errno = EBADF;
 229         return -1;
 230     }
 231
 232     --size; /* to allow space for the NUL terminator */
 233
 234     do {
 235         n = fp->end - fp->begin;
 236         if (n > size - copied) n = size - copied;
 237
 238         /* Look in the hFILE buffer for the delimiter */
 239         found = memchr(fp->begin, delim, n);
 240         if (found != NULL) {
 241             n = found - fp->begin + 1;
 242             memcpy(buffer + copied, fp->begin, n);
 243             buffer[n + copied] = '\0';
 244             fp->begin += n;
 245             return n + copied;
 246         }
 247
 248         /* No delimiter yet, copy as much as we can and refill if necessary */
 249         memcpy(buffer + copied, fp->begin, n);
 250         fp->begin += n;
 251         copied += n;
 252
 253         if (copied == size) { /* Output buffer full */
 254             buffer[copied] = '\0';
 255             return copied;
 256         }
 257
 258         got = refill_buffer(fp);
 259     } while (got > 0);
 260
 261     if (got < 0) return -1; /* Error on refill. */
 262
 263     buffer[copied] = '\0';  /* EOF, return anything that was copied. */
 264     return copied;
 265 }
 266
 267 char *hgets(char *buffer, int size, hFILE *fp)
 268 {
 269     if (size < 1) {
 270         fp->has_errno = errno = EINVAL;
 271         return NULL;
 272     }
 273     return hgetln(buffer, size, fp) > 0 ? buffer : NULL;
 274 }
 275
 276 ssize_t hpeek(hFILE *fp, void *buffer, size_t nbytes)
 277 {
 278     size_t n = fp->end - fp->begin;
 279     while (n < nbytes) {
 280         ssize_t ret = refill_buffer(fp);
 281         if (ret < 0) return ret;
 282         else if (ret == 0) break;
 283         else n += ret;
 284     }
 285
 286     if (n > nbytes) n = nbytes;
 287     memcpy(buffer, fp->begin, n);
 288     return n;
 289 }
 290
 291 /* Called only from hread(); when called, our buffer is empty and nread bytes
 292    have already been placed in the destination buffer.  */
 293 ssize_t hread2(hFILE *fp, void *destv, size_t nbytes, size_t nread)
 294 {
 295     const size_t capacity = fp->limit - fp->buffer;
 296     int buffer_invalidated = 0;
 297     char *dest = (char *) destv;
 298     dest += nread, nbytes -= nread;
 299
 300     // Read large requests directly into the destination buffer
 301     while (nbytes * 2 >= capacity && !fp->at_eof) {
 302         ssize_t n = fp->backend->read(fp, dest, nbytes);
 303         if (n < 0) { fp->has_errno = errno; return n; }
 304         else if (n == 0) fp->at_eof = 1;
 305         else buffer_invalidated = 1;
 306         fp->offset += n;
 307         dest += n, nbytes -= n;
 308         nread += n;
 309     }
 310
 311     if (buffer_invalidated) {
 312         // Our unread buffer is empty, so begin == end, but our already-read
 313         // buffer [buffer,begin) is likely non-empty and is no longer valid as
 314         // its contents are no longer adjacent to the file position indicator.
 315         // Discard it so that hseek() can't try to take advantage of it.
 316         fp->offset += fp->begin - fp->buffer;
 317         fp->begin = fp->end = fp->buffer;
 318     }
 319
 320     while (nbytes > 0 && !fp->at_eof) {
 321         size_t n;
 322         ssize_t ret = refill_buffer(fp);
 323         if (ret < 0) return ret;
 324
 325         n = fp->end - fp->begin;
 326         if (n > nbytes) n = nbytes;
 327         memcpy(dest, fp->begin, n);
 328         fp->begin += n;
 329         dest += n, nbytes -= n;
 330         nread += n;
 331     }
 332
 333     return nread;
 334 }
 335
 336 /* Flushes the write buffer, fp->[buffer,begin), out through the backend
 337    returning 0 on success or negative if an error occurred.  */
 338 static ssize_t flush_buffer(hFILE *fp)
 339 {
 340     const char *buffer = fp->buffer;
 341     while (buffer < fp->begin) {
 342         ssize_t n = fp->backend->write(fp, buffer, fp->begin - buffer);
 343         if (n < 0) { fp->has_errno = errno; return n; }
 344         buffer += n;
 345         fp->offset += n;
 346     }
 347
 348     fp->begin = fp->buffer;  // Leave the buffer empty
 349     return 0;
 350 }
 351
 352 int hflush(hFILE *fp)
 353 {
 354     if (flush_buffer(fp) < 0) return EOF;
 355     if (fp->backend->flush) {
 356         if (fp->backend->flush(fp) < 0) { fp->has_errno = errno; return EOF; }
 357     }
 358     return 0;
 359 }
 360
 361 /* Called only from hputc(), when our buffer is already full.  */
 362 int hputc2(int c, hFILE *fp)
 363 {
 364     if (flush_buffer(fp) < 0) return EOF;
 365     *(fp->begin++) = c;
 366     return c;
 367 }
 368
 369 /* Called only from hwrite() and hputs2(); when called, our buffer is full and
 370    ncopied bytes from the source have already been copied to our buffer.  */
 371 ssize_t hwrite2(hFILE *fp, const void *srcv, size_t totalbytes, size_t ncopied)
 372 {
 373     const char *src = (const char *) srcv;
 374     ssize_t ret;
 375     const size_t capacity = fp->limit - fp->buffer;
 376     size_t remaining = totalbytes - ncopied;
 377     src += ncopied;
 378
 379     ret = flush_buffer(fp);
 380     if (ret < 0) return ret;
 381
 382     // Write large blocks out directly from the source buffer
 383     while (remaining * 2 >= capacity) {
 384         ssize_t n = fp->backend->write(fp, src, remaining);
 385         if (n < 0) { fp->has_errno = errno; return n; }
 386         fp->offset += n;
 387         src += n, remaining -= n;
 388     }
 389
 390     // Just buffer any remaining characters
 391     memcpy(fp->begin, src, remaining);
 392     fp->begin += remaining;
 393
 394     return totalbytes;
 395 }
 396
 397 /* Called only from hputs(), when our buffer is already full.  */
 398 int hputs2(const char *text, size_t totalbytes, size_t ncopied, hFILE *fp)
 399 {
 400     return (hwrite2(fp, text, totalbytes, ncopied) >= 0)? 0 : EOF;
 401 }
 402
 403 off_t hseek(hFILE *fp, off_t offset, int whence)
 404 {
 405     off_t curpos, pos;
 406
 407     if (writebuffer_is_nonempty(fp)) {
 408         int ret = flush_buffer(fp);
 409         if (ret < 0) return ret;
 410     }
 411
 412     curpos = htell(fp);
 413
 414     // Relative offsets are given relative to the hFILE's stream position,
 415     // which may differ from the backend's physical position due to buffering
 416     // read-ahead.  Correct for this by converting to an absolute position.
 417     if (whence == SEEK_CUR) {
 418         if (curpos + offset < 0) {
 419             // Either a negative offset resulted in a position before the
 420             // start of the file, or we overflowed when given a positive offset
 421             fp->has_errno = errno = (offset < 0)? EINVAL : EOVERFLOW;
 422             return -1;
 423         }
 424
 425         whence = SEEK_SET;
 426         offset = curpos + offset;
 427     }
 428     // For fixed immobile buffers, convert everything else to SEEK_SET too
 429     // so that seeking can be avoided for all (within range) requests.
 430     else if (! fp->mobile && whence == SEEK_END) {
 431         size_t length = fp->end - fp->buffer;
 432         if (offset > 0 || -offset > length) {
 433             fp->has_errno = errno = EINVAL;
 434             return -1;
 435         }
 436
 437         whence = SEEK_SET;
 438         offset = length + offset;
 439     }
 440
 441     // Avoid seeking if the desired position is within our read buffer.
 442     // (But not when the next operation may be a write on a mobile buffer.)
 443     if (whence == SEEK_SET && (! fp->mobile || fp->readonly) &&
 444         offset >= fp->offset && offset - fp->offset <= fp->end - fp->buffer) {
 445         fp->begin = &fp->buffer[offset - fp->offset];
 446         return offset;
 447     }
 448
 449     pos = fp->backend->seek(fp, offset, whence);
 450     if (pos < 0) { fp->has_errno = errno; return pos; }
 451
 452     // Seeking succeeded, so discard any non-empty read buffer
 453     fp->begin = fp->end = fp->buffer;
 454     fp->at_eof = 0;
 455
 456     fp->offset = pos;
 457     return pos;
 458 }
 459
 460 int hclose(hFILE *fp)
 461 {
 462     int err = fp->has_errno;
 463
 464     if (writebuffer_is_nonempty(fp) && hflush(fp) < 0) err = fp->has_errno;
 465     if (fp->backend->close(fp) < 0) err = errno;
 466     hfile_destroy(fp);
 467
 468     if (err) {
 469         errno = err;
 470         return EOF;
 471     }
 472     else return 0;
 473 }
 474
 475 void hclose_abruptly(hFILE *fp)
 476 {
 477     int save = errno;
 478     if (fp->backend->close(fp) < 0) { /* Ignore subsequent errors */ }
 479     hfile_destroy(fp);
 480     errno = save;
 481 }
 482
 483
 484 /***************************
 485  * File descriptor backend *
 486  ***************************/
 487
 488 #ifndef _WIN32
 489 #include <sys/socket.h>
 490 #include <sys/stat.h>
 491 #define HAVE_STRUCT_STAT_ST_BLKSIZE
 492 #else
 493 #include <winsock2.h>
 494 #define HAVE_CLOSESOCKET
 495 #define HAVE_SETMODE
 496 #endif
 497 #include <fcntl.h>
 498 #include <unistd.h>
 499
 500 /* For Unix, it doesn't matter whether a file descriptor is a socket.
 501    However Windows insists on send()/recv() and its own closesocket()
 502    being used when fd happens to be a socket.  */
 503
 504 typedef struct {
 505     hFILE base;
 506     int fd;
 507     unsigned is_socket:1;
 508 } hFILE_fd;
 509
 510 static ssize_t fd_read(hFILE *fpv, void *buffer, size_t nbytes)
 511 {
 512     hFILE_fd *fp = (hFILE_fd *) fpv;
 513     ssize_t n;
 514     do {
 515         n = fp->is_socket? recv(fp->fd, buffer, nbytes, 0)
 516                          : read(fp->fd, buffer, nbytes);
 517     } while (n < 0 && errno == EINTR);
 518     return n;
 519 }
 520
 521 static ssize_t fd_write(hFILE *fpv, const void *buffer, size_t nbytes)
 522 {
 523     hFILE_fd *fp = (hFILE_fd *) fpv;
 524     ssize_t n;
 525     do {
 526         n = fp->is_socket?  send(fp->fd, buffer, nbytes, 0)
 527                          : write(fp->fd, buffer, nbytes);
 528     } while (n < 0 && errno == EINTR);
 529     return n;
 530 }
 531
 532 static off_t fd_seek(hFILE *fpv, off_t offset, int whence)
 533 {
 534     hFILE_fd *fp = (hFILE_fd *) fpv;
 535     return lseek(fp->fd, offset, whence);
 536 }
 537
 538 static int fd_flush(hFILE *fpv)
 539 {
 540     hFILE_fd *fp = (hFILE_fd *) fpv;
 541     int ret;
 542     do {
 543 #ifdef HAVE_FDATASYNC
 544         ret = fdatasync(fp->fd);
 545 #else
 546         ret = fsync(fp->fd);
 547 #endif
 548         // Ignore invalid-for-fsync(2) errors due to being, e.g., a pipe,
 549         // and operation-not-supported errors (Mac OS X)
 550         if (ret < 0 && (errno == EINVAL || errno == ENOTSUP)) ret = 0;
 551     } while (ret < 0 && errno == EINTR);
 552     return ret;
 553 }
 554
 555 static int fd_close(hFILE *fpv)
 556 {
 557     hFILE_fd *fp = (hFILE_fd *) fpv;
 558     int ret;
 559     do {
 560 #ifdef HAVE_CLOSESOCKET
 561         ret = fp->is_socket? closesocket(fp->fd) : close(fp->fd);
 562 #else
 563         ret = close(fp->fd);
 564 #endif
 565     } while (ret < 0 && errno == EINTR);
 566     return ret;
 567 }
 568
 569 static const struct hFILE_backend fd_backend =
 570 {
 571     fd_read, fd_write, fd_seek, fd_flush, fd_close
 572 };
 573
 574 static size_t blksize(int fd)
 575 {
 576 #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
 577     struct stat sbuf;
 578     if (fstat(fd, &sbuf) != 0) return 0;
 579     return sbuf.st_blksize;
 580 #else
 581     return 0;
 582 #endif
 583 }
 584
 585 static hFILE *hopen_fd(const char *filename, const char *mode)
 586 {
 587     hFILE_fd *fp = NULL;
 588     int fd = open(filename, hfile_oflags(mode), 0666);
 589     if (fd < 0) goto error;
 590
 591     fp = (hFILE_fd *) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
 592     if (fp == NULL) goto error;
 593
 594     fp->fd = fd;
 595     fp->is_socket = 0;
 596     fp->base.backend = &fd_backend;
 597     return &fp->base;
 598
 599 error:
 600     if (fd >= 0) { int save = errno; (void) close(fd); errno = save; }
 601     hfile_destroy((hFILE *) fp);
 602     return NULL;
 603 }
 604
 605 hFILE *hdopen(int fd, const char *mode)
 606 {
 607     hFILE_fd *fp = (hFILE_fd*) hfile_init(sizeof (hFILE_fd), mode, blksize(fd));
 608     if (fp == NULL) return NULL;
 609
 610     fp->fd = fd;
 611     fp->is_socket = (strchr(mode, 's') != NULL);
 612     fp->base.backend = &fd_backend;
 613     return &fp->base;
 614 }
 615
 616 static hFILE *hopen_fd_fileuri(const char *url, const char *mode)
 617 {
 618     if (strncmp(url, "file://localhost/", 17) == 0) url += 16;
 619     else if (strncmp(url, "file:///", 8) == 0) url += 7;
 620     else { errno = EPROTONOSUPPORT; return NULL; }
 621
 622     return hopen_fd(url, mode);
 623 }
 624
 625 static hFILE *hopen_fd_stdinout(const char *mode)
 626 {
 627     int fd = (strchr(mode, 'r') != NULL)? STDIN_FILENO : STDOUT_FILENO;
 628 #if defined HAVE_SETMODE && defined O_BINARY
 629     if (setmode(fd, O_BINARY) < 0) return NULL;
 630 #endif
 631     return hdopen(fd, mode);
 632 }
 633
 634 int hfile_oflags(const char *mode)
 635 {
 636     int rdwr = 0, flags = 0;
 637     const char *s;
 638     for (s = mode; *s; s++)
 639         switch (*s) {
 640         case 'r': rdwr = O_RDONLY;  break;
 641         case 'w': rdwr = O_WRONLY; flags |= O_CREAT | O_TRUNC;  break;
 642         case 'a': rdwr = O_WRONLY; flags |= O_CREAT | O_APPEND;  break;
 643         case '+': rdwr = O_RDWR;  break;
 644 #ifdef O_CLOEXEC
 645         case 'e': flags |= O_CLOEXEC;  break;
 646 #endif
 647 #ifdef O_EXCL
 648         case 'x': flags |= O_EXCL;  break;
 649 #endif
 650         default:  break;
 651         }
 652
 653 #ifdef O_BINARY
 654     flags |= O_BINARY;
 655 #endif
 656
 657     return rdwr | flags;
 658 }
 659
 660
 661 /*********************
 662  * In-memory backend *
 663  *********************/
 664
 665 #include "hts_internal.h"
 666
 667 typedef struct {
 668     hFILE base;
 669 } hFILE_mem;
 670
 671 static off_t mem_seek(hFILE *fpv, off_t offset, int whence)
 672 {
 673     errno = EINVAL;
 674     return -1;
 675 }
 676
 677 static int mem_close(hFILE *fpv)
 678 {
 679     return 0;
 680 }
 681
 682 static const struct hFILE_backend mem_backend =
 683 {
 684     NULL, NULL, mem_seek, NULL, mem_close
 685 };
 686
 687 static int cmp_prefix(const char *key, const char *s)
 688 {
 689     while (*key)
 690         if (tolower_c(*s) != *key) return +1;
 691         else s++, key++;
 692
 693     return 0;
 694 }
 695
 696 static hFILE *hopen_mem(const char *url, const char *mode)
 697 {
 698     size_t length, size;
 699     char *buffer;
 700     const char *data, *comma = strchr(url, ',');
 701     if (comma == NULL) { errno = EINVAL; return NULL; }
 702     data = comma+1;
 703
 704     // TODO Implement write modes
 705     if (strchr(mode, 'r') == NULL) { errno = EROFS; return NULL; }
 706
 707     if (comma - url >= 7 && cmp_prefix(";base64", &comma[-7]) == 0) {
 708         size = hts_base64_decoded_length(strlen(data));
 709         buffer = malloc(size);
 710         if (buffer == NULL) return NULL;
 711         hts_decode_base64(buffer, &length, data);
 712     }
 713     else {
 714         size = strlen(data) + 1;
 715         buffer = malloc(size);
 716         if (buffer == NULL) return NULL;
 717         hts_decode_percent(buffer, &length, data);
 718     }
 719
 720     hFILE_mem *fp = (hFILE_mem *)
 721         hfile_init_fixed(sizeof (hFILE_mem), mode, buffer, length, size);
 722     if (fp == NULL) { free(buffer); return NULL; }
 723
 724     fp->base.backend = &mem_backend;
 725     return &fp->base;
 726 }
 727
 728
 729 /*****************************************
 730  * Plugin and hopen() backend dispatcher *
 731  *****************************************/
 732
 733 #include "htslib/khash.h"
 734
 735 KHASH_MAP_INIT_STR(scheme_string, const struct hFILE_scheme_handler *)
 736 static khash_t(scheme_string) *schemes = NULL;
 737
 738 struct hFILE_plugin_list {
 739     struct hFILE_plugin plugin;
 740     struct hFILE_plugin_list *next;
 741 };
 742
 743 static struct hFILE_plugin_list *plugins = NULL;
 744 static pthread_mutex_t plugins_lock = PTHREAD_MUTEX_INITIALIZER;
 745
 746 static void hfile_exit()
 747 {
 748     pthread_mutex_lock(&plugins_lock);
 749
 750     kh_destroy(scheme_string, schemes);
 751
 752     while (plugins != NULL) {
 753         struct hFILE_plugin_list *p = plugins;
 754         if (p->plugin.destroy) p->plugin.destroy();
 755 #ifdef ENABLE_PLUGINS
 756         if (p->plugin.obj) close_plugin(p->plugin.obj);
 757 #endif
 758         plugins = p->next;
 759         free(p);
 760     }
 761
 762     pthread_mutex_unlock(&plugins_lock);
 763     pthread_mutex_destroy(&plugins_lock);
 764 }
 765
 766 static inline int priority(const struct hFILE_scheme_handler *handler)
 767 {
 768     return handler->priority % 1000;
 769 }
 770
 771 void hfile_add_scheme_handler(const char *scheme,
 772                               const struct hFILE_scheme_handler *handler)
 773 {
 774     int absent;
 775     khint_t k = kh_put(scheme_string, schemes, scheme, &absent);
 776     if (absent || priority(handler) > priority(kh_value(schemes, k))) {
 777         kh_value(schemes, k) = handler;
 778     }
 779 }
 780
 781 static int init_add_plugin(void *obj, int (*init)(struct hFILE_plugin *),
 782                            const char *pluginname)
 783 {
 784     struct hFILE_plugin_list *p = malloc (sizeof (struct hFILE_plugin_list));
 785     if (p == NULL) abort();
 786
 787     p->plugin.api_version = 1;
 788     p->plugin.obj = obj;
 789     p->plugin.name = NULL;
 790     p->plugin.destroy = NULL;
 791
 792     int ret = (*init)(&p->plugin);
 793
 794     if (ret != 0) {
 795         hts_log_debug("Initialisation failed for plugin \"%s\": %d", pluginname, ret);
 796         free(p);
 797         return ret;
 798     }
 799
 800     hts_log_debug("Loaded \"%s\"", pluginname);
 801
 802     p->next = plugins, plugins = p;
 803     return 0;
 804 }
 805
 806 static void load_hfile_plugins()
 807 {
 808     static const struct hFILE_scheme_handler
 809         data = { hopen_mem, hfile_always_local, "built-in", 80 },
 810         file = { hopen_fd_fileuri, hfile_always_local, "built-in", 80 };
 811
 812     schemes = kh_init(scheme_string);
 813     if (schemes == NULL) abort();
 814
 815     hfile_add_scheme_handler("data", &data);
 816     hfile_add_scheme_handler("file", &file);
 817     init_add_plugin(NULL, hfile_plugin_init_net, "knetfile");
 818
 819 #ifdef ENABLE_PLUGINS
 820     struct hts_path_itr path;
 821     const char *pluginname;
 822     hts_path_itr_setup(&path, NULL, NULL, "hfile_", 6, NULL, 0);
 823     while ((pluginname = hts_path_itr_next(&path)) != NULL) {
 824         void *obj;
 825         int (*init)(struct hFILE_plugin *) = (int (*)(struct hFILE_plugin *))
 826             load_plugin(&obj, pluginname, "hfile_plugin_init");
 827
 828         if (init) {
 829             if (init_add_plugin(obj, init, pluginname) != 0)
 830                 close_plugin(obj);
 831         }
 832     }
 833 #else
 834
 835 #ifdef HAVE_LIBCURL
 836     init_add_plugin(NULL, hfile_plugin_init_libcurl, "libcurl");
 837 #endif
 838 #ifdef ENABLE_GCS
 839     init_add_plugin(NULL, hfile_plugin_init_gcs, "gcs");
 840 #endif
 841 #ifdef ENABLE_S3
 842     init_add_plugin(NULL, hfile_plugin_init_s3, "s3");
 843 #endif
 844
 845 #endif
 846
 847     // In the unlikely event atexit() fails, it's better to succeed here and
 848     // carry on; then eventually when the program exits, we'll merely close
 849     // down the plugins uncleanly, as if we had aborted.
 850     (void) atexit(hfile_exit);
 851 }
 852
 853 /* A filename like "foo:bar" in which we don't recognise the scheme is
 854    either an ordinary file or an indication of a missing or broken plugin.
 855    Try to open it as an ordinary file; but if there's no such file, set
 856    errno distinctively to make the plugin issue apparent.  */
 857 static hFILE *hopen_unknown_scheme(const char *fname, const char *mode)
 858 {
 859     hFILE *fp = hopen_fd(fname, mode);
 860     if (fp == NULL && errno == ENOENT) errno = EPROTONOSUPPORT;
 861     return fp;
 862 }
 863
 864 /* Returns the appropriate handler, or NULL if the string isn't an URL.  */
 865 static const struct hFILE_scheme_handler *find_scheme_handler(const char *s)
 866 {
 867     static const struct hFILE_scheme_handler unknown_scheme =
 868         { hopen_unknown_scheme, hfile_always_local, "built-in", 0 };
 869
 870     char scheme[12];
 871     int i;
 872
 873     for (i = 0; i < sizeof scheme; i++)
 874         if (isalnum_c(s[i]) || s[i] == '+' || s[i] == '-' || s[i] == '.')
 875             scheme[i] = tolower_c(s[i]);
 876         else if (s[i] == ':') break;
 877         else return NULL;
 878
 879     if (i == 0 || i >= sizeof scheme) return NULL;
 880     scheme[i] = '\0';
 881
 882     pthread_mutex_lock(&plugins_lock);
 883     if (! schemes) load_hfile_plugins();
 884     pthread_mutex_unlock(&plugins_lock);
 885
 886     khint_t k = kh_get(scheme_string, schemes, scheme);
 887     return (k != kh_end(schemes))? kh_value(schemes, k) : &unknown_scheme;
 888 }
 889
 890 hFILE *hopen(const char *fname, const char *mode, ...)
 891 {
 892     const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);
 893     if (handler) {
 894         if (strchr(mode, ':') == NULL) return handler->open(fname, mode);
 895         else if (handler->priority >= 2000 && handler->vopen) {
 896             hFILE *fp;
 897             va_list arg;
 898             va_start(arg, mode);
 899             fp = handler->vopen(fname, mode, arg);
 900             va_end(arg);
 901             return fp;
 902         }
 903         else { errno = ENOTSUP; return NULL; }
 904     }
 905     else if (strcmp(fname, "-") == 0) return hopen_fd_stdinout(mode);
 906     else return hopen_fd(fname, mode);
 907 }
 908
 909 int hfile_always_local (const char *fname) { return 0; }
 910 int hfile_always_remote(const char *fname) { return 1; }
 911
 912 int hisremote(const char *fname)
 913 {
 914     const struct hFILE_scheme_handler *handler = find_scheme_handler(fname);
 915     return handler? handler->isremote(fname) : 0;
 916 }