xapian-core/common/io_utils.cc

   1 /** @file
   2  * @brief Wrappers for low-level POSIX I/O routines.
   3  */
   4 /* Copyright (C) 2004,2006,2007,2008,2009,2011,2012,2014,2015,2016 Olly Betts
   5  * Copyright (C) 2010 Richard Boulton
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "io_utils.h"
  25 #include "posixy_wrapper.h"
  26
  27 #include "safeunistd.h"
  28
  29 #include <cerrno>
  30 #include <cstring>
  31 #include <string>
  32
  33 #include <xapian/error.h>
  34
  35 #include "noreturn.h"
  36 #include "omassert.h"
  37 #include "str.h"
  38
  39 // Trying to include the correct headers with the correct defines set to
  40 // get pread() and pwrite() prototyped on every platform without breaking any
  41 // other platform is a real can of worms.  So instead we probe for what
  42 // prototypes (if any) are required in configure and put them into
  43 // PREAD_PROTOTYPE and PWRITE_PROTOTYPE.
  44 #if defined HAVE_PREAD && defined PREAD_PROTOTYPE
  45 PREAD_PROTOTYPE
  46 #endif
  47 #if defined HAVE_PWRITE && defined PWRITE_PROTOTYPE
  48 PWRITE_PROTOTYPE
  49 #endif
  50
  51 bool
  52 io_unlink(const std::string & filename)
  53 {
  54     if (posixy_unlink(filename.c_str()) == 0) {
  55         return true;
  56     }
  57     if (errno != ENOENT) {
  58         throw Xapian::DatabaseError(filename + ": delete failed", errno);
  59     }
  60     return false;
  61 }
  62
  63 // The smallest fd we want to use for a writable handle.
  64 const int MIN_WRITE_FD = 3;
  65
  66 int
  67 io_open_block_wr(const char * fname, bool anew)
  68 {
  69     // Use auto because on AIX O_CLOEXEC may be a 64-bit integer constant.
  70     auto flags = O_RDWR | O_BINARY | O_CLOEXEC;
  71     if (anew) flags |= O_CREAT | O_TRUNC;
  72     int fd = ::open(fname, flags, 0666);
  73     if (fd >= MIN_WRITE_FD || fd < 0) return fd;
  74
  75     // We want to avoid using fd < MIN_WRITE_FD, in case some other code in
  76     // the same process tries to write to stdout or stderr, which would end up
  77     // corrupting our database.
  78     int badfd = fd;
  79 #ifdef F_DUPFD_CLOEXEC
  80     // dup to the first unused fd >= MIN_WRITE_FD.
  81     fd = fcntl(badfd, F_DUPFD_CLOEXEC, MIN_WRITE_FD);
  82     // F_DUPFD_CLOEXEC may not be supported.
  83     if (fd < 0 && errno == EINVAL)
  84 #endif
  85 #ifdef F_DUPFD
  86     {
  87         fd = fcntl(badfd, F_DUPFD, MIN_WRITE_FD);
  88 # ifdef FD_CLOEXEC
  89         if (fd >= 0)
  90             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
  91 # endif
  92     }
  93     int save_errno = errno;
  94     (void)close(badfd);
  95     errno = save_errno;
  96 #else
  97     {
  98         char toclose[MIN_WRITE_FD];
  99         memset(toclose, 0, sizeof(toclose));
 100         fd = badfd;
 101         do {
 102             toclose[fd] = 1;
 103             fd = dup(fd);
 104         } while (fd >= 0 && fd < MIN_WRITE_FD);
 105         int save_errno = errno;
 106         for (badfd = 0; badfd != MIN_WRITE_FD; ++badfd)
 107             if (toclose[badfd])
 108                 close(badfd);
 109         if (fd < 0) {
 110             errno = save_errno;
 111         } else {
 112 # ifdef FD_CLOEXEC
 113             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
 114 # endif
 115         }
 116     }
 117 #endif
 118     Assert(fd >= MIN_WRITE_FD || fd < 0);
 119     return fd;
 120 }
 121
 122 size_t
 123 io_read(int fd, char * p, size_t n, size_t min)
 124 {
 125     size_t total = 0;
 126     while (n) {
 127         ssize_t c = read(fd, p, n);
 128         if (c <= 0) {
 129             if (c == 0) {
 130                 if (total >= min) break;
 131                 throw Xapian::DatabaseCorruptError("Couldn't read enough (EOF)");
 132             }
 133             if (errno == EINTR) continue;
 134             throw Xapian::DatabaseError("Error reading from file", errno);
 135         }
 136         p += c;
 137         total += c;
 138         n -= c;
 139     }
 140     return total;
 141 }
 142
 143 /** Write n bytes from block pointed to by p to file descriptor fd. */
 144 void
 145 io_write(int fd, const char * p, size_t n)
 146 {
 147     while (n) {
 148         ssize_t c = write(fd, p, n);
 149         if (c < 0) {
 150             if (errno == EINTR) continue;
 151             throw Xapian::DatabaseError("Error writing to file", errno);
 152         }
 153         p += c;
 154         n -= c;
 155     }
 156 }
 157
 158 XAPIAN_NORETURN(
 159         static void throw_block_error(const char * s, off_t b, int e = 0));
 160 static void
 161 throw_block_error(const char * s, off_t b, int e)
 162 {
 163     std::string m = s;
 164     m += str(b);
 165     throw Xapian::DatabaseError(m, e);
 166 }
 167
 168 #ifdef HAVE_POSIX_FADVISE
 169 bool
 170 io_readahead_block(int fd, size_t n, off_t b, off_t o)
 171 {
 172     o += b * n;
 173     // Assume that any failure is likely to also happen for another call with
 174     // the same fd.
 175     return posix_fadvise(fd, o, n, POSIX_FADV_WILLNEED) == 0;
 176 }
 177 #endif
 178
 179 void
 180 io_read_block(int fd, char * p, size_t n, off_t b, off_t o)
 181 {
 182     o += b * n;
 183     // Prefer pread if available since it's typically implemented as a
 184     // separate syscall, and that eliminates the overhead of an extra syscall
 185     // per block read.
 186 #ifdef HAVE_PREAD
 187     while (true) {
 188         ssize_t c = pread(fd, p, n, o);
 189         // We should get a full read most of the time, so streamline that case.
 190         if (usual(c == ssize_t(n)))
 191             return;
 192         // -1 is error, 0 is EOF
 193         if (c <= 0) {
 194             if (c == 0)
 195                 throw_block_error("EOF reading block ", b);
 196             // We get EINTR if the syscall was interrupted by a signal.
 197             // In this case we should retry the read.
 198             if (errno == EINTR) continue;
 199             throw_block_error("Error reading block ", b, errno);
 200         }
 201         p += c;
 202         n -= c;
 203         o += c;
 204     }
 205 #else
 206     if (rare(lseek(fd, o, SEEK_SET) < 0))
 207         throw_block_error("Error seeking to block ", b, errno);
 208     while (true) {
 209         ssize_t c = read(fd, p, n);
 210         // We should get a full read most of the time, so streamline that case.
 211         if (usual(c == ssize_t(n)))
 212             return;
 213         if (c <= 0) {
 214             if (c == 0)
 215                 throw_block_error("EOF reading block ", b);
 216             // We get EINTR if the syscall was interrupted by a signal.
 217             // In this case we should retry the read.
 218             if (errno == EINTR) continue;
 219             throw_block_error("Error reading block ", b, errno);
 220         }
 221         p += c;
 222         n -= c;
 223     }
 224 #endif
 225 }
 226
 227 void
 228 io_write_block(int fd, const char * p, size_t n, off_t b, off_t o)
 229 {
 230     o += b * n;
 231     // Prefer pwrite if available since it's typically implemented as a
 232     // separate syscall, and that eliminates the overhead of an extra syscall
 233     // per block write.
 234 #ifdef HAVE_PWRITE
 235     while (true) {
 236         ssize_t c = pwrite(fd, p, n, o);
 237         // We should get a full write most of the time, so streamline that case.
 238         if (usual(c == ssize_t(n)))
 239             return;
 240         if (c < 0) {
 241             // We get EINTR if the syscall was interrupted by a signal.
 242             // In this case we should retry the write.
 243             if (errno == EINTR) continue;
 244             throw_block_error("Error writing block ", b, errno);
 245         }
 246         p += c;
 247         n -= c;
 248         o += c;
 249     }
 250 #else
 251     if (rare(lseek(fd, o, SEEK_SET) < 0))
 252         throw_block_error("Error seeking to block ", b, errno);
 253     while (true) {
 254         ssize_t c = write(fd, p, n);
 255         // We should get a full write most of the time, so streamline that case.
 256         if (usual(c == ssize_t(n)))
 257             return;
 258         if (c < 0) {
 259             // We get EINTR if the syscall was interrupted by a signal.
 260             // In this case we should retry the write.
 261             if (errno == EINTR) continue;
 262             throw_block_error("Error writing block ", b, errno);
 263         }
 264         p += c;
 265         n -= c;
 266     }
 267 #endif
 268 }
 269
 270 bool
 271 io_tmp_rename(const std::string & tmp_file, const std::string & real_file)
 272 {
 273 #ifdef EXDEV
 274     // We retry on EXDEV a few times as some older Linux kernels are buggy and
 275     // fail with EXDEV when the two files are on the same device (as they
 276     // always ought to be when this function is used).  Don't retry forever in
 277     // case someone calls this with files on different devices.
 278     //
 279     // We're not sure exactly which kernels are buggy in this way, but there's
 280     // discussion here: https://www.spinics.net/lists/linux-nfs/msg17306.html
 281     //
 282     // Reported at: https://trac.xapian.org/ticket/698
 283     int retries = 5;
 284 retry:
 285 #endif
 286     if (posixy_rename(tmp_file.c_str(), real_file.c_str()) < 0) {
 287 #ifdef EXDEV
 288         if (errno == EXDEV && --retries > 0) goto retry;
 289 #endif
 290         // With NFS, rename() failing may just mean that the server crashed
 291         // after successfully renaming, but before reporting this, and then
 292         // the retried operation fails.  So we need to check if the source
 293         // file still exists, which we do by calling unlink(), since we want
 294         // to remove the temporary file anyway.
 295         int saved_errno = errno;
 296         if (unlink(tmp_file.c_str()) == 0 || errno != ENOENT) {
 297             errno = saved_errno;
 298             return false;
 299         }
 300     }
 301     return true;
 302 }