xapian-core/common/io_utils.cc

   1 /** @file
   2  * @brief Wrappers for low-level POSIX I/O routines.
   3  */
   4 /* Copyright (C) 2004,2006,2007,2008,2009,2011,2012,2014,2015,2016,2018 Olly Betts
   5  * Copyright (C) 2010 Richard Boulton
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "io_utils.h"
  25 #include "posixy_wrapper.h"
  26
  27 #include "safeunistd.h"
  28
  29 #include <cerrno>
  30 #include <cstring>
  31 #include <string>
  32
  33 #include <xapian/error.h>
  34
  35 #include "omassert.h"
  36 #include "str.h"
  37
  38 // Trying to include the correct headers with the correct defines set to
  39 // get pread() and pwrite() prototyped on every platform without breaking any
  40 // other platform is a real can of worms.  So instead we probe for what
  41 // prototypes (if any) are required in configure and put them into
  42 // PREAD_PROTOTYPE and PWRITE_PROTOTYPE.
  43 #if defined HAVE_PREAD && defined PREAD_PROTOTYPE
  44 PREAD_PROTOTYPE
  45 #endif
  46 #if defined HAVE_PWRITE && defined PWRITE_PROTOTYPE
  47 PWRITE_PROTOTYPE
  48 #endif
  49
  50 bool
  51 io_unlink(const std::string & filename)
  52 {
  53     if (posixy_unlink(filename.c_str()) == 0) {
  54         return true;
  55     }
  56     if (errno != ENOENT) {
  57         throw Xapian::DatabaseError(filename + ": delete failed", errno);
  58     }
  59     return false;
  60 }
  61
  62 // The smallest fd we want to use for a writable handle.
  63 const int MIN_WRITE_FD = 3;
  64
  65 int
  66 io_open_block_wr(const char * fname, bool anew)
  67 {
  68     // Use auto because on AIX O_CLOEXEC may be a 64-bit integer constant.
  69     auto flags = O_RDWR | O_BINARY | O_CLOEXEC;
  70     if (anew) flags |= O_CREAT | O_TRUNC;
  71     int fd = ::open(fname, flags, 0666);
  72     if (fd >= MIN_WRITE_FD || fd < 0) return fd;
  73
  74     // We want to avoid using fd < MIN_WRITE_FD, in case some other code in
  75     // the same process tries to write to stdout or stderr, which would end up
  76     // corrupting our database.
  77     int badfd = fd;
  78 #ifdef F_DUPFD_CLOEXEC
  79     // dup to the first unused fd >= MIN_WRITE_FD.
  80     fd = fcntl(badfd, F_DUPFD_CLOEXEC, MIN_WRITE_FD);
  81     // F_DUPFD_CLOEXEC may not be supported.
  82     if (fd < 0 && errno == EINVAL)
  83 #endif
  84 #ifdef F_DUPFD
  85     {
  86         fd = fcntl(badfd, F_DUPFD, MIN_WRITE_FD);
  87 # ifdef FD_CLOEXEC
  88         if (fd >= 0)
  89             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
  90 # endif
  91     }
  92     int save_errno = errno;
  93     (void)close(badfd);
  94     errno = save_errno;
  95 #else
  96     {
  97         char toclose[MIN_WRITE_FD];
  98         memset(toclose, 0, sizeof(toclose));
  99         fd = badfd;
 100         do {
 101             toclose[fd] = 1;
 102             fd = dup(fd);
 103         } while (fd >= 0 && fd < MIN_WRITE_FD);
 104         int save_errno = errno;
 105         for (badfd = 0; badfd != MIN_WRITE_FD; ++badfd)
 106             if (toclose[badfd])
 107                 close(badfd);
 108         if (fd < 0) {
 109             errno = save_errno;
 110         } else {
 111 # ifdef FD_CLOEXEC
 112             (void)fcntl(fd, F_SETFD, FD_CLOEXEC);
 113 # endif
 114         }
 115     }
 116 #endif
 117     Assert(fd >= MIN_WRITE_FD || fd < 0);
 118     return fd;
 119 }
 120
 121 size_t
 122 io_read(int fd, char * p, size_t n, size_t min)
 123 {
 124     size_t total = 0;
 125     while (n) {
 126         ssize_t c = read(fd, p, n);
 127         if (c <= 0) {
 128             if (c == 0) {
 129                 if (total >= min) break;
 130                 throw Xapian::DatabaseCorruptError("Couldn't read enough (EOF)");
 131             }
 132             if (errno == EINTR) continue;
 133             throw Xapian::DatabaseError("Error reading from file", errno);
 134         }
 135         p += c;
 136         total += c;
 137         n -= c;
 138     }
 139     return total;
 140 }
 141
 142 /** Write n bytes from block pointed to by p to file descriptor fd. */
 143 void
 144 io_write(int fd, const char * p, size_t n)
 145 {
 146     while (n) {
 147         ssize_t c = write(fd, p, n);
 148         if (c < 0) {
 149             if (errno == EINTR) continue;
 150             throw Xapian::DatabaseError("Error writing to file", errno);
 151         }
 152         p += c;
 153         n -= c;
 154     }
 155 }
 156
 157 size_t
 158 io_pread(int fd, char * p, size_t n, off_t o, size_t min)
 159 {
 160     size_t total = 0;
 161 #ifdef HAVE_PREAD
 162     while (true) {
 163         ssize_t c = pread(fd, p, n, o);
 164         // We should get a full read most of the time, so streamline that case.
 165         if (usual(c == ssize_t(n)))
 166             return total + n;
 167         // -1 is error, 0 is EOF
 168         if (c <= 0) {
 169             if (c == 0) {
 170                 if (min == 0)
 171                     return total;
 172                 throw Xapian::DatabaseError("EOF reading database");
 173             }
 174             // We get EINTR if the syscall was interrupted by a signal.
 175             // In this case we should retry the read.
 176             if (errno == EINTR) continue;
 177             throw Xapian::DatabaseError("Error reading database", errno);
 178         }
 179         total += c;
 180         if (total >= min)
 181             return total;
 182         p += c;
 183         n -= c;
 184         o += c;
 185     }
 186 #else
 187     if (rare(lseek(fd, o, SEEK_SET) < 0))
 188         throw Xapian::DatabaseError("Error seeking database", errno);
 189     while (true) {
 190         ssize_t c = read(fd, p, n);
 191         // We should get a full read most of the time, so streamline that case.
 192         if (usual(c == ssize_t(n)))
 193             return total + n;
 194         if (c <= 0) {
 195             if (c == 0) {
 196                 if (min == 0)
 197                     return total;
 198                 throw Xapian::DatabaseError("EOF reading database");
 199             }
 200             // We get EINTR if the syscall was interrupted by a signal.
 201             // In this case we should retry the read.
 202             if (errno == EINTR) continue;
 203             throw Xapian::DatabaseError("Error reading database", errno);
 204         }
 205         total += c;
 206         if (total >= min)
 207             return total;
 208         p += c;
 209         n -= c;
 210     }
 211 #endif
 212 }
 213
 214 void
 215 io_pwrite(int fd, const char * p, size_t n, off_t o)
 216 {
 217 #ifdef HAVE_PWRITE
 218     while (n) {
 219         ssize_t c = pwrite(fd, p, n, o);
 220         // We should get a full write most of the time, so streamline that
 221         // case.
 222         if (usual(c == ssize_t(n)))
 223             return;
 224         if (c < 0) {
 225             if (errno == EINTR) continue;
 226             throw Xapian::DatabaseError("Error writing to file", errno);
 227         }
 228         p += c;
 229         n -= c;
 230         o += c;
 231     }
 232 #else
 233     if (rare(lseek(fd, o, SEEK_SET) < 0))
 234         throw Xapian::DatabaseError("Error seeking database", errno);
 235     io_write(fd, p, n);
 236 #endif
 237 }
 238
 239 [[noreturn]]
 240 static void
 241 throw_block_error(const char * s, off_t b, int e = 0)
 242 {
 243     std::string m = s;
 244     m += str(b);
 245     throw Xapian::DatabaseError(m, e);
 246 }
 247
 248 #ifdef HAVE_POSIX_FADVISE
 249 bool
 250 io_readahead_block(int fd, size_t n, off_t b, off_t o)
 251 {
 252     o += b * n;
 253     // Assume that any failure is likely to also happen for another call with
 254     // the same fd.
 255     return posix_fadvise(fd, o, n, POSIX_FADV_WILLNEED) == 0;
 256 }
 257 #endif
 258
 259 void
 260 io_read_block(int fd, char * p, size_t n, off_t b, off_t o)
 261 {
 262     o += b * n;
 263     // Prefer pread if available since it's typically implemented as a
 264     // separate syscall, and that eliminates the overhead of an extra syscall
 265     // per block read.
 266 #ifdef HAVE_PREAD
 267     while (true) {
 268         ssize_t c = pread(fd, p, n, o);
 269         // We should get a full read most of the time, so streamline that case.
 270         if (usual(c == ssize_t(n)))
 271             return;
 272         // -1 is error, 0 is EOF
 273         if (c <= 0) {
 274             if (c == 0)
 275                 throw_block_error("EOF reading block ", b);
 276             // We get EINTR if the syscall was interrupted by a signal.
 277             // In this case we should retry the read.
 278             if (errno == EINTR) continue;
 279             throw_block_error("Error reading block ", b, errno);
 280         }
 281         p += c;
 282         n -= c;
 283         o += c;
 284     }
 285 #else
 286     if (rare(lseek(fd, o, SEEK_SET) < 0))
 287         throw_block_error("Error seeking to block ", b, errno);
 288     while (true) {
 289         ssize_t c = read(fd, p, n);
 290         // We should get a full read most of the time, so streamline that case.
 291         if (usual(c == ssize_t(n)))
 292             return;
 293         if (c <= 0) {
 294             if (c == 0)
 295                 throw_block_error("EOF reading block ", b);
 296             // We get EINTR if the syscall was interrupted by a signal.
 297             // In this case we should retry the read.
 298             if (errno == EINTR) continue;
 299             throw_block_error("Error reading block ", b, errno);
 300         }
 301         p += c;
 302         n -= c;
 303     }
 304 #endif
 305 }
 306
 307 void
 308 io_write_block(int fd, const char * p, size_t n, off_t b, off_t o)
 309 {
 310     o += b * n;
 311     // Prefer pwrite if available since it's typically implemented as a
 312     // separate syscall, and that eliminates the overhead of an extra syscall
 313     // per block write.
 314 #ifdef HAVE_PWRITE
 315     while (true) {
 316         ssize_t c = pwrite(fd, p, n, o);
 317         // We should get a full write most of the time, so streamline that case.
 318         if (usual(c == ssize_t(n)))
 319             return;
 320         if (c < 0) {
 321             // We get EINTR if the syscall was interrupted by a signal.
 322             // In this case we should retry the write.
 323             if (errno == EINTR) continue;
 324             throw_block_error("Error writing block ", b, errno);
 325         }
 326         p += c;
 327         n -= c;
 328         o += c;
 329     }
 330 #else
 331     if (rare(lseek(fd, o, SEEK_SET) < 0))
 332         throw_block_error("Error seeking to block ", b, errno);
 333     while (true) {
 334         ssize_t c = write(fd, p, n);
 335         // We should get a full write most of the time, so streamline that case.
 336         if (usual(c == ssize_t(n)))
 337             return;
 338         if (c < 0) {
 339             // We get EINTR if the syscall was interrupted by a signal.
 340             // In this case we should retry the write.
 341             if (errno == EINTR) continue;
 342             throw_block_error("Error writing block ", b, errno);
 343         }
 344         p += c;
 345         n -= c;
 346     }
 347 #endif
 348 }
 349
 350 bool
 351 io_tmp_rename(const std::string & tmp_file, const std::string & real_file)
 352 {
 353 #ifdef EXDEV
 354     // We retry on EXDEV a few times as some older Linux kernels are buggy and
 355     // fail with EXDEV when the two files are on the same device (as they
 356     // always ought to be when this function is used).  Don't retry forever in
 357     // case someone calls this with files on different devices.
 358     //
 359     // We're not sure exactly which kernels are buggy in this way, but there's
 360     // discussion here: https://www.spinics.net/lists/linux-nfs/msg17306.html
 361     //
 362     // Reported at: https://trac.xapian.org/ticket/698
 363     int retries = 5;
 364 retry:
 365 #endif
 366     if (posixy_rename(tmp_file.c_str(), real_file.c_str()) < 0) {
 367 #ifdef EXDEV
 368         if (errno == EXDEV && --retries > 0) goto retry;
 369 #endif
 370         // With NFS, rename() failing may just mean that the server crashed
 371         // after successfully renaming, but before reporting this, and then
 372         // the retried operation fails.  So we need to check if the source
 373         // file still exists, which we do by calling unlink(), since we want
 374         // to remove the temporary file anyway.
 375         int saved_errno = errno;
 376         if (unlink(tmp_file.c_str()) == 0 || errno != ENOENT) {
 377             errno = saved_errno;
 378             return false;
 379         }
 380     }
 381     return true;
 382 }