xapian-core/bin/xapian-replicate.cc

   1 /** @file
   2  * @brief Replicate a database from a master server to a local copy.
   3  */
   4 /* Copyright (C) 2008,2011,2012,2015 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "net/replicatetcpclient.h"
  25
  26 #include <xapian.h>
  27
  28 #include "gnu_getopt.h"
  29 #include "parseint.h"
  30 #include "stringutils.h"
  31 #include "safeunistd.h"
  32
  33 #include <iostream>
  34
  35 using namespace std;
  36
  37 #define PROG_NAME "xapian-replicate"
  38 #define PROG_DESC "Replicate a database from a master server to a local copy"
  39
  40 #define OPT_HELP 1
  41 #define OPT_VERSION 2
  42
  43 // Wait DEFAULT_INTERVAL seconds between updates unless --interval is passed.
  44 #define DEFAULT_INTERVAL 60
  45
  46 // Number of seconds before we assume that a reader will be closed.
  47 #define READER_CLOSE_TIME 30
  48
  49 // Socket level timeout (in seconds).
  50 #define DEFAULT_TIMEOUT 0
  51
  52 static void show_usage() {
  53     cout << "Usage: " PROG_NAME " [OPTIONS] DATABASE\n\n"
  54 "Options:\n"
  55 "  -h, --host=HOST     host to connect to (required)\n"
  56 "  -p, --port=PORT     port to connect to (required)\n"
  57 "  -m, --master=DB     replicate database DB from the master (default: DATABASE)\n"
  58 "  -i, --interval=N    wait N seconds between each connection to the master\n"
  59 "                      (default: " STRINGIZE(DEFAULT_INTERVAL) ")\n"
  60 "  -r, --reader-time=N wait N seconds to allow readers time to close before\n"
  61 "                      applying repeated changesets (default: " STRINGIZE(READER_CLOSE_TIME) ")\n"
  62 "  -t, --timeout=N     set socket timeouts (if supported) to N seconds; N=0 for\n"
  63 "                      no timeout (default: " STRINGIZE(DEFAULT_TIMEOUT) ")\n"
  64 "  -f, --force-copy    force a full copy of the database to be sent (and then\n"
  65 "                      replicate as normal)\n"
  66 "  -o, --one-shot      replicate only once and then exit\n"
  67 "  -q, --quiet         only report errors\n"
  68 "  -v, --verbose       be more verbose\n"
  69 "  --help              display this help and exit\n"
  70 "  --version           output version information and exit\n";
  71 }
  72
  73 int
  74 main(int argc, char **argv)
  75 {
  76     const char * opts = "h:p:m:i:r:t:ofqv";
  77     static const struct option long_opts[] = {
  78         {"host",        required_argument,      0, 'h'},
  79         {"port",        required_argument,      0, 'p'},
  80         {"master",      required_argument,      0, 'm'},
  81         {"interval",    required_argument,      0, 'i'},
  82         {"reader-time", required_argument,      0, 'r'},
  83         {"timeout",     required_argument,      0, 't'},
  84         {"one-shot",    no_argument,            0, 'o'},
  85         {"force-copy",  no_argument,            0, 'f'},
  86         {"quiet",       no_argument,            0, 'q'},
  87         {"verbose",     no_argument,            0, 'v'},
  88         {"help",        no_argument, 0, OPT_HELP},
  89         {"version",     no_argument, 0, OPT_VERSION},
  90         {NULL,          0, 0, 0}
  91     };
  92
  93     string host;
  94     int port = 0;
  95     string masterdb;
  96     int interval = DEFAULT_INTERVAL;
  97     bool one_shot = false;
  98     enum { NORMAL, VERBOSE, QUIET } verbosity = NORMAL;
  99     bool force_copy = false;
 100     int reader_close_time = READER_CLOSE_TIME;
 101     int timeout = DEFAULT_TIMEOUT;
 102
 103     int c;
 104     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 105         switch (c) {
 106             case 'h':
 107                 host.assign(optarg);
 108                 break;
 109             case 'p':
 110                 if (!parse_signed(optarg, port) ||
 111                     (port < 1 || port > 65535)) {
 112                     cerr << "Error: must specify a valid port number "
 113                             "(between 1 and 65535). \n";
 114                     exit(1);
 115                 }
 116                 break;
 117             case 'm':
 118                 masterdb.assign(optarg);
 119                 break;
 120             case 'i': {
 121                 unsigned int i_val;
 122                 if (!parse_unsigned(optarg, i_val)) {
 123                     cout << "Interval must be a non-negative integer\n";
 124                     show_usage();
 125                     exit(1);
 126                 }
 127                 interval = i_val;
 128                 break;
 129             }
 130             case 'r': {
 131                 unsigned int reader_time;
 132                 if (!parse_unsigned(optarg, reader_time)) {
 133                     cout << "reader close time must be a "
 134                             "non-negative integer\n";
 135                     show_usage();
 136                     exit(1);
 137                 }
 138                 reader_close_time = reader_time;
 139                 break;
 140             }
 141             case 't':
 142                 unsigned int socket_timeout;
 143                 if (!parse_unsigned(optarg, socket_timeout)) {
 144                     cout << "timeout must be a non-negative integer\n";
 145                     show_usage();
 146                     exit(1);
 147                 }
 148                 timeout = socket_timeout;
 149                 break;
 150             case 'f':
 151                 force_copy = true;
 152                 break;
 153             case 'o':
 154                 one_shot = true;
 155                 break;
 156             case 'q':
 157                 verbosity = QUIET;
 158                 break;
 159             case 'v':
 160                 verbosity = VERBOSE;
 161                 break;
 162             case OPT_HELP:
 163                 cout << PROG_NAME " - " PROG_DESC "\n\n";
 164                 show_usage();
 165                 exit(0);
 166             case OPT_VERSION:
 167                 cout << PROG_NAME " - " PACKAGE_STRING "\n";
 168                 exit(0);
 169             default:
 170                 show_usage();
 171                 exit(1);
 172         }
 173     }
 174
 175     if (argc - optind != 1) {
 176         show_usage();
 177         exit(1);
 178     }
 179
 180     if (host.empty()) {
 181         cout << "Host required - specify with --host=HOST\n\n";
 182         show_usage();
 183         exit(1);
 184     }
 185
 186     if (port == 0) {
 187         cout << "Port required - specify with --port=PORT\n\n";
 188         show_usage();
 189         exit(1);
 190     }
 191
 192     // Path to the database to create/update.
 193     string dbpath(argv[optind]);
 194
 195     if (masterdb.empty())
 196         masterdb = dbpath;
 197
 198     while (true) {
 199         try {
 200             if (verbosity == VERBOSE) {
 201                 cout << "Connecting to " << host << ":" << port << '\n';
 202             }
 203             ReplicateTcpClient client(host, port, 10.0, timeout);
 204             if (verbosity == VERBOSE) {
 205                 cout << "Getting update for " << dbpath << " from "
 206                      << masterdb << '\n';
 207             }
 208             Xapian::ReplicationInfo info;
 209             client.update_from_master(dbpath, masterdb, info,
 210                                       reader_close_time, force_copy);
 211             if (verbosity == VERBOSE) {
 212                 cout << "Update complete: "
 213                      << info.fullcopy_count << " copies, "
 214                      << info.changeset_count << " changesets, "
 215                      << (info.changed ? "new live database"
 216                                       : "no changes to live database")
 217                      << '\n';
 218             }
 219             if (verbosity != QUIET) {
 220                 if (info.fullcopy_count > 0 && !info.changed) {
 221                     cout <<
 222 "Replication using a full copy failed.  This usually means that the master\n"
 223 "database is changing too frequently.  Ensure that sufficient changesets are\n"
 224 "present by setting XAPIAN_MAX_CHANGESETS on the master.\n";
 225                 }
 226             }
 227             force_copy = false;
 228         } catch (const Xapian::NetworkError &error) {
 229             // Don't stop running if there's a network error - just log to
 230             // stderr and retry at next timeout.  This should make the client
 231             // robust against temporary network failures.
 232             cerr << argv[0] << ": " << error.get_description() << '\n';
 233
 234             // If we were running as a one-shot client though, we're going to
 235             // exit anyway, so let's make the return value reflect that there
 236             // was a failure.
 237             if (one_shot)
 238                 exit(1);
 239         } catch (const Xapian::Error &error) {
 240             cerr << argv[0] << ": " << error.get_description() << '\n';
 241             exit(1);
 242         } catch (const exception &e) {
 243             cerr << "Caught standard exception: " << e.what() << '\n';
 244             exit(1);
 245         } catch (...) {
 246             cerr << "Caught unknown exception\n";
 247             exit(1);
 248         }
 249         if (one_shot) break;
 250         sleep(interval);
 251     }
 252 }