xapian-core/bin/xapian-compact.cc

   1 /** @file
   2  * @brief Compact a database, or merge and compact several.
   3  */
   4 /* Copyright (C) 2003,2004,2005,2006,2007,2008,2009,2010,2015 Olly Betts
   5  * Copyright (C) 2008 Lemur Consulting Ltd
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include <xapian.h>
  26
  27 #include <cstdlib>
  28 #include <iostream>
  29
  30 #include "gnu_getopt.h"
  31
  32 using namespace std;
  33
  34 #define PROG_NAME "xapian-compact"
  35 #define PROG_DESC "Compact a database, or merge and compact several"
  36
  37 #define OPT_HELP 1
  38 #define OPT_VERSION 2
  39 #define OPT_NO_RENUMBER 3
  40
  41 static void show_usage() {
  42     cout << "Usage: " PROG_NAME " [OPTIONS] SOURCE_DATABASE... DESTINATION_DATABASE\n\n"
  43 "Options:\n"
  44 "  -b, --blocksize=B  Set the blocksize in bytes (e.g. 4096) or K (e.g. 4K)\n"
  45 "                     (must be between 2K and 64K and a power of 2, default 8K)\n"
  46 "  -n, --no-full      Disable full compaction\n"
  47 "  -F, --fuller       Enable fuller compaction (not recommended if you plan to\n"
  48 "                     update the compacted database)\n"
  49 "  -m, --multipass    If merging more than 3 databases, merge the postlists in\n"
  50 "                     multiple passes (which is generally faster but requires\n"
  51 "                     more disk space for temporary files)\n"
  52 "      --no-renumber  Preserve the numbering of document ids (useful if you have\n"
  53 "                     external references to them, or have set them to match\n"
  54 "                     unique ids from an external source).  Currently this\n"
  55 "                     option is only supported when merging databases if they\n"
  56 "                     have disjoint ranges of used document ids\n"
  57 "  -s, --single-file  Produce a single file database (not supported for chert)\n"
  58 "  --help             display this help and exit\n"
  59 "  --version          output version information and exit" << endl;
  60 }
  61
  62 class MyCompactor : public Xapian::Compactor {
  63     bool quiet;
  64
  65   public:
  66     MyCompactor() : quiet(false) { }
  67
  68     void set_quiet(bool quiet_) { quiet = quiet_; }
  69
  70     void set_status(const string & table, const string & status);
  71
  72     string
  73     resolve_duplicate_metadata(const string & key,
  74                                size_t n,
  75                                const string tags[]);
  76 };
  77
  78 void
  79 MyCompactor::set_status(const string & table, const string & status)
  80 {
  81     if (quiet)
  82         return;
  83     if (!status.empty())
  84         cout << '\r' << table << ": " << status << endl;
  85     else
  86         cout << table << " ..." << flush;
  87 }
  88
  89 string
  90 MyCompactor::resolve_duplicate_metadata(const string & key,
  91                                         size_t n,
  92                                         const string tags[])
  93 {
  94     (void)key;
  95     while (--n) {
  96         if (tags[0] != tags[n]) {
  97             cerr << "Warning: duplicate user metadata key with different tag value - picking value from first source database with a non-empty value" << endl;
  98             break;
  99         }
 100     }
 101     return tags[0];
 102 }
 103
 104 int
 105 main(int argc, char **argv)
 106 {
 107     const char * opts = "b:nFmqs";
 108     static const struct option long_opts[] = {
 109         {"fuller",      no_argument, 0, 'F'},
 110         {"no-full",     no_argument, 0, 'n'},
 111         {"multipass",   no_argument, 0, 'm'},
 112         {"blocksize",   required_argument, 0, 'b'},
 113         {"no-renumber", no_argument, 0, OPT_NO_RENUMBER},
 114         {"single-file", no_argument, 0, 's'},
 115         {"quiet",       no_argument, 0, 'q'},
 116         {"help",        no_argument, 0, OPT_HELP},
 117         {"version",     no_argument, 0, OPT_VERSION},
 118         {NULL,          0, 0, 0}
 119     };
 120
 121     MyCompactor compactor;
 122     Xapian::Compactor::compaction_level level = Xapian::Compactor::FULL;
 123     unsigned flags = 0;
 124     size_t block_size = 0;
 125
 126     int c;
 127     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 128         switch (c) {
 129             case 'b': {
 130                 char *p;
 131                 block_size = strtoul(optarg, &p, 10);
 132                 if (block_size <= 64 && (*p == 'K' || *p == 'k')) {
 133                     ++p;
 134                     block_size *= 1024;
 135                 }
 136                 if (*p || block_size < 2048 || block_size > 65536 ||
 137                     (block_size & (block_size - 1)) != 0) {
 138                     cerr << PROG_NAME": Bad value '" << optarg
 139                          << "' passed for blocksize, must be a power of 2 between 2K and 64K"
 140                          << endl;
 141                     exit(1);
 142                 }
 143                 break;
 144             }
 145             case 'n':
 146                 level = compactor.STANDARD;
 147                 break;
 148             case 'F':
 149                 level = compactor.FULLER;
 150                 break;
 151             case 'm':
 152                 flags |= Xapian::DBCOMPACT_MULTIPASS;
 153                 break;
 154             case OPT_NO_RENUMBER:
 155                 flags |= Xapian::DBCOMPACT_NO_RENUMBER;
 156                 break;
 157             case 's':
 158                 flags |= Xapian::DBCOMPACT_SINGLE_FILE;
 159                 break;
 160             case 'q':
 161                 compactor.set_quiet(true);
 162                 break;
 163             case OPT_HELP:
 164                 cout << PROG_NAME " - " PROG_DESC "\n\n";
 165                 show_usage();
 166                 exit(0);
 167             case OPT_VERSION:
 168                 cout << PROG_NAME " - " PACKAGE_STRING << endl;
 169                 exit(0);
 170             default:
 171                 show_usage();
 172                 exit(1);
 173         }
 174     }
 175
 176     if (argc - optind < 2) {
 177         show_usage();
 178         exit(1);
 179     }
 180
 181     // Path to the database to create.
 182     string destdir = argv[argc - 1];
 183
 184     try {
 185         Xapian::Database src;
 186         for (int i = optind; i < argc - 1; ++i) {
 187             src.add_database(Xapian::Database(argv[i]));
 188         }
 189         src.compact(destdir, level | flags, block_size, compactor);
 190     } catch (const Xapian::Error &error) {
 191         cerr << argv[0] << ": " << error.get_description() << endl;
 192         exit(1);
 193     } catch (const char * msg) {
 194         cerr << argv[0] << ": " << msg << endl;
 195         exit(1);
 196     }
 197 }