xapian-core/tests/api_query.cc

   1 /** @file
   2  * @brief Query-related tests.
   3  */
   4 /* Copyright (C) 2008,2009,2012,2013,2015,2016,2017,2019 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License as
   8  * published by the Free Software Foundation; either version 2 of the
   9  * License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  19  * USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "api_query.h"
  25
  26 #include <xapian.h>
  27
  28 #include "testsuite.h"
  29 #include "testutils.h"
  30
  31 #include "apitest.h"
  32
  33 using namespace std;
  34
  35 DEFINE_TESTCASE(queryterms1, !backend) {
  36     Xapian::Query query = Xapian::Query::MatchAll;
  37     /// Regression test - in 1.0.10 and earlier "" was included in the list.
  38     TEST(query.get_terms_begin() == query.get_terms_end());
  39     TEST(query.get_unique_terms_begin() == query.get_unique_terms_end());
  40     query = Xapian::Query(query.OP_AND_NOT, query, Xapian::Query("fair"));
  41     TEST_EQUAL(*query.get_terms_begin(), "fair");
  42     TEST_EQUAL(*query.get_unique_terms_begin(), "fair");
  43
  44     Xapian::QueryParser qp;
  45     Xapian::Query q = qp.parse_query("\"the the the\"");
  46     {
  47         auto t = q.get_terms_begin();
  48         size_t count = 0;
  49         while (t != q.get_terms_end()) {
  50             TEST_EQUAL(*t, "the");
  51             ++count;
  52             ++t;
  53         }
  54         TEST_EQUAL(count, 3);
  55     }
  56     {
  57         auto t = q.get_unique_terms_begin();
  58         size_t count = 0;
  59         while (t != q.get_unique_terms_end()) {
  60             TEST_EQUAL(*t, "the");
  61             ++count;
  62             ++t;
  63         }
  64         TEST_EQUAL(count, 1);
  65     }
  66 }
  67
  68 DEFINE_TESTCASE(matchall2, !backend) {
  69     TEST_STRINGS_EQUAL(Xapian::Query::MatchAll.get_description(),
  70                        "Query(<alldocuments>)");
  71 }
  72
  73 DEFINE_TESTCASE(matchnothing1, !backend) {
  74     TEST_STRINGS_EQUAL(Xapian::Query::MatchNothing.get_description(),
  75                        "Query()");
  76     vector<Xapian::Query> subqs;
  77     subqs.push_back(Xapian::Query("foo"));
  78     subqs.push_back(Xapian::Query::MatchNothing);
  79     Xapian::Query q(Xapian::Query::OP_AND, subqs.begin(), subqs.end());
  80     TEST_STRINGS_EQUAL(q.get_description(), "Query()");
  81
  82     Xapian::Query q2(Xapian::Query::OP_AND,
  83                      Xapian::Query("foo"), Xapian::Query::MatchNothing);
  84     TEST_STRINGS_EQUAL(q2.get_description(), "Query()");
  85
  86     Xapian::Query q3(Xapian::Query::OP_AND,
  87                      Xapian::Query::MatchNothing, Xapian::Query("foo"));
  88     TEST_STRINGS_EQUAL(q2.get_description(), "Query()");
  89
  90     Xapian::Query q4(Xapian::Query::OP_AND_MAYBE,
  91                      Xapian::Query("foo"), Xapian::Query::MatchNothing);
  92     TEST_STRINGS_EQUAL(q4.get_description(), "Query(foo)");
  93
  94     Xapian::Query q5(Xapian::Query::OP_AND_MAYBE,
  95                      Xapian::Query::MatchNothing, Xapian::Query("foo"));
  96     TEST_STRINGS_EQUAL(q5.get_description(), "Query()");
  97
  98     Xapian::Query q6(Xapian::Query::OP_AND_NOT,
  99                      Xapian::Query("foo"), Xapian::Query::MatchNothing);
 100     TEST_STRINGS_EQUAL(q6.get_description(), "Query(foo)");
 101
 102     Xapian::Query q7(Xapian::Query::OP_AND_NOT,
 103                      Xapian::Query::MatchNothing, Xapian::Query("foo"));
 104     TEST_STRINGS_EQUAL(q7.get_description(), "Query()");
 105 }
 106
 107 DEFINE_TESTCASE(overload1, !backend) {
 108     Xapian::Query q;
 109     q = Xapian::Query("foo") & Xapian::Query("bar");
 110     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo AND bar))");
 111
 112     // Test &= appends a same-type subquery (since Xapian 1.4.10).
 113     q &= Xapian::Query("baz");
 114     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo AND bar AND baz))");
 115     // But not if the RHS is the same query:
 116     q = Xapian::Query("foo") & Xapian::Query("bar");
 117 #ifdef __has_warning
 118 # if __has_warning("-Wself-assign-overloaded")
 119     // Suppress warning from newer clang about self-assignment so we can
 120     // test that self-assignment works!
 121 #  pragma clang diagnostic push
 122 #  pragma clang diagnostic ignored "-Wself-assign-overloaded"
 123 # endif
 124 #endif
 125     q &= q;
 126 #ifdef __has_warning
 127 # if __has_warning("-Wself-assign-overloaded")
 128 #  pragma clang diagnostic pop
 129 # endif
 130 #endif
 131     TEST_STRINGS_EQUAL(q.get_description(), "Query(((foo AND bar) AND (foo AND bar)))");
 132     {
 133         // Also not if the query has a refcount > 1.
 134         q = Xapian::Query("foo") & Xapian::Query("bar");
 135         Xapian::Query qcopy = q;
 136         qcopy &= Xapian::Query("baz");
 137         TEST_STRINGS_EQUAL(qcopy.get_description(), "Query(((foo AND bar) AND baz))");
 138         // And q shouldn't change.
 139         TEST_STRINGS_EQUAL(q.get_description(), "Query((foo AND bar))");
 140     }
 141     // Check that MatchNothing still results in MatchNothing:
 142     q = Xapian::Query("foo") & Xapian::Query("bar");
 143     q &= Xapian::Query::MatchNothing;
 144     TEST_STRINGS_EQUAL(q.get_description(), "Query()");
 145     // Check we don't combine for other operators:
 146     q = Xapian::Query("foo") | Xapian::Query("bar");
 147     q &= Xapian::Query("baz");
 148     TEST_STRINGS_EQUAL(q.get_description(), "Query(((foo OR bar) AND baz))");
 149
 150     // Test |= appends a same-type subquery (since Xapian 1.4.10).
 151     q = Xapian::Query("foo") | Xapian::Query("bar");
 152     q |= Xapian::Query("baz");
 153     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo OR bar OR baz))");
 154     // But not if the RHS is the same query:
 155     q = Xapian::Query("foo") | Xapian::Query("bar");
 156 #ifdef __has_warning
 157 # if __has_warning("-Wself-assign-overloaded")
 158     // Suppress warning from newer clang about self-assignment so we can
 159     // test that self-assignment works!
 160 #  pragma clang diagnostic push
 161 #  pragma clang diagnostic ignored "-Wself-assign-overloaded"
 162 # endif
 163 #endif
 164     q |= q;
 165 #ifdef __has_warning
 166 # if __has_warning("-Wself-assign-overloaded")
 167 #  pragma clang diagnostic pop
 168 # endif
 169 #endif
 170     TEST_STRINGS_EQUAL(q.get_description(), "Query(((foo OR bar) OR (foo OR bar)))");
 171     {
 172         // Also not if the query has a refcount > 1.
 173         q = Xapian::Query("foo") | Xapian::Query("bar");
 174         Xapian::Query qcopy = q;
 175         qcopy |= Xapian::Query("baz");
 176         TEST_STRINGS_EQUAL(qcopy.get_description(), "Query(((foo OR bar) OR baz))");
 177         // And q shouldn't change.
 178         TEST_STRINGS_EQUAL(q.get_description(), "Query((foo OR bar))");
 179     }
 180     // Check that MatchNothing still results in no change:
 181     q = Xapian::Query("foo") | Xapian::Query("bar");
 182     q |= Xapian::Query::MatchNothing;
 183     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo OR bar))");
 184     // Check we don't combine for other operators:
 185     q = Xapian::Query("foo") & Xapian::Query("bar");
 186     q |= Xapian::Query("baz");
 187     TEST_STRINGS_EQUAL(q.get_description(), "Query(((foo AND bar) OR baz))");
 188
 189     // Test ^= appends a same-type subquery (since Xapian 1.4.10).
 190     q = Xapian::Query("foo") ^ Xapian::Query("bar");
 191     q ^= Xapian::Query("baz");
 192     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo XOR bar XOR baz))");
 193     // But a query ^= itself gives an empty query.
 194     q = Xapian::Query("foo") ^ Xapian::Query("bar");
 195 #ifdef __has_warning
 196 # if __has_warning("-Wself-assign-overloaded")
 197     // Suppress warning from newer clang about self-assignment so we can
 198     // test that self-assignment works!
 199 #  pragma clang diagnostic push
 200 #  pragma clang diagnostic ignored "-Wself-assign-overloaded"
 201 # endif
 202 #endif
 203     q ^= q;
 204 #ifdef __has_warning
 205 # if __has_warning("-Wself-assign-overloaded")
 206 #  pragma clang diagnostic pop
 207 # endif
 208 #endif
 209     TEST_STRINGS_EQUAL(q.get_description(), "Query()");
 210     {
 211         // Even if the reference count > 1.
 212         q = Xapian::Query("foo") ^ Xapian::Query("bar");
 213         Xapian::Query qcopy = q;
 214         q ^= qcopy;
 215         TEST_STRINGS_EQUAL(q.get_description(), "Query()");
 216     }
 217     {
 218         // Also not if the query has a refcount > 1.
 219         q = Xapian::Query("foo") ^ Xapian::Query("bar");
 220         Xapian::Query qcopy = q;
 221         qcopy ^= Xapian::Query("baz");
 222         TEST_STRINGS_EQUAL(qcopy.get_description(), "Query(((foo XOR bar) XOR baz))");
 223         // And q shouldn't change.
 224         TEST_STRINGS_EQUAL(q.get_description(), "Query((foo XOR bar))");
 225     }
 226     // Check that MatchNothing still results in no change:
 227     q = Xapian::Query("foo") ^ Xapian::Query("bar");
 228     q ^= Xapian::Query::MatchNothing;
 229     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo XOR bar))");
 230     // Check we don't combine for other operators:
 231     q = Xapian::Query("foo") & Xapian::Query("bar");
 232     q ^= Xapian::Query("baz");
 233     TEST_STRINGS_EQUAL(q.get_description(), "Query(((foo AND bar) XOR baz))");
 234
 235     q = Xapian::Query("foo") &~ Xapian::Query("bar");
 236     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo AND_NOT bar))");
 237     // In 1.4.9 and earlier this gave (foo AND (<alldocuments> AND_NOT bar)).
 238     q = Xapian::Query("foo");
 239     q &= ~Xapian::Query("bar");
 240     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo AND_NOT bar))");
 241     q = ~Xapian::Query("bar");
 242     TEST_STRINGS_EQUAL(q.get_description(), "Query((<alldocuments> AND_NOT bar))");
 243     q = Xapian::Query("foo") & Xapian::Query::MatchNothing;
 244     TEST_STRINGS_EQUAL(q.get_description(), "Query()");
 245     q = Xapian::Query("foo") | Xapian::Query("bar");
 246     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo OR bar))");
 247     q = Xapian::Query("foo") | Xapian::Query::MatchNothing;
 248     TEST_STRINGS_EQUAL(q.get_description(), "Query(foo)");
 249     q = Xapian::Query("foo") ^ Xapian::Query("bar");
 250     TEST_STRINGS_EQUAL(q.get_description(), "Query((foo XOR bar))");
 251     q = Xapian::Query("foo") ^ Xapian::Query::MatchNothing;
 252     TEST_STRINGS_EQUAL(q.get_description(), "Query(foo)");
 253     q = 1.25 * (Xapian::Query("one") | Xapian::Query("two"));
 254     TEST_STRINGS_EQUAL(q.get_description(), "Query(1.25 * (one OR two))");
 255     q = (Xapian::Query("one") & Xapian::Query("two")) * 42;
 256     TEST_STRINGS_EQUAL(q.get_description(), "Query(42 * (one AND two))");
 257     q = Xapian::Query("one") / 2.0;
 258     TEST_STRINGS_EQUAL(q.get_description(), "Query(0.5 * one)");
 259 }
 260
 261 /** Regression test and feature test.
 262  *
 263  *  This threw AssertionError in 1.0.9 and earlier (bug#201) and gave valgrind
 264  *  errors in 1.0.11 and earlier (bug#349).
 265  *
 266  *  Currently the OR-subquery case is supported, other operators aren't.
 267  */
 268 DEFINE_TESTCASE(possubqueries1, writable) {
 269     Xapian::WritableDatabase db = get_writable_database();
 270     Xapian::Document doc;
 271     doc.add_posting("a", 1);
 272     doc.add_posting("b", 2);
 273     doc.add_posting("c", 3);
 274     db.add_document(doc);
 275
 276     Xapian::Query a_or_b(Xapian::Query::OP_OR,
 277                          Xapian::Query("a"),
 278                          Xapian::Query("b"));
 279     Xapian::Query near(Xapian::Query::OP_NEAR, a_or_b, a_or_b);
 280     // As of 1.3.0, we no longer rearrange queries at this point, so check
 281     // that we don't.
 282     TEST_STRINGS_EQUAL(near.get_description(),
 283                        "Query(((a OR b) NEAR 2 (a OR b)))");
 284     Xapian::Query phrase(Xapian::Query::OP_PHRASE, a_or_b, a_or_b);
 285     TEST_STRINGS_EQUAL(phrase.get_description(),
 286                        "Query(((a OR b) PHRASE 2 (a OR b)))");
 287
 288     Xapian::Query a_and_b(Xapian::Query::OP_AND,
 289                           Xapian::Query("a"),
 290                           Xapian::Query("b"));
 291     Xapian::Query a_near_b(Xapian::Query::OP_NEAR,
 292                            Xapian::Query("a"),
 293                            Xapian::Query("b"));
 294     Xapian::Query a_phrs_b(Xapian::Query::OP_PHRASE,
 295                            Xapian::Query("a"),
 296                            Xapian::Query("b"));
 297     Xapian::Query c("c");
 298
 299     // FIXME: The plan is to actually try to support the cases below, but
 300     // for now at least ensure they are cleanly rejected.
 301     Xapian::Enquire enq(db);
 302
 303     TEST_EXCEPTION(Xapian::UnimplementedError,
 304         Xapian::Query q(Xapian::Query::OP_NEAR, a_and_b, c);
 305         enq.set_query(q);
 306         (void)enq.get_mset(0, 10));
 307
 308     TEST_EXCEPTION(Xapian::UnimplementedError,
 309         Xapian::Query q(Xapian::Query::OP_NEAR, a_near_b, c);
 310         enq.set_query(q);
 311         (void)enq.get_mset(0, 10));
 312
 313     TEST_EXCEPTION(Xapian::UnimplementedError,
 314         Xapian::Query q(Xapian::Query::OP_NEAR, a_phrs_b, c);
 315         enq.set_query(q);
 316         (void)enq.get_mset(0, 10));
 317
 318     TEST_EXCEPTION(Xapian::UnimplementedError,
 319         Xapian::Query q(Xapian::Query::OP_PHRASE, a_and_b, c);
 320         enq.set_query(q);
 321         (void)enq.get_mset(0, 10));
 322
 323     TEST_EXCEPTION(Xapian::UnimplementedError,
 324         Xapian::Query q(Xapian::Query::OP_PHRASE, a_near_b, c);
 325         enq.set_query(q);
 326         (void)enq.get_mset(0, 10));
 327
 328     TEST_EXCEPTION(Xapian::UnimplementedError,
 329         Xapian::Query q(Xapian::Query::OP_PHRASE, a_phrs_b, c);
 330         enq.set_query(q);
 331         (void)enq.get_mset(0, 10));
 332 }
 333
 334 /// Test that XOR handles all remaining subqueries running out at the same
 335 //  time.
 336 DEFINE_TESTCASE(xor3, backend) {
 337     Xapian::Database db = get_database("apitest_simpledata");
 338
 339     static const char * const subqs[] = {
 340         "hack", "which", "paragraph", "is", "return"
 341     };
 342     // Document where the subqueries run out *does* match XOR:
 343     Xapian::Query q(Xapian::Query::OP_XOR, subqs, subqs + 5);
 344     Xapian::Enquire enq(db);
 345     enq.set_query(q);
 346     Xapian::MSet mset = enq.get_mset(0, 10);
 347
 348     TEST_EQUAL(mset.size(), 3);
 349     TEST_EQUAL(*mset[0], 4);
 350     TEST_EQUAL(*mset[1], 2);
 351     TEST_EQUAL(*mset[2], 3);
 352
 353     // Document where the subqueries run out *does not* match XOR:
 354     q = Xapian::Query(Xapian::Query::OP_XOR, subqs, subqs + 4);
 355     enq.set_query(q);
 356     mset = enq.get_mset(0, 10);
 357
 358     TEST_EQUAL(mset.size(), 4);
 359     TEST_EQUAL(*mset[0], 5);
 360     TEST_EQUAL(*mset[1], 4);
 361     TEST_EQUAL(*mset[2], 2);
 362     TEST_EQUAL(*mset[3], 3);
 363 }
 364
 365 /// Check encoding of non-UTF8 terms in query descriptions.
 366 DEFINE_TESTCASE(nonutf8termdesc1, !backend) {
 367     TEST_EQUAL(Xapian::Query("\xc0\x80\xf5\x80\x80\x80\xfe\xff").get_description(),
 368                "Query(\\xc0\\x80\\xf5\\x80\\x80\\x80\\xfe\\xff)");
 369     TEST_EQUAL(Xapian::Query(string("\x00\x1f", 2)).get_description(),
 370                "Query(\\x00\\x1f)");
 371     // Check that backslashes are encoded so output isn't ambiguous.
 372     TEST_EQUAL(Xapian::Query("back\\slash").get_description(),
 373                "Query(back\\x5cslash)");
 374     // Check that \x7f is escaped.
 375     TEST_EQUAL(Xapian::Query("D\x7f_\x7f~").get_description(),
 376                "Query(D\\x7f_\\x7f~)");
 377 }
 378
 379 /// Test introspection on Query objects.
 380 DEFINE_TESTCASE(queryintro1, !backend) {
 381     TEST_EQUAL(Xapian::Query::MatchAll.get_type(), Xapian::Query::LEAF_MATCH_ALL);
 382     TEST_EQUAL(Xapian::Query::MatchAll.get_num_subqueries(), 0);
 383     TEST_EQUAL(Xapian::Query::MatchNothing.get_type(), Xapian::Query::LEAF_MATCH_NOTHING);
 384     TEST_EQUAL(Xapian::Query::MatchNothing.get_num_subqueries(), 0);
 385
 386     Xapian::Query q;
 387     q = Xapian::Query(q.OP_AND_NOT, Xapian::Query::MatchAll, Xapian::Query("fair"));
 388     TEST_EQUAL(q.get_type(), q.OP_AND_NOT);
 389     TEST_EQUAL(q.get_num_subqueries(), 2);
 390     TEST_EQUAL(q.get_subquery(0).get_type(), q.LEAF_MATCH_ALL);
 391     TEST_EQUAL(q.get_subquery(1).get_type(), q.LEAF_TERM);
 392
 393     q = Xapian::Query("foo") & Xapian::Query("bar");
 394     TEST_EQUAL(q.get_type(), q.OP_AND);
 395
 396     q = Xapian::Query("foo") &~ Xapian::Query("bar");
 397     TEST_EQUAL(q.get_type(), q.OP_AND_NOT);
 398
 399     q = ~Xapian::Query("bar");
 400     TEST_EQUAL(q.get_type(), q.OP_AND_NOT);
 401
 402     q = Xapian::Query("foo") | Xapian::Query("bar");
 403     TEST_EQUAL(q.get_type(), q.OP_OR);
 404
 405     q = Xapian::Query("foo") ^ Xapian::Query("bar");
 406     TEST_EQUAL(q.get_type(), q.OP_XOR);
 407
 408     q = 1.25 * (Xapian::Query("one") | Xapian::Query("two"));
 409     TEST_EQUAL(q.get_type(), q.OP_SCALE_WEIGHT);
 410     TEST_EQUAL(q.get_num_subqueries(), 1);
 411     TEST_EQUAL(q.get_subquery(0).get_type(), q.OP_OR);
 412
 413     q = Xapian::Query("one") / 2.0;
 414     TEST_EQUAL(q.get_type(), q.OP_SCALE_WEIGHT);
 415     TEST_EQUAL(q.get_num_subqueries(), 1);
 416     TEST_EQUAL(q.get_subquery(0).get_type(), q.LEAF_TERM);
 417
 418     q = Xapian::Query(q.OP_NEAR, Xapian::Query("a"), Xapian::Query("b"));
 419     TEST_EQUAL(q.get_type(), q.OP_NEAR);
 420     TEST_EQUAL(q.get_num_subqueries(), 2);
 421     TEST_EQUAL(q.get_subquery(0).get_type(), q.LEAF_TERM);
 422     TEST_EQUAL(q.get_subquery(1).get_type(), q.LEAF_TERM);
 423
 424     q = Xapian::Query(q.OP_PHRASE, Xapian::Query("c"), Xapian::Query("d"));
 425     TEST_EQUAL(q.get_type(), q.OP_PHRASE);
 426     TEST_EQUAL(q.get_num_subqueries(), 2);
 427     TEST_EQUAL(q.get_subquery(0).get_type(), q.LEAF_TERM);
 428     TEST_EQUAL(q.get_subquery(1).get_type(), q.LEAF_TERM);
 429 }
 430
 431 /// Regression test for bug introduced in 1.3.1 and fixed in 1.3.3.
 432 //  We were incorrectly converting a term which indexed all docs and was used
 433 //  in an unweighted phrase into an all docs postlist, so check that this
 434 //  case actually works.
 435 DEFINE_TESTCASE(phrasealldocs1, backend) {
 436     Xapian::Database db = get_database("apitest_declen");
 437     Xapian::Query q;
 438     static const char * const phrase[] = { "this", "is", "the" };
 439     q = Xapian::Query(q.OP_AND_NOT,
 440             Xapian::Query("paragraph"),
 441             Xapian::Query(q.OP_PHRASE, phrase, phrase + 3));
 442     Xapian::Enquire enq(db);
 443     enq.set_query(q);
 444     Xapian::MSet mset = enq.get_mset(0, 10);
 445     TEST_EQUAL(mset.size(), 3);
 446 }
 447
 448 struct wildcard_testcase {
 449     const char * pattern;
 450     Xapian::termcount max_expansion;
 451     char max_type;
 452     const char * terms[4];
 453 };
 454
 455 #define WILDCARD_EXCEPTION { 0, 0, 0, "" }
 456 static const
 457 wildcard_testcase wildcard1_testcases[] = {
 458     // Tries to expand to 7 terms.
 459     { "th",     6, 'E', WILDCARD_EXCEPTION },
 460     { "thou",   1, 'E', { "though", 0, 0, 0 } },
 461     { "s",      2, 'F', { "say", "search", 0, 0 } },
 462     { "s",      2, 'M', { "simpl", "so", 0, 0 } }
 463 };
 464
 465 DEFINE_TESTCASE(wildcard1, backend) {
 466     // FIXME: The counting of terms the wildcard expands to is per subdatabase,
 467     // so the wildcard may expand to more terms than the limit if some aren't
 468     // in all subdatabases.  Also WILDCARD_LIMIT_MOST_FREQUENT uses the
 469     // frequency from the subdatabase, and so may select different terms in
 470     // each subdatabase.
 471     SKIP_TEST_FOR_BACKEND("multi");
 472     Xapian::Database db = get_database("apitest_simpledata");
 473     Xapian::Enquire enq(db);
 474     const Xapian::Query::op o = Xapian::Query::OP_WILDCARD;
 475
 476     for (auto&& test : wildcard1_testcases) {
 477         tout << test.pattern << endl;
 478         auto tend = test.terms + 4;
 479         while (tend[-1] == NULL) --tend;
 480         bool expect_exception = (tend - test.terms == 4 && tend[-1][0] == '\0');
 481         Xapian::Query q;
 482         if (test.max_type) {
 483             int max_type;
 484             switch (test.max_type) {
 485                 case 'E':
 486                     max_type = Xapian::Query::WILDCARD_LIMIT_ERROR;
 487                     break;
 488                 case 'F':
 489                     max_type = Xapian::Query::WILDCARD_LIMIT_FIRST;
 490                     break;
 491                 case 'M':
 492                     max_type = Xapian::Query::WILDCARD_LIMIT_MOST_FREQUENT;
 493                     break;
 494                 default:
 495                     FAIL_TEST("Unexpected max_type value");
 496             }
 497             q = Xapian::Query(o, test.pattern, test.max_expansion, max_type);
 498         } else {
 499             q = Xapian::Query(o, test.pattern, test.max_expansion);
 500         }
 501         enq.set_query(q);
 502         try {
 503             Xapian::MSet mset = enq.get_mset(0, 10);
 504             TEST(!expect_exception);
 505             q = Xapian::Query(q.OP_SYNONYM, test.terms, tend);
 506             enq.set_query(q);
 507             Xapian::MSet mset2 = enq.get_mset(0, 10);
 508             TEST_EQUAL(mset.size(), mset2.size());
 509             TEST(mset_range_is_same(mset, 0, mset2, 0, mset.size()));
 510         } catch (const Xapian::WildcardError &) {
 511             TEST(expect_exception);
 512         }
 513     }
 514 }
 515
 516 /// Regression test for #696, fixed in 1.3.4.
 517 DEFINE_TESTCASE(wildcard2, backend) {
 518     // FIXME: The counting of terms the wildcard expands to is per subdatabase,
 519     // so the wildcard may expand to more terms than the limit if some aren't
 520     // in all subdatabases.  Also WILDCARD_LIMIT_MOST_FREQUENT uses the
 521     // frequency from the subdatabase, and so may select different terms in
 522     // each subdatabase.
 523     SKIP_TEST_FOR_BACKEND("multi");
 524     Xapian::Database db = get_database("apitest_simpledata");
 525     Xapian::Enquire enq(db);
 526     const Xapian::Query::op o = Xapian::Query::OP_WILDCARD;
 527
 528     const int max_type = Xapian::Query::WILDCARD_LIMIT_MOST_FREQUENT;
 529     Xapian::Query q0(o, "w", 2, max_type);
 530     Xapian::Query q(o, "s", 2, max_type);
 531     Xapian::Query q2(o, "t", 2, max_type);
 532     q = Xapian::Query(q.OP_OR, q0, q);
 533     q = Xapian::Query(q.OP_OR, q, q2);
 534     enq.set_query(q);
 535     Xapian::MSet mset = enq.get_mset(0, 10);
 536     TEST_EQUAL(mset.size(), 6);
 537 }
 538
 539 DEFINE_TESTCASE(dualprefixwildcard1, backend) {
 540     Xapian::Database db = get_database("apitest_simpledata");
 541     Xapian::Query q(Xapian::Query::OP_SYNONYM,
 542                     Xapian::Query(Xapian::Query::OP_WILDCARD, "fo"),
 543                     Xapian::Query(Xapian::Query::OP_WILDCARD, "Sfo"));
 544     tout << q.get_description() << endl;
 545     Xapian::Enquire enq(db);
 546     enq.set_query(q);
 547     TEST_EQUAL(enq.get_mset(0, 5).size(), 2);
 548 }
 549
 550 struct positional_testcase {
 551     int window;
 552     const char * terms[4];
 553     Xapian::docid result;
 554 };
 555
 556 static const
 557 positional_testcase loosephrase1_testcases[] = {
 558     { 5, { "expect", "to", "mset", 0 }, 0 },
 559     { 5, { "word", "well", "the", 0 }, 2 },
 560     { 5, { "if", "word", "doesnt", 0 }, 0 },
 561     { 5, { "at", "line", "three", 0 }, 0 },
 562     { 5, { "paragraph", "other", "the", 0 }, 0 },
 563     { 5, { "other", "the", "with", 0 }, 0 }
 564 };
 565
 566 /// Regression test for bug fixed in 1.3.3 and 1.2.21.
 567 DEFINE_TESTCASE(loosephrase1, backend) {
 568     Xapian::Database db = get_database("apitest_simpledata");
 569     Xapian::Enquire enq(db);
 570
 571     for (auto&& test : loosephrase1_testcases) {
 572         auto tend = test.terms + 4;
 573         while (tend[-1] == NULL) --tend;
 574         auto OP_PHRASE = Xapian::Query::OP_PHRASE;
 575         Xapian::Query q(OP_PHRASE, test.terms, tend, test.window);
 576         enq.set_query(q);
 577         Xapian::MSet mset = enq.get_mset(0, 10);
 578         if (test.result == 0) {
 579             TEST(mset.empty());
 580         } else {
 581             TEST_EQUAL(mset.size(), 1);
 582             TEST_EQUAL(*mset[0], test.result);
 583         }
 584     }
 585 }
 586
 587 static const
 588 positional_testcase loosenear1_testcases[] = {
 589     { 4, { "test", "the", "with", 0 }, 1 },
 590     { 4, { "expect", "word", "the", 0 }, 2 },
 591     { 4, { "line", "be", "blank", 0 }, 1 },
 592     { 2, { "banana", "banana", 0, 0 }, 0 },
 593     { 3, { "banana", "banana", 0, 0 }, 0 },
 594     { 2, { "word", "word", 0, 0 }, 2 },
 595     { 4, { "work", "meant", "work", 0 }, 0 },
 596     { 4, { "this", "one", "yet", "one" }, 0 }
 597 };
 598
 599 /// Regression tests for bugs fixed in 1.3.3 and 1.2.21.
 600 DEFINE_TESTCASE(loosenear1, backend) {
 601     Xapian::Database db = get_database("apitest_simpledata");
 602     Xapian::Enquire enq(db);
 603
 604     for (auto&& test : loosenear1_testcases) {
 605         auto tend = test.terms + 4;
 606         while (tend[-1] == NULL) --tend;
 607         Xapian::Query q(Xapian::Query::OP_NEAR, test.terms, tend, test.window);
 608         enq.set_query(q);
 609         Xapian::MSet mset = enq.get_mset(0, 10);
 610         if (test.result == 0) {
 611             TEST(mset.empty());
 612         } else {
 613             TEST_EQUAL(mset.size(), 1);
 614             TEST_EQUAL(*mset[0], test.result);
 615         }
 616     }
 617 }
 618
 619 /// Regression test for bug fixed in 1.3.6 - the first case segfaulted in 1.3.x.
 620 DEFINE_TESTCASE(complexphrase1, backend) {
 621     Xapian::Database db = get_database("apitest_simpledata");
 622     Xapian::Enquire enq(db);
 623     Xapian::Query query(Xapian::Query::OP_PHRASE,
 624             Xapian::Query("a") | Xapian::Query("b"),
 625             Xapian::Query("i"));
 626     enq.set_query(query);
 627     TEST(enq.get_mset(0, 10).empty());
 628     Xapian::Query query2(Xapian::Query::OP_PHRASE,
 629             Xapian::Query("a") | Xapian::Query("b"),
 630             Xapian::Query("c"));
 631     enq.set_query(query2);
 632     TEST(enq.get_mset(0, 10).empty());
 633 }
 634
 635 /// Regression test for bug fixed in 1.3.6 - the first case segfaulted in 1.3.x.
 636 DEFINE_TESTCASE(complexnear1, backend) {
 637     Xapian::Database db = get_database("apitest_simpledata");
 638     Xapian::Enquire enq(db);
 639     Xapian::Query query(Xapian::Query::OP_NEAR,
 640             Xapian::Query("a") | Xapian::Query("b"),
 641             Xapian::Query("i"));
 642     enq.set_query(query);
 643     TEST(enq.get_mset(0, 10).empty());
 644     Xapian::Query query2(Xapian::Query::OP_NEAR,
 645             Xapian::Query("a") | Xapian::Query("b"),
 646             Xapian::Query("c"));
 647     enq.set_query(query2);
 648     TEST(enq.get_mset(0, 10).empty());
 649 }
 650
 651 /// Check subqueries of MatchAll, MatchNothing and PostingSource are supported.
 652 DEFINE_TESTCASE(complexphrase2, backend) {
 653     Xapian::Database db = get_database("apitest_simpledata");
 654     Xapian::Enquire enq(db);
 655     Xapian::ValueWeightPostingSource ps(0);
 656     Xapian::Query subqs[3] = {
 657         Xapian::Query(Xapian::Query::OP_PHRASE,
 658             Xapian::Query("a"),
 659             Xapian::Query(&ps)),
 660         Xapian::Query(Xapian::Query::OP_PHRASE,
 661             Xapian::Query("and"),
 662             Xapian::Query::MatchAll),
 663         Xapian::Query(Xapian::Query::OP_PHRASE,
 664             Xapian::Query("at"),
 665             Xapian::Query::MatchNothing)
 666     };
 667     Xapian::Query query(Xapian::Query::OP_OR, subqs, subqs + 3);
 668     enq.set_query(query);
 669     (void)enq.get_mset(0, 10);
 670 }
 671
 672 /// Check subqueries of MatchAll, MatchNothing and PostingSource are supported.
 673 DEFINE_TESTCASE(complexnear2, backend) {
 674     Xapian::Database db = get_database("apitest_simpledata");
 675     Xapian::Enquire enq(db);
 676     Xapian::ValueWeightPostingSource ps(0);
 677     Xapian::Query subqs[3] = {
 678         Xapian::Query(Xapian::Query::OP_NEAR,
 679             Xapian::Query("a"),
 680             Xapian::Query(&ps)),
 681         Xapian::Query(Xapian::Query::OP_NEAR,
 682             Xapian::Query("and"),
 683             Xapian::Query::MatchAll),
 684         Xapian::Query(Xapian::Query::OP_NEAR,
 685             Xapian::Query("at"),
 686             Xapian::Query::MatchNothing)
 687     };
 688     Xapian::Query query(Xapian::Query::OP_OR, subqs, subqs + 3);
 689     enq.set_query(query);
 690     (void)enq.get_mset(0, 10);
 691 }
 692
 693 /// A zero estimated number of matches broke the code to round the estimate.
 694 DEFINE_TESTCASE(zeroestimate1, backend) {
 695     Xapian::Enquire enquire(get_database("apitest_simpledata"));
 696     Xapian::Query phrase(Xapian::Query::OP_PHRASE,
 697                          Xapian::Query("absolute"),
 698                          Xapian::Query("rubbish"));
 699     enquire.set_query(phrase &~ Xapian::Query("queri"));
 700     Xapian::MSet mset = enquire.get_mset(0, 0);
 701     TEST_EQUAL(mset.get_matches_estimated(), 0);
 702 }
 703
 704 /// Feature test for OR under OP_PHRASE support added in 1.4.3.
 705 DEFINE_TESTCASE(complexphrase3, backend) {
 706     Xapian::Database db = get_database("apitest_simpledata");
 707     Xapian::Enquire enq(db);
 708     Xapian::Query query(Xapian::Query::OP_PHRASE,
 709             Xapian::Query("is") | Xapian::Query("as") | Xapian::Query("be"),
 710             Xapian::Query("a"));
 711     enq.set_query(query);
 712     mset_expect_order(enq.get_mset(0, 10), 1);
 713     Xapian::Query query2(Xapian::Query::OP_PHRASE,
 714             Xapian::Query("a"),
 715             Xapian::Query("is") | Xapian::Query("as") | Xapian::Query("be"));
 716     enq.set_query(query2);
 717     mset_expect_order(enq.get_mset(0, 10));
 718     Xapian::Query query3(Xapian::Query::OP_PHRASE,
 719             Xapian::Query("one") | Xapian::Query("with"),
 720             Xapian::Query("the") | Xapian::Query("of") | Xapian::Query("line"));
 721     enq.set_query(query3);
 722     mset_expect_order(enq.get_mset(0, 10), 1, 4, 5);
 723     Xapian::Query query4(Xapian::Query::OP_PHRASE,
 724             Xapian::Query("the") | Xapian::Query("of") | Xapian::Query("line"),
 725             Xapian::Query("one") | Xapian::Query("with"));
 726     enq.set_query(query4);
 727     mset_expect_order(enq.get_mset(0, 10));
 728 }
 729
 730 /// Feature test for OR under OP_NEAR support added in 1.4.3.
 731 DEFINE_TESTCASE(complexnear3, backend) {
 732     Xapian::Database db = get_database("apitest_simpledata");
 733     Xapian::Enquire enq(db);
 734     Xapian::Query query(Xapian::Query::OP_NEAR,
 735             Xapian::Query("is") | Xapian::Query("as") | Xapian::Query("be"),
 736             Xapian::Query("a"));
 737     enq.set_query(query);
 738     mset_expect_order(enq.get_mset(0, 10), 1);
 739     Xapian::Query query2(Xapian::Query::OP_NEAR,
 740             Xapian::Query("a"),
 741             Xapian::Query("is") | Xapian::Query("as") | Xapian::Query("be"));
 742     enq.set_query(query2);
 743     mset_expect_order(enq.get_mset(0, 10), 1);
 744     Xapian::Query query3(Xapian::Query::OP_NEAR,
 745             Xapian::Query("one") | Xapian::Query("with"),
 746             Xapian::Query("the") | Xapian::Query("of") | Xapian::Query("line"));
 747     enq.set_query(query3);
 748     mset_expect_order(enq.get_mset(0, 10), 1, 4, 5);
 749     Xapian::Query query4(Xapian::Query::OP_NEAR,
 750             Xapian::Query("the") | Xapian::Query("of") | Xapian::Query("line"),
 751             Xapian::Query("one") | Xapian::Query("with"));
 752     enq.set_query(query4);
 753     mset_expect_order(enq.get_mset(0, 10), 1, 4, 5);
 754 }
 755
 756 static void
 757 gen_subdbwithoutpos1_db(Xapian::WritableDatabase& db, const string&)
 758 {
 759     Xapian::Document doc;
 760     doc.add_term("this");
 761     doc.add_term("paragraph");
 762     doc.add_term("wibble", 5);
 763     db.add_document(doc);
 764 }
 765
 766 DEFINE_TESTCASE(subdbwithoutpos1, generated) {
 767     XFAIL_FOR_BACKEND("remote",
 768                       "Known but obscure remote bug which doesn't justify "
 769                       "protocol version bump");
 770     XFAIL_FOR_BACKEND("multi_remote",
 771                       "Known but obscure remote bug which doesn't justify "
 772                       "protocol version bump");
 773
 774     Xapian::Database db(get_database("apitest_simpledata"));
 775
 776     Xapian::Query q(Xapian::Query::OP_PHRASE,
 777                     Xapian::Query("this"),
 778                     Xapian::Query("paragraph"));
 779
 780     Xapian::Enquire enq1(db);
 781     enq1.set_query(q);
 782     Xapian::MSet mset1 = enq1.get_mset(0, 10);
 783     TEST_EQUAL(mset1.size(), 3);
 784
 785     Xapian::Database db2 =
 786         get_database("subdbwithoutpos1", gen_subdbwithoutpos1_db);
 787
 788     // If a database has no positional info, OP_PHRASE -> OP_AND.
 789     Xapian::Enquire enq2(db2);
 790     enq2.set_query(q);
 791     Xapian::MSet mset2 = enq2.get_mset(0, 10);
 792     TEST_EQUAL(mset2.size(), 1);
 793
 794     // If one sub-database in a combined database has no positional info but
 795     // other sub-databases do, then we shouldn't convert OP_PHRASE to OP_AND
 796     // (but prior to 1.4.3 we did).
 797     db.add_database(db2);
 798     Xapian::Enquire enq3(db);
 799     enq3.set_query(q);
 800     Xapian::MSet mset3 = enq3.get_mset(0, 10);
 801     TEST_EQUAL(mset3.size(), 3);
 802     // Regression test for bug introduced in 1.4.3 which led to a division by
 803     // zero and then (at least on Linux) we got 1% here.
 804     TEST_EQUAL(mset3[0].get_percent(), 100);
 805
 806     // Regression test for https://trac.xapian.org/ticket/752
 807     enq3.set_query((Xapian::Query("this") & q) | Xapian::Query("wibble"));
 808     mset3 = enq3.get_mset(0, 10);
 809     TEST_EQUAL(mset3.size(), 4);
 810 }
 811
 812 // Regression test for bug fixed in 1.4.4 and 1.2.25.
 813 DEFINE_TESTCASE(notandor1, backend) {
 814     Xapian::Database db(get_database("etext"));
 815     using Xapian::Query;
 816     Query q = Query("the") &~ (Query("friedrich") &
 817                                (Query("day") | Query("night")));
 818     Xapian::Enquire enq(db);
 819     enq.set_query(q);
 820
 821     Xapian::MSet mset = enq.get_mset(0, 10, db.get_doccount());
 822     TEST_EQUAL(mset.get_matches_estimated(), 344);
 823 }
 824
 825 // Regression test for bug fixed in git master before 1.5.0.
 826 DEFINE_TESTCASE(boolorbug1, backend) {
 827     Xapian::Database db(get_database("etext"));
 828     using Xapian::Query;
 829     Query q = Query("the") &~ Query(Query::OP_WILDCARD, "pru");
 830     Xapian::Enquire enq(db);
 831     enq.set_query(q);
 832
 833     Xapian::MSet mset = enq.get_mset(0, 10, db.get_doccount());
 834     // Due to a bug in BoolOrPostList this returned 330 results.
 835     TEST_EQUAL(mset.get_matches_estimated(), 331);
 836 }
 837
 838 // Regression test for bug introduced in 1.4.13 and fixed in 1.4.14.
 839 DEFINE_TESTCASE(hoistnotbug1, backend) {
 840     Xapian::Database db(get_database("etext"));
 841     using Xapian::Query;
 842     Query q(Query::OP_PHRASE, Query("the"), Query("king"));
 843     q &= ~Query("worldtornado");
 844     q &= Query("a");
 845     Xapian::Enquire enq(db);
 846     enq.set_query(q);
 847
 848     // This reliably fails before the fix in an assertion build, and may crash
 849     // in other builds.
 850     Xapian::MSet mset = enq.get_mset(0, 10, db.get_doccount());
 851     TEST_EQUAL(mset.get_matches_estimated(), 42);
 852 }
 853
 854 // Regression test for segfault optimising query on git master before 1.5.0.
 855 DEFINE_TESTCASE(emptynot1, backend) {
 856     Xapian::Database db(get_database("apitest_simpledata"));
 857     Xapian::Enquire enq(db);
 858     enq.set_weighting_scheme(Xapian::BoolWeight());
 859     Xapian::Query query = Xapian::Query("document") & Xapian::Query("api");
 860     // This range won't match anything, so collapses to MatchNothing as we
 861     // optimise the query.
 862     query = Xapian::Query(query.OP_AND_NOT,
 863                           query,
 864                           Xapian::Query(Xapian::Query::OP_VALUE_GE, 1234, "x"));
 865     enq.set_query(query);
 866     Xapian::MSet mset = enq.get_mset(0, 10);
 867     TEST_EQUAL(mset.size(), 1);
 868     // Essentially the same test but with a term which doesn't match anything
 869     // instead of a range.
 870     query = Xapian::Query("document") & Xapian::Query("api");
 871     query = Xapian::Query(query.OP_AND_NOT,
 872                           query,
 873                           Xapian::Query("nosuchterm"));
 874     enq.set_query(query);
 875     mset = enq.get_mset(0, 10);
 876     TEST_EQUAL(mset.size(), 1);
 877 }
 878
 879 // Similar case to emptynot1 but for OP_AND_MAYBE.  This case wasn't failing,
 880 // so this isn't a regression test, but we do want to ensure it works.
 881 DEFINE_TESTCASE(emptymaybe1, backend) {
 882     Xapian::Database db(get_database("apitest_simpledata"));
 883     Xapian::Enquire enq(db);
 884     enq.set_weighting_scheme(Xapian::BoolWeight());
 885     Xapian::Query query = Xapian::Query("document") & Xapian::Query("api");
 886     // This range won't match anything, so collapses to MatchNothing as we
 887     // optimise the query.
 888     query = Xapian::Query(query.OP_AND_MAYBE,
 889                           query,
 890                           Xapian::Query(Xapian::Query::OP_VALUE_GE, 1234, "x"));
 891     enq.set_query(query);
 892     Xapian::MSet mset = enq.get_mset(0, 10);
 893     TEST_EQUAL(mset.size(), 1);
 894     // Essentially the same test but with a term which doesn't match anything
 895     // instead of a range.
 896     query = Xapian::Query("document") & Xapian::Query("api");
 897     query = Xapian::Query(query.OP_AND_MAYBE,
 898                           query,
 899                           Xapian::Query("nosuchterm"));
 900     enq.set_query(query);
 901     mset = enq.get_mset(0, 10);
 902     TEST_EQUAL(mset.size(), 1);
 903 }
 904
 905 DEFINE_TESTCASE(phraseweightcheckbug1, backend) {
 906     Xapian::Database db(get_database("phraseweightcheckbug1"));
 907     Xapian::Enquire enq(db);
 908     static const char* const words[] = {"hello", "world"};
 909     Xapian::Query query{Xapian::Query::OP_PHRASE, begin(words), end(words), 2};
 910     query = Xapian::Query(query.OP_OR, query, Xapian::Query("most"));
 911     tout << query.get_description() << '\n';
 912     enq.set_query(query);
 913     Xapian::MSet mset = enq.get_mset(0, 3);
 914     TEST_EQUAL(mset.size(), 3);
 915 }
 916
 917 DEFINE_TESTCASE(orphanedhint1, backend) {
 918     Xapian::Database db(get_database("apitest_simpledata"));
 919     Xapian::Enquire enq(db);
 920     auto OP_WILDCARD = Xapian::Query::OP_WILDCARD;
 921     Xapian::Query query = Xapian::Query(OP_WILDCARD, "doc") &
 922                           Xapian::Query(OP_WILDCARD, "xyzzy");
 923     query |= Xapian::Query("test");
 924     tout << query.get_description() << '\n';
 925     enq.set_query(query);
 926     Xapian::MSet mset = enq.get_mset(0, 3);
 927     TEST_EQUAL(mset.size(), 1);
 928 }