xapian-core/api/postingsource.cc

   1 /** @file
   2  * @brief External sources of posting information
   3  */
   4 /* Copyright (C) 2008-2024 Olly Betts
   5  * Copyright (C) 2008,2009 Lemur Consulting Ltd
   6  * Copyright (C) 2010 Richard Boulton
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "xapian/postingsource.h"
  26
  27 #include "backends/databaseinternal.h"
  28 #include "backends/documentinternal.h"
  29 #include "matcher/postlisttree.h"
  30
  31 #include "xapian/document.h"
  32 #include "xapian/error.h"
  33 #include "xapian/queryparser.h" // For sortable_unserialise().
  34
  35 #include "omassert.h"
  36 #include "pack.h"
  37 #include "serialise-double.h"
  38 #include "str.h"
  39
  40 #include <cfloat>
  41 #include <memory>
  42
  43 using namespace std;
  44
  45 namespace Xapian {
  46
  47 PostingSource::~PostingSource() { }
  48
  49 double
  50 PostingSource::get_weight() const
  51 {
  52     return 0;
  53 }
  54
  55 void
  56 PostingSource::skip_to(Xapian::docid did, double min_wt)
  57 {
  58     while (!at_end() && get_docid() < did) {
  59         next(min_wt);
  60     }
  61 }
  62
  63 bool
  64 PostingSource::check(Xapian::docid did, double min_wt)
  65 {
  66     skip_to(did, min_wt);
  67     return true;
  68 }
  69
  70 PostingSource *
  71 PostingSource::clone() const
  72 {
  73     return NULL;
  74 }
  75
  76 string
  77 PostingSource::name() const
  78 {
  79     return string();
  80 }
  81
  82 string
  83 PostingSource::serialise() const
  84 {
  85     throw Xapian::UnimplementedError("serialise() not supported for this PostingSource");
  86 }
  87
  88 PostingSource *
  89 PostingSource::unserialise(const string &) const
  90 {
  91     throw Xapian::UnimplementedError("unserialise() not supported for this PostingSource");
  92 }
  93
  94 PostingSource *
  95 PostingSource::unserialise_with_registry(const std::string &s,
  96                                          const Registry &) const
  97 {
  98     return unserialise(s);
  99 }
 100
 101 void
 102 PostingSource::reset(const Database& db, Xapian::doccount)
 103 {
 104     init(db);
 105 }
 106
 107 void
 108 PostingSource::init(const Database&)
 109 {
 110     const char* msg = "Either PostingSource::reset() or PostingSource::init() "
 111                       "must be overridden";
 112     throw Xapian::InvalidOperationError(msg);
 113 }
 114
 115 string
 116 PostingSource::get_description() const
 117 {
 118     return "Xapian::PostingSource subclass";
 119 }
 120
 121 Xapian::doccount
 122 ValuePostingSource::get_termfreq_min() const
 123 {
 124     return termfreq_min;
 125 }
 126
 127 Xapian::doccount
 128 ValuePostingSource::get_termfreq_est() const
 129 {
 130     return termfreq_est;
 131 }
 132
 133 Xapian::doccount
 134 ValuePostingSource::get_termfreq_max() const
 135 {
 136     return termfreq_max;
 137 }
 138
 139 void
 140 ValuePostingSource::next(double min_wt)
 141 {
 142     if (!started) {
 143         started = true;
 144         value_it = db.valuestream_begin(slot);
 145     } else {
 146         ++value_it;
 147     }
 148
 149     if (value_it == db.valuestream_end(slot)) return;
 150
 151     if (min_wt > get_maxweight()) {
 152         value_it = db.valuestream_end(slot);
 153         return;
 154     }
 155 }
 156
 157 void
 158 ValuePostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 159 {
 160     if (!started) {
 161         started = true;
 162         value_it = db.valuestream_begin(slot);
 163
 164         if (value_it == db.valuestream_end(slot)) return;
 165     }
 166
 167     if (min_wt > get_maxweight()) {
 168         value_it = db.valuestream_end(slot);
 169         return;
 170     }
 171     value_it.skip_to(min_docid);
 172 }
 173
 174 bool
 175 ValuePostingSource::check(Xapian::docid min_docid, double min_wt)
 176 {
 177     if (!started) {
 178         started = true;
 179         value_it = db.valuestream_begin(slot);
 180
 181         if (value_it == db.valuestream_end(slot)) return true;
 182     }
 183
 184     if (min_wt > get_maxweight()) {
 185         value_it = db.valuestream_end(slot);
 186         return true;
 187     }
 188     return value_it.check(min_docid);
 189 }
 190
 191 bool
 192 ValuePostingSource::at_end() const
 193 {
 194     return started && value_it == db.valuestream_end(slot);
 195 }
 196
 197 Xapian::docid
 198 ValuePostingSource::get_docid() const
 199 {
 200     return value_it.get_docid();
 201 }
 202
 203 void
 204 ValuePostingSource::reset(const Database& db_, Xapian::doccount)
 205 {
 206     db = db_;
 207     started = false;
 208     set_maxweight(DBL_MAX);
 209     termfreq_max = db.get_value_freq(slot);
 210     termfreq_est = termfreq_max;
 211     termfreq_min = termfreq_max;
 212 }
 213
 214 string
 215 ValuePostingSource::get_description() const
 216 {
 217     string desc("Xapian::ValuePostingSource(slot=");
 218     desc += str(get_slot());
 219     desc += ")";
 220     return desc;
 221 }
 222
 223
 224 ValueWeightPostingSource::ValueWeightPostingSource(Xapian::valueno slot_)
 225         : ValuePostingSource(slot_)
 226 {
 227 }
 228
 229 double
 230 ValueWeightPostingSource::get_weight() const
 231 {
 232     Assert(!at_end());
 233     Assert(get_started());
 234     return sortable_unserialise(get_value());
 235 }
 236
 237 ValueWeightPostingSource *
 238 ValueWeightPostingSource::clone() const
 239 {
 240     return new ValueWeightPostingSource(get_slot());
 241 }
 242
 243 string
 244 ValueWeightPostingSource::name() const
 245 {
 246     return string("Xapian::ValueWeightPostingSource");
 247 }
 248
 249 string
 250 ValueWeightPostingSource::serialise() const
 251 {
 252     string result;
 253     pack_uint_last(result, get_slot());
 254     return result;
 255 }
 256
 257 ValueWeightPostingSource *
 258 ValueWeightPostingSource::unserialise(const string &s) const
 259 {
 260     const char * p = s.data();
 261     const char * end = p + s.size();
 262
 263     Xapian::valueno new_slot;
 264     if (!unpack_uint_last(&p, end, &new_slot)) {
 265         unpack_throw_serialisation_error(p);
 266     }
 267
 268     return new ValueWeightPostingSource(new_slot);
 269 }
 270
 271 void
 272 ValueWeightPostingSource::reset(const Database& db_,
 273                                 Xapian::doccount shard_index)
 274 {
 275     ValuePostingSource::reset(db_, shard_index);
 276
 277     string upper_bound = get_database().get_value_upper_bound(get_slot());
 278     if (upper_bound.empty()) {
 279         // This should only happen if there are no entries, in which case the
 280         // maxweight is 0.
 281         set_maxweight(0.0);
 282     } else {
 283         set_maxweight(sortable_unserialise(upper_bound));
 284     }
 285 }
 286
 287 string
 288 ValueWeightPostingSource::get_description() const
 289 {
 290     string desc("Xapian::ValueWeightPostingSource(slot=");
 291     desc += str(get_slot());
 292     desc += ")";
 293     return desc;
 294 }
 295
 296
 297 ValueMapPostingSource::ValueMapPostingSource(Xapian::valueno slot_)
 298         : ValuePostingSource(slot_),
 299           default_weight(0.0),
 300           max_weight_in_map(0.0)
 301 {
 302 }
 303
 304 void
 305 ValueMapPostingSource::add_mapping(const string & key, double wt)
 306 {
 307     weight_map[key] = wt;
 308     max_weight_in_map = max(wt, max_weight_in_map);
 309 }
 310
 311 void
 312 ValueMapPostingSource::clear_mappings()
 313 {
 314     weight_map.clear();
 315     max_weight_in_map = 0.0;
 316 }
 317
 318 void
 319 ValueMapPostingSource::set_default_weight(double wt)
 320 {
 321     default_weight = wt;
 322 }
 323
 324 double
 325 ValueMapPostingSource::get_weight() const
 326 {
 327     map<string, double>::const_iterator wit = weight_map.find(get_value());
 328     if (wit == weight_map.end()) {
 329         return default_weight;
 330     }
 331     return wit->second;
 332 }
 333
 334 ValueMapPostingSource *
 335 ValueMapPostingSource::clone() const
 336 {
 337     unique_ptr<ValueMapPostingSource> res(
 338             new ValueMapPostingSource(get_slot()));
 339     map<string, double>::const_iterator i;
 340     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 341         res->add_mapping(i->first, i->second);
 342     }
 343     res->set_default_weight(default_weight);
 344     return res.release();
 345 }
 346
 347 string
 348 ValueMapPostingSource::name() const
 349 {
 350     return string("Xapian::ValueMapPostingSource");
 351 }
 352
 353 string
 354 ValueMapPostingSource::serialise() const
 355 {
 356     string result;
 357     pack_uint(result, get_slot());
 358     result += serialise_double(default_weight);
 359
 360     map<string, double>::const_iterator i;
 361     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 362         pack_string(result, i->first);
 363         result.append(serialise_double(i->second));
 364     }
 365
 366     return result;
 367 }
 368
 369 ValueMapPostingSource *
 370 ValueMapPostingSource::unserialise(const string &s) const
 371 {
 372     const char * p = s.data();
 373     const char * end = p + s.size();
 374
 375     Xapian::valueno new_slot;
 376     if (!unpack_uint(&p, end, &new_slot)) {
 377         unpack_throw_serialisation_error(p);
 378     }
 379     unique_ptr<ValueMapPostingSource> res(new ValueMapPostingSource(new_slot));
 380     res->set_default_weight(unserialise_double(&p, end));
 381     while (p != end) {
 382         string key;
 383         if (!unpack_string(&p, end, key)) {
 384             unpack_throw_serialisation_error(p);
 385         }
 386         res->add_mapping(key, unserialise_double(&p, end));
 387     }
 388     return res.release();
 389 }
 390
 391 void
 392 ValueMapPostingSource::reset(const Database& db_, Xapian::doccount shard_index)
 393 {
 394     ValuePostingSource::reset(db_, shard_index);
 395     set_maxweight(max(max_weight_in_map, default_weight));
 396 }
 397
 398 string
 399 ValueMapPostingSource::get_description() const
 400 {
 401     string desc("Xapian::ValueMapPostingSource(slot=");
 402     desc += str(get_slot());
 403     desc += ")";
 404     return desc;
 405 }
 406
 407 FixedWeightPostingSource::FixedWeightPostingSource(double wt)
 408     : started(false)
 409 {
 410     // The weight is fixed at wt, so that's the maxweight too.  So just store wt
 411     // as the maxweight and we can read it from there when we need it.
 412     set_maxweight(wt);
 413 }
 414
 415 Xapian::doccount
 416 FixedWeightPostingSource::get_termfreq_min() const
 417 {
 418     return termfreq;
 419 }
 420
 421 Xapian::doccount
 422 FixedWeightPostingSource::get_termfreq_est() const
 423 {
 424     return termfreq;
 425 }
 426
 427 Xapian::doccount
 428 FixedWeightPostingSource::get_termfreq_max() const
 429 {
 430     return termfreq;
 431 }
 432
 433 double
 434 FixedWeightPostingSource::get_weight() const
 435 {
 436     return get_maxweight();
 437 }
 438
 439 void
 440 FixedWeightPostingSource::next(double min_wt)
 441 {
 442     if (!started) {
 443         started = true;
 444         it = db.postlist_begin(string());
 445     } else {
 446         ++it;
 447     }
 448
 449     if (it == db.postlist_end(string())) return;
 450
 451     if (check_docid) {
 452         it.skip_to(check_docid + 1);
 453         check_docid = 0;
 454     }
 455
 456     if (min_wt > get_maxweight()) {
 457         it = db.postlist_end(string());
 458     }
 459 }
 460
 461 void
 462 FixedWeightPostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 463 {
 464     if (!started) {
 465         started = true;
 466         it = db.postlist_begin(string());
 467
 468         if (it == db.postlist_end(string())) return;
 469     }
 470
 471     if (check_docid) {
 472         if (min_docid < check_docid)
 473             min_docid = check_docid + 1;
 474         check_docid = 0;
 475     }
 476
 477     if (min_wt > get_maxweight()) {
 478         it = db.postlist_end(string());
 479         return;
 480     }
 481     it.skip_to(min_docid);
 482 }
 483
 484 bool
 485 FixedWeightPostingSource::check(Xapian::docid min_docid, double)
 486 {
 487     // We're guaranteed not to be called if the document doesn't
 488     // exist, so just remember the docid passed, and return true.
 489     check_docid = min_docid;
 490     return true;
 491 }
 492
 493 bool
 494 FixedWeightPostingSource::at_end() const
 495 {
 496     if (check_docid != 0) return false;
 497     return started && it == db.postlist_end(string());
 498 }
 499
 500 Xapian::docid
 501 FixedWeightPostingSource::get_docid() const
 502 {
 503     if (check_docid != 0) return check_docid;
 504     return *it;
 505 }
 506
 507 FixedWeightPostingSource *
 508 FixedWeightPostingSource::clone() const
 509 {
 510     return new FixedWeightPostingSource(get_maxweight());
 511 }
 512
 513 string
 514 FixedWeightPostingSource::name() const
 515 {
 516     return string("Xapian::FixedWeightPostingSource");
 517 }
 518
 519 string
 520 FixedWeightPostingSource::serialise() const
 521 {
 522     return serialise_double(get_maxweight());
 523 }
 524
 525 FixedWeightPostingSource *
 526 FixedWeightPostingSource::unserialise(const string &s) const
 527 {
 528     const char * p = s.data();
 529     const char * s_end = p + s.size();
 530     double new_wt = unserialise_double(&p, s_end);
 531     if (p != s_end) {
 532         throw Xapian::NetworkError("Bad serialised FixedWeightPostingSource - junk at end");
 533     }
 534     return new FixedWeightPostingSource(new_wt);
 535 }
 536
 537 void
 538 FixedWeightPostingSource::reset(const Xapian::Database& db_, Xapian::doccount)
 539 {
 540     db = db_;
 541     termfreq = db_.get_doccount();
 542     started = false;
 543     check_docid = 0;
 544 }
 545
 546 string
 547 FixedWeightPostingSource::get_description() const
 548 {
 549     string desc("Xapian::FixedWeightPostingSource(wt=");
 550     desc += str(get_maxweight());
 551     desc += ")";
 552     return desc;
 553 }
 554
 555 }