xapian-core/api/postingsource.cc

   1 /** @file
   2  * @brief External sources of posting information
   3  */
   4 /* Copyright (C) 2008-2022 Olly Betts
   5  * Copyright (C) 2008,2009 Lemur Consulting Ltd
   6  * Copyright (C) 2010 Richard Boulton
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  21  */
  22
  23 #include <config.h>
  24
  25 // We need to be able to set deprecated members of ValuePostingSource.
  26 #define XAPIAN_DEPRECATED(X) X
  27 #include "xapian/postingsource.h"
  28
  29 #include "autoptr.h"
  30
  31 #include "backends/database.h"
  32 #include "backends/document.h"
  33 #include "matcher/multimatch.h"
  34
  35 #include "xapian/document.h"
  36 #include "xapian/error.h"
  37 #include "xapian/queryparser.h" // For sortable_unserialise().
  38
  39 #include "omassert.h"
  40 #include "net/length.h"
  41 #include "serialise-double.h"
  42 #include "str.h"
  43
  44 #include <cfloat>
  45
  46 using namespace std;
  47
  48 namespace Xapian {
  49
  50 PostingSource::~PostingSource() { }
  51
  52 void
  53 PostingSource::set_maxweight(double max_weight)
  54 {
  55     if (usual(matcher_)) {
  56         MultiMatch * multimatch = static_cast<MultiMatch*>(matcher_);
  57         multimatch->recalc_maxweight();
  58     }
  59     max_weight_ = max_weight;
  60 }
  61
  62 double
  63 PostingSource::get_weight() const
  64 {
  65     return 0;
  66 }
  67
  68 void
  69 PostingSource::skip_to(Xapian::docid did, double min_wt)
  70 {
  71     while (!at_end() && get_docid() < did) {
  72         next(min_wt);
  73     }
  74 }
  75
  76 bool
  77 PostingSource::check(Xapian::docid did, double min_wt)
  78 {
  79     skip_to(did, min_wt);
  80     return true;
  81 }
  82
  83 PostingSource *
  84 PostingSource::clone() const
  85 {
  86     return NULL;
  87 }
  88
  89 string
  90 PostingSource::name() const
  91 {
  92     return string();
  93 }
  94
  95 string
  96 PostingSource::serialise() const
  97 {
  98     throw Xapian::UnimplementedError("serialise() not supported for this PostingSource");
  99 }
 100
 101 PostingSource *
 102 PostingSource::unserialise(const string &) const
 103 {
 104     throw Xapian::UnimplementedError("unserialise() not supported for this PostingSource");
 105 }
 106
 107 PostingSource *
 108 PostingSource::unserialise_with_registry(const std::string &s,
 109                                          const Registry &) const
 110 {
 111     return unserialise(s);
 112 }
 113
 114 string
 115 PostingSource::get_description() const
 116 {
 117     return "Xapian::PostingSource subclass";
 118 }
 119
 120
 121 ValuePostingSource::ValuePostingSource(Xapian::valueno slot_)
 122         : real_slot(slot_),
 123           db(real_db),
 124           slot(real_slot),
 125           value_it(real_value_it),
 126           started(real_started),
 127           termfreq_min(real_termfreq_min),
 128           termfreq_est(real_termfreq_est),
 129           termfreq_max(real_termfreq_max)
 130 {
 131 }
 132
 133 Xapian::doccount
 134 ValuePostingSource::get_termfreq_min() const
 135 {
 136     return real_termfreq_min;
 137 }
 138
 139 Xapian::doccount
 140 ValuePostingSource::get_termfreq_est() const
 141 {
 142     return real_termfreq_est;
 143 }
 144
 145 Xapian::doccount
 146 ValuePostingSource::get_termfreq_max() const
 147 {
 148     return real_termfreq_max;
 149 }
 150
 151 void
 152 ValuePostingSource::next(double min_wt)
 153 {
 154     if (!real_started) {
 155         real_started = true;
 156         real_value_it = real_db.valuestream_begin(real_slot);
 157     } else {
 158         ++real_value_it;
 159     }
 160
 161     if (real_value_it == real_db.valuestream_end(real_slot)) return;
 162
 163     if (min_wt > get_maxweight()) {
 164         real_value_it = real_db.valuestream_end(real_slot);
 165         return;
 166     }
 167 }
 168
 169 void
 170 ValuePostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 171 {
 172     if (!real_started) {
 173         real_started = true;
 174         real_value_it = real_db.valuestream_begin(real_slot);
 175
 176         if (real_value_it == real_db.valuestream_end(real_slot)) return;
 177     }
 178
 179     if (min_wt > get_maxweight()) {
 180         real_value_it = real_db.valuestream_end(real_slot);
 181         return;
 182     }
 183     real_value_it.skip_to(min_docid);
 184 }
 185
 186 bool
 187 ValuePostingSource::check(Xapian::docid min_docid, double min_wt)
 188 {
 189     if (!real_started) {
 190         real_started = true;
 191         real_value_it = real_db.valuestream_begin(real_slot);
 192
 193         if (real_value_it == real_db.valuestream_end(real_slot)) return true;
 194     }
 195
 196     if (min_wt > get_maxweight()) {
 197         real_value_it = real_db.valuestream_end(real_slot);
 198         return true;
 199     }
 200     return real_value_it.check(min_docid);
 201 }
 202
 203 bool
 204 ValuePostingSource::at_end() const
 205 {
 206     return real_started && real_value_it == real_db.valuestream_end(real_slot);
 207 }
 208
 209 Xapian::docid
 210 ValuePostingSource::get_docid() const
 211 {
 212     return real_value_it.get_docid();
 213 }
 214
 215 void
 216 ValuePostingSource::init(const Database & db_)
 217 {
 218     real_db = db_;
 219     real_started = false;
 220     set_maxweight(DBL_MAX);
 221     real_termfreq_max = real_db.get_value_freq(real_slot);
 222     real_termfreq_est = real_termfreq_max;
 223     real_termfreq_min = real_termfreq_max;
 224 }
 225
 226
 227 ValueWeightPostingSource::ValueWeightPostingSource(Xapian::valueno slot_)
 228         : ValuePostingSource(slot_)
 229 {
 230 }
 231
 232 double
 233 ValueWeightPostingSource::get_weight() const
 234 {
 235     Assert(!at_end());
 236     Assert(get_started());
 237     return sortable_unserialise(get_value());
 238 }
 239
 240 ValueWeightPostingSource *
 241 ValueWeightPostingSource::clone() const
 242 {
 243     return new ValueWeightPostingSource(get_slot());
 244 }
 245
 246 string
 247 ValueWeightPostingSource::name() const
 248 {
 249     return string("Xapian::ValueWeightPostingSource");
 250 }
 251
 252 string
 253 ValueWeightPostingSource::serialise() const
 254 {
 255     return encode_length(get_slot());
 256 }
 257
 258 ValueWeightPostingSource *
 259 ValueWeightPostingSource::unserialise(const string &s) const
 260 {
 261     const char * p = s.data();
 262     const char * end = p + s.size();
 263
 264     Xapian::valueno new_slot;
 265     decode_length(&p, end, new_slot);
 266     if (p != end) {
 267         throw Xapian::NetworkError("Bad serialised ValueWeightPostingSource - junk at end");
 268     }
 269
 270     return new ValueWeightPostingSource(new_slot);
 271 }
 272
 273 void
 274 ValueWeightPostingSource::init(const Database & db_)
 275 {
 276     ValuePostingSource::init(db_);
 277
 278     string upper_bound = get_database().get_value_upper_bound(get_slot());
 279     if (upper_bound.empty()) {
 280         // This should only happen if there are no entries, in which case the
 281         // maxweight is 0.
 282         set_maxweight(0.0);
 283     } else {
 284         set_maxweight(sortable_unserialise(upper_bound));
 285     }
 286 }
 287
 288 string
 289 ValueWeightPostingSource::get_description() const
 290 {
 291     string desc("Xapian::ValueWeightPostingSource(slot=");
 292     desc += str(get_slot());
 293     desc += ")";
 294     return desc;
 295 }
 296
 297
 298 ValueMapPostingSource::ValueMapPostingSource(Xapian::valueno slot_)
 299         : ValuePostingSource(slot_),
 300           default_weight(0.0),
 301           max_weight_in_map(0.0)
 302 {
 303 }
 304
 305 void
 306 ValueMapPostingSource::add_mapping(const string & key, double wt)
 307 {
 308     weight_map[key] = wt;
 309     max_weight_in_map = max(wt, max_weight_in_map);
 310 }
 311
 312 void
 313 ValueMapPostingSource::clear_mappings()
 314 {
 315     weight_map.clear();
 316     max_weight_in_map = 0.0;
 317 }
 318
 319 void
 320 ValueMapPostingSource::set_default_weight(double wt)
 321 {
 322     default_weight = wt;
 323 }
 324
 325 double
 326 ValueMapPostingSource::get_weight() const
 327 {
 328     map<string, double>::const_iterator wit = weight_map.find(get_value());
 329     if (wit == weight_map.end()) {
 330         return default_weight;
 331     }
 332     return wit->second;
 333 }
 334
 335 ValueMapPostingSource *
 336 ValueMapPostingSource::clone() const
 337 {
 338     AutoPtr<ValueMapPostingSource> res(new ValueMapPostingSource(get_slot()));
 339     map<string, double>::const_iterator i;
 340     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 341         res->add_mapping(i->first, i->second);
 342     }
 343     res->set_default_weight(default_weight);
 344     return res.release();
 345 }
 346
 347 string
 348 ValueMapPostingSource::name() const
 349 {
 350     return string("Xapian::ValueMapPostingSource");
 351 }
 352
 353 string
 354 ValueMapPostingSource::serialise() const
 355 {
 356     string result = encode_length(get_slot());
 357     result += serialise_double(default_weight);
 358
 359     map<string, double>::const_iterator i;
 360     for (i = weight_map.begin(); i != weight_map.end(); ++i) {
 361         result.append(encode_length(i->first.size()));
 362         result.append(i->first);
 363         result.append(serialise_double(i->second));
 364     }
 365
 366     return result;
 367 }
 368
 369 ValueMapPostingSource *
 370 ValueMapPostingSource::unserialise(const string &s) const
 371 {
 372     const char * p = s.data();
 373     const char * end = p + s.size();
 374
 375     Xapian::valueno new_slot;
 376     decode_length(&p, end, new_slot);
 377     AutoPtr<ValueMapPostingSource> res(new ValueMapPostingSource(new_slot));
 378     res->set_default_weight(unserialise_double(&p, end));
 379     while (p != end) {
 380         size_t keylen;
 381         decode_length_and_check(&p, end, keylen);
 382         string key(p, keylen);
 383         p += keylen;
 384         res->add_mapping(key, unserialise_double(&p, end));
 385     }
 386     return res.release();
 387 }
 388
 389 void
 390 ValueMapPostingSource::init(const Database & db_)
 391 {
 392     ValuePostingSource::init(db_);
 393     set_maxweight(max(max_weight_in_map, default_weight));
 394 }
 395
 396 string
 397 ValueMapPostingSource::get_description() const
 398 {
 399     string desc("Xapian::ValueMapPostingSource(slot=");
 400     desc += str(get_slot());
 401     desc += ")";
 402     return desc;
 403 }
 404
 405 FixedWeightPostingSource::FixedWeightPostingSource(double wt)
 406     : started(false)
 407 {
 408     // The weight is fixed at wt, so that's the maxweight too.  So just store wt
 409     // as the maxweight and we can read it from there when we need it.
 410     set_maxweight(wt);
 411 }
 412
 413 Xapian::doccount
 414 FixedWeightPostingSource::get_termfreq_min() const
 415 {
 416     return termfreq;
 417 }
 418
 419 Xapian::doccount
 420 FixedWeightPostingSource::get_termfreq_est() const
 421 {
 422     return termfreq;
 423 }
 424
 425 Xapian::doccount
 426 FixedWeightPostingSource::get_termfreq_max() const
 427 {
 428     return termfreq;
 429 }
 430
 431 double
 432 FixedWeightPostingSource::get_weight() const
 433 {
 434     return get_maxweight();
 435 }
 436
 437 void
 438 FixedWeightPostingSource::next(double min_wt)
 439 {
 440     if (!started) {
 441         started = true;
 442         it = db.postlist_begin(string());
 443     } else {
 444         ++it;
 445     }
 446
 447     if (it == db.postlist_end(string())) return;
 448
 449     if (check_docid) {
 450         it.skip_to(check_docid + 1);
 451         check_docid = 0;
 452     }
 453
 454     if (min_wt > get_maxweight()) {
 455         it = db.postlist_end(string());
 456     }
 457 }
 458
 459 void
 460 FixedWeightPostingSource::skip_to(Xapian::docid min_docid, double min_wt)
 461 {
 462     if (!started) {
 463         started = true;
 464         it = db.postlist_begin(string());
 465
 466         if (it == db.postlist_end(string())) return;
 467     }
 468
 469     if (check_docid) {
 470         if (min_docid < check_docid)
 471             min_docid = check_docid + 1;
 472         check_docid = 0;
 473     }
 474
 475     if (min_wt > get_maxweight()) {
 476         it = db.postlist_end(string());
 477         return;
 478     }
 479     it.skip_to(min_docid);
 480 }
 481
 482 bool
 483 FixedWeightPostingSource::check(Xapian::docid min_docid, double)
 484 {
 485     // We're guaranteed not to be called if the document doesn't
 486     // exist, so just remember the docid passed, and return true.
 487     check_docid = min_docid;
 488     return true;
 489 }
 490
 491 bool
 492 FixedWeightPostingSource::at_end() const
 493 {
 494     if (check_docid != 0) return false;
 495     return started && it == db.postlist_end(string());
 496 }
 497
 498 Xapian::docid
 499 FixedWeightPostingSource::get_docid() const
 500 {
 501     if (check_docid != 0) return check_docid;
 502     return *it;
 503 }
 504
 505 FixedWeightPostingSource *
 506 FixedWeightPostingSource::clone() const
 507 {
 508     return new FixedWeightPostingSource(get_maxweight());
 509 }
 510
 511 string
 512 FixedWeightPostingSource::name() const
 513 {
 514     return string("Xapian::FixedWeightPostingSource");
 515 }
 516
 517 string
 518 FixedWeightPostingSource::serialise() const
 519 {
 520     return serialise_double(get_maxweight());
 521 }
 522
 523 FixedWeightPostingSource *
 524 FixedWeightPostingSource::unserialise(const string &s) const
 525 {
 526     const char * p = s.data();
 527     const char * s_end = p + s.size();
 528     double new_wt = unserialise_double(&p, s_end);
 529     if (p != s_end) {
 530         throw Xapian::NetworkError("Bad serialised FixedWeightPostingSource - junk at end");
 531     }
 532     return new FixedWeightPostingSource(new_wt);
 533 }
 534
 535 void
 536 FixedWeightPostingSource::init(const Xapian::Database & db_)
 537 {
 538     db = db_;
 539     termfreq = db_.get_doccount();
 540     started = false;
 541     check_docid = 0;
 542 }
 543
 544 string
 545 FixedWeightPostingSource::get_description() const
 546 {
 547     string desc("Xapian::FixedWeightPostingSource(wt=");
 548     desc += str(get_maxweight());
 549     desc += ")";
 550     return desc;
 551 }
 552
 553 }