xapian-core/weight/bb2weight.cc

   1 /** @file
   2  * @brief Xapian::BB2Weight class - the BB2 weighting scheme of the DFR framework.
   3  */
   4 /* Copyright (C) 2013,2014 Aarsh Shah
   5  * Copyright (C) 2014,2015,2016,2017,2024 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  20  */
  21
  22 #include <config.h>
  23
  24 #include "xapian/weight.h"
  25
  26 #include "weightinternal.h"
  27
  28 #include "serialise-double.h"
  29
  30 #include "xapian/error.h"
  31
  32 #include <cmath>
  33
  34 using namespace std;
  35
  36 namespace Xapian {
  37
  38 static double stirling_value(double difference, double y, double stirling_constant)
  39 {
  40     return ((y + 0.5) * (stirling_constant - log2(y)) + (difference * stirling_constant));
  41 }
  42
  43 BB2Weight::BB2Weight(double c) : param_c(c)
  44 {
  45     if (param_c <= 0)
  46         throw Xapian::InvalidArgumentError("Parameter c is invalid");
  47     need_stat(AVERAGE_LENGTH);
  48     need_stat(DOC_LENGTH);
  49     need_stat(DOC_LENGTH_MIN);
  50     need_stat(DOC_LENGTH_MAX);
  51     need_stat(COLLECTION_SIZE);
  52     need_stat(COLLECTION_FREQ);
  53     need_stat(WDF);
  54     need_stat(WDF_MAX);
  55     need_stat(WQF);
  56     need_stat(TERMFREQ);
  57 }
  58
  59 BB2Weight *
  60 BB2Weight::clone() const
  61 {
  62     return new BB2Weight(param_c);
  63 }
  64
  65 void
  66 BB2Weight::init(double factor)
  67 {
  68     if (factor == 0.0) {
  69         // This object is for the term-independent contribution, and that's
  70         // always zero for this scheme.
  71         return;
  72     }
  73
  74     double wdfn_upper = get_wdf_upper_bound();
  75
  76     if (wdfn_upper == 0) {
  77         upper_bound = 0.0;
  78         return;
  79     }
  80
  81     c_product_avlen = param_c * get_average_length();
  82     double wdfn_lower(1.0);
  83     wdfn_lower *= log2(1 + c_product_avlen / get_doclength_upper_bound());
  84     wdfn_upper *= log2(1 + c_product_avlen / get_doclength_lower_bound());
  85
  86     double F = get_collection_freq();
  87
  88     // Clamp wdfn to at most (F - 1) to avoid ill-defined log calculations in
  89     // stirling_value().
  90     if (rare(wdfn_lower >= F - 1))
  91         wdfn_upper = F - 1;
  92     if (rare(wdfn_upper >= F - 1))
  93         wdfn_upper = F - 1;
  94
  95     B_constant = get_wqf() * factor * (F + 1.0) / get_termfreq();
  96
  97     // Clamp N to at least 2 to avoid ill-defined log calculations in
  98     // stirling_value().
  99     double N = rare(get_collection_size() <= 2) ? 2.0 : double(get_collection_size());
 100
 101     wt = -1.0 / log(2.0) - log2(N - 1.0);
 102     stirling_constant_1 = log2(N + F - 1.0);
 103     stirling_constant_2 = log2(F);
 104
 105     // Maximize the Stirling value to be used in the upper bound.
 106     // Calculate the individual terms keeping the maximization of Stirling value
 107     // in mind.
 108     double y_min = F - wdfn_upper;
 109     double y_max = N + F - wdfn_lower - 2.0;
 110
 111     double stirling_max = stirling_value(wdfn_upper + 1.0, y_max,
 112                                          stirling_constant_1) -
 113                           stirling_value(wdfn_lower, y_min,
 114                                          stirling_constant_2);
 115
 116     double B_max = B_constant / (wdfn_lower + 1.0);
 117     upper_bound = B_max * (wt + stirling_max);
 118     if (rare(upper_bound < 0.0))
 119         upper_bound = 0.0;
 120 }
 121
 122 string
 123 BB2Weight::name() const
 124 {
 125     return "bb2";
 126 }
 127
 128 string
 129 BB2Weight::serialise() const
 130 {
 131     return serialise_double(param_c);
 132 }
 133
 134 BB2Weight *
 135 BB2Weight::unserialise(const string & s) const
 136 {
 137     const char *ptr = s.data();
 138     const char *end = ptr + s.size();
 139     double c = unserialise_double(&ptr, end);
 140     if (rare(ptr != end))
 141         throw Xapian::SerialisationError("Extra data in BB2Weight::unserialise()");
 142     return new BB2Weight(c);
 143 }
 144
 145 double
 146 BB2Weight::get_sumpart(Xapian::termcount wdf, Xapian::termcount len,
 147                        Xapian::termcount, Xapian::termcount) const
 148 {
 149     if (wdf == 0) return 0.0;
 150
 151     double wdfn = wdf * log2(1 + c_product_avlen / len);
 152
 153     double F = get_collection_freq();
 154
 155     // Clamp wdfn to at most (F - 1) to avoid ill-defined log calculations in
 156     // stirling_value().
 157     if (rare(wdfn >= F - 1))
 158         wdfn = F - 1;
 159
 160     // Clamp N to at least 2 to avoid ill-defined log calculations in
 161     // stirling_value().
 162     Xapian::doccount N = get_collection_size();
 163     Xapian::doccount N_less_2 = rare(N <= 2) ? 0 : N - 2;
 164
 165     double y2 = F - wdfn;
 166     double y1 = N_less_2 + y2;
 167     double stirling = stirling_value(wdfn + 1.0, y1, stirling_constant_1) -
 168                       stirling_value(wdfn, y2, stirling_constant_2);
 169
 170     double B = B_constant / (wdfn + 1.0);
 171     double final_weight = B * (wt + stirling);
 172     if (rare(final_weight < 0.0))
 173         final_weight = 0.0;
 174     return final_weight;
 175 }
 176
 177 double
 178 BB2Weight::get_maxpart() const
 179 {
 180     return upper_bound;
 181 }
 182
 183 [[noreturn]]
 184 static inline void
 185 parameter_error(const char* message, const char* params)
 186 {
 187     Xapian::Weight::Internal::parameter_error(message, "bb2", params);
 188 }
 189
 190 BB2Weight*
 191 BB2Weight::create_from_parameters(const char* params) const
 192 {
 193     const char* p = params;
 194     if (*p == '\0')
 195         return new Xapian::BB2Weight();
 196     double c = 1.0;
 197     if (!Xapian::Weight::Internal::double_param(&p, &c))
 198         parameter_error("Parameter is invalid", params);
 199     if (*p)
 200         parameter_error("Extra data after parameter", params);
 201     return new Xapian::BB2Weight(c);
 202 }
 203
 204 }