components/variations/variations_seed_simulator_unittest.cc

   1 // Copyright 2014 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/variations/variations_seed_simulator.h"
   6
   7 #include <map>
   8
   9 #include "base/strings/stringprintf.h"
  10 #include "components/variations/processed_study.h"
  11 #include "components/variations/proto/study.pb.h"
  12 #include "components/variations/variations_associated_data.h"
  13 #include "testing/gtest/include/gtest/gtest.h"
  14
  15 namespace variations {
  16
  17 namespace {
  18
  19 // An implementation of EntropyProvider that always returns a specific entropy
  20 // value, regardless of field trial.
  21 class TestEntropyProvider : public base::FieldTrial::EntropyProvider {
  22  public:
  23   explicit TestEntropyProvider(double entropy_value)
  24       : entropy_value_(entropy_value) {}
  25   ~TestEntropyProvider() override {}
  26
  27   // base::FieldTrial::EntropyProvider implementation:
  28   double GetEntropyForTrial(const std::string& trial_name,
  29                             uint32 randomization_seed) const override {
  30     return entropy_value_;
  31   }
  32
  33  private:
  34   const double entropy_value_;
  35
  36   DISALLOW_COPY_AND_ASSIGN(TestEntropyProvider);
  37 };
  38
  39 // Creates and activates a single-group field trial with name |trial_name| and
  40 // group |group_name| and variations |params| (if not NULL).
  41 void CreateTrial(const std::string& trial_name,
  42                  const std::string& group_name,
  43                  const std::map<std::string, std::string>* params) {
  44   base::FieldTrialList::CreateFieldTrial(trial_name, group_name);
  45   if (params != NULL)
  46     AssociateVariationParams(trial_name, group_name, *params);
  47   base::FieldTrialList::FindFullName(trial_name);
  48 }
  49
  50 // Creates a study with the given |study_name| and |consistency|.
  51 Study CreateStudy(const std::string& study_name,
  52                   Study_Consistency consistency) {
  53   Study study;
  54   study.set_name(study_name);
  55   study.set_consistency(consistency);
  56   return study;
  57 }
  58
  59 // Adds an experiment to |study| with the specified |experiment_name| and
  60 // |probability| values and sets it as the study's default experiment.
  61 Study_Experiment* AddExperiment(const std::string& experiment_name,
  62                                 int probability,
  63                                 Study* study) {
  64   Study_Experiment* experiment = study->add_experiment();
  65   experiment->set_name(experiment_name);
  66   experiment->set_probability_weight(probability);
  67   study->set_default_experiment_name(experiment_name);
  68   return experiment;
  69 }
  70
  71 // Add an experiment param with |param_name| and |param_value| to |experiment|.
  72 Study_Experiment_Param* AddExperimentParam(const std::string& param_name,
  73                                            const std::string& param_value,
  74                                            Study_Experiment* experiment) {
  75   Study_Experiment_Param* param = experiment->add_param();
  76   param->set_name(param_name);
  77   param->set_value(param_value);
  78   return param;
  79 }
  80
  81 }  // namespace
  82
  83 class VariationsSeedSimulatorTest : public ::testing::Test {
  84  public:
  85   VariationsSeedSimulatorTest() : field_trial_list_(NULL) {
  86   }
  87
  88   ~VariationsSeedSimulatorTest() override {
  89     // Ensure that the maps are cleared between tests, since they are stored as
  90     // process singletons.
  91     testing::ClearAllVariationIDs();
  92     testing::ClearAllVariationParams();
  93   }
  94
  95   // Uses a VariationsSeedSimulator to simulate the differences between
  96   // |studies| and the current field trial state.
  97   VariationsSeedSimulator::Result SimulateDifferences(
  98       const std::vector<ProcessedStudy>& studies) {
  99     TestEntropyProvider provider(0.5);
 100     VariationsSeedSimulator seed_simulator(provider);
 101     return seed_simulator.ComputeDifferences(studies);
 102   }
 103
 104   // Simulates the differences between |study| and the current field trial
 105   // state, returning a string like "1 2 3", where 1 is the number of regular
 106   // group changes, 2 is the number of "kill best effort" group changes and 3
 107   // is the number of "kill critical" group changes.
 108   std::string SimulateStudyDifferences(const Study* study) {
 109     std::vector<ProcessedStudy> studies;
 110     if (!ProcessedStudy::ValidateAndAppendStudy(study, false, &studies))
 111       return "invalid study";
 112     return ConvertSimulationResultToString(SimulateDifferences(studies));
 113   }
 114
 115   // Simulates the differences between expired |study| and the current field
 116   // trial state, returning a string like "1 2 3", where 1 is the number of
 117   // regular group changes, 2 is the number of "kill best effort" group changes
 118   // and 3 is the number of "kill critical" group changes.
 119   std::string SimulateStudyDifferencesExpired(const Study* study) {
 120     std::vector<ProcessedStudy> studies;
 121     if (!ProcessedStudy::ValidateAndAppendStudy(study, true, &studies))
 122       return "invalid study";
 123     if (!studies[0].is_expired())
 124       return "not expired";
 125     return ConvertSimulationResultToString(SimulateDifferences(studies));
 126   }
 127
 128   // Formats |result| as a string with format "1 2 3", where 1 is the number of
 129   // regular group changes, 2 is the number of "kill best effort" group changes
 130   // and 3 is the number of "kill critical" group changes.
 131   std::string ConvertSimulationResultToString(
 132       const VariationsSeedSimulator::Result& result) {
 133     return base::StringPrintf("%d %d %d",
 134                               result.normal_group_change_count,
 135                               result.kill_best_effort_group_change_count,
 136                               result.kill_critical_group_change_count);
 137   }
 138
 139  private:
 140   base::FieldTrialList field_trial_list_;
 141
 142   DISALLOW_COPY_AND_ASSIGN(VariationsSeedSimulatorTest);
 143 };
 144
 145 TEST_F(VariationsSeedSimulatorTest, PermanentNoChanges) {
 146   CreateTrial("A", "B", NULL);
 147
 148   std::vector<ProcessedStudy> processed_studies;
 149   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 150   Study_Experiment* experiment = AddExperiment("B", 100, &study);
 151
 152   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 153
 154   experiment->set_type(Study_Experiment_Type_NORMAL);
 155   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 156   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 157   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 158   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 159   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 160   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 161   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 162 }
 163
 164 TEST_F(VariationsSeedSimulatorTest, PermanentGroupChange) {
 165   CreateTrial("A", "B", NULL);
 166
 167   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 168   Study_Experiment* experiment = AddExperiment("C", 100, &study);
 169
 170   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 171
 172   // Changing "C" group type should not affect the type of change. (Since the
 173   // type is evaluated for the "old" group.)
 174   experiment->set_type(Study_Experiment_Type_NORMAL);
 175   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 176   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 177   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 178   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 179   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 180   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 181   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 182 }
 183
 184 TEST_F(VariationsSeedSimulatorTest, PermanentExpired) {
 185   CreateTrial("A", "B", NULL);
 186
 187   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 188   Study_Experiment* experiment = AddExperiment("B", 1, &study);
 189   AddExperiment("C", 0, &study);
 190
 191   // There should be a difference because the study is expired, which should
 192   // result in the default group "C" being chosen.
 193   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
 194
 195   experiment->set_type(Study_Experiment_Type_NORMAL);
 196   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
 197   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 198   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
 199   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 200   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
 201   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 202   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
 203 }
 204
 205 TEST_F(VariationsSeedSimulatorTest, SessionRandomized) {
 206   CreateTrial("A", "B", NULL);
 207
 208   Study study = CreateStudy("A", Study_Consistency_SESSION);
 209   Study_Experiment* experiment = AddExperiment("B", 1, &study);
 210   AddExperiment("C", 1, &study);
 211   AddExperiment("D", 1, &study);
 212
 213   // There should be no differences, since a session randomized study can result
 214   // in any of the groups being chosen on startup.
 215   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 216
 217   experiment->set_type(Study_Experiment_Type_NORMAL);
 218   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 219   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 220   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 221   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 222   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 223   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 224   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 225 }
 226
 227 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupRemoved) {
 228   CreateTrial("A", "B", NULL);
 229
 230   Study study = CreateStudy("A", Study_Consistency_SESSION);
 231   AddExperiment("C", 1, &study);
 232   AddExperiment("D", 1, &study);
 233
 234   // There should be a difference since there is no group "B" in the new config.
 235   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 236 }
 237
 238 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedGroupProbabilityZero) {
 239   CreateTrial("A", "B", NULL);
 240
 241   Study study = CreateStudy("A", Study_Consistency_SESSION);
 242   Study_Experiment* experiment = AddExperiment("B", 0, &study);
 243   AddExperiment("C", 1, &study);
 244   AddExperiment("D", 1, &study);
 245
 246   // There should be a difference since group "B" has probability 0.
 247   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 248
 249   experiment->set_type(Study_Experiment_Type_NORMAL);
 250   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 251   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 252   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 253   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 254   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
 255   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 256   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
 257 }
 258
 259 TEST_F(VariationsSeedSimulatorTest, SessionRandomizedExpired) {
 260   CreateTrial("A", "B", NULL);
 261
 262   Study study = CreateStudy("A", Study_Consistency_SESSION);
 263   Study_Experiment* experiment = AddExperiment("B", 1, &study);
 264   AddExperiment("C", 1, &study);
 265   AddExperiment("D", 1, &study);
 266
 267   // There should be a difference because the study is expired, which should
 268   // result in the default group "D" being chosen.
 269   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
 270
 271   experiment->set_type(Study_Experiment_Type_NORMAL);
 272   EXPECT_EQ("1 0 0", SimulateStudyDifferencesExpired(&study));
 273   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 274   EXPECT_EQ("0 0 0", SimulateStudyDifferencesExpired(&study));
 275   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 276   EXPECT_EQ("0 1 0", SimulateStudyDifferencesExpired(&study));
 277   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 278   EXPECT_EQ("0 0 1", SimulateStudyDifferencesExpired(&study));
 279 }
 280
 281 TEST_F(VariationsSeedSimulatorTest, ParamsUnchanged) {
 282   std::map<std::string, std::string> params;
 283   params["p1"] = "x";
 284   params["p2"] = "y";
 285   params["p3"] = "z";
 286   CreateTrial("A", "B", &params);
 287
 288   std::vector<ProcessedStudy> processed_studies;
 289   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 290   Study_Experiment* experiment = AddExperiment("B", 100, &study);
 291   AddExperimentParam("p2", "y", experiment);
 292   AddExperimentParam("p1", "x", experiment);
 293   AddExperimentParam("p3", "z", experiment);
 294
 295   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 296
 297   experiment->set_type(Study_Experiment_Type_NORMAL);
 298   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 299   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 300   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 301   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 302   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 303   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 304   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 305 }
 306
 307 TEST_F(VariationsSeedSimulatorTest, ParamsChanged) {
 308   std::map<std::string, std::string> params;
 309   params["p1"] = "x";
 310   params["p2"] = "y";
 311   params["p3"] = "z";
 312   CreateTrial("A", "B", &params);
 313
 314   std::vector<ProcessedStudy> processed_studies;
 315   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 316   Study_Experiment* experiment = AddExperiment("B", 100, &study);
 317   AddExperimentParam("p2", "test", experiment);
 318   AddExperimentParam("p1", "x", experiment);
 319   AddExperimentParam("p3", "z", experiment);
 320
 321   // The param lists differ.
 322   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 323
 324   experiment->set_type(Study_Experiment_Type_NORMAL);
 325   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 326   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 327   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 328   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 329   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
 330   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 331   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
 332 }
 333
 334 TEST_F(VariationsSeedSimulatorTest, ParamsRemoved) {
 335   std::map<std::string, std::string> params;
 336   params["p1"] = "x";
 337   params["p2"] = "y";
 338   params["p3"] = "z";
 339   CreateTrial("A", "B", &params);
 340
 341   std::vector<ProcessedStudy> processed_studies;
 342   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 343   Study_Experiment* experiment = AddExperiment("B", 100, &study);
 344
 345   // The current group has params, but the new config doesn't have any.
 346   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 347
 348   experiment->set_type(Study_Experiment_Type_NORMAL);
 349   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 350   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 351   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 352   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 353   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
 354   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 355   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
 356 }
 357
 358 TEST_F(VariationsSeedSimulatorTest, ParamsAdded) {
 359   CreateTrial("A", "B", NULL);
 360
 361   std::vector<ProcessedStudy> processed_studies;
 362   Study study = CreateStudy("A", Study_Consistency_PERMANENT);
 363   Study_Experiment* experiment = AddExperiment("B", 100, &study);
 364   AddExperimentParam("p2", "y", experiment);
 365   AddExperimentParam("p1", "x", experiment);
 366   AddExperimentParam("p3", "z", experiment);
 367
 368   // The current group has no params, but the config has added some.
 369   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 370
 371   experiment->set_type(Study_Experiment_Type_NORMAL);
 372   EXPECT_EQ("1 0 0", SimulateStudyDifferences(&study));
 373   experiment->set_type(Study_Experiment_Type_IGNORE_CHANGE);
 374   EXPECT_EQ("0 0 0", SimulateStudyDifferences(&study));
 375   experiment->set_type(Study_Experiment_Type_KILL_BEST_EFFORT);
 376   EXPECT_EQ("0 1 0", SimulateStudyDifferences(&study));
 377   experiment->set_type(Study_Experiment_Type_KILL_CRITICAL);
 378   EXPECT_EQ("0 0 1", SimulateStudyDifferences(&study));
 379 }
 380
 381 }  // namespace variations