components/variations/variations_seed_processor_unittest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/variations/variations_seed_processor.h"
   6
   7 #include <map>
   8 #include <vector>
   9
  10 #include "base/bind.h"
  11 #include "base/command_line.h"
  12 #include "base/strings/string_split.h"
  13 #include "base/strings/utf_string_conversions.h"
  14 #include "components/variations/processed_study.h"
  15 #include "components/variations/variations_associated_data.h"
  16 #include "testing/gtest/include/gtest/gtest.h"
  17
  18 namespace variations {
  19
  20 namespace {
  21
  22 // Converts |time| to Study proto format.
  23 int64 TimeToProtoTime(const base::Time& time) {
  24   return (time - base::Time::UnixEpoch()).InSeconds();
  25 }
  26
  27 // Constants for testing associating command line flags with trial groups.
  28 const char kFlagStudyName[] = "flag_test_trial";
  29 const char kFlagGroup1Name[] = "flag_group1";
  30 const char kFlagGroup2Name[] = "flag_group2";
  31 const char kNonFlagGroupName[] = "non_flag_group";
  32 const char kForcingFlag1[] = "flag_test1";
  33 const char kForcingFlag2[] = "flag_test2";
  34
  35 const VariationID kExperimentId = 123;
  36
  37 // Adds an experiment to |study| with the specified |name| and |probability|.
  38 Study_Experiment* AddExperiment(const std::string& name, int probability,
  39                                 Study* study) {
  40   Study_Experiment* experiment = study->add_experiment();
  41   experiment->set_name(name);
  42   experiment->set_probability_weight(probability);
  43   return experiment;
  44 }
  45
  46 // Populates |study| with test data used for testing associating command line
  47 // flags with trials groups. The study will contain three groups, a default
  48 // group that isn't associated with a flag, and two other groups, both
  49 // associated with different flags.
  50 Study CreateStudyWithFlagGroups(int default_group_probability,
  51                                 int flag_group1_probability,
  52                                 int flag_group2_probability) {
  53   DCHECK_GE(default_group_probability, 0);
  54   DCHECK_GE(flag_group1_probability, 0);
  55   DCHECK_GE(flag_group2_probability, 0);
  56   Study study;
  57   study.set_name(kFlagStudyName);
  58   study.set_default_experiment_name(kNonFlagGroupName);
  59
  60   AddExperiment(kNonFlagGroupName, default_group_probability, &study);
  61   AddExperiment(kFlagGroup1Name, flag_group1_probability, &study)
  62       ->set_forcing_flag(kForcingFlag1);
  63   AddExperiment(kFlagGroup2Name, flag_group2_probability, &study)
  64       ->set_forcing_flag(kForcingFlag2);
  65
  66   return study;
  67 }
  68
  69 // Tests whether a field trial is active (i.e. group() has been called on it).
  70 bool IsFieldTrialActive(const std::string& trial_name) {
  71   base::FieldTrial::ActiveGroups active_groups;
  72   base::FieldTrialList::GetActiveFieldTrialGroups(&active_groups);
  73   for (size_t i = 0; i < active_groups.size(); ++i) {
  74     if (active_groups[i].trial_name == trial_name)
  75       return true;
  76   }
  77   return false;
  78 }
  79
  80 class TestOverrideStringCallback {
  81  public:
  82   typedef std::map<uint32_t, base::string16> OverrideMap;
  83
  84   TestOverrideStringCallback()
  85       : callback_(base::Bind(&TestOverrideStringCallback::Override,
  86                              base::Unretained(this))) {}
  87
  88   virtual ~TestOverrideStringCallback() {}
  89
  90   const VariationsSeedProcessor::UIStringOverrideCallback& callback() const {
  91     return callback_;
  92   }
  93
  94   const OverrideMap& overrides() const { return overrides_; }
  95
  96  private:
  97   void Override(uint32_t hash, const base::string16& string) {
  98     overrides_[hash] = string;
  99   }
 100
 101   VariationsSeedProcessor::UIStringOverrideCallback callback_;
 102   OverrideMap overrides_;
 103
 104   DISALLOW_COPY_AND_ASSIGN(TestOverrideStringCallback);
 105 };
 106
 107 }  // namespace
 108
 109 class VariationsSeedProcessorTest : public ::testing::Test {
 110  public:
 111   VariationsSeedProcessorTest() {
 112   }
 113
 114   ~VariationsSeedProcessorTest() override {
 115     // Ensure that the maps are cleared between tests, since they are stored as
 116     // process singletons.
 117     testing::ClearAllVariationIDs();
 118     testing::ClearAllVariationParams();
 119   }
 120
 121   bool CreateTrialFromStudy(const Study* study) {
 122     ProcessedStudy processed_study;
 123     if (processed_study.Init(study, false)) {
 124       VariationsSeedProcessor().CreateTrialFromStudy(
 125           processed_study, override_callback_.callback());
 126       return true;
 127     }
 128     return false;
 129   }
 130
 131  protected:
 132   TestOverrideStringCallback override_callback_;
 133
 134  private:
 135   DISALLOW_COPY_AND_ASSIGN(VariationsSeedProcessorTest);
 136 };
 137
 138 TEST_F(VariationsSeedProcessorTest, AllowForceGroupAndVariationId) {
 139   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 140
 141   base::FieldTrialList field_trial_list(NULL);
 142
 143   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 144   study.mutable_experiment(1)->set_google_web_experiment_id(kExperimentId);
 145
 146   EXPECT_TRUE(CreateTrialFromStudy(&study));
 147   EXPECT_EQ(kFlagGroup1Name,
 148             base::FieldTrialList::FindFullName(kFlagStudyName));
 149
 150   VariationID id = GetGoogleVariationID(GOOGLE_WEB_PROPERTIES, kFlagStudyName,
 151                                         kFlagGroup1Name);
 152   EXPECT_EQ(kExperimentId, id);
 153 }
 154
 155 // Test that the group for kForcingFlag1 is forced.
 156 TEST_F(VariationsSeedProcessorTest, ForceGroupWithFlag1) {
 157   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 158
 159   base::FieldTrialList field_trial_list(NULL);
 160
 161   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 162   EXPECT_TRUE(CreateTrialFromStudy(&study));
 163   EXPECT_EQ(kFlagGroup1Name,
 164             base::FieldTrialList::FindFullName(kFlagStudyName));
 165 }
 166
 167 // Test that the group for kForcingFlag2 is forced.
 168 TEST_F(VariationsSeedProcessorTest, ForceGroupWithFlag2) {
 169   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag2);
 170
 171   base::FieldTrialList field_trial_list(NULL);
 172
 173   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 174   EXPECT_TRUE(CreateTrialFromStudy(&study));
 175   EXPECT_EQ(kFlagGroup2Name,
 176             base::FieldTrialList::FindFullName(kFlagStudyName));
 177 }
 178
 179 TEST_F(VariationsSeedProcessorTest, ForceGroup_ChooseFirstGroupWithFlag) {
 180   // Add the flag to the command line arguments so the flag group is forced.
 181   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 182   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag2);
 183
 184   base::FieldTrialList field_trial_list(NULL);
 185
 186   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 187   EXPECT_TRUE(CreateTrialFromStudy(&study));
 188   EXPECT_EQ(kFlagGroup1Name,
 189             base::FieldTrialList::FindFullName(kFlagStudyName));
 190 }
 191
 192 TEST_F(VariationsSeedProcessorTest, ForceGroup_DontChooseGroupWithFlag) {
 193   base::FieldTrialList field_trial_list(NULL);
 194
 195   // The two flag groups are given high probability, which would normally make
 196   // them very likely to be chosen. They won't be chosen since flag groups are
 197   // never chosen when their flag isn't present.
 198   Study study = CreateStudyWithFlagGroups(1, 999, 999);
 199   EXPECT_TRUE(CreateTrialFromStudy(&study));
 200   EXPECT_EQ(kNonFlagGroupName,
 201             base::FieldTrialList::FindFullName(kFlagStudyName));
 202 }
 203
 204 TEST_F(VariationsSeedProcessorTest,
 205        NonExpiredStudyPrioritizedOverExpiredStudy) {
 206   VariationsSeedProcessor seed_processor;
 207
 208   const std::string kTrialName = "A";
 209   const std::string kGroup1Name = "Group1";
 210
 211   VariationsSeed seed;
 212   Study* study1 = seed.add_study();
 213   study1->set_name(kTrialName);
 214   study1->set_default_experiment_name("Default");
 215   AddExperiment(kGroup1Name, 100, study1);
 216   AddExperiment("Default", 0, study1);
 217   Study* study2 = seed.add_study();
 218   *study2 = *study1;
 219   ASSERT_EQ(seed.study(0).name(), seed.study(1).name());
 220
 221   const base::Time year_ago =
 222       base::Time::Now() - base::TimeDelta::FromDays(365);
 223
 224   const base::Version version("20.0.0.0");
 225
 226   // Check that adding [expired, non-expired] activates the non-expired one.
 227   ASSERT_EQ(std::string(), base::FieldTrialList::FindFullName(kTrialName));
 228   {
 229     base::FieldTrialList field_trial_list(NULL);
 230     study1->set_expiry_date(TimeToProtoTime(year_ago));
 231     seed_processor.CreateTrialsFromSeed(seed,
 232                                         "en-CA",
 233                                         base::Time::Now(),
 234                                         version,
 235                                         Study_Channel_STABLE,
 236                                         Study_FormFactor_DESKTOP,
 237                                         "",
 238                                         "",
 239                                         override_callback_.callback());
 240     EXPECT_EQ(kGroup1Name, base::FieldTrialList::FindFullName(kTrialName));
 241   }
 242
 243   // Check that adding [non-expired, expired] activates the non-expired one.
 244   ASSERT_EQ(std::string(), base::FieldTrialList::FindFullName(kTrialName));
 245   {
 246     base::FieldTrialList field_trial_list(NULL);
 247     study1->clear_expiry_date();
 248     study2->set_expiry_date(TimeToProtoTime(year_ago));
 249     seed_processor.CreateTrialsFromSeed(seed,
 250                                         "en-CA",
 251                                         base::Time::Now(),
 252                                         version,
 253                                         Study_Channel_STABLE,
 254                                         Study_FormFactor_DESKTOP,
 255                                         "",
 256                                         "",
 257                                         override_callback_.callback());
 258     EXPECT_EQ(kGroup1Name, base::FieldTrialList::FindFullName(kTrialName));
 259   }
 260 }
 261
 262 TEST_F(VariationsSeedProcessorTest, OverrideUIStrings) {
 263   base::FieldTrialList field_trial_list(NULL);
 264
 265   Study study;
 266   study.set_name("Study1");
 267   study.set_default_experiment_name("B");
 268   study.set_activation_type(Study_ActivationType_ACTIVATION_AUTO);
 269
 270   Study_Experiment* experiment1 = AddExperiment("A", 0, &study);
 271   Study_Experiment_OverrideUIString* override =
 272       experiment1->add_override_ui_string();
 273
 274   override->set_name_hash(1234);
 275   override->set_value("test");
 276
 277   Study_Experiment* experiment2 = AddExperiment("B", 1, &study);
 278
 279   EXPECT_TRUE(CreateTrialFromStudy(&study));
 280
 281   const TestOverrideStringCallback::OverrideMap& overrides =
 282       override_callback_.overrides();
 283
 284   EXPECT_TRUE(overrides.empty());
 285
 286   study.set_name("Study2");
 287   experiment1->set_probability_weight(1);
 288   experiment2->set_probability_weight(0);
 289
 290   EXPECT_TRUE(CreateTrialFromStudy(&study));
 291
 292   EXPECT_EQ(1u, overrides.size());
 293   TestOverrideStringCallback::OverrideMap::const_iterator it =
 294       overrides.find(1234);
 295   EXPECT_EQ(base::ASCIIToUTF16("test"), it->second);
 296 }
 297
 298 TEST_F(VariationsSeedProcessorTest, OverrideUIStringsWithForcingFlag) {
 299   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 300   ASSERT_EQ(kForcingFlag1, study.experiment(1).forcing_flag());
 301
 302   study.set_activation_type(Study_ActivationType_ACTIVATION_AUTO);
 303   Study_Experiment_OverrideUIString* override =
 304       study.mutable_experiment(1)->add_override_ui_string();
 305   override->set_name_hash(1234);
 306   override->set_value("test");
 307
 308   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 309   base::FieldTrialList field_trial_list(NULL);
 310   EXPECT_TRUE(CreateTrialFromStudy(&study));
 311   EXPECT_EQ(kFlagGroup1Name, base::FieldTrialList::FindFullName(study.name()));
 312
 313   const TestOverrideStringCallback::OverrideMap& overrides =
 314       override_callback_.overrides();
 315   EXPECT_EQ(1u, overrides.size());
 316   TestOverrideStringCallback::OverrideMap::const_iterator it =
 317       overrides.find(1234);
 318   EXPECT_EQ(base::ASCIIToUTF16("test"), it->second);
 319 }
 320
 321 TEST_F(VariationsSeedProcessorTest, ValidateStudy) {
 322   Study study;
 323   study.set_default_experiment_name("def");
 324   AddExperiment("abc", 100, &study);
 325   Study_Experiment* default_group = AddExperiment("def", 200, &study);
 326
 327   ProcessedStudy processed_study;
 328   EXPECT_TRUE(processed_study.Init(&study, false));
 329   EXPECT_EQ(300, processed_study.total_probability());
 330   EXPECT_FALSE(processed_study.all_assignments_to_one_group());
 331
 332   // Min version checks.
 333   study.mutable_filter()->set_min_version("1.2.3.*");
 334   EXPECT_TRUE(processed_study.Init(&study, false));
 335   study.mutable_filter()->set_min_version("1.*.3");
 336   EXPECT_FALSE(processed_study.Init(&study, false));
 337   study.mutable_filter()->set_min_version("1.2.3");
 338   EXPECT_TRUE(processed_study.Init(&study, false));
 339
 340   // Max version checks.
 341   study.mutable_filter()->set_max_version("2.3.4.*");
 342   EXPECT_TRUE(processed_study.Init(&study, false));
 343   study.mutable_filter()->set_max_version("*.3");
 344   EXPECT_FALSE(processed_study.Init(&study, false));
 345   study.mutable_filter()->set_max_version("2.3.4");
 346   EXPECT_TRUE(processed_study.Init(&study, false));
 347
 348   study.clear_default_experiment_name();
 349   EXPECT_FALSE(processed_study.Init(&study, false));
 350
 351   study.set_default_experiment_name("xyz");
 352   EXPECT_FALSE(processed_study.Init(&study, false));
 353
 354   study.set_default_experiment_name("def");
 355   default_group->clear_name();
 356   EXPECT_FALSE(processed_study.Init(&study, false));
 357
 358   default_group->set_name("def");
 359   EXPECT_TRUE(processed_study.Init(&study, false));
 360   Study_Experiment* repeated_group = study.add_experiment();
 361   repeated_group->set_name("abc");
 362   repeated_group->set_probability_weight(1);
 363   EXPECT_FALSE(processed_study.Init(&study, false));
 364 }
 365
 366 TEST_F(VariationsSeedProcessorTest, ProcessedStudyAllAssignmentsToOneGroup) {
 367   Study study;
 368   study.set_default_experiment_name("def");
 369   AddExperiment("def", 100, &study);
 370
 371   ProcessedStudy processed_study;
 372   EXPECT_TRUE(processed_study.Init(&study, false));
 373   EXPECT_TRUE(processed_study.all_assignments_to_one_group());
 374
 375   AddExperiment("abc", 0, &study);
 376   AddExperiment("flag", 0, &study)->set_forcing_flag(kForcingFlag1);
 377   EXPECT_TRUE(processed_study.Init(&study, false));
 378   EXPECT_TRUE(processed_study.all_assignments_to_one_group());
 379
 380   AddExperiment("xyz", 1, &study);
 381   EXPECT_TRUE(processed_study.Init(&study, false));
 382   EXPECT_FALSE(processed_study.all_assignments_to_one_group());
 383
 384   // Try with default group and first group being at 0.
 385   Study study2;
 386   study2.set_default_experiment_name("def");
 387   AddExperiment("def", 0, &study2);
 388   AddExperiment("xyz", 34, &study2);
 389   EXPECT_TRUE(processed_study.Init(&study2, false));
 390   EXPECT_TRUE(processed_study.all_assignments_to_one_group());
 391   AddExperiment("abc", 12, &study2);
 392   EXPECT_TRUE(processed_study.Init(&study2, false));
 393   EXPECT_FALSE(processed_study.all_assignments_to_one_group());
 394 }
 395
 396 TEST_F(VariationsSeedProcessorTest, VariationParams) {
 397   base::FieldTrialList field_trial_list(NULL);
 398
 399   Study study;
 400   study.set_name("Study1");
 401   study.set_default_experiment_name("B");
 402
 403   Study_Experiment* experiment1 = AddExperiment("A", 1, &study);
 404   Study_Experiment_Param* param = experiment1->add_param();
 405   param->set_name("x");
 406   param->set_value("y");
 407
 408   Study_Experiment* experiment2 = AddExperiment("B", 0, &study);
 409
 410   EXPECT_TRUE(CreateTrialFromStudy(&study));
 411   EXPECT_EQ("y", GetVariationParamValue("Study1", "x"));
 412
 413   study.set_name("Study2");
 414   experiment1->set_probability_weight(0);
 415   experiment2->set_probability_weight(1);
 416   EXPECT_TRUE(CreateTrialFromStudy(&study));
 417   EXPECT_EQ(std::string(), GetVariationParamValue("Study2", "x"));
 418 }
 419
 420 TEST_F(VariationsSeedProcessorTest, VariationParamsWithForcingFlag) {
 421   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 422   ASSERT_EQ(kForcingFlag1, study.experiment(1).forcing_flag());
 423   Study_Experiment_Param* param = study.mutable_experiment(1)->add_param();
 424   param->set_name("x");
 425   param->set_value("y");
 426
 427   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 428   base::FieldTrialList field_trial_list(NULL);
 429   EXPECT_TRUE(CreateTrialFromStudy(&study));
 430   EXPECT_EQ(kFlagGroup1Name, base::FieldTrialList::FindFullName(study.name()));
 431   EXPECT_EQ("y", GetVariationParamValue(study.name(), "x"));
 432 }
 433
 434 TEST_F(VariationsSeedProcessorTest, StartsActive) {
 435   base::FieldTrialList field_trial_list(NULL);
 436
 437   VariationsSeed seed;
 438   Study* study1 = seed.add_study();
 439   study1->set_name("A");
 440   study1->set_default_experiment_name("Default");
 441   AddExperiment("AA", 100, study1);
 442   AddExperiment("Default", 0, study1);
 443
 444   Study* study2 = seed.add_study();
 445   study2->set_name("B");
 446   study2->set_default_experiment_name("Default");
 447   AddExperiment("BB", 100, study2);
 448   AddExperiment("Default", 0, study2);
 449   study2->set_activation_type(Study_ActivationType_ACTIVATION_AUTO);
 450
 451   Study* study3 = seed.add_study();
 452   study3->set_name("C");
 453   study3->set_default_experiment_name("Default");
 454   AddExperiment("CC", 100, study3);
 455   AddExperiment("Default", 0, study3);
 456   study3->set_activation_type(Study_ActivationType_ACTIVATION_EXPLICIT);
 457
 458   VariationsSeedProcessor seed_processor;
 459   seed_processor.CreateTrialsFromSeed(seed,
 460                                       "en-CA",
 461                                       base::Time::Now(),
 462                                       base::Version("20.0.0.0"),
 463                                       Study_Channel_STABLE,
 464                                       Study_FormFactor_DESKTOP,
 465                                       "",
 466                                       "",
 467                                       override_callback_.callback());
 468
 469   // Non-specified and ACTIVATION_EXPLICIT should not start active, but
 470   // ACTIVATION_AUTO should.
 471   EXPECT_FALSE(IsFieldTrialActive("A"));
 472   EXPECT_TRUE(IsFieldTrialActive("B"));
 473   EXPECT_FALSE(IsFieldTrialActive("C"));
 474
 475   EXPECT_EQ("AA", base::FieldTrialList::FindFullName("A"));
 476   EXPECT_EQ("BB", base::FieldTrialList::FindFullName("B"));
 477   EXPECT_EQ("CC", base::FieldTrialList::FindFullName("C"));
 478
 479   // Now, all studies should be active.
 480   EXPECT_TRUE(IsFieldTrialActive("A"));
 481   EXPECT_TRUE(IsFieldTrialActive("B"));
 482   EXPECT_TRUE(IsFieldTrialActive("C"));
 483 }
 484
 485 TEST_F(VariationsSeedProcessorTest, StartsActiveWithFlag) {
 486   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 487
 488   base::FieldTrialList field_trial_list(NULL);
 489
 490   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 491   study.set_activation_type(Study_ActivationType_ACTIVATION_AUTO);
 492
 493   EXPECT_TRUE(CreateTrialFromStudy(&study));
 494   EXPECT_TRUE(IsFieldTrialActive(kFlagStudyName));
 495
 496   EXPECT_EQ(kFlagGroup1Name,
 497             base::FieldTrialList::FindFullName(kFlagStudyName));
 498 }
 499
 500 TEST_F(VariationsSeedProcessorTest, ForcingFlagAlreadyForced) {
 501   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 502   ASSERT_EQ(kNonFlagGroupName, study.experiment(0).name());
 503   Study_Experiment_Param* param = study.mutable_experiment(0)->add_param();
 504   param->set_name("x");
 505   param->set_value("y");
 506   study.mutable_experiment(0)->set_google_web_experiment_id(kExperimentId);
 507
 508   base::FieldTrialList field_trial_list(NULL);
 509   base::FieldTrialList::CreateFieldTrial(kFlagStudyName, kNonFlagGroupName);
 510
 511   base::CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 512   EXPECT_TRUE(CreateTrialFromStudy(&study));
 513   // The previously forced experiment should still hold.
 514   EXPECT_EQ(kNonFlagGroupName,
 515             base::FieldTrialList::FindFullName(study.name()));
 516
 517   // Check that params and experiment ids correspond.
 518   EXPECT_EQ("y", GetVariationParamValue(study.name(), "x"));
 519   VariationID id = GetGoogleVariationID(GOOGLE_WEB_PROPERTIES, kFlagStudyName,
 520                                         kNonFlagGroupName);
 521   EXPECT_EQ(kExperimentId, id);
 522 }
 523
 524 }  // namespace variations