components/variations/variations_seed_processor_unittest.cc

   1 // Copyright 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "components/variations/variations_seed_processor.h"
   6
   7 #include <vector>
   8
   9 #include "base/command_line.h"
  10 #include "base/strings/string_split.h"
  11 #include "components/variations/variations_associated_data.h"
  12 #include "testing/gtest/include/gtest/gtest.h"
  13
  14 namespace chrome_variations {
  15
  16 namespace {
  17
  18 // Converts |time| to Study proto format.
  19 int64 TimeToProtoTime(const base::Time& time) {
  20   return (time - base::Time::UnixEpoch()).InSeconds();
  21 }
  22
  23 // Constants for testing associating command line flags with trial groups.
  24 const char kFlagStudyName[] = "flag_test_trial";
  25 const char kFlagGroup1Name[] = "flag_group1";
  26 const char kFlagGroup2Name[] = "flag_group2";
  27 const char kNonFlagGroupName[] = "non_flag_group";
  28 const char kForcingFlag1[] = "flag_test1";
  29 const char kForcingFlag2[] = "flag_test2";
  30
  31 // Adds an experiment to |study| with the specified |name| and |probability|.
  32 Study_Experiment* AddExperiment(const std::string& name, int probability,
  33                                 Study* study) {
  34   Study_Experiment* experiment = study->add_experiment();
  35   experiment->set_name(name);
  36   experiment->set_probability_weight(probability);
  37   return experiment;
  38 }
  39
  40 // Populates |study| with test data used for testing associating command line
  41 // flags with trials groups. The study will contain three groups, a default
  42 // group that isn't associated with a flag, and two other groups, both
  43 // associated with different flags.
  44 Study CreateStudyWithFlagGroups(int default_group_probability,
  45                                 int flag_group1_probability,
  46                                 int flag_group2_probability) {
  47   DCHECK_GE(default_group_probability, 0);
  48   DCHECK_GE(flag_group1_probability, 0);
  49   DCHECK_GE(flag_group2_probability, 0);
  50   Study study;
  51   study.set_name(kFlagStudyName);
  52   study.set_default_experiment_name(kNonFlagGroupName);
  53
  54   AddExperiment(kNonFlagGroupName, default_group_probability, &study);
  55   AddExperiment(kFlagGroup1Name, flag_group1_probability, &study)
  56       ->set_forcing_flag(kForcingFlag1);
  57   AddExperiment(kFlagGroup2Name, flag_group2_probability, &study)
  58       ->set_forcing_flag(kForcingFlag2);
  59
  60   return study;
  61 }
  62
  63 }  // namespace
  64
  65 TEST(VariationsSeedProcessorTest, CheckStudyChannel) {
  66   VariationsSeedProcessor seed_processor;
  67
  68   const Study_Channel channels[] = {
  69     Study_Channel_CANARY,
  70     Study_Channel_DEV,
  71     Study_Channel_BETA,
  72     Study_Channel_STABLE,
  73   };
  74   bool channel_added[arraysize(channels)] = { 0 };
  75
  76   Study_Filter filter;
  77
  78   // Check in the forwarded order. The loop cond is <= arraysize(channels)
  79   // instead of < so that the result of adding the last channel gets checked.
  80   for (size_t i = 0; i <= arraysize(channels); ++i) {
  81     for (size_t j = 0; j < arraysize(channels); ++j) {
  82       const bool expected = channel_added[j] || filter.channel_size() == 0;
  83       const bool result = seed_processor.CheckStudyChannel(filter, channels[j]);
  84       EXPECT_EQ(expected, result) << "Case " << i << "," << j << " failed!";
  85     }
  86
  87     if (i < arraysize(channels)) {
  88       filter.add_channel(channels[i]);
  89       channel_added[i] = true;
  90     }
  91   }
  92
  93   // Do the same check in the reverse order.
  94   filter.clear_channel();
  95   memset(&channel_added, 0, sizeof(channel_added));
  96   for (size_t i = 0; i <= arraysize(channels); ++i) {
  97     for (size_t j = 0; j < arraysize(channels); ++j) {
  98       const bool expected = channel_added[j] || filter.channel_size() == 0;
  99       const bool result = seed_processor.CheckStudyChannel(filter, channels[j]);
 100       EXPECT_EQ(expected, result) << "Case " << i << "," << j << " failed!";
 101     }
 102
 103     if (i < arraysize(channels)) {
 104       const int index = arraysize(channels) - i - 1;
 105       filter.add_channel(channels[index]);
 106       channel_added[index] = true;
 107     }
 108   }
 109 }
 110
 111 TEST(VariationsSeedProcessorTest, CheckStudyLocale) {
 112   VariationsSeedProcessor seed_processor;
 113
 114   struct {
 115     const char* filter_locales;
 116     bool en_us_result;
 117     bool en_ca_result;
 118     bool fr_result;
 119   } test_cases[] = {
 120     {"en-US", true, false, false},
 121     {"en-US,en-CA,fr", true, true, true},
 122     {"en-US,en-CA,en-GB", true, true, false},
 123     {"en-GB,en-CA,en-US", true, true, false},
 124     {"ja,kr,vi", false, false, false},
 125     {"fr-CA", false, false, false},
 126     {"", true, true, true},
 127   };
 128
 129   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(test_cases); ++i) {
 130     std::vector<std::string> filter_locales;
 131     Study_Filter filter;
 132     base::SplitString(test_cases[i].filter_locales, ',', &filter_locales);
 133     for (size_t j = 0; j < filter_locales.size(); ++j)
 134       filter.add_locale(filter_locales[j]);
 135     EXPECT_EQ(test_cases[i].en_us_result,
 136               seed_processor.CheckStudyLocale(filter, "en-US"));
 137     EXPECT_EQ(test_cases[i].en_ca_result,
 138               seed_processor.CheckStudyLocale(filter, "en-CA"));
 139     EXPECT_EQ(test_cases[i].fr_result,
 140               seed_processor.CheckStudyLocale(filter, "fr"));
 141   }
 142 }
 143
 144 TEST(VariationsSeedProcessorTest, CheckStudyPlatform) {
 145   VariationsSeedProcessor seed_processor;
 146
 147   const Study_Platform platforms[] = {
 148     Study_Platform_PLATFORM_WINDOWS,
 149     Study_Platform_PLATFORM_MAC,
 150     Study_Platform_PLATFORM_LINUX,
 151     Study_Platform_PLATFORM_CHROMEOS,
 152     Study_Platform_PLATFORM_ANDROID,
 153     Study_Platform_PLATFORM_IOS,
 154   };
 155   ASSERT_EQ(Study_Platform_Platform_ARRAYSIZE,
 156             static_cast<int>(arraysize(platforms)));
 157   bool platform_added[arraysize(platforms)] = { 0 };
 158
 159   Study_Filter filter;
 160
 161   // Check in the forwarded order. The loop cond is <= arraysize(platforms)
 162   // instead of < so that the result of adding the last channel gets checked.
 163   for (size_t i = 0; i <= arraysize(platforms); ++i) {
 164     for (size_t j = 0; j < arraysize(platforms); ++j) {
 165       const bool expected = platform_added[j] || filter.platform_size() == 0;
 166       const bool result = seed_processor.CheckStudyPlatform(filter,
 167                                                             platforms[j]);
 168       EXPECT_EQ(expected, result) << "Case " << i << "," << j << " failed!";
 169     }
 170
 171     if (i < arraysize(platforms)) {
 172       filter.add_platform(platforms[i]);
 173       platform_added[i] = true;
 174     }
 175   }
 176
 177   // Do the same check in the reverse order.
 178   filter.clear_platform();
 179   memset(&platform_added, 0, sizeof(platform_added));
 180   for (size_t i = 0; i <= arraysize(platforms); ++i) {
 181     for (size_t j = 0; j < arraysize(platforms); ++j) {
 182       const bool expected = platform_added[j] || filter.platform_size() == 0;
 183       const bool result = seed_processor.CheckStudyPlatform(filter,
 184                                                             platforms[j]);
 185       EXPECT_EQ(expected, result) << "Case " << i << "," << j << " failed!";
 186     }
 187
 188     if (i < arraysize(platforms)) {
 189       const int index = arraysize(platforms) - i - 1;
 190       filter.add_platform(platforms[index]);
 191       platform_added[index] = true;
 192     }
 193   }
 194 }
 195
 196 TEST(VariationsSeedProcessorTest, CheckStudyStartDate) {
 197   VariationsSeedProcessor seed_processor;
 198
 199   const base::Time now = base::Time::Now();
 200   const base::TimeDelta delta = base::TimeDelta::FromHours(1);
 201   const struct {
 202     const base::Time start_date;
 203     bool expected_result;
 204   } start_test_cases[] = {
 205     { now - delta, true },
 206     { now, true },
 207     { now + delta, false },
 208   };
 209
 210   Study_Filter filter;
 211
 212   // Start date not set should result in true.
 213   EXPECT_TRUE(seed_processor.CheckStudyStartDate(filter, now));
 214
 215   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(start_test_cases); ++i) {
 216     filter.set_start_date(TimeToProtoTime(start_test_cases[i].start_date));
 217     const bool result = seed_processor.CheckStudyStartDate(filter, now);
 218     EXPECT_EQ(start_test_cases[i].expected_result, result)
 219         << "Case " << i << " failed!";
 220   }
 221 }
 222
 223 TEST(VariationsSeedProcessorTest, CheckStudyVersion) {
 224   VariationsSeedProcessor seed_processor;
 225
 226   const struct {
 227     const char* min_version;
 228     const char* version;
 229     bool expected_result;
 230   } min_test_cases[] = {
 231     { "1.2.2", "1.2.3", true },
 232     { "1.2.3", "1.2.3", true },
 233     { "1.2.4", "1.2.3", false },
 234     { "1.3.2", "1.2.3", false },
 235     { "2.1.2", "1.2.3", false },
 236     { "0.3.4", "1.2.3", true },
 237     // Wildcards.
 238     { "1.*", "1.2.3", true },
 239     { "1.2.*", "1.2.3", true },
 240     { "1.2.3.*", "1.2.3", true },
 241     { "1.2.4.*", "1.2.3", false },
 242     { "2.*", "1.2.3", false },
 243     { "0.3.*", "1.2.3", true },
 244   };
 245
 246   const struct {
 247     const char* max_version;
 248     const char* version;
 249     bool expected_result;
 250   } max_test_cases[] = {
 251     { "1.2.2", "1.2.3", false },
 252     { "1.2.3", "1.2.3", true },
 253     { "1.2.4", "1.2.3", true },
 254     { "2.1.1", "1.2.3", true },
 255     { "2.1.1", "2.3.4", false },
 256     // Wildcards
 257     { "2.1.*", "2.3.4", false },
 258     { "2.*", "2.3.4", true },
 259     { "2.3.*", "2.3.4", true },
 260     { "2.3.4.*", "2.3.4", true },
 261     { "2.3.4.0.*", "2.3.4", true },
 262     { "2.4.*", "2.3.4", true },
 263     { "1.3.*", "2.3.4", false },
 264     { "1.*", "2.3.4", false },
 265   };
 266
 267   Study_Filter filter;
 268
 269   // Min/max version not set should result in true.
 270   EXPECT_TRUE(seed_processor.CheckStudyVersion(filter, base::Version("1.2.3")));
 271
 272   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(min_test_cases); ++i) {
 273     filter.set_min_version(min_test_cases[i].min_version);
 274     const bool result =
 275         seed_processor.CheckStudyVersion(filter,
 276                                          Version(min_test_cases[i].version));
 277     EXPECT_EQ(min_test_cases[i].expected_result, result) <<
 278         "Min. version case " << i << " failed!";
 279   }
 280   filter.clear_min_version();
 281
 282   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(max_test_cases); ++i) {
 283     filter.set_max_version(max_test_cases[i].max_version);
 284     const bool result =
 285         seed_processor.CheckStudyVersion(filter,
 286                                          Version(max_test_cases[i].version));
 287     EXPECT_EQ(max_test_cases[i].expected_result, result) <<
 288         "Max version case " << i << " failed!";
 289   }
 290
 291   // Check intersection semantics.
 292   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(min_test_cases); ++i) {
 293     for (size_t j = 0; j < ARRAYSIZE_UNSAFE(max_test_cases); ++j) {
 294       filter.set_min_version(min_test_cases[i].min_version);
 295       filter.set_max_version(max_test_cases[j].max_version);
 296
 297       if (!min_test_cases[i].expected_result) {
 298         const bool result =
 299             seed_processor.CheckStudyVersion(
 300                 filter, Version(min_test_cases[i].version));
 301         EXPECT_FALSE(result) << "Case " << i << "," << j << " failed!";
 302       }
 303
 304       if (!max_test_cases[j].expected_result) {
 305         const bool result =
 306             seed_processor.CheckStudyVersion(
 307                 filter, Version(max_test_cases[j].version));
 308         EXPECT_FALSE(result) << "Case " << i << "," << j << " failed!";
 309       }
 310     }
 311   }
 312 }
 313
 314 // Test that the group for kForcingFlag1 is forced.
 315 TEST(VariationsSeedProcessorTest, ForceGroupWithFlag1) {
 316   CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 317
 318   base::FieldTrialList field_trial_list(NULL);
 319
 320   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 321   VariationsSeedProcessor().CreateTrialFromStudy(study, false);
 322
 323   EXPECT_EQ(kFlagGroup1Name,
 324             base::FieldTrialList::FindFullName(kFlagStudyName));
 325 }
 326
 327 // Test that the group for kForcingFlag2 is forced.
 328 TEST(VariationsSeedProcessorTest, ForceGroupWithFlag2) {
 329   CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag2);
 330
 331   base::FieldTrialList field_trial_list(NULL);
 332
 333   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 334   VariationsSeedProcessor().CreateTrialFromStudy(study, false);
 335
 336   EXPECT_EQ(kFlagGroup2Name,
 337             base::FieldTrialList::FindFullName(kFlagStudyName));
 338 }
 339
 340 TEST(VariationsSeedProcessorTest, ForceGroup_ChooseFirstGroupWithFlag) {
 341   // Add the flag to the command line arguments so the flag group is forced.
 342   CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag1);
 343   CommandLine::ForCurrentProcess()->AppendSwitch(kForcingFlag2);
 344
 345   base::FieldTrialList field_trial_list(NULL);
 346
 347   Study study = CreateStudyWithFlagGroups(100, 0, 0);
 348   VariationsSeedProcessor().CreateTrialFromStudy(study, false);
 349
 350   EXPECT_EQ(kFlagGroup1Name,
 351             base::FieldTrialList::FindFullName(kFlagStudyName));
 352 }
 353
 354 TEST(VariationsSeedProcessorTest, ForceGroup_DontChooseGroupWithFlag) {
 355   base::FieldTrialList field_trial_list(NULL);
 356
 357   // The two flag groups are given high probability, which would normally make
 358   // them very likely to be chosen. They won't be chosen since flag groups are
 359   // never chosen when their flag isn't present.
 360   Study study = CreateStudyWithFlagGroups(1, 999, 999);
 361   VariationsSeedProcessor().CreateTrialFromStudy(study, false);
 362   EXPECT_EQ(kNonFlagGroupName,
 363             base::FieldTrialList::FindFullName(kFlagStudyName));
 364 }
 365
 366 TEST(VariationsSeedProcessorTest, IsStudyExpired) {
 367   VariationsSeedProcessor seed_processor;
 368
 369   const base::Time now = base::Time::Now();
 370   const base::TimeDelta delta = base::TimeDelta::FromHours(1);
 371   const struct {
 372     const base::Time expiry_date;
 373     bool expected_result;
 374   } expiry_test_cases[] = {
 375     { now - delta, true },
 376     { now, true },
 377     { now + delta, false },
 378   };
 379
 380   Study study;
 381
 382   // Expiry date not set should result in false.
 383   EXPECT_FALSE(seed_processor.IsStudyExpired(study, now));
 384
 385   for (size_t i = 0; i < ARRAYSIZE_UNSAFE(expiry_test_cases); ++i) {
 386     study.set_expiry_date(TimeToProtoTime(expiry_test_cases[i].expiry_date));
 387     const bool result = seed_processor.IsStudyExpired(study, now);
 388     EXPECT_EQ(expiry_test_cases[i].expected_result, result)
 389         << "Case " << i << " failed!";
 390   }
 391 }
 392
 393 TEST(VariationsSeedProcessorTest, NonExpiredStudyPrioritizedOverExpiredStudy) {
 394   VariationsSeedProcessor seed_processor;
 395
 396   const std::string kTrialName = "A";
 397   const std::string kGroup1Name = "Group1";
 398
 399   TrialsSeed seed;
 400   Study* study1 = seed.add_study();
 401   study1->set_name(kTrialName);
 402   study1->set_default_experiment_name("Default");
 403   AddExperiment(kGroup1Name, 100, study1);
 404   AddExperiment("Default", 0, study1);
 405   Study* study2 = seed.add_study();
 406   *study2 = *study1;
 407   ASSERT_EQ(seed.study(0).name(), seed.study(1).name());
 408
 409   const base::Time year_ago =
 410       base::Time::Now() - base::TimeDelta::FromDays(365);
 411
 412   const base::Version version("20.0.0.0");
 413
 414   // Check that adding [expired, non-expired] activates the non-expired one.
 415   ASSERT_EQ(std::string(), base::FieldTrialList::FindFullName(kTrialName));
 416   {
 417     base::FieldTrialList field_trial_list(NULL);
 418     study1->set_expiry_date(TimeToProtoTime(year_ago));
 419     seed_processor.CreateTrialsFromSeed(seed, "en-CA", base::Time::Now(),
 420                                         version, Study_Channel_STABLE);
 421     EXPECT_EQ(kGroup1Name, base::FieldTrialList::FindFullName(kTrialName));
 422   }
 423
 424   // Check that adding [non-expired, expired] activates the non-expired one.
 425   ASSERT_EQ(std::string(), base::FieldTrialList::FindFullName(kTrialName));
 426   {
 427     base::FieldTrialList field_trial_list(NULL);
 428     study1->clear_expiry_date();
 429     study2->set_expiry_date(TimeToProtoTime(year_ago));
 430     seed_processor.CreateTrialsFromSeed(seed, "en-CA", base::Time::Now(),
 431                                         version, Study_Channel_STABLE);
 432     EXPECT_EQ(kGroup1Name, base::FieldTrialList::FindFullName(kTrialName));
 433   }
 434 }
 435
 436 TEST(VariationsSeedProcessorTest, ValidateStudy) {
 437   VariationsSeedProcessor seed_processor;
 438
 439   Study study;
 440   study.set_default_experiment_name("def");
 441   AddExperiment("abc", 100, &study);
 442   Study_Experiment* default_group = AddExperiment("def", 200, &study);
 443
 444   base::FieldTrial::Probability total_probability = 0;
 445   bool valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 446       study, &total_probability);
 447   EXPECT_TRUE(valid);
 448   EXPECT_EQ(300, total_probability);
 449
 450   // Min version checks.
 451   study.mutable_filter()->set_min_version("1.2.3.*");
 452   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 453       study, &total_probability);
 454   EXPECT_TRUE(valid);
 455   study.mutable_filter()->set_min_version("1.*.3");
 456   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 457       study, &total_probability);
 458   EXPECT_FALSE(valid);
 459   study.mutable_filter()->set_min_version("1.2.3");
 460   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 461       study, &total_probability);
 462   EXPECT_TRUE(valid);
 463
 464   // Max version checks.
 465   study.mutable_filter()->set_max_version("2.3.4.*");
 466   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 467       study, &total_probability);
 468   EXPECT_TRUE(valid);
 469   study.mutable_filter()->set_max_version("*.3");
 470   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 471       study, &total_probability);
 472   EXPECT_FALSE(valid);
 473   study.mutable_filter()->set_max_version("2.3.4");
 474   valid = seed_processor.ValidateStudyAndComputeTotalProbability(
 475       study, &total_probability);
 476   EXPECT_TRUE(valid);
 477
 478   study.clear_default_experiment_name();
 479   valid = seed_processor.ValidateStudyAndComputeTotalProbability(study,
 480       &total_probability);
 481   EXPECT_FALSE(valid);
 482
 483   study.set_default_experiment_name("xyz");
 484   valid = seed_processor.ValidateStudyAndComputeTotalProbability(study,
 485       &total_probability);
 486   EXPECT_FALSE(valid);
 487
 488   study.set_default_experiment_name("def");
 489   default_group->clear_name();
 490   valid = seed_processor.ValidateStudyAndComputeTotalProbability(study,
 491       &total_probability);
 492   EXPECT_FALSE(valid);
 493
 494   default_group->set_name("def");
 495   valid = seed_processor.ValidateStudyAndComputeTotalProbability(study,
 496       &total_probability);
 497   ASSERT_TRUE(valid);
 498   Study_Experiment* repeated_group = study.add_experiment();
 499   repeated_group->set_name("abc");
 500   repeated_group->set_probability_weight(1);
 501   valid = seed_processor.ValidateStudyAndComputeTotalProbability(study,
 502       &total_probability);
 503   EXPECT_FALSE(valid);
 504 }
 505
 506 TEST(VariationsSeedProcessorTest, VariationParams) {
 507   base::FieldTrialList field_trial_list(NULL);
 508   VariationsSeedProcessor seed_processor;
 509
 510   Study study;
 511   study.set_name("Study1");
 512   study.set_default_experiment_name("B");
 513
 514   Study_Experiment* experiment1 = AddExperiment("A", 1, &study);
 515   Study_Experiment_Param* param = experiment1->add_param();
 516   param->set_name("x");
 517   param->set_value("y");
 518
 519   Study_Experiment* experiment2 = AddExperiment("B", 0, &study);
 520
 521   seed_processor.CreateTrialFromStudy(study, false);
 522   EXPECT_EQ("y", GetVariationParamValue("Study1", "x"));
 523
 524   study.set_name("Study2");
 525   experiment1->set_probability_weight(0);
 526   experiment2->set_probability_weight(1);
 527   seed_processor.CreateTrialFromStudy(study, false);
 528   EXPECT_EQ(std::string(), GetVariationParamValue("Study2", "x"));
 529 }
 530
 531 }  // namespace chrome_variations