QUIC - cleanup changes to sync chromium tree with internal source.
[chromium-blink-merge.git] / base / metrics / field_trial.h
blob93521288b7af4386a2a370b8b8e8658c852a6c20
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 // FieldTrial is a class for handling details of statistical experiments
6 // performed by actual users in the field (i.e., in a shipped or beta product).
7 // All code is called exclusively on the UI thread currently.
8 //
9 // The simplest example is an experiment to see whether one of two options
10 // produces "better" results across our user population. In that scenario, UMA
11 // data is uploaded to aggregate the test results, and this FieldTrial class
12 // manages the state of each such experiment (state == which option was
13 // pseudo-randomly selected).
15 // States are typically generated randomly, either based on a one time
16 // randomization (which will yield the same results, in terms of selecting
17 // the client for a field trial or not, for every run of the program on a
18 // given machine), or by a session randomization (generated each time the
19 // application starts up, but held constant during the duration of the
20 // process).
22 //------------------------------------------------------------------------------
23 // Example: Suppose we have an experiment involving memory, such as determining
24 // the impact of some pruning algorithm.
25 // We assume that we already have a histogram of memory usage, such as:
27 // UMA_HISTOGRAM_COUNTS("Memory.RendererTotal", count);
29 // Somewhere in main thread initialization code, we'd probably define an
30 // instance of a FieldTrial, with code such as:
32 // // FieldTrials are reference counted, and persist automagically until
33 // // process teardown, courtesy of their automatic registration in
34 // // FieldTrialList.
35 // // Note: This field trial will run in Chrome instances compiled through
36 // // 8 July, 2015, and after that all instances will be in "StandardMem".
37 // scoped_refptr<base::FieldTrial> trial(
38 // base::FieldTrialList::FactoryGetFieldTrial(
39 // "MemoryExperiment", 1000, "StandardMem", 2015, 7, 8,
40 // base::FieldTrial::ONE_TIME_RANDOMIZED, NULL));
42 // const int high_mem_group =
43 // trial->AppendGroup("HighMem", 20); // 2% in HighMem group.
44 // const int low_mem_group =
45 // trial->AppendGroup("LowMem", 20); // 2% in LowMem group.
46 // // Take action depending of which group we randomly land in.
47 // if (trial->group() == high_mem_group)
48 // SetPruningAlgorithm(kType1); // Sample setting of browser state.
49 // else if (trial->group() == low_mem_group)
50 // SetPruningAlgorithm(kType2); // Sample alternate setting.
52 //------------------------------------------------------------------------------
54 #ifndef BASE_METRICS_FIELD_TRIAL_H_
55 #define BASE_METRICS_FIELD_TRIAL_H_
57 #include <map>
58 #include <set>
59 #include <string>
60 #include <vector>
62 #include "base/base_export.h"
63 #include "base/gtest_prod_util.h"
64 #include "base/memory/ref_counted.h"
65 #include "base/observer_list_threadsafe.h"
66 #include "base/synchronization/lock.h"
67 #include "base/time/time.h"
69 namespace base {
71 class FieldTrialList;
73 class BASE_EXPORT FieldTrial : public RefCounted<FieldTrial> {
74 public:
75 typedef int Probability; // Probability type for being selected in a trial.
77 // Specifies the persistence of the field trial group choice.
78 enum RandomizationType {
79 // One time randomized trials will persist the group choice between
80 // restarts, which is recommended for most trials, especially those that
81 // change user visible behavior.
82 ONE_TIME_RANDOMIZED,
83 // Session randomized trials will roll the dice to select a group on every
84 // process restart.
85 SESSION_RANDOMIZED,
88 // EntropyProvider is an interface for providing entropy for one-time
89 // randomized (persistent) field trials.
90 class BASE_EXPORT EntropyProvider {
91 public:
92 virtual ~EntropyProvider();
94 // Returns a double in the range of [0, 1) to be used for the dice roll for
95 // the specified field trial. If |randomization_seed| is not 0, it will be
96 // used in preference to |trial_name| for generating the entropy by entropy
97 // providers that support it. A given instance should always return the same
98 // value given the same input |trial_name| and |randomization_seed| values.
99 virtual double GetEntropyForTrial(const std::string& trial_name,
100 uint32 randomization_seed) const = 0;
103 // A pair representing a Field Trial and its selected group.
104 struct ActiveGroup {
105 std::string trial_name;
106 std::string group_name;
109 // A triplet representing a FieldTrial, its selected group and whether it's
110 // active.
111 struct FieldTrialState {
112 std::string trial_name;
113 std::string group_name;
114 bool activated;
117 typedef std::vector<ActiveGroup> ActiveGroups;
119 // A return value to indicate that a given instance has not yet had a group
120 // assignment (and hence is not yet participating in the trial).
121 static const int kNotFinalized;
123 // Disables this trial, meaning it always determines the default group
124 // has been selected. May be called immediately after construction, or
125 // at any time after initialization (should not be interleaved with
126 // AppendGroup calls). Once disabled, there is no way to re-enable a
127 // trial.
128 // TODO(mad): http://code.google.com/p/chromium/issues/detail?id=121446
129 // This doesn't properly reset to Default when a group was forced.
130 void Disable();
132 // Establish the name and probability of the next group in this trial.
133 // Sometimes, based on construction randomization, this call may cause the
134 // provided group to be *THE* group selected for use in this instance.
135 // The return value is the group number of the new group.
136 int AppendGroup(const std::string& name, Probability group_probability);
138 // Return the name of the FieldTrial (excluding the group name).
139 const std::string& trial_name() const { return trial_name_; }
141 // Return the randomly selected group number that was assigned, and notify
142 // any/all observers that this finalized group number has presumably been used
143 // (queried), and will never change. Note that this will force an instance to
144 // participate, and make it illegal to attempt to probabilistically add any
145 // other groups to the trial.
146 int group();
148 // If the group's name is empty, a string version containing the group number
149 // is used as the group name. This causes a winner to be chosen if none was.
150 const std::string& group_name();
152 // Set the field trial as forced, meaning that it was setup earlier than
153 // the hard coded registration of the field trial to override it.
154 // This allows the code that was hard coded to register the field trial to
155 // still succeed even though the field trial has already been registered.
156 // This must be called after appending all the groups, since we will make
157 // the group choice here. Note that this is a NOOP for already forced trials.
158 // And, as the rest of the FieldTrial code, this is not thread safe and must
159 // be done from the UI thread.
160 void SetForced();
162 // Enable benchmarking sets field trials to a common setting.
163 static void EnableBenchmarking();
165 // Creates a FieldTrial object with the specified parameters, to be used for
166 // simulation of group assignment without actually affecting global field
167 // trial state in the running process. Group assignment will be done based on
168 // |entropy_value|, which must have a range of [0, 1).
170 // Note: Using this function will not register the field trial globally in the
171 // running process - for that, use FieldTrialList::FactoryGetFieldTrial().
173 // The ownership of the returned FieldTrial is transfered to the caller which
174 // is responsible for deref'ing it (e.g. by using scoped_refptr<FieldTrial>).
175 static FieldTrial* CreateSimulatedFieldTrial(
176 const std::string& trial_name,
177 Probability total_probability,
178 const std::string& default_group_name,
179 double entropy_value);
181 private:
182 // Allow tests to access our innards for testing purposes.
183 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Registration);
184 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, AbsoluteProbabilities);
185 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, RemainingProbability);
186 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, FiftyFiftyProbability);
187 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, MiddleProbabilities);
188 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, OneWinner);
189 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DisableProbability);
190 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroups);
191 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, AllGroups);
192 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, ActiveGroupsNotFinalized);
193 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, Save);
194 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SaveAll);
195 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DuplicateRestore);
196 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOff);
197 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedTurnFeatureOn);
198 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_Default);
199 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, SetForcedChangeDefault_NonDefault);
200 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, FloatBoundariesGiveEqualGroupSizes);
201 FRIEND_TEST_ALL_PREFIXES(FieldTrialTest, DoesNotSurpassTotalProbability);
203 friend class base::FieldTrialList;
205 friend class RefCounted<FieldTrial>;
207 // This is the group number of the 'default' group when a choice wasn't forced
208 // by a call to FieldTrialList::CreateFieldTrial. It is kept private so that
209 // consumers don't use it by mistake in cases where the group was forced.
210 static const int kDefaultGroupNumber;
212 // Creates a field trial with the specified parameters. Group assignment will
213 // be done based on |entropy_value|, which must have a range of [0, 1).
214 FieldTrial(const std::string& trial_name,
215 Probability total_probability,
216 const std::string& default_group_name,
217 double entropy_value);
218 virtual ~FieldTrial();
220 // Return the default group name of the FieldTrial.
221 std::string default_group_name() const { return default_group_name_; }
223 // Marks this trial as having been registered with the FieldTrialList. Must be
224 // called no more than once and before any |group()| calls have occurred.
225 void SetTrialRegistered();
227 // Sets the chosen group name and number.
228 void SetGroupChoice(const std::string& group_name, int number);
230 // Ensures that a group is chosen, if it hasn't yet been. The field trial
231 // might yet be disabled, so this call will *not* notify observers of the
232 // status.
233 void FinalizeGroupChoice();
235 // Returns the trial name and selected group name for this field trial via
236 // the output parameter |active_group|, but only if the group has already
237 // been chosen and has been externally observed via |group()| and the trial
238 // has not been disabled. In that case, true is returned and |active_group|
239 // is filled in; otherwise, the result is false and |active_group| is left
240 // untouched.
241 bool GetActiveGroup(ActiveGroup* active_group) const;
243 // Returns the trial name and selected group name for this field trial via
244 // the output parameter |field_trial_state|, but only if the trial has not
245 // been disabled. In that case, true is returned and |field_trial_state| is
246 // filled in; otherwise, the result is false and |field_trial_state| is left
247 // untouched.
248 bool GetState(FieldTrialState* field_trial_state) const;
250 // Returns the group_name. A winner need not have been chosen.
251 std::string group_name_internal() const { return group_name_; }
253 // The name of the field trial, as can be found via the FieldTrialList.
254 const std::string trial_name_;
256 // The maximum sum of all probabilities supplied, which corresponds to 100%.
257 // This is the scaling factor used to adjust supplied probabilities.
258 const Probability divisor_;
260 // The name of the default group.
261 const std::string default_group_name_;
263 // The randomly selected probability that is used to select a group (or have
264 // the instance not participate). It is the product of divisor_ and a random
265 // number between [0, 1).
266 Probability random_;
268 // Sum of the probabilities of all appended groups.
269 Probability accumulated_group_probability_;
271 // The number that will be returned by the next AppendGroup() call.
272 int next_group_number_;
274 // The pseudo-randomly assigned group number.
275 // This is kNotFinalized if no group has been assigned.
276 int group_;
278 // A textual name for the randomly selected group. Valid after |group()|
279 // has been called.
280 std::string group_name_;
282 // When enable_field_trial_ is false, field trial reverts to the 'default'
283 // group.
284 bool enable_field_trial_;
286 // When forced_ is true, we return the chosen group from AppendGroup when
287 // appropriate.
288 bool forced_;
290 // Specifies whether the group choice has been reported to observers.
291 bool group_reported_;
293 // Whether this trial is registered with the global FieldTrialList and thus
294 // should notify it when its group is queried.
295 bool trial_registered_;
297 // When benchmarking is enabled, field trials all revert to the 'default'
298 // group.
299 static bool enable_benchmarking_;
301 DISALLOW_COPY_AND_ASSIGN(FieldTrial);
304 //------------------------------------------------------------------------------
305 // Class with a list of all active field trials. A trial is active if it has
306 // been registered, which includes evaluating its state based on its probaility.
307 // Only one instance of this class exists.
308 class BASE_EXPORT FieldTrialList {
309 public:
310 // Specifies whether field trials should be activated (marked as "used"), when
311 // created using |CreateTrialsFromString()|. Has no effect on trials that are
312 // prefixed with |kActivationMarker|, which will always be activated."
313 enum FieldTrialActivationMode {
314 DONT_ACTIVATE_TRIALS,
315 ACTIVATE_TRIALS,
318 // Define a separator character to use when creating a persistent form of an
319 // instance. This is intended for use as a command line argument, passed to a
320 // second process to mimic our state (i.e., provide the same group name).
321 static const char kPersistentStringSeparator; // Currently a slash.
323 // Define a marker character to be used as a prefix to a trial name on the
324 // command line which forces its activation.
325 static const char kActivationMarker; // Currently an asterisk.
327 // Year that is guaranteed to not be expired when instantiating a field trial
328 // via |FactoryGetFieldTrial()|. Set to two years from the build date.
329 static int kNoExpirationYear;
331 // Observer is notified when a FieldTrial's group is selected.
332 class BASE_EXPORT Observer {
333 public:
334 // Notify observers when FieldTrials's group is selected.
335 virtual void OnFieldTrialGroupFinalized(const std::string& trial_name,
336 const std::string& group_name) = 0;
338 protected:
339 virtual ~Observer();
342 // This singleton holds the global list of registered FieldTrials.
344 // To support one-time randomized field trials, specify a non-NULL
345 // |entropy_provider| which should be a source of uniformly distributed
346 // entropy values. Takes ownership of |entropy_provider|. If one time
347 // randomization is not desired, pass in NULL for |entropy_provider|.
348 explicit FieldTrialList(const FieldTrial::EntropyProvider* entropy_provider);
350 // Destructor Release()'s references to all registered FieldTrial instances.
351 ~FieldTrialList();
353 // Get a FieldTrial instance from the factory.
355 // |name| is used to register the instance with the FieldTrialList class,
356 // and can be used to find the trial (only one trial can be present for each
357 // name). |default_group_name| is the name of the default group which will
358 // be chosen if none of the subsequent appended groups get to be chosen.
359 // |default_group_number| can receive the group number of the default group as
360 // AppendGroup returns the number of the subsequence groups. |trial_name| and
361 // |default_group_name| may not be empty but |default_group_number| can be
362 // NULL if the value is not needed.
364 // Group probabilities that are later supplied must sum to less than or equal
365 // to the |total_probability|. Arguments |year|, |month| and |day_of_month|
366 // specify the expiration time. If the build time is after the expiration time
367 // then the field trial reverts to the 'default' group.
369 // Use this static method to get a startup-randomized FieldTrial or a
370 // previously created forced FieldTrial.
371 static FieldTrial* FactoryGetFieldTrial(
372 const std::string& trial_name,
373 FieldTrial::Probability total_probability,
374 const std::string& default_group_name,
375 const int year,
376 const int month,
377 const int day_of_month,
378 FieldTrial::RandomizationType randomization_type,
379 int* default_group_number);
381 // Same as FactoryGetFieldTrial(), but allows specifying a custom seed to be
382 // used on one-time randomized field trials (instead of a hash of the trial
383 // name, which is used otherwise or if |randomization_seed| has value 0). The
384 // |randomization_seed| value (other than 0) should never be the same for two
385 // trials, else this would result in correlated group assignments.
386 // Note: Using a custom randomization seed is only supported by the
387 // PermutedEntropyProvider (which is used when UMA is not enabled).
388 static FieldTrial* FactoryGetFieldTrialWithRandomizationSeed(
389 const std::string& trial_name,
390 FieldTrial::Probability total_probability,
391 const std::string& default_group_name,
392 const int year,
393 const int month,
394 const int day_of_month,
395 FieldTrial::RandomizationType randomization_type,
396 uint32 randomization_seed,
397 int* default_group_number);
399 // The Find() method can be used to test to see if a named Trial was already
400 // registered, or to retrieve a pointer to it from the global map.
401 static FieldTrial* Find(const std::string& name);
403 // Returns the group number chosen for the named trial, or
404 // FieldTrial::kNotFinalized if the trial does not exist.
405 static int FindValue(const std::string& name);
407 // Returns the group name chosen for the named trial, or the empty string if
408 // the trial does not exist. The first call of this function on a given field
409 // trial will mark it as active, so that its state will be reported with usage
410 // metrics, crashes, etc.
411 static std::string FindFullName(const std::string& name);
413 // Returns true if the named trial has been registered.
414 static bool TrialExists(const std::string& name);
416 // Creates a persistent representation of active FieldTrial instances for
417 // resurrection in another process. This allows randomization to be done in
418 // one process, and secondary processes can be synchronized on the result.
419 // The resulting string contains the name and group name pairs of all
420 // registered FieldTrials for which the group has been chosen and externally
421 // observed (via |group()|) and which have not been disabled, with "/" used
422 // to separate all names and to terminate the string. This string is parsed
423 // by |CreateTrialsFromString()|.
424 static void StatesToString(std::string* output);
426 // Creates a persistent representation of all FieldTrial instances for
427 // resurrection in another process. This allows randomization to be done in
428 // one process, and secondary processes can be synchronized on the result.
429 // The resulting string contains the name and group name pairs of all
430 // registered FieldTrials which have not been disabled, with "/" used
431 // to separate all names and to terminate the string. All activated trials
432 // have their name prefixed with "*". This string is parsed by
433 // |CreateTrialsFromString()|.
434 static void AllStatesToString(std::string* output);
436 // Fills in the supplied vector |active_groups| (which must be empty when
437 // called) with a snapshot of all registered FieldTrials for which the group
438 // has been chosen and externally observed (via |group()|) and which have
439 // not been disabled.
440 static void GetActiveFieldTrialGroups(
441 FieldTrial::ActiveGroups* active_groups);
443 // Use a state string (re: StatesToString()) to augment the current list of
444 // field trials to include the supplied trials, and using a 100% probability
445 // for each trial, force them to have the same group string. This is commonly
446 // used in a non-browser process, to carry randomly selected state in a
447 // browser process into this non-browser process, but could also be invoked
448 // through a command line argument to the browser process. The created field
449 // trials are all marked as "used" for the purposes of active trial reporting
450 // if |mode| is ACTIVATE_TRIALS, otherwise each trial will be marked as "used"
451 // if it is prefixed with |kActivationMarker|. Trial names in
452 // |ignored_trial_names| are ignored when parsing |prior_trials|.
453 static bool CreateTrialsFromString(
454 const std::string& prior_trials,
455 FieldTrialActivationMode mode,
456 const std::set<std::string>& ignored_trial_names);
458 // Create a FieldTrial with the given |name| and using 100% probability for
459 // the FieldTrial, force FieldTrial to have the same group string as
460 // |group_name|. This is commonly used in a non-browser process, to carry
461 // randomly selected state in a browser process into this non-browser process.
462 // It returns NULL if there is a FieldTrial that is already registered with
463 // the same |name| but has different finalized group string (|group_name|).
464 static FieldTrial* CreateFieldTrial(const std::string& name,
465 const std::string& group_name);
467 // Add an observer to be notified when a field trial is irrevocably committed
468 // to being part of some specific field_group (and hence the group_name is
469 // also finalized for that field_trial).
470 static void AddObserver(Observer* observer);
472 // Remove an observer.
473 static void RemoveObserver(Observer* observer);
475 // Notify all observers that a group has been finalized for |field_trial|.
476 static void NotifyFieldTrialGroupSelection(FieldTrial* field_trial);
478 // Return the number of active field trials.
479 static size_t GetFieldTrialCount();
481 private:
482 // A map from FieldTrial names to the actual instances.
483 typedef std::map<std::string, FieldTrial*> RegistrationMap;
485 // If one-time randomization is enabled, returns a weak pointer to the
486 // corresponding EntropyProvider. Otherwise, returns NULL.
487 static const FieldTrial::EntropyProvider*
488 GetEntropyProviderForOneTimeRandomization();
490 // Helper function should be called only while holding lock_.
491 FieldTrial* PreLockedFind(const std::string& name);
493 // Register() stores a pointer to the given trial in a global map.
494 // This method also AddRef's the indicated trial.
495 // This should always be called after creating a new FieldTrial instance.
496 static void Register(FieldTrial* trial);
498 static FieldTrialList* global_; // The singleton of this class.
500 // This will tell us if there is an attempt to register a field
501 // trial or check if one-time randomization is enabled without
502 // creating the FieldTrialList. This is not an error, unless a
503 // FieldTrialList is created after that.
504 static bool used_without_global_;
506 // Lock for access to registered_.
507 base::Lock lock_;
508 RegistrationMap registered_;
510 // Entropy provider to be used for one-time randomized field trials. If NULL,
511 // one-time randomization is not supported.
512 scoped_ptr<const FieldTrial::EntropyProvider> entropy_provider_;
514 // List of observers to be notified when a group is selected for a FieldTrial.
515 scoped_refptr<ObserverListThreadSafe<Observer> > observer_list_;
517 DISALLOW_COPY_AND_ASSIGN(FieldTrialList);
520 } // namespace base
522 #endif // BASE_METRICS_FIELD_TRIAL_H_