chrome/test/nacl/nacl_browsertest_uma.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "base/test/histogram_tester.h"
   6 #include "chrome/test/nacl/nacl_browsertest_util.h"
   7 #include "components/nacl/browser/nacl_browser.h"
   8 #include "components/nacl/renderer/platform_info.h"
   9 #include "components/nacl/renderer/ppb_nacl_private.h"
  10 #include "content/public/test/browser_test_utils.h"
  11 #include "native_client/src/trusted/service_runtime/nacl_error_code.h"
  12
  13 namespace {
  14
  15 void CheckPNaClLoadUMAs(base::HistogramTester& histograms,
  16                         const std::string& compiler_suffix) {
  17   // The histograms that do not vary by compiler (so no compiler_suffix).
  18   histograms.ExpectTotalCount("NaCl.Perf.PNaClLoadTime.LoadLinker", 1);
  19   histograms.ExpectTotalCount("NaCl.Perf.PNaClLoadTime.LinkTime", 1);
  20   histograms.ExpectTotalCount("NaCl.Perf.Size.Manifest", 1);
  21   histograms.ExpectTotalCount("NaCl.Perf.Size.Pexe", 1);
  22   // The histograms that vary by compiler.
  23   histograms.ExpectTotalCount("NaCl.Options.PNaCl.OptLevel" + compiler_suffix,
  24                               1);
  25   histograms.ExpectTotalCount(
  26       "NaCl.Perf.Size.PNaClTranslatedNexe" + compiler_suffix, 1);
  27   histograms.ExpectTotalCount(
  28       "NaCl.Perf.Size.PexeNexeSizePct" + compiler_suffix, 1);
  29   histograms.ExpectTotalCount(
  30       "NaCl.Perf.PNaClLoadTime.LoadCompiler" + compiler_suffix, 1);
  31   histograms.ExpectTotalCount(
  32       "NaCl.Perf.PNaClLoadTime.CompileTime" + compiler_suffix, 1);
  33   histograms.ExpectTotalCount(
  34       "NaCl.Perf.PNaClLoadTime.CompileKBPerSec" + compiler_suffix, 1);
  35   histograms.ExpectTotalCount(
  36       "NaCl.Perf.PNaClLoadTime.PctCompiledWhenFullyDownloaded" +
  37           compiler_suffix,
  38       1);
  39   histograms.ExpectTotalCount(
  40       "NaCl.Perf.PNaClLoadTime.TotalUncachedTime" + compiler_suffix, 1);
  41   histograms.ExpectTotalCount(
  42       "NaCl.Perf.PNaClLoadTime.TotalUncachedKBPerSec" + compiler_suffix, 1);
  43   histograms.ExpectTotalCount("NaCl.Perf.PNaClCache.IsHit" + compiler_suffix,
  44                               1);
  45 }
  46
  47 NACL_BROWSER_TEST_F(NaClBrowserTest, SuccessfulLoadUMA, {
  48   base::HistogramTester histograms;
  49   // Load a NaCl module to generate UMA data.
  50   RunLoadTest(FILE_PATH_LITERAL("nacl_load_test.html"));
  51
  52   // Make sure histograms from child processes have been accumulated in the
  53   // browser brocess.
  54   content::FetchHistogramsFromChildProcesses();
  55
  56   // Did the plugin report success?
  57   histograms.ExpectUniqueSample("NaCl.LoadStatus.Plugin",
  58                                 PP_NACL_ERROR_LOAD_SUCCESS, 1);
  59
  60   // Did the sel_ldr report success?
  61   histograms.ExpectUniqueSample("NaCl.LoadStatus.SelLdr",
  62                                 LOAD_OK, 1);
  63
  64   // Check validation cache usage:
  65   if (IsAPnaclTest()) {
  66     // Should have received 5 validation queries:
  67     // - Three for the IRT: the app and both of the translator nexes use it.
  68     // - Two for the two PNaCl translator nexes.
  69     // The PNaCl app nexe comes from a delete-on-close temp file, so it
  70     // doesn't have a stable identity for validation caching. Overall, there
  71     // are 3 eligible nexes. The first 3 queries for these eligible nexes
  72     // are misses, and the latter two of the IRT queries are hits.
  73     histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
  74                                nacl::NaClBrowser::CACHE_MISS, 3);
  75     histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
  76                                nacl::NaClBrowser::CACHE_HIT, 2);
  77     // Should have received a cache setting afterwards (IRT set only once).
  78     histograms.ExpectUniqueSample("NaCl.ValidationCache.Set",
  79                                   nacl::NaClBrowser::CACHE_HIT, 3);
  80   } else {
  81     // For the open-web, only the IRT is considered a "safe" and
  82     // identity-cachable file. The nexes and .so files are not.
  83     // Should have one cache query for the IRT.
  84     histograms.ExpectUniqueSample("NaCl.ValidationCache.Query",
  85                                   nacl::NaClBrowser::CACHE_MISS, 1);
  86     // Should have received a cache setting afterwards for the IRT.
  87     histograms.ExpectUniqueSample("NaCl.ValidationCache.Set",
  88                                   nacl::NaClBrowser::CACHE_HIT, 1);
  89   }
  90
  91   // Make sure we have other important histograms.
  92   if (!IsAPnaclTest()) {
  93     histograms.ExpectTotalCount("NaCl.Perf.StartupTime.LoadModule", 1);
  94     histograms.ExpectTotalCount("NaCl.Perf.StartupTime.Total", 1);
  95     histograms.ExpectTotalCount("NaCl.Perf.Size.Manifest", 1);
  96     histograms.ExpectTotalCount("NaCl.Perf.Size.Nexe", 1);
  97   } else {
  98     // There should be the total (suffix-free), and the LLC bucket.
  99     // Subzero is tested separately.
 100     CheckPNaClLoadUMAs(histograms, "");
 101     CheckPNaClLoadUMAs(histograms, ".LLC");
 102   }
 103 })
 104
 105 // Test that a successful load adds stats to Subzero buckets.
 106 IN_PROC_BROWSER_TEST_F(NaClBrowserTestPnaclSubzero, SuccessfulLoadUMA) {
 107   // Only test where Subzero is supported.
 108   if (strcmp(nacl::GetSandboxArch(), "x86-32") != 0)
 109     return;
 110
 111   base::HistogramTester histograms;
 112   // Run a load test that uses the -O0 NMF option.
 113   RunLoadTest(FILE_PATH_LITERAL("pnacl_options.html?use_nmf=o_0"));
 114
 115   // Make sure histograms from child processes have been accumulated in the
 116   // browser brocess.
 117   content::FetchHistogramsFromChildProcesses();
 118
 119   // Did the plugin report success?
 120   histograms.ExpectUniqueSample("NaCl.LoadStatus.Plugin",
 121                                 PP_NACL_ERROR_LOAD_SUCCESS, 1);
 122
 123   // Did the sel_ldr report success?
 124   histograms.ExpectUniqueSample("NaCl.LoadStatus.SelLdr", LOAD_OK, 1);
 125
 126   // There should be the total (suffix-free), and the Subzero bucket.
 127   CheckPNaClLoadUMAs(histograms, "");
 128   CheckPNaClLoadUMAs(histograms, ".Subzero");
 129 }
 130
 131 class NaClBrowserTestNewlibVcacheExtension:
 132       public NaClBrowserTestNewlibExtension {
 133  public:
 134   base::FilePath::StringType Variant() override {
 135     return FILE_PATH_LITERAL("extension_vcache_test/newlib");
 136   }
 137 };
 138
 139 IN_PROC_BROWSER_TEST_F(NaClBrowserTestNewlibVcacheExtension,
 140                        ValidationCacheOfMainNexe) {
 141   base::HistogramTester histograms;
 142   // Hardcoded extension AppID that corresponds to the hardcoded
 143   // public key in the manifest.json file. We need to load the extension
 144   // nexe from the same origin, so we can't just try to load the extension
 145   // nexe as a mime-type handler from a non-extension URL.
 146   base::FilePath::StringType full_url =
 147       FILE_PATH_LITERAL("chrome-extension://cbcdidchbppangcjoddlpdjlenngjldk/")
 148       FILE_PATH_LITERAL("extension_validation_cache.html");
 149   RunNaClIntegrationTest(full_url, true);
 150
 151   // Make sure histograms from child processes have been accumulated in the
 152   // browser brocess.
 153   content::FetchHistogramsFromChildProcesses();
 154   // Should have received 2 validation queries (one for IRT and one for NEXE),
 155   // and responded with a miss.
 156   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 157                                nacl::NaClBrowser::CACHE_MISS, 2);
 158   // TOTAL should then be 2 queries so far.
 159   histograms.ExpectTotalCount("NaCl.ValidationCache.Query", 2);
 160   // Should have received a cache setting afterwards for IRT and nexe.
 161   histograms.ExpectBucketCount("NaCl.ValidationCache.Set",
 162                                nacl::NaClBrowser::CACHE_HIT, 2);
 163
 164   // Load it again to hit the cache.
 165   RunNaClIntegrationTest(full_url, true);
 166   content::FetchHistogramsFromChildProcesses();
 167   // Should have received 2 more validation queries later (IRT and NEXE),
 168   // and responded with a hit.
 169   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 170                                nacl::NaClBrowser::CACHE_HIT, 2);
 171   // TOTAL should then be 4 queries now.
 172   histograms.ExpectTotalCount("NaCl.ValidationCache.Query", 4);
 173   // Still only 2 settings.
 174   histograms.ExpectTotalCount("NaCl.ValidationCache.Set", 2);
 175 }
 176
 177 class NaClBrowserTestGLibcVcacheExtension:
 178       public NaClBrowserTestGLibcExtension {
 179  public:
 180   base::FilePath::StringType Variant() override {
 181     return FILE_PATH_LITERAL("extension_vcache_test/glibc");
 182   }
 183 };
 184
 185 IN_PROC_BROWSER_TEST_F(NaClBrowserTestGLibcVcacheExtension,
 186                        MAYBE_GLIBC(ValidationCacheOfMainNexe)) {
 187   // Make sure histograms from child processes have been accumulated in the
 188   // browser process.
 189   base::HistogramTester histograms;
 190   // Hardcoded extension AppID that corresponds to the hardcoded
 191   // public key in the manifest.json file. We need to load the extension
 192   // nexe from the same origin, so we can't just try to load the extension
 193   // nexe as a mime-type handler from a non-extension URL.
 194   base::FilePath::StringType full_url =
 195       FILE_PATH_LITERAL("chrome-extension://cbcdidchbppangcjoddlpdjlenngjldk/")
 196       FILE_PATH_LITERAL("extension_validation_cache.html");
 197   RunNaClIntegrationTest(full_url, true);
 198
 199   // Should have received 9 validation queries, which respond with misses:
 200   //   - the IRT
 201   //   - ld.so (the initial nexe)
 202   //   - main.nexe
 203   //   - libppapi_cpp.so
 204   //   - libpthread.so.9b15f6a6
 205   //   - libstdc++.so.6
 206   //   - libgcc_s.so.1
 207   //   - libc.so.9b15f6a6
 208   //   - libm.so.9b15f6a6
 209   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 210                                nacl::NaClBrowser::CACHE_MISS, 9);
 211   // TOTAL should then be 9 queries so far.
 212   histograms.ExpectTotalCount("NaCl.ValidationCache.Query", 9);
 213   // Should have received a cache setting afterwards for IRT and nexe.
 214   histograms.ExpectBucketCount("NaCl.ValidationCache.Set",
 215                                nacl::NaClBrowser::CACHE_HIT, 9);
 216
 217   // Load it again to hit the cache.
 218   RunNaClIntegrationTest(full_url, true);
 219   // Should have received 9 more validation queries and responded with hits.
 220   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 221                                nacl::NaClBrowser::CACHE_HIT, 9);
 222   histograms.ExpectTotalCount("NaCl.ValidationCache.Query", 18);
 223   histograms.ExpectTotalCount("NaCl.ValidationCache.Set", 9);
 224 }
 225
 226 // Test that validation for the 2 PNaCl translator nexes can be cached.
 227 IN_PROC_BROWSER_TEST_F(NaClBrowserTestPnacl,
 228                        ValidationCacheOfTranslatorNexes) {
 229   base::HistogramTester histograms;
 230   // Run a load test w/ one pexe cache identity.
 231   RunLoadTest(FILE_PATH_LITERAL("pnacl_options.html?use_nmf=o_0"));
 232
 233   content::FetchHistogramsFromChildProcesses();
 234   // Should have received 5 validation queries:
 235   // - Three for the IRT: the app and both of the translator nexes use it.
 236   // - Two for the two PNaCl translator nexes.
 237   // - The PNaCl app nexe comes from a delete-on-close temp file, so it
 238   //   doesn't have a stable identity for validation caching.
 239   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 240                                nacl::NaClBrowser::CACHE_MISS, 3);
 241   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 242                                nacl::NaClBrowser::CACHE_HIT, 2);
 243   // Should have received a cache setting afterwards.
 244   histograms.ExpectUniqueSample("NaCl.ValidationCache.Set",
 245                                nacl::NaClBrowser::CACHE_HIT, 3);
 246
 247   // Load the same pexe, but with a different cache identity.
 248   // This means that translation will actually be redone,
 249   // forcing the translators to be loaded a second time (but now with
 250   // cache hits!)
 251   RunLoadTest(FILE_PATH_LITERAL("pnacl_options.html?use_nmf=o_2"));
 252
 253   // Should now have 5 more queries on top of the previous ones.
 254   histograms.ExpectTotalCount("NaCl.ValidationCache.Query", 10);
 255   // With the extra queries being cache hits.
 256   histograms.ExpectBucketCount("NaCl.ValidationCache.Query",
 257                                nacl::NaClBrowser::CACHE_HIT, 7);
 258   // No extra cache settings.
 259   histograms.ExpectUniqueSample("NaCl.ValidationCache.Set",
 260                                 nacl::NaClBrowser::CACHE_HIT, 3);
 261 }
 262
 263
 264 // TODO(ncbray) convert the rest of nacl_uma.py (currently in the NaCl repo.)
 265 // Test validation failures and crashes.
 266
 267 }  // namespace