lldb/test/API/commands/statistics/basic/TestStats.py

   1 import json
   2 import os
   3 import re
   4
   5 import lldb
   6 from lldbsuite.test.decorators import *
   7 from lldbsuite.test.lldbtest import *
   8 from lldbsuite.test import lldbutil
   9
  10
  11 class TestCase(TestBase):
  12     NO_DEBUG_INFO_TESTCASE = True
  13
  14     def test_enable_disable(self):
  15         """
  16         Test "statistics disable" and "statistics enable". These don't do
  17         anything anymore for cheap to gather statistics. In the future if
  18         statistics are expensive to gather, we can enable the feature inside
  19         of LLDB and test that enabling and disabling stops expesive information
  20         from being gathered.
  21         """
  22         self.build()
  23         target = self.createTestTarget()
  24
  25         self.expect(
  26             "statistics disable",
  27             substrs=["need to enable statistics before disabling"],
  28             error=True,
  29         )
  30         self.expect("statistics enable")
  31         self.expect("statistics enable", substrs=["already enabled"], error=True)
  32         self.expect("statistics disable")
  33         self.expect(
  34             "statistics disable",
  35             substrs=["need to enable statistics before disabling"],
  36             error=True,
  37         )
  38
  39     def verify_key_in_dict(self, key, d, description):
  40         self.assertIn(
  41             key, d, 'make sure key "%s" is in dictionary %s' % (key, description)
  42         )
  43
  44     def verify_key_not_in_dict(self, key, d, description):
  45         self.assertNotIn(
  46             key, d, 'make sure key "%s" is in dictionary %s' % (key, description)
  47         )
  48
  49     def verify_keys(self, dict, description, keys_exist, keys_missing=None):
  50         """
  51         Verify that all keys in "keys_exist" list are top level items in
  52         "dict", and that all keys in "keys_missing" do not exist as top
  53         level items in "dict".
  54         """
  55         if keys_exist:
  56             for key in keys_exist:
  57                 self.verify_key_in_dict(key, dict, description)
  58         if keys_missing:
  59             for key in keys_missing:
  60                 self.verify_key_not_in_dict(key, dict, description)
  61
  62     def verify_success_fail_count(self, stats, key, num_successes, num_fails):
  63         self.verify_key_in_dict(key, stats, 'stats["%s"]' % (key))
  64         success_fail_dict = stats[key]
  65         self.assertEqual(
  66             success_fail_dict["successes"], num_successes, "make sure success count"
  67         )
  68         self.assertEqual(
  69             success_fail_dict["failures"], num_fails, "make sure success count"
  70         )
  71
  72     def get_target_stats(self, debug_stats):
  73         if "targets" in debug_stats:
  74             return debug_stats["targets"][0]
  75         return None
  76
  77     def get_command_stats(self, debug_stats):
  78         if "commands" in debug_stats:
  79             return debug_stats["commands"]
  80         return None
  81
  82     def test_expressions_frame_var_counts(self):
  83         self.build()
  84         lldbutil.run_to_source_breakpoint(
  85             self, "// break here", lldb.SBFileSpec("main.cpp")
  86         )
  87
  88         self.expect("expr patatino", substrs=["27"])
  89         stats = self.get_target_stats(self.get_stats())
  90         self.verify_success_fail_count(stats, "expressionEvaluation", 1, 0)
  91         self.expect(
  92             "expr doesnt_exist",
  93             error=True,
  94             substrs=["undeclared identifier 'doesnt_exist'"],
  95         )
  96         # Doesn't successfully execute.
  97         self.expect("expr int *i = nullptr; *i", error=True)
  98         # Interpret an integer as an array with 3 elements is a failure for
  99         # the "expr" command, but the expression evaluation will succeed and
 100         # be counted as a success even though the "expr" options will for the
 101         # command to fail. It is more important to track expression evaluation
 102         # from all sources instead of just through the command, so this was
 103         # changed. If we want to track command success and fails, we can do
 104         # so using another metric.
 105         self.expect(
 106             "expr -Z 3 -- 1",
 107             error=True,
 108             substrs=["expression cannot be used with --element-count"],
 109         )
 110         # We should have gotten 3 new failures and the previous success.
 111         stats = self.get_target_stats(self.get_stats())
 112         self.verify_success_fail_count(stats, "expressionEvaluation", 2, 2)
 113
 114         self.expect("statistics enable")
 115         # 'frame var' with enabled statistics will change stats.
 116         self.expect("frame var", substrs=["27"])
 117         stats = self.get_target_stats(self.get_stats())
 118         self.verify_success_fail_count(stats, "frameVariable", 1, 0)
 119
 120         # Test that "stopCount" is available when the process has run
 121         self.assertIn("stopCount", stats, 'ensure "stopCount" is in target JSON')
 122         self.assertGreater(
 123             stats["stopCount"], 0, 'make sure "stopCount" is greater than zero'
 124         )
 125
 126     def test_default_no_run(self):
 127         """Test "statistics dump" without running the target.
 128
 129         When we don't run the target, we expect to not see any 'firstStopTime'
 130         or 'launchOrAttachTime' top level keys that measure the launch or
 131         attach of the target.
 132
 133         Output expected to be something like:
 134
 135         (lldb) statistics dump
 136         {
 137           "memory" : {...},
 138           "modules" : [...],
 139           "targets" : [
 140             {
 141                 "targetCreateTime": 0.26566899599999999,
 142                 "expressionEvaluation": {
 143                     "failures": 0,
 144                     "successes": 0
 145                 },
 146                 "frameVariable": {
 147                     "failures": 0,
 148                     "successes": 0
 149                 },
 150                 "moduleIdentifiers": [...],
 151             }
 152           ],
 153           "totalDebugInfoByteSize": 182522234,
 154           "totalDebugInfoIndexTime": 2.33343,
 155           "totalDebugInfoParseTime": 8.2121400240000071,
 156           "totalSymbolTableParseTime": 0.123,
 157           "totalSymbolTableIndexTime": 0.234,
 158         }
 159         """
 160         self.build()
 161         target = self.createTestTarget()
 162         debug_stats = self.get_stats()
 163         debug_stat_keys = [
 164             "memory",
 165             "modules",
 166             "targets",
 167             "totalSymbolTableParseTime",
 168             "totalSymbolTableIndexTime",
 169             "totalSymbolTablesLoadedFromCache",
 170             "totalSymbolTablesSavedToCache",
 171             "totalDebugInfoByteSize",
 172             "totalDebugInfoIndexTime",
 173             "totalDebugInfoIndexLoadedFromCache",
 174             "totalDebugInfoIndexSavedToCache",
 175             "totalDebugInfoParseTime",
 176         ]
 177         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 178         stats = debug_stats["targets"][0]
 179         keys_exist = [
 180             "expressionEvaluation",
 181             "frameVariable",
 182             "moduleIdentifiers",
 183             "targetCreateTime",
 184         ]
 185         keys_missing = ["firstStopTime", "launchOrAttachTime"]
 186         self.verify_keys(stats, '"stats"', keys_exist, keys_missing)
 187         self.assertGreater(stats["targetCreateTime"], 0.0)
 188
 189     def test_default_with_run(self):
 190         """Test "statistics dump" when running the target to a breakpoint.
 191
 192         When we run the target, we expect to see 'launchOrAttachTime' and
 193         'firstStopTime' top level keys.
 194
 195         Output expected to be something like:
 196
 197         (lldb) statistics dump
 198         {
 199           "memory" : {...},
 200           "modules" : [...],
 201           "targets" : [
 202                 {
 203                     "firstStopTime": 0.34164492800000001,
 204                     "launchOrAttachTime": 0.31969605400000001,
 205                     "moduleIdentifiers": [...],
 206                     "targetCreateTime": 0.0040863039999999998
 207                     "expressionEvaluation": {
 208                         "failures": 0,
 209                         "successes": 0
 210                     },
 211                     "frameVariable": {
 212                         "failures": 0,
 213                         "successes": 0
 214                     },
 215                 }
 216             ],
 217             "totalDebugInfoByteSize": 182522234,
 218             "totalDebugInfoIndexTime": 2.33343,
 219             "totalDebugInfoParseTime": 8.2121400240000071,
 220             "totalSymbolTableParseTime": 0.123,
 221             "totalSymbolTableIndexTime": 0.234,
 222         }
 223
 224         """
 225         self.build()
 226         target = self.createTestTarget()
 227         lldbutil.run_to_source_breakpoint(
 228             self, "// break here", lldb.SBFileSpec("main.cpp")
 229         )
 230         debug_stats = self.get_stats()
 231         debug_stat_keys = [
 232             "memory",
 233             "modules",
 234             "targets",
 235             "totalSymbolTableParseTime",
 236             "totalSymbolTableIndexTime",
 237             "totalSymbolTablesLoadedFromCache",
 238             "totalSymbolTablesSavedToCache",
 239             "totalDebugInfoByteSize",
 240             "totalDebugInfoIndexTime",
 241             "totalDebugInfoIndexLoadedFromCache",
 242             "totalDebugInfoIndexSavedToCache",
 243             "totalDebugInfoParseTime",
 244         ]
 245         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 246         stats = debug_stats["targets"][0]
 247         keys_exist = [
 248             "expressionEvaluation",
 249             "firstStopTime",
 250             "frameVariable",
 251             "launchOrAttachTime",
 252             "moduleIdentifiers",
 253             "targetCreateTime",
 254             "summaryProviderStatistics",
 255         ]
 256         self.verify_keys(stats, '"stats"', keys_exist, None)
 257         self.assertGreater(stats["firstStopTime"], 0.0)
 258         self.assertGreater(stats["launchOrAttachTime"], 0.0)
 259         self.assertGreater(stats["targetCreateTime"], 0.0)
 260
 261     def test_memory(self):
 262         """
 263         Test "statistics dump" and the memory information.
 264         """
 265         self.build()
 266         exe = self.getBuildArtifact("a.out")
 267         target = self.createTestTarget(file_path=exe)
 268         debug_stats = self.get_stats()
 269         debug_stat_keys = [
 270             "memory",
 271             "modules",
 272             "targets",
 273             "totalSymbolTableParseTime",
 274             "totalSymbolTableIndexTime",
 275             "totalSymbolTablesLoadedFromCache",
 276             "totalSymbolTablesSavedToCache",
 277             "totalDebugInfoParseTime",
 278             "totalDebugInfoIndexTime",
 279             "totalDebugInfoIndexLoadedFromCache",
 280             "totalDebugInfoIndexSavedToCache",
 281             "totalDebugInfoByteSize",
 282         ]
 283         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 284
 285         memory = debug_stats["memory"]
 286         memory_keys = [
 287             "strings",
 288         ]
 289         self.verify_keys(memory, '"memory"', memory_keys, None)
 290
 291         strings = memory["strings"]
 292         strings_keys = [
 293             "bytesTotal",
 294             "bytesUsed",
 295             "bytesUnused",
 296         ]
 297         self.verify_keys(strings, '"strings"', strings_keys, None)
 298
 299     def find_module_in_metrics(self, path, stats):
 300         modules = stats["modules"]
 301         for module in modules:
 302             if module["path"] == path:
 303                 return module
 304         return None
 305
 306     def find_module_by_id_in_metrics(self, id, stats):
 307         modules = stats["modules"]
 308         for module in modules:
 309             if module["identifier"] == id:
 310                 return module
 311         return None
 312
 313     def test_modules(self):
 314         """
 315         Test "statistics dump" and the module information.
 316         """
 317         self.build()
 318         exe = self.getBuildArtifact("a.out")
 319         target = self.createTestTarget(file_path=exe)
 320         debug_stats = self.get_stats()
 321         debug_stat_keys = [
 322             "memory",
 323             "modules",
 324             "targets",
 325             "totalSymbolTableParseTime",
 326             "totalSymbolTableIndexTime",
 327             "totalSymbolTablesLoadedFromCache",
 328             "totalSymbolTablesSavedToCache",
 329             "totalDebugInfoParseTime",
 330             "totalDebugInfoIndexTime",
 331             "totalDebugInfoIndexLoadedFromCache",
 332             "totalDebugInfoIndexSavedToCache",
 333             "totalDebugInfoByteSize",
 334         ]
 335         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 336         stats = debug_stats["targets"][0]
 337         keys_exist = [
 338             "moduleIdentifiers",
 339         ]
 340         self.verify_keys(stats, '"stats"', keys_exist, None)
 341         exe_module = self.find_module_in_metrics(exe, debug_stats)
 342         module_keys = [
 343             "debugInfoByteSize",
 344             "debugInfoIndexLoadedFromCache",
 345             "debugInfoIndexTime",
 346             "debugInfoIndexSavedToCache",
 347             "debugInfoParseTime",
 348             "identifier",
 349             "path",
 350             "symbolTableIndexTime",
 351             "symbolTableLoadedFromCache",
 352             "symbolTableParseTime",
 353             "symbolTableSavedToCache",
 354             "triple",
 355             "uuid",
 356         ]
 357         self.assertNotEqual(exe_module, None)
 358         self.verify_keys(exe_module, 'module dict for "%s"' % (exe), module_keys)
 359
 360     def test_commands(self):
 361         """
 362         Test "statistics dump" and the command information.
 363         """
 364         self.build()
 365         exe = self.getBuildArtifact("a.out")
 366         target = self.createTestTarget(file_path=exe)
 367
 368         interp = self.dbg.GetCommandInterpreter()
 369         result = lldb.SBCommandReturnObject()
 370         interp.HandleCommand("target list", result)
 371         interp.HandleCommand("target list", result)
 372
 373         debug_stats = self.get_stats()
 374
 375         command_stats = self.get_command_stats(debug_stats)
 376         self.assertNotEqual(command_stats, None)
 377         self.assertEqual(command_stats["target list"], 2)
 378
 379     def test_breakpoints(self):
 380         """Test "statistics dump"
 381
 382         Output expected to be something like:
 383
 384         {
 385           "memory" : {...},
 386           "modules" : [...],
 387           "targets" : [
 388                 {
 389                     "firstStopTime": 0.34164492800000001,
 390                     "launchOrAttachTime": 0.31969605400000001,
 391                     "moduleIdentifiers": [...],
 392                     "targetCreateTime": 0.0040863039999999998
 393                     "expressionEvaluation": {
 394                         "failures": 0,
 395                         "successes": 0
 396                     },
 397                     "frameVariable": {
 398                         "failures": 0,
 399                         "successes": 0
 400                     },
 401                     "breakpoints": [
 402                         {
 403                             "details": {...},
 404                             "id": 1,
 405                             "resolveTime": 2.65438675
 406                         },
 407                         {
 408                             "details": {...},
 409                             "id": 2,
 410                             "resolveTime": 4.3632581669999997
 411                         }
 412                     ]
 413                 }
 414             ],
 415             "totalDebugInfoByteSize": 182522234,
 416             "totalDebugInfoIndexTime": 2.33343,
 417             "totalDebugInfoParseTime": 8.2121400240000071,
 418             "totalSymbolTableParseTime": 0.123,
 419             "totalSymbolTableIndexTime": 0.234,
 420             "totalBreakpointResolveTime": 7.0176449170000001
 421         }
 422
 423         """
 424         self.build()
 425         target = self.createTestTarget()
 426         self.runCmd("b main.cpp:7")
 427         self.runCmd("b a_function")
 428         debug_stats = self.get_stats()
 429         debug_stat_keys = [
 430             "memory",
 431             "modules",
 432             "targets",
 433             "totalSymbolTableParseTime",
 434             "totalSymbolTableIndexTime",
 435             "totalSymbolTablesLoadedFromCache",
 436             "totalSymbolTablesSavedToCache",
 437             "totalDebugInfoParseTime",
 438             "totalDebugInfoIndexTime",
 439             "totalDebugInfoIndexLoadedFromCache",
 440             "totalDebugInfoIndexSavedToCache",
 441             "totalDebugInfoByteSize",
 442         ]
 443         self.verify_keys(debug_stats, '"debug_stats"', debug_stat_keys, None)
 444         target_stats = debug_stats["targets"][0]
 445         keys_exist = [
 446             "breakpoints",
 447             "expressionEvaluation",
 448             "frameVariable",
 449             "targetCreateTime",
 450             "moduleIdentifiers",
 451             "totalBreakpointResolveTime",
 452             "summaryProviderStatistics",
 453         ]
 454         self.verify_keys(target_stats, '"stats"', keys_exist, None)
 455         self.assertGreater(target_stats["totalBreakpointResolveTime"], 0.0)
 456         breakpoints = target_stats["breakpoints"]
 457         bp_keys_exist = [
 458             "details",
 459             "id",
 460             "internal",
 461             "numLocations",
 462             "numResolvedLocations",
 463             "resolveTime",
 464         ]
 465         for breakpoint in breakpoints:
 466             self.verify_keys(
 467                 breakpoint, 'target_stats["breakpoints"]', bp_keys_exist, None
 468             )
 469
 470     @skipUnlessDarwin
 471     @no_debug_info_test
 472     def test_dsym_binary_has_symfile_in_stats(self):
 473         """
 474         Test that if our executable has a stand alone dSYM file containing
 475         debug information, that the dSYM file path is listed as a key/value
 476         pair in the "a.out" binaries module stats. Also verify the the main
 477         executable's module statistics has a debug info size that is greater
 478         than zero as the dSYM contains debug info.
 479         """
 480         self.build(debug_info="dsym")
 481         exe_name = "a.out"
 482         exe = self.getBuildArtifact(exe_name)
 483         dsym = self.getBuildArtifact(exe_name + ".dSYM")
 484         # Make sure the executable file exists after building.
 485         self.assertTrue(os.path.exists(exe))
 486         # Make sure the dSYM file exists after building.
 487         self.assertTrue(os.path.isdir(dsym))
 488
 489         # Create the target
 490         target = self.createTestTarget(file_path=exe)
 491
 492         debug_stats = self.get_stats()
 493
 494         exe_stats = self.find_module_in_metrics(exe, debug_stats)
 495         # If we have a dSYM file, there should be a key/value pair in the module
 496         # statistics and the path should match the dSYM file path in the build
 497         # artifacts.
 498         self.assertIn("symbolFilePath", exe_stats)
 499         stats_dsym = exe_stats["symbolFilePath"]
 500
 501         # Make sure main executable's module info has debug info size that is
 502         # greater than zero as the dSYM file and main executable work together
 503         # in the lldb.SBModule class to provide the data.
 504         self.assertGreater(exe_stats["debugInfoByteSize"], 0)
 505
 506         # The "dsym" variable contains the bundle directory for the dSYM, while
 507         # the "stats_dsym" will have the
 508         self.assertIn(dsym, stats_dsym)
 509         # Since we have a dSYM file, we should not be loading DWARF from the .o
 510         # files and the .o file module identifiers should NOT be in the module
 511         # statistics.
 512         self.assertNotIn("symbolFileModuleIdentifiers", exe_stats)
 513
 514     @skipUnlessDarwin
 515     @no_debug_info_test
 516     def test_no_dsym_binary_has_symfile_identifiers_in_stats(self):
 517         """
 518         Test that if our executable loads debug info from the .o files,
 519         that the module statistics contains a 'symbolFileModuleIdentifiers'
 520         key which is a list of module identifiers, and verify that the
 521         module identifier can be used to find the .o file's module stats.
 522         Also verify the the main executable's module statistics has a debug
 523         info size that is zero, as the main executable itself has no debug
 524         info, but verify that the .o files have debug info size that is
 525         greater than zero. This test ensures that we don't double count
 526         debug info.
 527         """
 528         self.build(debug_info="dwarf")
 529         exe_name = "a.out"
 530         exe = self.getBuildArtifact(exe_name)
 531         dsym = self.getBuildArtifact(exe_name + ".dSYM")
 532         # Make sure the executable file exists after building.
 533         self.assertTrue(os.path.exists(exe))
 534         # Make sure the dSYM file doesn't exist after building.
 535         self.assertFalse(os.path.isdir(dsym))
 536
 537         # Create the target
 538         target = self.createTestTarget(file_path=exe)
 539
 540         # Force the 'main.o' .o file's DWARF to be loaded so it will show up
 541         # in the stats.
 542         self.runCmd("b main.cpp:7")
 543
 544         debug_stats = self.get_stats("--all-targets")
 545
 546         exe_stats = self.find_module_in_metrics(exe, debug_stats)
 547         # If we don't have a dSYM file, there should not be a key/value pair in
 548         # the module statistics.
 549         self.assertNotIn("symbolFilePath", exe_stats)
 550
 551         # Make sure main executable's module info has debug info size that is
 552         # zero as there is no debug info in the main executable, only in the
 553         # .o files. The .o files will also only be loaded if something causes
 554         # them to be loaded, so we set a breakpoint to force the .o file debug
 555         # info to be loaded.
 556         self.assertEqual(exe_stats["debugInfoByteSize"], 0)
 557
 558         # When we don't have a dSYM file, the SymbolFileDWARFDebugMap class
 559         # should create modules for each .o file that contains DWARF that the
 560         # symbol file creates, so we need to verify that we have a valid module
 561         # identifier for main.o that is we should not be loading DWARF from the .o
 562         # files and the .o file module identifiers should NOT be in the module
 563         # statistics.
 564         self.assertIn("symbolFileModuleIdentifiers", exe_stats)
 565
 566         symfileIDs = exe_stats["symbolFileModuleIdentifiers"]
 567         for symfileID in symfileIDs:
 568             o_module = self.find_module_by_id_in_metrics(symfileID, debug_stats)
 569             self.assertNotEqual(o_module, None)
 570             # Make sure each .o file has some debug info bytes.
 571             self.assertGreater(o_module["debugInfoByteSize"], 0)
 572
 573     @skipUnlessDarwin
 574     @no_debug_info_test
 575     def test_had_frame_variable_errors(self):
 576         """
 577         Test that if we have frame variable errors that we see this in the
 578         statistics for the module that had issues.
 579         """
 580         self.build(debug_info="dwarf")
 581         exe_name = "a.out"
 582         exe = self.getBuildArtifact(exe_name)
 583         dsym = self.getBuildArtifact(exe_name + ".dSYM")
 584         main_obj = self.getBuildArtifact("main.o")
 585         # Make sure the executable file exists after building.
 586         self.assertTrue(os.path.exists(exe))
 587         # Make sure the dSYM file doesn't exist after building.
 588         self.assertFalse(os.path.isdir(dsym))
 589         # Make sure the main.o object file exists after building.
 590         self.assertTrue(os.path.exists(main_obj))
 591
 592         # Delete the main.o file that contains the debug info so we force an
 593         # error when we run to main and try to get variables
 594         os.unlink(main_obj)
 595
 596         (target, process, thread, bkpt) = lldbutil.run_to_name_breakpoint(self, "main")
 597
 598         # Get stats and verify we had errors.
 599         stats = self.get_stats()
 600         exe_stats = self.find_module_in_metrics(exe, stats)
 601         self.assertIsNotNone(exe_stats)
 602
 603         # Make sure we have "debugInfoHadVariableErrors" variable that is set to
 604         # false before failing to get local variables due to missing .o file.
 605         self.assertFalse(exe_stats["debugInfoHadVariableErrors"])
 606
 607         # Verify that the top level statistic that aggregates the number of
 608         # modules with debugInfoHadVariableErrors is zero
 609         self.assertEqual(stats["totalModuleCountWithVariableErrors"], 0)
 610
 611         # Try and fail to get variables
 612         vars = thread.GetFrameAtIndex(0).GetVariables(True, True, False, True)
 613
 614         # Make sure we got an error back that indicates that variables were not
 615         # available
 616         self.assertTrue(vars.GetError().Fail())
 617
 618         # Get stats and verify we had errors.
 619         stats = self.get_stats()
 620         exe_stats = self.find_module_in_metrics(exe, stats)
 621         self.assertIsNotNone(exe_stats)
 622
 623         # Make sure we have "hadFrameVariableErrors" variable that is set to
 624         # true after failing to get local variables due to missing .o file.
 625         self.assertTrue(exe_stats["debugInfoHadVariableErrors"])
 626
 627         # Verify that the top level statistic that aggregates the number of
 628         # modules with debugInfoHadVariableErrors is greater than zero
 629         self.assertGreater(stats["totalModuleCountWithVariableErrors"], 0)
 630
 631     def test_transcript_happy_path(self):
 632         """
 633         Test "statistics dump" and the transcript information.
 634         """
 635         self.build()
 636         exe = self.getBuildArtifact("a.out")
 637         target = self.createTestTarget(file_path=exe)
 638         self.runCmd("settings set interpreter.save-transcript true")
 639         self.runCmd("version")
 640
 641         # Verify the output of a first "statistics dump"
 642         debug_stats = self.get_stats("--transcript true")
 643         self.assertIn("transcript", debug_stats)
 644         transcript = debug_stats["transcript"]
 645         self.assertEqual(len(transcript), 2)
 646         self.assertEqual(transcript[0]["commandName"], "version")
 647         self.assertEqual(transcript[1]["commandName"], "statistics dump")
 648         # The first "statistics dump" in the transcript should have no output
 649         self.assertNotIn("output", transcript[1])
 650
 651         # Verify the output of a second "statistics dump"
 652         debug_stats = self.get_stats("--transcript true")
 653         self.assertIn("transcript", debug_stats)
 654         transcript = debug_stats["transcript"]
 655         self.assertEqual(len(transcript), 3)
 656         self.assertEqual(transcript[0]["commandName"], "version")
 657         self.assertEqual(transcript[1]["commandName"], "statistics dump")
 658         # The first "statistics dump" in the transcript should have output now
 659         self.assertIn("output", transcript[1])
 660         self.assertEqual(transcript[2]["commandName"], "statistics dump")
 661         # The second "statistics dump" in the transcript should have no output
 662         self.assertNotIn("output", transcript[2])
 663
 664     def verify_stats(self, stats, expectation, options):
 665         for field_name in expectation:
 666             idx = field_name.find(".")
 667             if idx == -1:
 668                 # `field_name` is a top-level field
 669                 exists = field_name in stats
 670                 should_exist = expectation[field_name]
 671                 should_exist_string = "" if should_exist else "not "
 672                 self.assertEqual(
 673                     exists,
 674                     should_exist,
 675                     f"'{field_name}' should {should_exist_string}exist for 'statistics dump{options}'",
 676                 )
 677             else:
 678                 # `field_name` is a string of "<top-level field>.<second-level field>"
 679                 top_level_field_name = field_name[0:idx]
 680                 second_level_field_name = field_name[idx + 1 :]
 681                 for top_level_field in (
 682                     stats[top_level_field_name] if top_level_field_name in stats else {}
 683                 ):
 684                     exists = second_level_field_name in top_level_field
 685                     should_exist = expectation[field_name]
 686                     should_exist_string = "" if should_exist else "not "
 687                     self.assertEqual(
 688                         exists,
 689                         should_exist,
 690                         f"'{field_name}' should {should_exist_string}exist for 'statistics dump{options}'",
 691                     )
 692
 693     def get_test_cases_for_sections_existence(self):
 694         should_always_exist_or_not = {
 695             "totalDebugInfoEnabled": True,
 696             "memory": True,
 697         }
 698         test_cases = [
 699             {  # Everything mode
 700                 "command_options": "",
 701                 "api_options": {},
 702                 "expect": {
 703                     "commands": True,
 704                     "targets": True,
 705                     "targets.moduleIdentifiers": True,
 706                     "targets.breakpoints": True,
 707                     "targets.expressionEvaluation": True,
 708                     "targets.frameVariable": True,
 709                     "targets.totalSharedLibraryEventHitCount": True,
 710                     "modules": True,
 711                     "transcript": True,
 712                 },
 713             },
 714             {  # Summary mode
 715                 "command_options": " --summary",
 716                 "api_options": {
 717                     "SetSummaryOnly": True,
 718                 },
 719                 "expect": {
 720                     "commands": False,
 721                     "targets": True,
 722                     "targets.moduleIdentifiers": False,
 723                     "targets.breakpoints": False,
 724                     "targets.expressionEvaluation": False,
 725                     "targets.frameVariable": False,
 726                     "targets.totalSharedLibraryEventHitCount": True,
 727                     "modules": False,
 728                     "transcript": False,
 729                 },
 730             },
 731             {  # Summary mode with targets
 732                 "command_options": " --summary --targets=true",
 733                 "api_options": {
 734                     "SetSummaryOnly": True,
 735                     "SetIncludeTargets": True,
 736                 },
 737                 "expect": {
 738                     "commands": False,
 739                     "targets": True,
 740                     "targets.moduleIdentifiers": False,
 741                     "targets.breakpoints": False,
 742                     "targets.expressionEvaluation": False,
 743                     "targets.frameVariable": False,
 744                     "targets.totalSharedLibraryEventHitCount": True,
 745                     "modules": False,
 746                     "transcript": False,
 747                 },
 748             },
 749             {  # Summary mode without targets
 750                 "command_options": " --summary --targets=false",
 751                 "api_options": {
 752                     "SetSummaryOnly": True,
 753                     "SetIncludeTargets": False,
 754                 },
 755                 "expect": {
 756                     "commands": False,
 757                     "targets": False,
 758                     "modules": False,
 759                     "transcript": False,
 760                 },
 761             },
 762             {  # Summary mode with modules
 763                 "command_options": " --summary --modules=true",
 764                 "api_options": {
 765                     "SetSummaryOnly": True,
 766                     "SetIncludeModules": True,
 767                 },
 768                 "expect": {
 769                     "commands": False,
 770                     "targets": True,
 771                     "targets.moduleIdentifiers": False,
 772                     "targets.breakpoints": False,
 773                     "targets.expressionEvaluation": False,
 774                     "targets.frameVariable": False,
 775                     "targets.totalSharedLibraryEventHitCount": True,
 776                     "modules": True,
 777                     "transcript": False,
 778                 },
 779             },
 780             {  # Default mode without modules and transcript
 781                 "command_options": " --modules=false --transcript=false",
 782                 "api_options": {
 783                     "SetIncludeModules": False,
 784                     "SetIncludeTranscript": False,
 785                 },
 786                 "expect": {
 787                     "commands": True,
 788                     "targets": True,
 789                     "targets.moduleIdentifiers": False,
 790                     "targets.breakpoints": True,
 791                     "targets.expressionEvaluation": True,
 792                     "targets.frameVariable": True,
 793                     "targets.totalSharedLibraryEventHitCount": True,
 794                     "modules": False,
 795                     "transcript": False,
 796                 },
 797             },
 798             {  # Default mode without modules
 799                 "command_options": " --modules=false",
 800                 "api_options": {
 801                     "SetIncludeModules": False,
 802                 },
 803                 "expect": {
 804                     "commands": True,
 805                     "targets": True,
 806                     "targets.moduleIdentifiers": False,
 807                     "targets.breakpoints": True,
 808                     "targets.expressionEvaluation": True,
 809                     "targets.frameVariable": True,
 810                     "targets.totalSharedLibraryEventHitCount": True,
 811                     "modules": False,
 812                     "transcript": True,
 813                 },
 814             },
 815         ]
 816         return (should_always_exist_or_not, test_cases)
 817
 818     def test_sections_existence_through_command(self):
 819         """
 820         Test "statistics dump" and the existence of sections when different
 821         options are given through the command line (CLI or HandleCommand).
 822         """
 823         self.build()
 824         exe = self.getBuildArtifact("a.out")
 825         target = self.createTestTarget(file_path=exe)
 826
 827         # Create some transcript so that it can be tested.
 828         self.runCmd("settings set interpreter.save-transcript true")
 829         self.runCmd("version")
 830         self.runCmd("b main")
 831         # Then disable transcript so that it won't change during verification
 832         self.runCmd("settings set interpreter.save-transcript false")
 833
 834         # Expectation
 835         (
 836             should_always_exist_or_not,
 837             test_cases,
 838         ) = self.get_test_cases_for_sections_existence()
 839
 840         # Verification
 841         for test_case in test_cases:
 842             options = test_case["command_options"]
 843             # Get statistics dump result
 844             stats = self.get_stats(options)
 845             # Verify that each field should exist (or not)
 846             expectation = {**should_always_exist_or_not, **test_case["expect"]}
 847             self.verify_stats(stats, expectation, options)
 848
 849     def test_sections_existence_through_api(self):
 850         """
 851         Test "statistics dump" and the existence of sections when different
 852         options are given through the public API.
 853         """
 854         self.build()
 855         exe = self.getBuildArtifact("a.out")
 856         target = self.createTestTarget(file_path=exe)
 857
 858         # Create some transcript so that it can be tested.
 859         self.runCmd("settings set interpreter.save-transcript true")
 860         self.runCmd("version")
 861         self.runCmd("b main")
 862         # But disable transcript so that it won't change during verification
 863         self.runCmd("settings set interpreter.save-transcript false")
 864
 865         # Expectation
 866         (
 867             should_always_exist_or_not,
 868             test_cases,
 869         ) = self.get_test_cases_for_sections_existence()
 870
 871         # Verification
 872         for test_case in test_cases:
 873             # Create options
 874             options = test_case["api_options"]
 875             sb_options = lldb.SBStatisticsOptions()
 876             for method_name, param_value in options.items():
 877                 getattr(sb_options, method_name)(param_value)
 878             # Get statistics dump result
 879             stream = lldb.SBStream()
 880             target.GetStatistics(sb_options).GetAsJSON(stream)
 881             stats = json.loads(stream.GetData())
 882             # Verify that each field should exist (or not)
 883             expectation = {**should_always_exist_or_not, **test_case["expect"]}
 884             self.verify_stats(stats, expectation, options)
 885
 886     def test_order_of_options_do_not_matter(self):
 887         """
 888         Test "statistics dump" and the order of options.
 889         """
 890         self.build()
 891         exe = self.getBuildArtifact("a.out")
 892         target = self.createTestTarget(file_path=exe)
 893
 894         # Create some transcript so that it can be tested.
 895         self.runCmd("settings set interpreter.save-transcript true")
 896         self.runCmd("version")
 897         self.runCmd("b main")
 898         # Then disable transcript so that it won't change during verification
 899         self.runCmd("settings set interpreter.save-transcript false")
 900
 901         # The order of the following options shouldn't matter
 902         test_cases = [
 903             (" --summary", " --targets=true"),
 904             (" --summary", " --targets=false"),
 905             (" --summary", " --modules=true"),
 906             (" --summary", " --modules=false"),
 907             (" --summary", " --transcript=true"),
 908             (" --summary", " --transcript=false"),
 909         ]
 910
 911         # Verification
 912         for options in test_cases:
 913             debug_stats_0 = self.get_stats(options[0] + options[1])
 914             debug_stats_1 = self.get_stats(options[1] + options[0])
 915             # Redact all numbers
 916             debug_stats_0 = re.sub(r"\d+", "0", json.dumps(debug_stats_0))
 917             debug_stats_1 = re.sub(r"\d+", "0", json.dumps(debug_stats_1))
 918             # Verify that the two output are the same
 919             self.assertEqual(
 920                 debug_stats_0,
 921                 debug_stats_1,
 922                 f"The order of options '{options[0]}' and '{options[1]}' should not matter",
 923             )
 924
 925     @skipIfWindows
 926     def test_summary_statistics_providers(self):
 927         """
 928         Test summary timing statistics is included in statistics dump when
 929         a type with a summary provider exists, and is evaluated.
 930         """
 931
 932         self.build()
 933         target = self.createTestTarget()
 934         lldbutil.run_to_source_breakpoint(
 935             self, "// stop here", lldb.SBFileSpec("main.cpp")
 936         )
 937         self.expect("frame var", substrs=["hello world"])
 938         stats = self.get_target_stats(self.get_stats())
 939         self.assertIn("summaryProviderStatistics", stats)
 940         summary_providers = stats["summaryProviderStatistics"]
 941         # We don't want to take a dependency on the type name, so we just look
 942         # for string and that it was called once.
 943         summary_provider_str = str(summary_providers)
 944         self.assertIn("string", summary_provider_str)
 945         self.assertIn("'count': 1", summary_provider_str)
 946         self.assertIn("'totalTime':", summary_provider_str)
 947         # We may hit the std::string C++ provider, or a summary provider string
 948         self.assertIn("'type':", summary_provider_str)
 949         self.assertTrue(
 950             "c++" in summary_provider_str or "string" in summary_provider_str
 951         )
 952
 953         self.runCmd("continue")
 954         self.runCmd("command script import BoxFormatter.py")
 955         self.expect("frame var", substrs=["box = [27]"])
 956         stats = self.get_target_stats(self.get_stats())
 957         self.assertIn("summaryProviderStatistics", stats)
 958         summary_providers = stats["summaryProviderStatistics"]
 959         summary_provider_str = str(summary_providers)
 960         self.assertIn("BoxFormatter.summary", summary_provider_str)
 961         self.assertIn("'count': 1", summary_provider_str)
 962         self.assertIn("'totalTime':", summary_provider_str)
 963         self.assertIn("'type': 'python'", summary_provider_str)
 964
 965     @skipIfWindows
 966     def test_summary_statistics_providers_vec(self):
 967         """
 968         Test summary timing statistics is included in statistics dump when
 969         a type with a summary provider exists, and is evaluated. This variation
 970         tests that vector recurses into it's child type.
 971         """
 972         self.build()
 973         target = self.createTestTarget()
 974         lldbutil.run_to_source_breakpoint(
 975             self, "// stop vector", lldb.SBFileSpec("main.cpp")
 976         )
 977         self.expect(
 978             "frame var", substrs=["int_vec", "double_vec", "[0] = 1", "[7] = 8"]
 979         )
 980         stats = self.get_target_stats(self.get_stats())
 981         self.assertIn("summaryProviderStatistics", stats)
 982         summary_providers = stats["summaryProviderStatistics"]
 983         summary_provider_str = str(summary_providers)
 984         self.assertIn("'count': 2", summary_provider_str)
 985         self.assertIn("'totalTime':", summary_provider_str)
 986         self.assertIn("'type':", summary_provider_str)
 987         # We may hit the std::vector C++ provider, or a summary provider string
 988         if "c++" in summary_provider_str:
 989             self.assertIn("std::vector", summary_provider_str)
 990
 991     @skipIfWindows
 992     def test_multiple_targets(self):
 993         """
 994         Test statistics dump only reports the stats from current target and
 995         "statistics dump --all-targets" includes all target stats.
 996         """
 997         da = {"CXX_SOURCES": "main.cpp", "EXE": self.getBuildArtifact("a.out")}
 998         self.build(dictionary=da)
 999         self.addTearDownCleanup(dictionary=da)
1000
1001         db = {"CXX_SOURCES": "second.cpp", "EXE": self.getBuildArtifact("second.out")}
1002         self.build(dictionary=db)
1003         self.addTearDownCleanup(dictionary=db)
1004
1005         main_exe = self.getBuildArtifact("a.out")
1006         second_exe = self.getBuildArtifact("second.out")
1007
1008         (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
1009             self, "// break here", lldb.SBFileSpec("main.cpp"), None, "a.out"
1010         )
1011         debugger_stats1 = self.get_stats()
1012         self.assertIsNotNone(self.find_module_in_metrics(main_exe, debugger_stats1))
1013         self.assertIsNone(self.find_module_in_metrics(second_exe, debugger_stats1))
1014
1015         (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(
1016             self, "// break here", lldb.SBFileSpec("second.cpp"), None, "second.out"
1017         )
1018         debugger_stats2 = self.get_stats()
1019         self.assertIsNone(self.find_module_in_metrics(main_exe, debugger_stats2))
1020         self.assertIsNotNone(self.find_module_in_metrics(second_exe, debugger_stats2))
1021
1022         all_targets_stats = self.get_stats("--all-targets")
1023         self.assertIsNotNone(self.find_module_in_metrics(main_exe, all_targets_stats))
1024         self.assertIsNotNone(self.find_module_in_metrics(second_exe, all_targets_stats))