tools/perf/scripts/python/gecko.py

   1 # gecko.py - Convert perf record output to Firefox's gecko profile format
   2 # SPDX-License-Identifier: GPL-2.0
   3 #
   4 # The script converts perf.data to Gecko Profile Format,
   5 # which can be read by https://profiler.firefox.com/.
   6 #
   7 # Usage:
   8 #
   9 #     perf record -a -g -F 99 sleep 60
  10 #     perf script report gecko
  11 #
  12 # Combined:
  13 #
  14 #     perf script gecko -F 99 -a sleep 60
  15
  16 import os
  17 import sys
  18 import time
  19 import json
  20 import string
  21 import random
  22 import argparse
  23 import threading
  24 import webbrowser
  25 import urllib.parse
  26 from os import system
  27 from functools import reduce
  28 from dataclasses import dataclass, field
  29 from http.server import HTTPServer, SimpleHTTPRequestHandler, test
  30 from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any
  31
  32 # Add the Perf-Trace-Util library to the Python path
  33 sys.path.append(os.environ['PERF_EXEC_PATH'] + \
  34         '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
  35
  36 from perf_trace_context import *
  37 from Core import *
  38
  39 StringID = int
  40 StackID = int
  41 FrameID = int
  42 CategoryID = int
  43 Milliseconds = float
  44
  45 # start_time is intialiazed only once for the all event traces.
  46 start_time = None
  47
  48 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425
  49 # Follow Brendan Gregg's Flamegraph convention: orange for kernel and yellow for user space by default.
  50 CATEGORIES = None
  51
  52 # The product name is used by the profiler UI to show the Operating system and Processor.
  53 PRODUCT = os.popen('uname -op').read().strip()
  54
  55 # store the output file
  56 output_file = None
  57
  58 # Here key = tid, value = Thread
  59 tid_to_thread = dict()
  60
  61 # The HTTP server is used to serve the profile to the profiler UI.
  62 http_server_thread = None
  63
  64 # The category index is used by the profiler UI to show the color of the flame graph.
  65 USER_CATEGORY_INDEX = 0
  66 KERNEL_CATEGORY_INDEX = 1
  67
  68 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156
  69 class Frame(NamedTuple):
  70         string_id: StringID
  71         relevantForJS: bool
  72         innerWindowID: int
  73         implementation: None
  74         optimizations: None
  75         line: None
  76         column: None
  77         category: CategoryID
  78         subcategory: int
  79
  80 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216
  81 class Stack(NamedTuple):
  82         prefix_id: Optional[StackID]
  83         frame_id: FrameID
  84
  85 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90
  86 class Sample(NamedTuple):
  87         stack_id: Optional[StackID]
  88         time_ms: Milliseconds
  89         responsiveness: int
  90
  91 @dataclass
  92 class Thread:
  93         """A builder for a profile of the thread.
  94
  95         Attributes:
  96                 comm: Thread command-line (name).
  97                 pid: process ID of containing process.
  98                 tid: thread ID.
  99                 samples: Timeline of profile samples.
 100                 frameTable: interned stack frame ID -> stack frame.
 101                 stringTable: interned string ID -> string.
 102                 stringMap: interned string -> string ID.
 103                 stackTable: interned stack ID -> stack.
 104                 stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID.
 105                 frameMap: Stack Frame string -> interned Frame ID.
 106                 comm: str
 107                 pid: int
 108                 tid: int
 109                 samples: List[Sample] = field(default_factory=list)
 110                 frameTable: List[Frame] = field(default_factory=list)
 111                 stringTable: List[str] = field(default_factory=list)
 112                 stringMap: Dict[str, int] = field(default_factory=dict)
 113                 stackTable: List[Stack] = field(default_factory=list)
 114                 stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict)
 115                 frameMap: Dict[str, int] = field(default_factory=dict)
 116         """
 117         comm: str
 118         pid: int
 119         tid: int
 120         samples: List[Sample] = field(default_factory=list)
 121         frameTable: List[Frame] = field(default_factory=list)
 122         stringTable: List[str] = field(default_factory=list)
 123         stringMap: Dict[str, int] = field(default_factory=dict)
 124         stackTable: List[Stack] = field(default_factory=list)
 125         stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict)
 126         frameMap: Dict[str, int] = field(default_factory=dict)
 127
 128         def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int:
 129                 """Gets a matching stack, or saves the new stack. Returns a Stack ID."""
 130                 key = f"{frame_id}" if prefix_id is None else f"{frame_id},{prefix_id}"
 131                 # key = (prefix_id, frame_id)
 132                 stack_id = self.stackMap.get(key)
 133                 if stack_id is None:
 134                         # return stack_id
 135                         stack_id = len(self.stackTable)
 136                         self.stackTable.append(Stack(prefix_id=prefix_id, frame_id=frame_id))
 137                         self.stackMap[key] = stack_id
 138                 return stack_id
 139
 140         def _intern_string(self, string: str) -> int:
 141                 """Gets a matching string, or saves the new string. Returns a String ID."""
 142                 string_id = self.stringMap.get(string)
 143                 if string_id is not None:
 144                         return string_id
 145                 string_id = len(self.stringTable)
 146                 self.stringTable.append(string)
 147                 self.stringMap[string] = string_id
 148                 return string_id
 149
 150         def _intern_frame(self, frame_str: str) -> int:
 151                 """Gets a matching stack frame, or saves the new frame. Returns a Frame ID."""
 152                 frame_id = self.frameMap.get(frame_str)
 153                 if frame_id is not None:
 154                         return frame_id
 155                 frame_id = len(self.frameTable)
 156                 self.frameMap[frame_str] = frame_id
 157                 string_id = self._intern_string(frame_str)
 158
 159                 symbol_name_to_category = KERNEL_CATEGORY_INDEX if frame_str.find('kallsyms') != -1 \
 160                 or frame_str.find('/vmlinux') != -1 \
 161                 or frame_str.endswith('.ko)') \
 162                 else USER_CATEGORY_INDEX
 163
 164                 self.frameTable.append(Frame(
 165                         string_id=string_id,
 166                         relevantForJS=False,
 167                         innerWindowID=0,
 168                         implementation=None,
 169                         optimizations=None,
 170                         line=None,
 171                         column=None,
 172                         category=symbol_name_to_category,
 173                         subcategory=None,
 174                 ))
 175                 return frame_id
 176
 177         def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None:
 178                 """Add a timestamped stack trace sample to the thread builder.
 179                 Args:
 180                         comm: command-line (name) of the thread at this sample
 181                         stack: sampled stack frames. Root first, leaf last.
 182                         time_ms: timestamp of sample in milliseconds.
 183                 """
 184                 # Ihreads may not set their names right after they are created.
 185                 # Instead, they might do it later. In such situations, to use the latest name they have set.
 186                 if self.comm != comm:
 187                         self.comm = comm
 188
 189                 prefix_stack_id = reduce(lambda prefix_id, frame: self._intern_stack
 190                                                 (self._intern_frame(frame), prefix_id), stack, None)
 191                 if prefix_stack_id is not None:
 192                         self.samples.append(Sample(stack_id=prefix_stack_id,
 193                                                                         time_ms=time_ms,
 194                                                                         responsiveness=0))
 195
 196         def _to_json_dict(self) -> Dict:
 197                 """Converts current Thread to GeckoThread JSON format."""
 198                 # Gecko profile format is row-oriented data as List[List],
 199                 # And a schema for interpreting each index.
 200                 # Schema:
 201                 # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md
 202                 # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230
 203                 return {
 204                         "tid": self.tid,
 205                         "pid": self.pid,
 206                         "name": self.comm,
 207                         # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51
 208                         "markers": {
 209                                 "schema": {
 210                                         "name": 0,
 211                                         "startTime": 1,
 212                                         "endTime": 2,
 213                                         "phase": 3,
 214                                         "category": 4,
 215                                         "data": 5,
 216                                 },
 217                                 "data": [],
 218                         },
 219
 220                         # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90
 221                         "samples": {
 222                                 "schema": {
 223                                         "stack": 0,
 224                                         "time": 1,
 225                                         "responsiveness": 2,
 226                                 },
 227                                 "data": self.samples
 228                         },
 229
 230                         # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156
 231                         "frameTable": {
 232                                 "schema": {
 233                                         "location": 0,
 234                                         "relevantForJS": 1,
 235                                         "innerWindowID": 2,
 236                                         "implementation": 3,
 237                                         "optimizations": 4,
 238                                         "line": 5,
 239                                         "column": 6,
 240                                         "category": 7,
 241                                         "subcategory": 8,
 242                                 },
 243                                 "data": self.frameTable,
 244                         },
 245
 246                         # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216
 247                         "stackTable": {
 248                                 "schema": {
 249                                         "prefix": 0,
 250                                         "frame": 1,
 251                                 },
 252                                 "data": self.stackTable,
 253                         },
 254                         "stringTable": self.stringTable,
 255                         "registerTime": 0,
 256                         "unregisterTime": None,
 257                         "processType": "default",
 258                 }
 259
 260 # Uses perf script python interface to parse each
 261 # event and store the data in the thread builder.
 262 def process_event(param_dict: Dict) -> None:
 263         global start_time
 264         global tid_to_thread
 265         time_stamp = (param_dict['sample']['time'] // 1000) / 1000
 266         pid = param_dict['sample']['pid']
 267         tid = param_dict['sample']['tid']
 268         comm = param_dict['comm']
 269
 270         # Start time is the time of the first sample
 271         if not start_time:
 272                 start_time = time_stamp
 273
 274         # Parse and append the callchain of the current sample into a stack.
 275         stack = []
 276         if param_dict['callchain']:
 277                 for call in param_dict['callchain']:
 278                         if 'sym' not in call:
 279                                 continue
 280                         stack.append(f'{call["sym"]["name"]} (in {call["dso"]})')
 281                 if len(stack) != 0:
 282                         # Reverse the stack, as root come first and the leaf at the end.
 283                         stack = stack[::-1]
 284
 285         # During perf record if -g is not used, the callchain is not available.
 286         # In that case, the symbol and dso are available in the event parameters.
 287         else:
 288                 func = param_dict['symbol'] if 'symbol' in param_dict else '[unknown]'
 289                 dso = param_dict['dso'] if 'dso' in param_dict else '[unknown]'
 290                 stack.append(f'{func} (in {dso})')
 291
 292         # Add sample to the specific thread.
 293         thread = tid_to_thread.get(tid)
 294         if thread is None:
 295                 thread = Thread(comm=comm, pid=pid, tid=tid)
 296                 tid_to_thread[tid] = thread
 297         thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp)
 298
 299 def trace_begin() -> None:
 300         global output_file
 301         if (output_file is None):
 302                 print("Staring Firefox Profiler on your default browser...")
 303                 global http_server_thread
 304                 http_server_thread = threading.Thread(target=test, args=(CORSRequestHandler, HTTPServer,))
 305                 http_server_thread.daemon = True
 306                 http_server_thread.start()
 307
 308 # Trace_end runs at the end and will be used to aggregate
 309 # the data into the final json object and print it out to stdout.
 310 def trace_end() -> None:
 311         global output_file
 312         threads = [thread._to_json_dict() for thread in tid_to_thread.values()]
 313
 314         # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305
 315         gecko_profile_with_meta = {
 316                 "meta": {
 317                         "interval": 1,
 318                         "processType": 0,
 319                         "product": PRODUCT,
 320                         "stackwalk": 1,
 321                         "debug": 0,
 322                         "gcpoison": 0,
 323                         "asyncstack": 1,
 324                         "startTime": start_time,
 325                         "shutdownTime": None,
 326                         "version": 24,
 327                         "presymbolicated": True,
 328                         "categories": CATEGORIES,
 329                         "markerSchema": [],
 330                         },
 331                 "libs": [],
 332                 "threads": threads,
 333                 "processes": [],
 334                 "pausedRanges": [],
 335         }
 336         # launch the profiler on local host if not specified --save-only args, otherwise print to file
 337         if (output_file is None):
 338                 output_file = 'gecko_profile.json'
 339                 with open(output_file, 'w') as f:
 340                         json.dump(gecko_profile_with_meta, f, indent=2)
 341                 launchFirefox(output_file)
 342                 time.sleep(1)
 343                 print(f'[ perf gecko: Captured and wrote into {output_file} ]')
 344         else:
 345                 print(f'[ perf gecko: Captured and wrote into {output_file} ]')
 346                 with open(output_file, 'w') as f:
 347                         json.dump(gecko_profile_with_meta, f, indent=2)
 348
 349 # Used to enable Cross-Origin Resource Sharing (CORS) for requests coming from 'https://profiler.firefox.com', allowing it to access resources from this server.
 350 class CORSRequestHandler(SimpleHTTPRequestHandler):
 351         def end_headers (self):
 352                 self.send_header('Access-Control-Allow-Origin', 'https://profiler.firefox.com')
 353                 SimpleHTTPRequestHandler.end_headers(self)
 354
 355 # start a local server to serve the gecko_profile.json file to the profiler.firefox.com
 356 def launchFirefox(file):
 357         safe_string = urllib.parse.quote_plus(f'http://localhost:8000/{file}')
 358         url = 'https://profiler.firefox.com/from-url/' + safe_string
 359         webbrowser.open(f'{url}')
 360
 361 def main() -> None:
 362         global output_file
 363         global CATEGORIES
 364         parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format which can be uploaded to profiler.firefox.com for visualization")
 365
 366         # Add the command-line options
 367         # Colors must be defined according to this:
 368         # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css
 369         parser.add_argument('--user-color', default='yellow', help='Color for the User category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta'])
 370         parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta'])
 371         # If --save-only is specified, the output will be saved to a file instead of opening Firefox's profiler directly.
 372         parser.add_argument('--save-only', help='Save the output to a file instead of opening Firefox\'s profiler')
 373
 374         # Parse the command-line arguments
 375         args = parser.parse_args()
 376         # Access the values provided by the user
 377         user_color = args.user_color
 378         kernel_color = args.kernel_color
 379         output_file = args.save_only
 380
 381         CATEGORIES = [
 382                 {
 383                         "name": 'User',
 384                         "color": user_color,
 385                         "subcategories": ['Other']
 386                 },
 387                 {
 388                         "name": 'Kernel',
 389                         "color": kernel_color,
 390                         "subcategories": ['Other']
 391                 },
 392         ]
 393
 394 if __name__ == '__main__':
 395         main()