tools/metrics/actions/extract_actions.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Extract UserMetrics "actions" strings from the Chrome source.
   8
   9 This program generates the list of known actions we expect to see in the
  10 user behavior logs.  It walks the Chrome source, looking for calls to
  11 UserMetrics functions, extracting actions and warning on improper calls,
  12 as well as generating the lists of possible actions in situations where
  13 there are many possible actions.
  14
  15 See also:
  16   base/metrics/user_metrics.h
  17
  18 After extracting all actions, the content will go through a pretty print
  19 function to make sure it's well formatted. If the file content needs to be
  20 changed, a window will be prompted asking for user's consent. The old version
  21 will also be saved in a backup file.
  22 """
  23
  24 __author__ = 'evanm (Evan Martin)'
  25
  26 from HTMLParser import HTMLParser
  27 import logging
  28 import os
  29 import re
  30 import shutil
  31 import sys
  32 from xml.dom import minidom
  33
  34 import print_style
  35
  36 sys.path.insert(1, os.path.join(sys.path[0], '..', '..', 'python'))
  37 from google import path_utils
  38
  39 # Import the metrics/common module for pretty print xml.
  40 sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
  41 import presubmit_util
  42 import diff_util
  43 import pretty_print_xml
  44
  45 USER_METRICS_ACTION_RE = re.compile(r"""
  46   [^a-zA-Z]                   # Preceded by a non-alphabetical character.
  47   UserMetricsAction           # Name of the function.
  48   \(                          # Opening parenthesis.
  49   \s*                         # Any amount of whitespace, including new lines.
  50   (.+?)                       # A sequence of characters for the param.
  51   \)                          # Closing parenthesis.
  52   """,
  53   re.VERBOSE | re.DOTALL      # Verbose syntax and makes . also match new lines.
  54 )
  55 COMPUTED_ACTION_RE = re.compile(r'RecordComputedAction')
  56 QUOTED_STRING_RE = re.compile(r'\"(.+?)\"')
  57
  58 # Files that are known to use content::RecordComputedAction(), which means
  59 # they require special handling code in this script.
  60 # To add a new file, add it to this list and add the appropriate logic to
  61 # generate the known actions to AddComputedActions() below.
  62 KNOWN_COMPUTED_USERS = (
  63   'back_forward_menu_model.cc',
  64   'options_page_view.cc',
  65   'render_view_host.cc',  # called using webkit identifiers
  66   'user_metrics.cc',  # method definition
  67   'new_tab_ui.cc',  # most visited clicks 1-9
  68   'extension_metrics_module.cc', # extensions hook for user metrics
  69   'language_options_handler_common.cc', # languages and input methods in CrOS
  70   'cros_language_options_handler.cc', # languages and input methods in CrOS
  71   'about_flags.cc', # do not generate a warning; see AddAboutFlagsActions()
  72   'external_metrics.cc',  # see AddChromeOSActions()
  73   'core_options_handler.cc',  # see AddWebUIActions()
  74   'browser_render_process_host.cc',  # see AddRendererActions()
  75   'render_thread_impl.cc',  # impl of RenderThread::RecordComputedAction()
  76   'render_process_host_impl.cc',  # browser side impl for
  77                                   # RenderThread::RecordComputedAction()
  78   'mock_render_thread.cc',  # mock of RenderThread::RecordComputedAction()
  79   'ppb_pdf_impl.cc',  # see AddClosedSourceActions()
  80   'pepper_pdf_host.cc',  # see AddClosedSourceActions()
  81 )
  82
  83 # Language codes used in Chrome. The list should be updated when a new
  84 # language is added to app/l10n_util.cc, as follows:
  85 #
  86 # % (cat app/l10n_util.cc | \
  87 #    perl -n0e 'print $1 if /kAcceptLanguageList.*?\{(.*?)\}/s' | \
  88 #    perl -nle 'print $1, if /"(.*)"/'; echo 'es-419') | \
  89 #   sort | perl -pe "s/(.*)\n/'\$1', /" | \
  90 #   fold -w75 -s | perl -pe 's/^/  /;s/ $//'; echo
  91 #
  92 # The script extracts language codes from kAcceptLanguageList, but es-419
  93 # (Spanish in Latin America) is an exception.
  94 LANGUAGE_CODES = (
  95   'af', 'am', 'ar', 'az', 'be', 'bg', 'bh', 'bn', 'br', 'bs', 'ca', 'co',
  96   'cs', 'cy', 'da', 'de', 'de-AT', 'de-CH', 'de-DE', 'el', 'en', 'en-AU',
  97   'en-CA', 'en-GB', 'en-NZ', 'en-US', 'en-ZA', 'eo', 'es', 'es-419', 'et',
  98   'eu', 'fa', 'fi', 'fil', 'fo', 'fr', 'fr-CA', 'fr-CH', 'fr-FR', 'fy',
  99   'ga', 'gd', 'gl', 'gn', 'gu', 'ha', 'haw', 'he', 'hi', 'hr', 'hu', 'hy',
 100   'ia', 'id', 'is', 'it', 'it-CH', 'it-IT', 'ja', 'jw', 'ka', 'kk', 'km',
 101   'kn', 'ko', 'ku', 'ky', 'la', 'ln', 'lo', 'lt', 'lv', 'mk', 'ml', 'mn',
 102   'mo', 'mr', 'ms', 'mt', 'nb', 'ne', 'nl', 'nn', 'no', 'oc', 'om', 'or',
 103   'pa', 'pl', 'ps', 'pt', 'pt-BR', 'pt-PT', 'qu', 'rm', 'ro', 'ru', 'sd',
 104   'sh', 'si', 'sk', 'sl', 'sn', 'so', 'sq', 'sr', 'st', 'su', 'sv', 'sw',
 105   'ta', 'te', 'tg', 'th', 'ti', 'tk', 'to', 'tr', 'tt', 'tw', 'ug', 'uk',
 106   'ur', 'uz', 'vi', 'xh', 'yi', 'yo', 'zh', 'zh-CN', 'zh-TW', 'zu',
 107 )
 108
 109 # Input method IDs used in Chrome OS. The list should be updated when a
 110 # new input method is added to
 111 # chromeos/ime/input_methods.txt in the Chrome tree, as
 112 # follows:
 113 #
 114 # % sort chromeos/ime/input_methods.txt | \
 115 #   perl -ne "print \"'\$1', \" if /^([^#]+?)\s/" | \
 116 #   fold -w75 -s | perl -pe 's/^/  /;s/ $//'; echo
 117 #
 118 # The script extracts input method IDs from input_methods.txt.
 119 INPUT_METHOD_IDS = (
 120   'xkb:am:phonetic:arm', 'xkb:be::fra', 'xkb:be::ger', 'xkb:be::nld',
 121   'xkb:bg::bul', 'xkb:bg:phonetic:bul', 'xkb:br::por', 'xkb:by::bel',
 122   'xkb:ca::fra', 'xkb:ca:eng:eng', 'xkb:ca:multix:fra', 'xkb:ch::ger',
 123   'xkb:ch:fr:fra', 'xkb:cz::cze', 'xkb:cz:qwerty:cze', 'xkb:de::ger',
 124   'xkb:de:neo:ger', 'xkb:dk::dan', 'xkb:ee::est', 'xkb:es::spa',
 125   'xkb:es:cat:cat', 'xkb:fi::fin', 'xkb:fr::fra', 'xkb:gb:dvorak:eng',
 126   'xkb:gb:extd:eng', 'xkb:ge::geo', 'xkb:gr::gre', 'xkb:hr::scr',
 127   'xkb:hu::hun', 'xkb:il::heb', 'xkb:is::ice', 'xkb:it::ita', 'xkb:jp::jpn',
 128   'xkb:latam::spa', 'xkb:lt::lit', 'xkb:lv:apostrophe:lav', 'xkb:mn::mon',
 129   'xkb:no::nob', 'xkb:pl::pol', 'xkb:pt::por', 'xkb:ro::rum', 'xkb:rs::srp',
 130   'xkb:ru::rus', 'xkb:ru:phonetic:rus', 'xkb:se::swe', 'xkb:si::slv',
 131   'xkb:sk::slo', 'xkb:tr::tur', 'xkb:ua::ukr', 'xkb:us::eng',
 132   'xkb:us:altgr-intl:eng', 'xkb:us:colemak:eng', 'xkb:us:dvorak:eng',
 133   'xkb:us:intl:eng',
 134 )
 135
 136 # The path to the root of the repository.
 137 REPOSITORY_ROOT = os.path.join(path_utils.ScriptDir(), '..', '..', '..')
 138
 139 number_of_files_total = 0
 140
 141 # Tags that need to be inserted to each 'action' tag and their default content.
 142 TAGS = {'description': 'Please enter the description of the metric.',
 143         'owner': ('Please list the metric\'s owners. Add more owner tags as '
 144                   'needed.')}
 145
 146
 147 def AddComputedActions(actions):
 148   """Add computed actions to the actions list.
 149
 150   Arguments:
 151     actions: set of actions to add to.
 152   """
 153
 154   # Actions for back_forward_menu_model.cc.
 155   for dir in ('BackMenu_', 'ForwardMenu_'):
 156     actions.add(dir + 'ShowFullHistory')
 157     actions.add(dir + 'Popup')
 158     for i in range(1, 20):
 159       actions.add(dir + 'HistoryClick' + str(i))
 160       actions.add(dir + 'ChapterClick' + str(i))
 161
 162   # Actions for new_tab_ui.cc.
 163   for i in range(1, 10):
 164     actions.add('MostVisited%d' % i)
 165
 166   # Actions for language_options_handler.cc (Chrome OS specific).
 167   for input_method_id in INPUT_METHOD_IDS:
 168     actions.add('LanguageOptions_DisableInputMethod_%s' % input_method_id)
 169     actions.add('LanguageOptions_EnableInputMethod_%s' % input_method_id)
 170   for language_code in LANGUAGE_CODES:
 171     actions.add('LanguageOptions_UiLanguageChange_%s' % language_code)
 172     actions.add('LanguageOptions_SpellCheckLanguageChange_%s' % language_code)
 173
 174 def AddWebKitEditorActions(actions):
 175   """Add editor actions from editor_client_impl.cc.
 176
 177   Arguments:
 178     actions: set of actions to add to.
 179   """
 180   action_re = re.compile(r'''\{ [\w']+, +\w+, +"(.*)" +\},''')
 181
 182   editor_file = os.path.join(REPOSITORY_ROOT, 'webkit', 'api', 'src',
 183                              'EditorClientImpl.cc')
 184   for line in open(editor_file):
 185     match = action_re.search(line)
 186     if match:  # Plain call to RecordAction
 187       actions.add(match.group(1))
 188
 189 def AddClosedSourceActions(actions):
 190   """Add actions that are in code which is not checked out by default
 191
 192   Arguments
 193     actions: set of actions to add to.
 194   """
 195   actions.add('PDF.FitToHeightButton')
 196   actions.add('PDF.FitToWidthButton')
 197   actions.add('PDF.LoadFailure')
 198   actions.add('PDF.LoadSuccess')
 199   actions.add('PDF.PreviewDocumentLoadFailure')
 200   actions.add('PDF.PrintButton')
 201   actions.add('PDF.PrintPage')
 202   actions.add('PDF.SaveButton')
 203   actions.add('PDF.ZoomFromBrowser')
 204   actions.add('PDF.ZoomInButton')
 205   actions.add('PDF.ZoomOutButton')
 206   actions.add('PDF_Unsupported_3D')
 207   actions.add('PDF_Unsupported_Attachment')
 208   actions.add('PDF_Unsupported_Bookmarks')
 209   actions.add('PDF_Unsupported_Digital_Signature')
 210   actions.add('PDF_Unsupported_Movie')
 211   actions.add('PDF_Unsupported_Portfolios_Packages')
 212   actions.add('PDF_Unsupported_Rights_Management')
 213   actions.add('PDF_Unsupported_Screen')
 214   actions.add('PDF_Unsupported_Shared_Form')
 215   actions.add('PDF_Unsupported_Shared_Review')
 216   actions.add('PDF_Unsupported_Sound')
 217   actions.add('PDF_Unsupported_XFA')
 218
 219 def AddAndroidActions(actions):
 220   """Add actions that are used by Chrome on Android.
 221
 222   Arguments
 223     actions: set of actions to add to.
 224   """
 225   actions.add('Cast_Sender_CastDeviceSelected');
 226   actions.add('Cast_Sender_CastEnterFullscreen');
 227   actions.add('Cast_Sender_CastMediaType');
 228   actions.add('Cast_Sender_CastPlayRequested');
 229   actions.add('Cast_Sender_YouTubeDeviceSelected');
 230   actions.add('DataReductionProxy_PromoDisplayed');
 231   actions.add('DataReductionProxy_PromoLearnMore');
 232   actions.add('DataReductionProxy_TurnedOn');
 233   actions.add('DataReductionProxy_TurnedOnFromPromo');
 234   actions.add('DataReductionProxy_TurnedOff');
 235   actions.add('MobileActionBarShown')
 236   actions.add('MobileBeamCallbackSuccess')
 237   actions.add('MobileBeamInvalidAppState')
 238   actions.add('MobileBreakpadUploadAttempt')
 239   actions.add('MobileBreakpadUploadFailure')
 240   actions.add('MobileBreakpadUploadSuccess')
 241   actions.add('MobileContextMenuCopyImageLinkAddress')
 242   actions.add('MobileContextMenuCopyLinkAddress')
 243   actions.add('MobileContextMenuCopyLinkText')
 244   actions.add('MobileContextMenuDownloadImage')
 245   actions.add('MobileContextMenuDownloadLink')
 246   actions.add('MobileContextMenuDownloadVideo')
 247   actions.add('MobileContextMenuImage')
 248   actions.add('MobileContextMenuLink')
 249   actions.add('MobileContextMenuOpenImageInNewTab')
 250   actions.add('MobileContextMenuOpenLink')
 251   actions.add('MobileContextMenuOpenLinkInIncognito')
 252   actions.add('MobileContextMenuOpenLinkInNewTab')
 253   actions.add('MobileContextMenuSaveImage')
 254   actions.add('MobileContextMenuSearchByImage')
 255   actions.add('MobileContextMenuShareLink')
 256   actions.add('MobileContextMenuText')
 257   actions.add('MobileContextMenuVideo')
 258   actions.add('MobileContextMenuViewImage')
 259   actions.add('MobileFirstEditInOmnibox')
 260   actions.add('MobileFocusedFakeboxOnNtp')
 261   actions.add('MobileFocusedOmniboxNotOnNtp')
 262   actions.add('MobileFocusedOmniboxOnNtp')
 263   actions.add('MobileFreAttemptSignIn')
 264   actions.add('MobileFreSignInSuccessful')
 265   actions.add('MobileFreSkipSignIn')
 266   actions.add('MobileMenuAddToBookmarks')
 267   actions.add('MobileMenuAddToHomescreen')
 268   actions.add('MobileMenuAllBookmarks')
 269   actions.add('MobileMenuBack')
 270   actions.add('MobileMenuCloseAllTabs')
 271   actions.add('MobileMenuCloseTab')
 272   actions.add('MobileMenuDirectShare')
 273   actions.add('MobileMenuFeedback')
 274   actions.add('MobileMenuFindInPage')
 275   actions.add('MobileMenuForward')
 276   actions.add('MobileMenuFullscreen')
 277   actions.add('MobileMenuHistory')
 278   actions.add('MobileMenuNewIncognitoTab')
 279   actions.add('MobileMenuNewTab')
 280   actions.add('MobileMenuOpenTabs')
 281   actions.add('MobileMenuPrint')
 282   actions.add('MobileMenuQuit')
 283   actions.add('MobileMenuReload')
 284   actions.add('MobileMenuRequestDesktopSite')
 285   actions.add('MobileMenuSettings')
 286   actions.add('MobileMenuShare')
 287   actions.add('MobileMenuShow')
 288   actions.add('MobileNTPBookmark')
 289   actions.add('MobileNTPForeignSession')
 290   actions.add('MobileNTPMostVisited')
 291   actions.add('MobileNTPRecentlyClosed')
 292   actions.add('MobileNTPSwitchToBookmarks')
 293   actions.add('MobileNTPSwitchToIncognito')
 294   actions.add('MobileNTPSwitchToMostVisited')
 295   actions.add('MobileNTPSwitchToOpenTabs')
 296   actions.add('MobileNewTabOpened')
 297   actions.add('MobileOmniboxSearch')
 298   actions.add('MobileOmniboxVoiceSearch')
 299   actions.add('MobileOmniboxRefineSuggestion')
 300   actions.add('MobilePageLoaded')
 301   actions.add('MobilePageLoadedDesktopUserAgent')
 302   actions.add('MobilePageLoadedWithKeyboard')
 303   actions.add('MobilePullGestureReload')
 304   actions.add('MobileReceivedExternalIntent')
 305   actions.add('MobileRendererCrashed')
 306   actions.add('MobileShortcutAllBookmarks')
 307   actions.add('MobileShortcutFindInPage')
 308   actions.add('MobileShortcutNewIncognitoTab')
 309   actions.add('MobileShortcutNewTab')
 310   actions.add('MobileSideSwipeFinished')
 311   actions.add('MobileStackViewCloseTab')
 312   actions.add('MobileStackViewSwipeCloseTab')
 313   actions.add('MobileTabClobbered')
 314   actions.add('MobileTabClosed')
 315   actions.add('MobileTabStripCloseTab')
 316   actions.add('MobileTabStripNewTab')
 317   actions.add('MobileTabSwitched')
 318   actions.add('MobileToolbarBack')
 319   actions.add('MobileToolbarForward')
 320   actions.add('MobileToolbarNewTab')
 321   actions.add('MobileToolbarReload')
 322   actions.add('MobileToolbarShowMenu')
 323   actions.add('MobileToolbarShowStackView')
 324   actions.add('MobileToolbarStackViewNewTab')
 325   actions.add('MobileToolbarToggleBookmark')
 326   actions.add('MobileUsingMenuByHwButtonTap')
 327   actions.add('MobileUsingMenuBySwButtonDragging')
 328   actions.add('MobileUsingMenuBySwButtonTap')
 329   actions.add('SystemBack')
 330   actions.add('SystemBackForNavigation')
 331
 332 def AddAboutFlagsActions(actions):
 333   """This parses the experimental feature flags for UMA actions.
 334
 335   Arguments:
 336     actions: set of actions to add to.
 337   """
 338   about_flags = os.path.join(REPOSITORY_ROOT, 'chrome', 'browser',
 339                              'about_flags.cc')
 340   flag_name_re = re.compile(r'\s*"([0-9a-zA-Z\-_]+)",\s*// FLAGS:RECORD_UMA')
 341   for line in open(about_flags):
 342     match = flag_name_re.search(line)
 343     if match:
 344       actions.add("AboutFlags_" + match.group(1))
 345     # If the line contains the marker but was not matched by the regex, put up
 346     # an error if the line is not a comment.
 347     elif 'FLAGS:RECORD_UMA' in line and line[0:2] != '//':
 348       print >>sys.stderr, 'WARNING: This line is marked for recording ' + \
 349           'about:flags metrics, but is not in the proper format:\n' + line
 350
 351 def AddBookmarkManagerActions(actions):
 352   """Add actions that are used by BookmarkManager.
 353
 354   Arguments
 355     actions: set of actions to add to.
 356   """
 357   actions.add('BookmarkManager_Command_AddPage')
 358   actions.add('BookmarkManager_Command_Copy')
 359   actions.add('BookmarkManager_Command_Cut')
 360   actions.add('BookmarkManager_Command_Delete')
 361   actions.add('BookmarkManager_Command_Edit')
 362   actions.add('BookmarkManager_Command_Export')
 363   actions.add('BookmarkManager_Command_Import')
 364   actions.add('BookmarkManager_Command_NewFolder')
 365   actions.add('BookmarkManager_Command_OpenIncognito')
 366   actions.add('BookmarkManager_Command_OpenInNewTab')
 367   actions.add('BookmarkManager_Command_OpenInNewWindow')
 368   actions.add('BookmarkManager_Command_OpenInSame')
 369   actions.add('BookmarkManager_Command_Paste')
 370   actions.add('BookmarkManager_Command_ShowInFolder')
 371   actions.add('BookmarkManager_Command_Sort')
 372   actions.add('BookmarkManager_Command_UndoDelete')
 373   actions.add('BookmarkManager_Command_UndoGlobal')
 374   actions.add('BookmarkManager_Command_UndoNone')
 375
 376   actions.add('BookmarkManager_NavigateTo_BookmarkBar')
 377   actions.add('BookmarkManager_NavigateTo_Mobile')
 378   actions.add('BookmarkManager_NavigateTo_Other')
 379   actions.add('BookmarkManager_NavigateTo_Recent')
 380   actions.add('BookmarkManager_NavigateTo_Search')
 381   actions.add('BookmarkManager_NavigateTo_SubFolder')
 382
 383 def AddChromeOSActions(actions):
 384   """Add actions reported by non-Chrome processes in Chrome OS.
 385
 386   Arguments:
 387     actions: set of actions to add to.
 388   """
 389   # Actions sent by Chrome OS update engine.
 390   actions.add('Updater.ServerCertificateChanged')
 391   actions.add('Updater.ServerCertificateFailed')
 392
 393   # Actions sent by Chrome OS cryptohome.
 394   actions.add('Cryptohome.PKCS11InitFail')
 395
 396 def AddExtensionActions(actions):
 397   """Add actions reported by extensions via chrome.metricsPrivate API.
 398
 399   Arguments:
 400     actions: set of actions to add to.
 401   """
 402   # Actions sent by Chrome OS File Browser.
 403   actions.add('FileBrowser.CreateNewFolder')
 404   actions.add('FileBrowser.PhotoEditor.Edit')
 405   actions.add('FileBrowser.PhotoEditor.View')
 406   actions.add('FileBrowser.SuggestApps.ShowDialog')
 407
 408   # Actions sent by Google Now client.
 409   actions.add('GoogleNow.MessageClicked')
 410   actions.add('GoogleNow.ButtonClicked0')
 411   actions.add('GoogleNow.ButtonClicked1')
 412   actions.add('GoogleNow.Dismissed')
 413
 414   # Actions sent by Chrome Connectivity Diagnostics.
 415   actions.add('ConnectivityDiagnostics.LaunchSource.OfflineChromeOS')
 416   actions.add('ConnectivityDiagnostics.LaunchSource.WebStore')
 417   actions.add('ConnectivityDiagnostics.UA.LogsShown')
 418   actions.add('ConnectivityDiagnostics.UA.PassingTestsShown')
 419   actions.add('ConnectivityDiagnostics.UA.SettingsShown')
 420   actions.add('ConnectivityDiagnostics.UA.TestResultExpanded')
 421   actions.add('ConnectivityDiagnostics.UA.TestSuiteRun')
 422
 423   # Actions sent by 'Ok Google' Hotwording.
 424   actions.add('Hotword.HotwordTrigger')
 425
 426
 427 class InvalidStatementException(Exception):
 428   """Indicates an invalid statement was found."""
 429
 430
 431 class ActionNameFinder:
 432   """Helper class to find action names in source code file."""
 433
 434   def __init__(self, path, contents):
 435     self.__path = path
 436     self.__pos = 0
 437     self.__contents = contents
 438
 439   def FindNextAction(self):
 440     """Finds the next action name in the file.
 441
 442     Returns:
 443       The name of the action found or None if there are no more actions.
 444     Raises:
 445       InvalidStatementException if the next action statement is invalid
 446       and could not be parsed. There may still be more actions in the file,
 447       so FindNextAction() can continue to be called to find following ones.
 448     """
 449     match = USER_METRICS_ACTION_RE.search(self.__contents, pos=self.__pos)
 450     if not match:
 451       return None
 452     match_start = match.start()
 453     self.__pos = match.end()
 454     match = QUOTED_STRING_RE.match(match.group(1))
 455     if not match:
 456       self._RaiseException(match_start, self.__pos)
 457     return match.group(1)
 458
 459   def _RaiseException(self, match_start, match_end):
 460     """Raises an InvalidStatementException for the specified code range."""
 461     line_number = self.__contents.count('\n', 0, match_start) + 1
 462     # Add 1 to |match_start| since the RE checks the preceding character.
 463     statement = self.__contents[match_start + 1:match_end]
 464     raise InvalidStatementException(
 465       '%s uses UserMetricsAction incorrectly on line %d:\n%s' %
 466       (self.__path, line_number, statement))
 467
 468
 469 def GrepForActions(path, actions):
 470   """Grep a source file for calls to UserMetrics functions.
 471
 472   Arguments:
 473     path: path to the file
 474     actions: set of actions to add to
 475   """
 476   global number_of_files_total
 477   number_of_files_total = number_of_files_total + 1
 478
 479   finder = ActionNameFinder(path, open(path).read())
 480   while True:
 481     try:
 482       action_name = finder.FindNextAction()
 483       if not action_name:
 484         break
 485       actions.add(action_name)
 486     except InvalidStatementException, e:
 487       logging.warning(str(e))
 488
 489   line_number = 0
 490   for line in open(path):
 491     line_number = line_number + 1
 492     if COMPUTED_ACTION_RE.search(line):
 493       # Warn if this file shouldn't be calling RecordComputedAction.
 494       if os.path.basename(path) not in KNOWN_COMPUTED_USERS:
 495         logging.warning('%s has RecordComputedAction statement on line %d' %
 496                         (path, line_number))
 497
 498 class WebUIActionsParser(HTMLParser):
 499   """Parses an HTML file, looking for all tags with a 'metric' attribute.
 500   Adds user actions corresponding to any metrics found.
 501
 502   Arguments:
 503     actions: set of actions to add to
 504   """
 505   def __init__(self, actions):
 506     HTMLParser.__init__(self)
 507     self.actions = actions
 508
 509   def handle_starttag(self, tag, attrs):
 510     # We only care to examine tags that have a 'metric' attribute.
 511     attrs = dict(attrs)
 512     if not 'metric' in attrs:
 513       return
 514
 515     # Boolean metrics have two corresponding actions.  All other metrics have
 516     # just one corresponding action.  By default, we check the 'dataType'
 517     # attribute.
 518     is_boolean = ('dataType' in attrs and attrs['dataType'] == 'boolean')
 519     if 'type' in attrs and attrs['type'] in ('checkbox', 'radio'):
 520       if attrs['type'] == 'checkbox':
 521         is_boolean = True
 522       else:
 523         # Radio buttons are boolean if and only if their values are 'true' or
 524         # 'false'.
 525         assert(attrs['type'] == 'radio')
 526         if 'value' in attrs and attrs['value'] in ['true', 'false']:
 527           is_boolean = True
 528
 529     if is_boolean:
 530       self.actions.add(attrs['metric'] + '_Enable')
 531       self.actions.add(attrs['metric'] + '_Disable')
 532     else:
 533       self.actions.add(attrs['metric'])
 534
 535 def GrepForWebUIActions(path, actions):
 536   """Grep a WebUI source file for elements with associated metrics.
 537
 538   Arguments:
 539     path: path to the file
 540     actions: set of actions to add to
 541   """
 542   close_called = False
 543   try:
 544     parser = WebUIActionsParser(actions)
 545     parser.feed(open(path).read())
 546     # An exception can be thrown by parser.close(), so do it in the try to
 547     # ensure the path of the file being parsed gets printed if that happens.
 548     close_called = True
 549     parser.close()
 550   except Exception, e:
 551     print "Error encountered for path %s" % path
 552     raise e
 553   finally:
 554     if not close_called:
 555       parser.close()
 556
 557 def WalkDirectory(root_path, actions, extensions, callback):
 558   for path, dirs, files in os.walk(root_path):
 559     if '.svn' in dirs:
 560       dirs.remove('.svn')
 561     if '.git' in dirs:
 562       dirs.remove('.git')
 563     for file in files:
 564       ext = os.path.splitext(file)[1]
 565       if ext in extensions:
 566         callback(os.path.join(path, file), actions)
 567
 568 def AddLiteralActions(actions):
 569   """Add literal actions specified via calls to UserMetrics functions.
 570
 571   Arguments:
 572     actions: set of actions to add to.
 573   """
 574   EXTENSIONS = ('.cc', '.mm', '.c', '.m')
 575
 576   # Walk the source tree to process all .cc files.
 577   ash_root = os.path.normpath(os.path.join(REPOSITORY_ROOT, 'ash'))
 578   WalkDirectory(ash_root, actions, EXTENSIONS, GrepForActions)
 579   chrome_root = os.path.normpath(os.path.join(REPOSITORY_ROOT, 'chrome'))
 580   WalkDirectory(chrome_root, actions, EXTENSIONS, GrepForActions)
 581   content_root = os.path.normpath(os.path.join(REPOSITORY_ROOT, 'content'))
 582   WalkDirectory(content_root, actions, EXTENSIONS, GrepForActions)
 583   components_root = os.path.normpath(os.path.join(REPOSITORY_ROOT,
 584                     'components'))
 585   WalkDirectory(components_root, actions, EXTENSIONS, GrepForActions)
 586   net_root = os.path.normpath(os.path.join(REPOSITORY_ROOT, 'net'))
 587   WalkDirectory(net_root, actions, EXTENSIONS, GrepForActions)
 588   webkit_root = os.path.normpath(os.path.join(REPOSITORY_ROOT, 'webkit'))
 589   WalkDirectory(os.path.join(webkit_root, 'glue'), actions, EXTENSIONS,
 590                 GrepForActions)
 591   WalkDirectory(os.path.join(webkit_root, 'port'), actions, EXTENSIONS,
 592                 GrepForActions)
 593
 594 def AddWebUIActions(actions):
 595   """Add user actions defined in WebUI files.
 596
 597   Arguments:
 598     actions: set of actions to add to.
 599   """
 600   resources_root = os.path.join(REPOSITORY_ROOT, 'chrome', 'browser',
 601                                 'resources')
 602   WalkDirectory(resources_root, actions, ('.html'), GrepForWebUIActions)
 603
 604 def AddHistoryPageActions(actions):
 605   """Add actions that are used in History page.
 606
 607   Arguments
 608     actions: set of actions to add to.
 609   """
 610   actions.add('HistoryPage_BookmarkStarClicked')
 611   actions.add('HistoryPage_EntryMenuRemoveFromHistory')
 612   actions.add('HistoryPage_EntryLinkClick')
 613   actions.add('HistoryPage_EntryLinkRightClick')
 614   actions.add('HistoryPage_SearchResultClick')
 615   actions.add('HistoryPage_EntryMenuShowMoreFromSite')
 616   actions.add('HistoryPage_NewestHistoryClick')
 617   actions.add('HistoryPage_NewerHistoryClick')
 618   actions.add('HistoryPage_OlderHistoryClick')
 619   actions.add('HistoryPage_Search')
 620   actions.add('HistoryPage_InitClearBrowsingData')
 621   actions.add('HistoryPage_RemoveSelected')
 622   actions.add('HistoryPage_SearchResultRemove')
 623   actions.add('HistoryPage_ConfirmRemoveSelected')
 624   actions.add('HistoryPage_CancelRemoveSelected')
 625
 626 def AddAutomaticResetBannerActions(actions):
 627   """Add actions that are used for the automatic profile settings reset banners
 628   in chrome://settings.
 629
 630   Arguments
 631     actions: set of actions to add to.
 632   """
 633   # These actions relate to the the automatic settings reset banner shown as
 634   # a result of the reset prompt.
 635   actions.add('AutomaticReset_WebUIBanner_BannerShown')
 636   actions.add('AutomaticReset_WebUIBanner_ManuallyClosed')
 637   actions.add('AutomaticReset_WebUIBanner_ResetClicked')
 638
 639   # These actions relate to the the automatic settings reset banner shown as
 640   # a result of settings hardening.
 641   actions.add('AutomaticSettingsReset_WebUIBanner_BannerShown')
 642   actions.add('AutomaticSettingsReset_WebUIBanner_ManuallyClosed')
 643   actions.add('AutomaticSettingsReset_WebUIBanner_LearnMoreClicked')
 644   actions.add('AutomaticSettingsReset_WebUIBanner_ResetClicked')
 645
 646
 647 class Error(Exception):
 648   pass
 649
 650
 651 def _ExtractText(parent_dom, tag_name):
 652   """Extract the text enclosed by |tag_name| under |parent_dom|
 653
 654   Args:
 655     parent_dom: The parent Element under which text node is searched for.
 656     tag_name: The name of the tag which contains a text node.
 657
 658   Returns:
 659     A (list of) string enclosed by |tag_name| under |parent_dom|.
 660   """
 661   texts = []
 662   for child_dom in parent_dom.getElementsByTagName(tag_name):
 663     text_dom = child_dom.childNodes
 664     if text_dom.length != 1:
 665       raise Error('More than 1 child node exists under %s' % tag_name)
 666     if text_dom[0].nodeType != minidom.Node.TEXT_NODE:
 667       raise Error('%s\'s child node is not a text node.' % tag_name)
 668     texts.append(text_dom[0].data)
 669   return texts
 670
 671
 672 class Action(object):
 673   def __init__(self, name, description, owners, obsolete=None):
 674     self.name = name
 675     self.description = description
 676     self.owners = owners
 677     self.obsolete = obsolete
 678
 679
 680 def ParseActionFile(file_content):
 681   """Parse the XML data currently stored in the file.
 682
 683   Args:
 684     file_content: a string containing the action XML file content.
 685
 686   Returns:
 687     (actions, actions_dict) actions is a set with all user actions' names.
 688     actions_dict is a dict from user action name to Action object.
 689   """
 690   dom = minidom.parseString(file_content)
 691
 692   comment_nodes = []
 693   # Get top-level comments. It is assumed that all comments are placed before
 694   # <acionts> tag. Therefore the loop will stop if it encounters a non-comment
 695   # node.
 696   for node in dom.childNodes:
 697     if node.nodeType == minidom.Node.COMMENT_NODE:
 698       comment_nodes.append(node)
 699     else:
 700       break
 701
 702   actions = set()
 703   actions_dict = {}
 704   # Get each user action data.
 705   for action_dom in dom.getElementsByTagName('action'):
 706     action_name = action_dom.getAttribute('name')
 707     actions.add(action_name)
 708
 709     owners = _ExtractText(action_dom, 'owner')
 710     # There is only one description for each user action. Get the first element
 711     # of the returned list.
 712     description_list = _ExtractText(action_dom, 'description')
 713     if len(description_list) > 1:
 714       logging.error('user actions "%s" has more than one descriptions. Exactly '
 715                     'one description is needed for each user action. Please '
 716                     'fix.', action_name)
 717       sys.exit(1)
 718     description = description_list[0] if description_list else None
 719     # There is at most one obsolete tag for each user action.
 720     obsolete_list = _ExtractText(action_dom, 'obsolete')
 721     if len(obsolete_list) > 1:
 722       logging.error('user actions "%s" has more than one obsolete tag. At most '
 723                     'one obsolete tag can be added for each user action. Please'
 724                     ' fix.', action_name)
 725       sys.exit(1)
 726     obsolete = obsolete_list[0] if obsolete_list else None
 727     actions_dict[action_name] = Action(action_name, description, owners,
 728                                        obsolete)
 729   return actions, actions_dict, comment_nodes
 730
 731
 732 def _CreateActionTag(doc, action_name, action_object):
 733   """Create a new action tag.
 734
 735   Format of an action tag:
 736   <action name="name">
 737     <owner>Owner</owner>
 738     <description>Description.</description>
 739     <obsolete>Deprecated.</obsolete>
 740   </action>
 741
 742   <obsolete> is an optional tag. It's added to user actions that are no longer
 743   used any more.
 744
 745   If action_name is in actions_dict, the values to be inserted are based on the
 746   corresponding Action object. If action_name is not in actions_dict, the
 747   default value from TAGS is used.
 748
 749   Args:
 750     doc: The document under which the new action tag is created.
 751     action_name: The name of an action.
 752     action_object: An action object representing the data to be inserted.
 753
 754   Returns:
 755     An action tag Element with proper children elements.
 756   """
 757   action_dom = doc.createElement('action')
 758   action_dom.setAttribute('name', action_name)
 759
 760   # Create owner tag.
 761   if action_object and action_object.owners:
 762     # If owners for this action is not None, use the stored value. Otherwise,
 763     # use the default value.
 764     for owner in action_object.owners:
 765       owner_dom = doc.createElement('owner')
 766       owner_dom.appendChild(doc.createTextNode(owner))
 767       action_dom.appendChild(owner_dom)
 768   else:
 769     # Use default value.
 770     owner_dom = doc.createElement('owner')
 771     owner_dom.appendChild(doc.createTextNode(TAGS.get('owner', '')))
 772     action_dom.appendChild(owner_dom)
 773
 774   # Create description tag.
 775   description_dom = doc.createElement('description')
 776   action_dom.appendChild(description_dom)
 777   if action_object and action_object.description:
 778     # If description for this action is not None, use the store value.
 779     # Otherwise, use the default value.
 780     description_dom.appendChild(doc.createTextNode(
 781         action_object.description))
 782   else:
 783     description_dom.appendChild(doc.createTextNode(
 784         TAGS.get('description', '')))
 785
 786   # Create obsolete tag.
 787   if action_object and action_object.obsolete:
 788     obsolete_dom = doc.createElement('obsolete')
 789     action_dom.appendChild(obsolete_dom)
 790     obsolete_dom.appendChild(doc.createTextNode(
 791         action_object.obsolete))
 792
 793   return action_dom
 794
 795
 796 def PrettyPrint(actions, actions_dict, comment_nodes=[]):
 797   """Given a list of action data, create a well-printed minidom document.
 798
 799   Args:
 800     actions: A list of action names.
 801     actions_dict: A mappting from action name to Action object.
 802
 803   Returns:
 804     A well-printed minidom document that represents the input action data.
 805   """
 806   doc = minidom.Document()
 807
 808   # Attach top-level comments.
 809   for node in comment_nodes:
 810     doc.appendChild(node)
 811
 812   actions_element = doc.createElement('actions')
 813   doc.appendChild(actions_element)
 814
 815   # Attach action node based on updated |actions|.
 816   for action in sorted(actions):
 817     actions_element.appendChild(
 818         _CreateActionTag(doc, action, actions_dict.get(action, None)))
 819
 820   return print_style.GetPrintStyle().PrettyPrintNode(doc)
 821
 822
 823 def UpdateXml(original_xml):
 824   actions, actions_dict, comment_nodes = ParseActionFile(original_xml)
 825
 826   AddComputedActions(actions)
 827   # TODO(fmantek): bring back webkit editor actions.
 828   # AddWebKitEditorActions(actions)
 829   AddAboutFlagsActions(actions)
 830   AddWebUIActions(actions)
 831
 832   AddLiteralActions(actions)
 833
 834   # print "Scanned {0} number of files".format(number_of_files_total)
 835   # print "Found {0} entries".format(len(actions))
 836
 837   AddAndroidActions(actions)
 838   AddAutomaticResetBannerActions(actions)
 839   AddBookmarkManagerActions(actions)
 840   AddChromeOSActions(actions)
 841   AddClosedSourceActions(actions)
 842   AddExtensionActions(actions)
 843   AddHistoryPageActions(actions)
 844
 845   return PrettyPrint(actions, actions_dict, comment_nodes)
 846
 847
 848 def main(argv):
 849   presubmit_util.DoPresubmitMain(argv, 'actions.xml', 'actions.old.xml',
 850                                  'extract_actions.py', UpdateXml)
 851
 852 if '__main__' == __name__:
 853   sys.exit(main(sys.argv))