tools/gen_keyboard_overlay_data/gen_keyboard_overlay_data.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Generate keyboard layout and hotkey data for the keyboard overlay.
   7
   8 This script fetches data from the keyboard layout and hotkey data spreadsheet,
   9 and output the data depending on the option.
  10
  11   --cc: Rewrites a part of C++ code in
  12       chrome/browser/chromeos/webui/keyboard_overlay_ui.cc
  13
  14   --grd: Rewrites a part of grd messages in
  15       chrome/app/generated_resources.grd
  16
  17   --js: Rewrites the entire JavaScript code in
  18       chrome/browser/resources/keyboard_overlay/keyboard_overlay_data.js
  19
  20 These options can be specified at the same time.
  21
  22 e.g.
  23 python gen_keyboard_overlay_data.py --cc --grd --js
  24
  25 The output directory of the generated files can be changed with --outdir.
  26
  27 e.g. (This will generate tmp/keyboard_overlay.js)
  28 python gen_keyboard_overlay_data.py --outdir=tmp --js
  29 """
  30
  31 import cStringIO
  32 import datetime
  33 import gdata.spreadsheet.service
  34 import getpass
  35 import json
  36 import optparse
  37 import os
  38 import re
  39 import sys
  40
  41 MODIFIER_SHIFT = 1 << 0
  42 MODIFIER_CTRL = 1 << 1
  43 MODIFIER_ALT = 1 << 2
  44
  45 KEYBOARD_GLYPH_SPREADSHEET_KEY = '0Ao3KldW9piwEdExLbGR6TmZ2RU9aUjFCMmVxWkVqVmc'
  46 HOTKEY_SPREADSHEET_KEY = '0AqzoqbAMLyEPdE1RQXdodk1qVkFyTWtQbUxROVM1cXc'
  47 CC_OUTDIR = 'chrome/browser/ui/webui/chromeos'
  48 CC_FILENAME = 'keyboard_overlay_ui.cc'
  49 GRD_OUTDIR = 'chrome/app'
  50 GRD_FILENAME = 'chromeos_strings.grdp'
  51 JS_OUTDIR = 'chrome/browser/resources/chromeos'
  52 JS_FILENAME = 'keyboard_overlay_data.js'
  53 CC_START = r'IDS_KEYBOARD_OVERLAY_INSTRUCTIONS_HIDE },'
  54 CC_END = r'};'
  55 GRD_START = r'  <!-- BEGIN GENERATED KEYBOARD OVERLAY STRINGS -->'
  56 GRD_END = r'  <!-- END GENERATED KEYBOARD OVERLAY STRINGS -->'
  57
  58 LABEL_MAP = {
  59   'glyph_arrow_down': 'down',
  60   'glyph_arrow_left': 'left',
  61   'glyph_arrow_right': 'right',
  62   'glyph_arrow_up': 'up',
  63   'glyph_back': 'back',
  64   'glyph_backspace': 'backspace',
  65   'glyph_brightness_down': 'bright down',
  66   'glyph_brightness_up': 'bright up',
  67   'glyph_enter': 'enter',
  68   'glyph_forward': 'forward',
  69   'glyph_fullscreen': 'full screen',
  70   # Kana/Eisu key on Japanese keyboard
  71   'glyph_ime': u'\u304b\u306a\u0020\u002f\u0020\u82f1\u6570',
  72   'glyph_lock': 'lock',
  73   'glyph_overview': 'switch window',
  74   'glyph_power': 'power',
  75   'glyph_right': 'right',
  76   'glyph_reload': 'reload',
  77   'glyph_search': 'search',
  78   'glyph_shift': 'shift',
  79   'glyph_tab': 'tab',
  80   'glyph_tools': 'tools',
  81   'glyph_volume_down': 'vol. down',
  82   'glyph_volume_mute': 'mute',
  83   'glyph_volume_up': 'vol. up',
  84 };
  85
  86 INPUT_METHOD_ID_TO_OVERLAY_ID = {
  87   'm17n:ar:kbd': 'ar',
  88   'm17n:fa:isiri': 'ar',
  89   'm17n:hi:itrans': 'hi',
  90   'm17n:th:kesmanee': 'th',
  91   'm17n:th:pattachote': 'th',
  92   'm17n:th:tis820': 'th',
  93   'm17n:vi:tcvn': 'vi',
  94   'm17n:vi:telex': 'vi',
  95   'm17n:vi:viqr': 'vi',
  96   'm17n:vi:vni': 'vi',
  97   'm17n:zh:cangjie': 'zh_TW',
  98   'm17n:zh:quick': 'zh_TW',
  99   'mozc': 'en_US',
 100   'mozc-chewing': 'zh_TW',
 101   'mozc-dv': 'en_US_dvorak',
 102   'mozc-hangul': 'ko',
 103   'mozc-jp': 'ja',
 104   'pinyin': 'zh_CN',
 105   'pinyin-dv': 'en_US_dvorak',
 106   'xkb:be::fra': 'fr',
 107   'xkb:be::ger': 'de',
 108   'xkb:be::nld': 'nl',
 109   'xkb:bg::bul': 'bg',
 110   'xkb:bg:phonetic:bul': 'bg',
 111   'xkb:br::por': 'pt_BR',
 112   'xkb:ca::fra': 'fr_CA',
 113   'xkb:ca:eng:eng': 'ca',
 114   'xkb:ch::ger': 'de',
 115   'xkb:ch:fr:fra': 'fr',
 116   'xkb:cz::cze': 'cs',
 117   'xkb:de::ger': 'de',
 118   'xkb:de:neo:ger': 'de_neo',
 119   'xkb:dk::dan': 'da',
 120   'xkb:ee::est': 'et',
 121   'xkb:es::spa': 'es',
 122   'xkb:es:cat:cat': 'ca',
 123   'xkb:fi::fin': 'fi',
 124   'xkb:fr::fra': 'fr',
 125   'xkb:gb:dvorak:eng': 'en_GB_dvorak',
 126   'xkb:gb:extd:eng': 'en_GB',
 127   'xkb:gr::gre': 'el',
 128   'xkb:hr::scr': 'hr',
 129   'xkb:hu::hun': 'hu',
 130   'xkb:il::heb': 'iw',
 131   'xkb:it::ita': 'it',
 132   'xkb:jp::jpn': 'ja',
 133   'xkb:kr:kr104:kor': 'ko',
 134   'xkb:latam::spa': 'es_419',
 135   'xkb:lt::lit': 'lt',
 136   'xkb:lv:apostrophe:lav': 'lv',
 137   'xkb:no::nob': 'no',
 138   'xkb:pl::pol': 'pl',
 139   'xkb:pt::por': 'pt_PT',
 140   'xkb:ro::rum': 'ro',
 141   'xkb:rs::srp': 'sr',
 142   'xkb:ru::rus': 'ru',
 143   'xkb:ru:phonetic:rus': 'ru',
 144   'xkb:se::swe': 'sv',
 145   'xkb:si::slv': 'sl',
 146   'xkb:sk::slo': 'sk',
 147   'xkb:tr::tur': 'tr',
 148   'xkb:ua::ukr': 'uk',
 149   'xkb:us::eng': 'en_US',
 150   'xkb:us:altgr-intl:eng': 'en_US_altgr_intl',
 151   'xkb:us:colemak:eng': 'en_US_colemak',
 152   'xkb:us:dvorak:eng': 'en_US_dvorak',
 153   'xkb:us:intl:eng': 'en_US_intl',
 154   'zinnia-japanese': 'ja',
 155 }
 156
 157 # The file was first generated in 2012 and we have a policy of not updating
 158 # copyright dates.
 159 COPYRIGHT_HEADER=\
 160 """// Copyright (c) 2012 The Chromium Authors. All rights reserved.
 161 // Use of this source code is governed by a BSD-style license that can be
 162 // found in the LICENSE file.
 163
 164 // This is a generated file but may contain local modifications. See
 165 // src/tools/gen_keyboard_overlay_data/gen_keyboard_overlay_data.py --help
 166 """
 167
 168 # A snippet for grd file
 169 GRD_SNIPPET_TEMPLATE="""  <message name="%s" desc="%s">
 170     %s
 171   </message>
 172 """
 173
 174 # A snippet for C++ file
 175 CC_SNIPPET_TEMPLATE="""  { "%s", %s },
 176 """
 177
 178
 179 def SplitBehavior(behavior):
 180   """Splits the behavior to compose a message or i18n-content value.
 181
 182   Examples:
 183     'Activate last tab' => ['Activate', 'last', 'tab']
 184     'Close tab' => ['Close', 'tab']
 185   """
 186   return [x for x in re.split('[ ()"-.,]', behavior) if len(x) > 0]
 187
 188
 189 def ToMessageName(behavior):
 190   """Composes a message name for grd file.
 191
 192   Examples:
 193     'Activate last tab' => IDS_KEYBOARD_OVERLAY_ACTIVATE_LAST_TAB
 194     'Close tab' => IDS_KEYBOARD_OVERLAY_CLOSE_TAB
 195   """
 196   segments = [segment.upper() for segment in SplitBehavior(behavior)]
 197   return 'IDS_KEYBOARD_OVERLAY_' + ('_'.join(segments))
 198
 199
 200 def ToMessageDesc(description):
 201   """Composes a message description for grd file."""
 202   message_desc = 'The text in the keyboard overlay to explain the shortcut'
 203   if description:
 204     message_desc = '%s (%s).' % (message_desc, description)
 205   else:
 206     message_desc += '.'
 207   return message_desc
 208
 209
 210 def Toi18nContent(behavior):
 211   """Composes a i18n-content value for HTML/JavaScript files.
 212
 213   Examples:
 214     'Activate last tab' => keyboardOverlayActivateLastTab
 215     'Close tab' => keyboardOverlayCloseTab
 216   """
 217   segments = [segment.lower() for segment in SplitBehavior(behavior)]
 218   result = 'keyboardOverlay'
 219   for segment in segments:
 220     result += segment[0].upper() + segment[1:]
 221   return result
 222
 223
 224 def ToKeys(hotkey):
 225   """Converts the action value to shortcut keys used from JavaScript.
 226
 227   Examples:
 228     'Ctrl - 9' => '9<>CTRL'
 229     'Ctrl - Shift - Tab' => 'tab<>CTRL<>SHIFT'
 230   """
 231   values = hotkey.split(' - ')
 232   modifiers = sorted(value.upper() for value in values
 233                      if value in ['Shift', 'Ctrl', 'Alt', 'Search'])
 234   keycode = [value.lower() for value in values
 235              if value not in ['Shift', 'Ctrl', 'Alt', 'Search']]
 236   # The keys which are highlighted even without modifier keys.
 237   base_keys = ['backspace', 'power']
 238   if not modifiers and (keycode and keycode[0] not in base_keys):
 239     return None
 240   return '<>'.join(keycode + modifiers)
 241
 242
 243 def ParseOptions():
 244   """Parses the input arguemnts and returns options."""
 245   # default_username = os.getusername() + '@google.com';
 246   default_username = '%s@google.com' % os.environ.get('USER')
 247   parser = optparse.OptionParser()
 248   parser.add_option('--key', dest='key',
 249                     help='The key of the spreadsheet (required).')
 250   parser.add_option('--username', dest='username',
 251                     default=default_username,
 252                     help='Your user name (default: %s).' % default_username)
 253   parser.add_option('--password', dest='password',
 254                     help='Your password.')
 255   parser.add_option('--account_type', default='GOOGLE', dest='account_type',
 256                     help='Account type used for gdata login (default: GOOGLE)')
 257   parser.add_option('--js', dest='js', default=False, action='store_true',
 258                     help='Output js file.')
 259   parser.add_option('--grd', dest='grd', default=False, action='store_true',
 260                     help='Output resource file.')
 261   parser.add_option('--cc', dest='cc', default=False, action='store_true',
 262                     help='Output cc file.')
 263   parser.add_option('--outdir', dest='outdir', default=None,
 264                     help='Specify the directory files are generated.')
 265   (options, unused_args) = parser.parse_args()
 266
 267   if not options.username.endswith('google.com'):
 268     print 'google.com account is necessary to use this script.'
 269     sys.exit(-1)
 270
 271   if (not (options.js or options.grd or options.cc)):
 272     print 'Either --js, --grd, or --cc needs to be specified.'
 273     sys.exit(-1)
 274
 275   # Get the password from the terminal, if needed.
 276   if not options.password:
 277     options.password = getpass.getpass(
 278         'Application specific password for %s: ' % options.username)
 279   return options
 280
 281
 282 def InitClient(options):
 283   """Initializes the spreadsheet client."""
 284   client = gdata.spreadsheet.service.SpreadsheetsService()
 285   client.email = options.username
 286   client.password = options.password
 287   client.source = 'Spread Sheet'
 288   client.account_type = options.account_type
 289   print 'Logging in as %s (%s)' % (client.email, client.account_type)
 290   client.ProgrammaticLogin()
 291   return client
 292
 293
 294 def PrintDiffs(message, lhs, rhs):
 295   """Prints the differences between |lhs| and |rhs|."""
 296   dif = set(lhs).difference(rhs)
 297   if dif:
 298     print message, ', '.join(dif)
 299
 300
 301 def FetchSpreadsheetFeeds(client, key, sheets, cols):
 302   """Fetch feeds from the spreadsheet.
 303
 304   Args:
 305     client: A spreadsheet client to be used for fetching data.
 306     key: A key string of the spreadsheet to be fetched.
 307     sheets: A list of the sheet names to read data from.
 308     cols: A list of columns to read data from.
 309   """
 310   worksheets_feed = client.GetWorksheetsFeed(key)
 311   print 'Fetching data from the worksheet: %s' % worksheets_feed.title.text
 312   worksheets_data = {}
 313   titles = []
 314   for entry in worksheets_feed.entry:
 315     worksheet_id = entry.id.text.split('/')[-1]
 316     list_feed = client.GetListFeed(key, worksheet_id)
 317     list_data = []
 318     # Hack to deal with sheet names like 'sv (Copy of fl)'
 319     title = list_feed.title.text.split('(')[0].strip()
 320     titles.append(title)
 321     if title not in sheets:
 322       continue
 323     print 'Reading data from the sheet: %s' % list_feed.title.text
 324     for i, entry in enumerate(list_feed.entry):
 325       line_data = {}
 326       for k in entry.custom:
 327         if (k not in cols) or (not entry.custom[k].text):
 328           continue
 329         line_data[k] = entry.custom[k].text
 330       list_data.append(line_data)
 331     worksheets_data[title] = list_data
 332   PrintDiffs('Exist only on the spreadsheet: ', titles, sheets)
 333   PrintDiffs('Specified but do not exist on the spreadsheet: ', sheets, titles)
 334   return worksheets_data
 335
 336
 337 def FetchKeyboardGlyphData(client):
 338   """Fetches the keyboard glyph data from the spreadsheet."""
 339   glyph_cols = ['scancode', 'p0', 'p1', 'p2', 'p3', 'p4', 'p5', 'p6', 'p7',
 340                 'p8', 'p9', 'label', 'format', 'notes']
 341   keyboard_glyph_data = FetchSpreadsheetFeeds(
 342       client, KEYBOARD_GLYPH_SPREADSHEET_KEY,
 343       INPUT_METHOD_ID_TO_OVERLAY_ID.values(), glyph_cols)
 344   ret = {}
 345   for lang in keyboard_glyph_data:
 346     ret[lang] = {}
 347     keys = {}
 348     for line in keyboard_glyph_data[lang]:
 349       scancode = line.get('scancode')
 350       if (not scancode) and line.get('notes'):
 351         ret[lang]['layoutName'] = line['notes']
 352         continue
 353       del line['scancode']
 354       if 'notes' in line:
 355         del line['notes']
 356       if 'label' in line:
 357         line['label'] = LABEL_MAP.get(line['label'], line['label'])
 358       keys[scancode] = line
 359     # Add a label to space key
 360     if '39' not in keys:
 361       keys['39'] = {'label': 'space'}
 362     ret[lang]['keys'] = keys
 363   return ret
 364
 365
 366 def FetchLayoutsData(client):
 367   """Fetches the keyboard glyph data from the spreadsheet."""
 368   layout_names = ['U_layout', 'J_layout', 'E_layout', 'B_layout']
 369   cols = ['scancode', 'x', 'y', 'w', 'h']
 370   layouts = FetchSpreadsheetFeeds(client, KEYBOARD_GLYPH_SPREADSHEET_KEY,
 371                                   layout_names, cols)
 372   ret = {}
 373   for layout_name, layout in layouts.items():
 374     ret[layout_name[0]] = []
 375     for row in layout:
 376       line = []
 377       for col in cols:
 378         value = row.get(col)
 379         if not value:
 380           line.append('')
 381         else:
 382           if col != 'scancode':
 383             value = float(value)
 384           line.append(value)
 385       ret[layout_name[0]].append(line)
 386   return ret
 387
 388
 389 def FetchHotkeyData(client):
 390   """Fetches the hotkey data from the spreadsheet."""
 391   hotkey_sheet = ['Cross Platform Behaviors']
 392   hotkey_cols = ['behavior', 'context', 'kind', 'actionctrlctrlcmdonmac',
 393                  'chromeos', 'descriptionfortranslation']
 394   hotkey_data = FetchSpreadsheetFeeds(client, HOTKEY_SPREADSHEET_KEY,
 395                                       hotkey_sheet, hotkey_cols)
 396   action_to_id = {}
 397   id_to_behavior = {}
 398   # (behavior, action)
 399   result = []
 400   for line in hotkey_data['Cross Platform Behaviors']:
 401     if (not line.get('chromeos')) or (line.get('kind') != 'Key'):
 402       continue
 403     action = ToKeys(line['actionctrlctrlcmdonmac'])
 404     if not action:
 405       continue
 406     behavior = line['behavior'].strip()
 407     description = line.get('descriptionfortranslation')
 408     result.append((behavior, action, description))
 409   return result
 410
 411
 412 def UniqueBehaviors(hotkey_data):
 413   """Retrieves a sorted list of unique behaviors from |hotkey_data|."""
 414   return sorted(set((behavior, description) for (behavior, _, description)
 415                     in hotkey_data),
 416                 cmp=lambda x, y: cmp(ToMessageName(x[0]), ToMessageName(y[0])))
 417
 418
 419 def GetPath(path_from_src):
 420   """Returns the absolute path of the specified path."""
 421   path = os.path.join(os.path.dirname(__file__), '../..', path_from_src)
 422   if not os.path.isfile(path):
 423     print 'WARNING: %s does not exist. Maybe moved or renamed?' % path
 424   return path
 425
 426
 427 def OutputFile(outpath, snippet):
 428   """Output the snippet into the specified path."""
 429   out = file(outpath, 'w')
 430   out.write(COPYRIGHT_HEADER + '\n')
 431   out.write(snippet)
 432   print 'Output ' + os.path.normpath(outpath)
 433
 434
 435 def RewriteFile(start, end, original_dir, original_filename, snippet,
 436                 outdir=None):
 437   """Replaces a part of the specified file with snippet and outputs it."""
 438   original_path = GetPath(os.path.join(original_dir, original_filename))
 439   original = file(original_path, 'r')
 440   original_content = original.read()
 441   original.close()
 442   if outdir:
 443     outpath = os.path.join(outdir, original_filename)
 444   else:
 445     outpath = original_path
 446   out = file(outpath, 'w')
 447   rx = re.compile(r'%s\n.*?%s\n' % (re.escape(start), re.escape(end)),
 448                   re.DOTALL)
 449   new_content = re.sub(rx, '%s\n%s%s\n' % (start, snippet, end),
 450                        original_content)
 451   out.write(new_content)
 452   out.close()
 453   print 'Output ' + os.path.normpath(outpath)
 454
 455
 456 def OutputJson(keyboard_glyph_data, hotkey_data, layouts, var_name, outdir):
 457   """Outputs the keyboard overlay data as a JSON file."""
 458   action_to_id = {}
 459   for (behavior, action, _) in hotkey_data:
 460     i18nContent = Toi18nContent(behavior)
 461     action_to_id[action] = i18nContent
 462   data = {'keyboardGlyph': keyboard_glyph_data,
 463           'shortcut': action_to_id,
 464           'layouts': layouts,
 465           'inputMethodIdToOverlayId': INPUT_METHOD_ID_TO_OVERLAY_ID}
 466
 467   if not outdir:
 468     outdir = JS_OUTDIR
 469   outpath = GetPath(os.path.join(outdir, JS_FILENAME))
 470   json_data =  json.dumps(data, sort_keys=True, indent=2)
 471   # Remove redundant spaces after ','
 472   json_data = json_data.replace(', \n', ',\n')
 473   # Replace double quotes with single quotes to avoid lint warnings.
 474   json_data = json_data.replace('\"', '\'')
 475   snippet = 'var %s = %s;\n' % (var_name, json_data)
 476   OutputFile(outpath, snippet)
 477
 478
 479 def OutputGrd(hotkey_data, outdir):
 480   """Outputs a part of messages in the grd file."""
 481   snippet = cStringIO.StringIO()
 482   for (behavior, description) in UniqueBehaviors(hotkey_data):
 483     # Do not generate message for 'Show wrench menu'. It is handled manually
 484     # based on branding.
 485     if behavior == 'Show wrench menu':
 486       continue
 487     snippet.write(GRD_SNIPPET_TEMPLATE %
 488                   (ToMessageName(behavior), ToMessageDesc(description),
 489                    behavior))
 490
 491   RewriteFile(GRD_START, GRD_END, GRD_OUTDIR, GRD_FILENAME, snippet.getvalue(),
 492               outdir)
 493
 494
 495 def OutputCC(hotkey_data, outdir):
 496   """Outputs a part of code in the C++ file."""
 497   snippet = cStringIO.StringIO()
 498   for (behavior, _) in UniqueBehaviors(hotkey_data):
 499     message_name = ToMessageName(behavior)
 500     output = CC_SNIPPET_TEMPLATE % (Toi18nContent(behavior), message_name)
 501     # Break the line if the line is longer than 80 characters
 502     if len(output) > 80:
 503       output = output.replace(' ' + message_name, '\n    %s' % message_name)
 504     snippet.write(output)
 505
 506   RewriteFile(CC_START, CC_END, CC_OUTDIR, CC_FILENAME, snippet.getvalue(),
 507               outdir)
 508
 509
 510 def main():
 511   options = ParseOptions()
 512   client = InitClient(options)
 513   hotkey_data = FetchHotkeyData(client)
 514
 515   if options.js:
 516     keyboard_glyph_data = FetchKeyboardGlyphData(client)
 517
 518   if options.js:
 519     layouts = FetchLayoutsData(client)
 520     OutputJson(keyboard_glyph_data, hotkey_data, layouts, 'keyboardOverlayData',
 521                options.outdir)
 522   if options.grd:
 523     OutputGrd(hotkey_data, options.outdir)
 524   if options.cc:
 525     OutputCC(hotkey_data, options.outdir)
 526
 527
 528 if __name__ == '__main__':
 529   main()