remoting/webapp/build-webapp.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Creates a directory with with the unpacked contents of the remoting webapp.
   7
   8 The directory will contain a copy-of or a link-to to all remoting webapp
   9 resources.  This includes HTML/JS and any plugin binaries. The script also
  10 massages resulting files appropriately with host plugin data. Finally,
  11 a zip archive for all of the above is produced.
  12 """
  13
  14 # Python 2.5 compatibility
  15 from __future__ import with_statement
  16
  17 import argparse
  18 import io
  19 import os
  20 import platform
  21 import re
  22 import shutil
  23 import subprocess
  24 import sys
  25 import time
  26 import zipfile
  27
  28 # Update the module path, assuming that this script is in src/remoting/webapp,
  29 # and that the google_api_keys module is in src/google_apis. Note that
  30 # sys.path[0] refers to the directory containing this script.
  31 if __name__ == '__main__':
  32   sys.path.append(
  33       os.path.abspath(os.path.join(sys.path[0], '../../google_apis')))
  34 import google_api_keys
  35
  36
  37 def findAndReplace(filepath, findString, replaceString):
  38   """Does a search and replace on the contents of a file."""
  39   oldFilename = os.path.basename(filepath) + '.old'
  40   oldFilepath = os.path.join(os.path.dirname(filepath), oldFilename)
  41   os.rename(filepath, oldFilepath)
  42   with open(oldFilepath) as input:
  43     with open(filepath, 'w') as output:
  44       for s in input:
  45         output.write(s.replace(findString, replaceString))
  46   os.remove(oldFilepath)
  47
  48
  49 def createZip(zip_path, directory):
  50   """Creates a zipfile at zip_path for the given directory."""
  51   zipfile_base = os.path.splitext(os.path.basename(zip_path))[0]
  52   zip = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
  53   for (root, dirs, files) in os.walk(directory):
  54     for f in files:
  55       full_path = os.path.join(root, f)
  56       rel_path = os.path.relpath(full_path, directory)
  57       zip.write(full_path, os.path.join(zipfile_base, rel_path))
  58   zip.close()
  59
  60
  61 def replaceString(destination, placeholder, value):
  62   findAndReplace(os.path.join(destination, 'plugin_settings.js'),
  63                  "'" + placeholder + "'", "'" + value + "'")
  64
  65
  66 def replaceBool(destination, placeholder, value):
  67   # Look for a "!!" in the source code so the expession we're
  68   # replacing looks like a boolean to the compiler.  A single "!"
  69   # would satisfy the compiler but might confused human readers.
  70   findAndReplace(os.path.join(destination, 'plugin_settings.js'),
  71                  "!!'" + placeholder + "'", 'true' if value else 'false')
  72
  73
  74 def parseBool(boolStr):
  75   """Tries to parse a string as a boolean value.
  76
  77   Returns a bool on success; raises ValueError on failure.
  78   """
  79   lower = boolStr.lower()
  80   if lower in ['0', 'false']: return False
  81   if lower in ['1', 'true']: return True
  82   raise ValueError('not a boolean string {!r}'.format(boolStr))
  83
  84
  85 def getenvBool(name, defaultValue):
  86   """Gets an environment value as a boolean."""
  87   rawValue = os.environ.get(name)
  88   if rawValue is None:
  89     return defaultValue
  90   try:
  91     return parseBool(rawValue)
  92   except ValueError:
  93     raise Exception('Value of ${} must be boolean!'.format(name))
  94
  95
  96 def processJinjaTemplate(input_file, include_paths, output_file, context):
  97   jinja2_path = os.path.normpath(
  98       os.path.join(os.path.abspath(__file__),
  99                    '../../../third_party/jinja2'))
 100   sys.path.append(os.path.split(jinja2_path)[0])
 101   import jinja2
 102   (template_path, template_name) = os.path.split(input_file)
 103   include_paths = [template_path] + include_paths
 104   env = jinja2.Environment(loader=jinja2.FileSystemLoader(include_paths))
 105   template = env.get_template(template_name)
 106   rendered = template.render(context)
 107   io.open(output_file, 'w', encoding='utf-8').write(rendered)
 108
 109
 110 def buildWebApp(buildtype, version, destination, zip_path,
 111                 manifest_template, webapp_type, appid, app_client_id, app_name,
 112                 app_description, app_capabilities, manifest_key, files,
 113                 files_listfile, locales_listfile, jinja_paths,
 114                 service_environment, use_gcd):
 115   """Does the main work of building the webapp directory and zipfile.
 116
 117   Args:
 118     buildtype: the type of build ("Official", "Release" or "Dev").
 119     destination: A string with path to directory where the webapp will be
 120                  written.
 121     zipfile: A string with path to the zipfile to create containing the
 122              contents of |destination|.
 123     manifest_template: jinja2 template file for manifest.
 124     webapp_type: webapp type:
 125                  For DesktopRemoting: "desktop"
 126                  For AppRemoting: "app_remoting" or "shared_module"
 127     appid: A string with the Remoting Application Id (only used for app
 128            remoting webapps). If supplied, it defaults to using the
 129            test API server.
 130     app_client_id: The OAuth2 client ID for the webapp.
 131     app_name: A string with the name of the application.
 132     app_description: A string with the description of the application.
 133     app_capabilities: A set of strings naming the capabilities that should be
 134                       enabled for this application.
 135     manifest_key: The manifest key for the webapp.
 136     files: An array of strings listing the paths for resources to include
 137            in this webapp.
 138     files_listfile: The name of a file containing a list of files, one per
 139                     line, identifying the resources to include in this webapp.
 140                     This is an alternate to specifying the files directly via
 141                     the 'files' option. The files listed in this file are
 142                     appended to the files passed via the 'files' option, if any.
 143     locales_listfile: The name of a file containing a list of locales, one per
 144                       line, which are copied, along with their directory
 145                       structure, from the _locales directory down.
 146     jinja_paths: An array of paths to search for {%include} directives in
 147                  addition to the directory containing the manifest template.
 148     service_environment: Used to point the webapp to one of the
 149                          dev/test/staging/vendor/prod/prod-testing environments
 150     use_gcd: True if GCD support should be enabled.
 151   """
 152
 153   # Load the locales files from the locales_listfile.
 154   if not locales_listfile:
 155     raise Exception('You must specify a locales_listfile')
 156   locales = []
 157   with open(locales_listfile) as input:
 158     for s in input:
 159       locales.append(s.rstrip())
 160
 161   # Load the files from the files_listfile.
 162   if files_listfile:
 163     with open(files_listfile) as input:
 164       for s in input:
 165         files.append(s.rstrip())
 166
 167   # Ensure a fresh directory.
 168   try:
 169     shutil.rmtree(destination)
 170   except OSError:
 171     if os.path.exists(destination):
 172       raise
 173     else:
 174       pass
 175   os.makedirs(destination, 0775)
 176
 177   if buildtype != 'Official' and buildtype != 'Release' and buildtype != 'Dev':
 178     raise Exception('Unknown buildtype: ' + buildtype)
 179
 180   jinja_context = {
 181     'webapp_type': webapp_type,
 182     'buildtype': buildtype,
 183   }
 184
 185   # Copy all the files.
 186   for current_file in files:
 187     destination_file = os.path.join(destination, os.path.basename(current_file))
 188
 189     # Process *.jinja2 files as jinja2 templates
 190     if current_file.endswith(".jinja2"):
 191       destination_file = destination_file[:-len(".jinja2")]
 192       processJinjaTemplate(current_file, jinja_paths,
 193                            destination_file, jinja_context)
 194     else:
 195       shutil.copy2(current_file, destination_file)
 196
 197   # Copy all the locales, preserving directory structure
 198   destination_locales = os.path.join(destination, '_locales')
 199   os.mkdir(destination_locales, 0775)
 200   remoting_locales = os.path.join(destination, 'remoting_locales')
 201   os.mkdir(remoting_locales, 0775)
 202   for current_locale in locales:
 203     extension = os.path.splitext(current_locale)[1]
 204     if extension == '.json':
 205       locale_id = os.path.split(os.path.split(current_locale)[0])[1]
 206       destination_dir = os.path.join(destination_locales, locale_id)
 207       destination_file = os.path.join(destination_dir,
 208                                       os.path.split(current_locale)[1])
 209       os.mkdir(destination_dir, 0775)
 210       shutil.copy2(current_locale, destination_file)
 211     elif extension == '.pak':
 212       destination_file = os.path.join(remoting_locales,
 213                                       os.path.split(current_locale)[1])
 214       shutil.copy2(current_locale, destination_file)
 215     else:
 216       raise Exception('Unknown extension: ' + current_locale)
 217
 218   is_app_remoting_webapp = webapp_type == 'app_remoting'
 219   is_app_remoting_shared_module = webapp_type == 'shared_module'
 220   is_app_remoting = is_app_remoting_webapp or is_app_remoting_shared_module
 221   is_prod_service_environment = service_environment == 'vendor' or \
 222                                 service_environment == 'prod' or \
 223                                 service_environment == 'prod-testing'
 224   is_desktop_remoting = not is_app_remoting
 225
 226   # Allow host names for google services/apis to be overriden via env vars.
 227   oauth2AccountsHost = os.environ.get(
 228       'OAUTH2_ACCOUNTS_HOST', 'https://accounts.google.com')
 229   oauth2ApiHost = os.environ.get(
 230       'OAUTH2_API_HOST', 'https://www.googleapis.com')
 231   directoryApiHost = os.environ.get(
 232       'DIRECTORY_API_HOST', 'https://www.googleapis.com')
 233   remotingApiHost = os.environ.get(
 234       'REMOTING_API_HOST', 'https://remoting-pa.googleapis.com')
 235
 236   if is_app_remoting:
 237     appRemotingApiHost = os.environ.get(
 238         'APP_REMOTING_API_HOST', None)
 239
 240   if is_app_remoting_webapp:
 241     appRemotingApplicationId = os.environ.get(
 242         'APP_REMOTING_APPLICATION_ID', None)
 243
 244     # Release/Official builds are special because they are what we will upload
 245     # to the web store.  The checks below will validate that prod builds are
 246     # being generated correctly (no overrides) and with the correct buildtype.
 247     # They also verify that folks are not accidentally building dev/test/staging
 248     # apps for release (no impersonation) instead of dev.
 249     if is_prod_service_environment and buildtype == 'Dev':
 250       raise Exception("Prod environment cannot be built for 'dev' builds")
 251
 252     if buildtype != 'Dev':
 253       if not is_prod_service_environment:
 254         raise Exception('Invalid service_environment targeted for '
 255                         + buildtype + ': ' + service_environment)
 256       if appid != None:
 257         raise Exception('Cannot pass in an appid for '
 258                         + buildtype + ' builds: ' + service_environment)
 259       if appRemotingApiHost != None:
 260         raise Exception('Cannot set APP_REMOTING_API_HOST env var for '
 261                         + buildtype + ' builds')
 262       if appRemotingApplicationId != None:
 263         raise Exception('Cannot set APP_REMOTING_APPLICATION_ID env var for '
 264                         + buildtype + ' builds')
 265
 266     # If an Application ID was set (either from service_environment variable or
 267     # from a command line argument), hardcode it, otherwise get it at runtime.
 268     effectiveAppId = appRemotingApplicationId or appid
 269     if effectiveAppId:
 270       appRemotingApplicationId = "'" + effectiveAppId + "'"
 271     else:
 272       appRemotingApplicationId = "chrome.i18n.getMessage('@@extension_id')"
 273     findAndReplace(os.path.join(destination, 'arv_main.js'),
 274                    "'APP_REMOTING_APPLICATION_ID'", appRemotingApplicationId)
 275
 276   oauth2BaseUrl = oauth2AccountsHost + '/o/oauth2'
 277   oauth2ApiBaseUrl = oauth2ApiHost + '/oauth2'
 278   directoryApiBaseUrl = directoryApiHost + '/chromoting/v1'
 279   telemetryApiBaseUrl = remotingApiHost + '/v1/events'
 280
 281   if is_app_remoting:
 282     # Set the apiary endpoint and then set the endpoint version
 283     if not appRemotingApiHost:
 284       if is_prod_service_environment:
 285         appRemotingApiHost = 'https://www.googleapis.com'
 286       else:
 287         appRemotingApiHost = 'https://www-googleapis-test.sandbox.google.com'
 288
 289     # TODO(garykac) Currently, the shared module is always set up for the
 290     # dev service_environment. Update build so that the dev environment can
 291     # be controlled by the app stub rather than hard-coded into the shared
 292     # module.
 293     if service_environment == 'dev' or is_app_remoting_shared_module:
 294       appRemotingServicePath = '/appremoting/v1beta1_dev'
 295     elif service_environment == 'test':
 296       appRemotingServicePath = '/appremoting/v1beta1'
 297     elif service_environment == 'staging':
 298       appRemotingServicePath = '/appremoting/v1beta1_staging'
 299     elif service_environment == 'vendor':
 300       appRemotingServicePath = '/appremoting/v1beta1_vendor'
 301     elif service_environment == 'prod':
 302       appRemotingServicePath = '/appremoting/v1beta1'
 303     elif service_environment == 'prod-testing':
 304       appRemotingServicePath = '/appremoting/v1beta1_prod_testing'
 305     else:
 306       raise Exception('Unknown service environment: ' + service_environment)
 307     appRemotingApiBaseUrl = appRemotingApiHost + appRemotingServicePath
 308   else:
 309     appRemotingApiBaseUrl = ''
 310
 311   # TODO(garykac) replaceString (et al.) implictly update plugin_settings.js,
 312   # which doesn't exist for the app stub. We need to move app-specific
 313   # AppRemoting options into arv_main.js.
 314   if not is_app_remoting_webapp:
 315     replaceBool(destination, 'USE_GCD', use_gcd)
 316     replaceString(destination, 'OAUTH2_BASE_URL', oauth2BaseUrl)
 317     replaceString(destination, 'OAUTH2_API_BASE_URL', oauth2ApiBaseUrl)
 318     replaceString(destination, 'DIRECTORY_API_BASE_URL', directoryApiBaseUrl)
 319     replaceString(destination, 'TELEMETRY_API_BASE_URL', telemetryApiBaseUrl)
 320     if is_app_remoting:
 321       replaceString(destination, 'APP_REMOTING_API_BASE_URL',
 322                     appRemotingApiBaseUrl)
 323
 324   # Substitute hosts in the manifest's CSP list.
 325   # Ensure we list the API host only once if it's the same for multiple APIs.
 326   googleApiHosts = ' '.join(set([oauth2ApiHost, directoryApiHost]))
 327
 328   # WCS and the OAuth trampoline are both hosted on talkgadget. Split them into
 329   # separate suffix/prefix variables to allow for wildcards in manifest.json.
 330   talkGadgetHostSuffix = os.environ.get(
 331       'TALK_GADGET_HOST_SUFFIX', 'talkgadget.google.com')
 332   talkGadgetHostPrefix = os.environ.get(
 333       'TALK_GADGET_HOST_PREFIX', 'https://chromoting-client.')
 334   oauth2RedirectHostPrefix = os.environ.get(
 335       'OAUTH2_REDIRECT_HOST_PREFIX', 'https://chromoting-oauth.')
 336
 337   # Use a wildcard in the manifest.json host specs if the prefixes differ.
 338   talkGadgetHostJs = talkGadgetHostPrefix + talkGadgetHostSuffix
 339   talkGadgetBaseUrl = talkGadgetHostJs + '/talkgadget'
 340   if talkGadgetHostPrefix == oauth2RedirectHostPrefix:
 341     talkGadgetHostJson = talkGadgetHostJs
 342   else:
 343     talkGadgetHostJson = 'https://*.' + talkGadgetHostSuffix
 344
 345   # Set the correct OAuth2 redirect URL.
 346   oauth2RedirectHostJs = oauth2RedirectHostPrefix + talkGadgetHostSuffix
 347   oauth2RedirectHostJson = talkGadgetHostJson
 348   oauth2RedirectPath = '/talkgadget/oauth/chrome-remote-desktop'
 349   oauth2RedirectBaseUrlJs = oauth2RedirectHostJs + oauth2RedirectPath
 350   oauth2RedirectBaseUrlJson = oauth2RedirectHostJson + oauth2RedirectPath
 351   if buildtype == 'Official':
 352     oauth2RedirectUrlJs = ("'" + oauth2RedirectBaseUrlJs +
 353                            "/rel/' + chrome.i18n.getMessage('@@extension_id')")
 354     oauth2RedirectUrlJson = oauth2RedirectBaseUrlJson + '/rel/*'
 355   else:
 356     oauth2RedirectUrlJs = "'" + oauth2RedirectBaseUrlJs + "/dev'"
 357     oauth2RedirectUrlJson = oauth2RedirectBaseUrlJson + '/dev*'
 358   thirdPartyAuthUrlJs = oauth2RedirectBaseUrlJs + '/thirdpartyauth'
 359   thirdPartyAuthUrlJson = oauth2RedirectBaseUrlJson + '/thirdpartyauth*'
 360   xmppServer = os.environ.get('XMPP_SERVER', 'talk.google.com:443')
 361
 362   if not is_app_remoting_webapp:
 363     replaceString(destination, 'TALK_GADGET_URL', talkGadgetBaseUrl)
 364     findAndReplace(os.path.join(destination, 'plugin_settings.js'),
 365                    "'OAUTH2_REDIRECT_URL'", oauth2RedirectUrlJs)
 366
 367     # Configure xmpp server and directory bot settings in the plugin.
 368     xmpp_server_user_tls = getenvBool('XMPP_SERVER_USE_TLS', True)
 369     if (buildtype != 'Dev' and not xmpp_server_user_tls):
 370       raise Exception('TLS can must be enabled in non Dev builds.')
 371
 372     replaceBool(
 373         destination, 'XMPP_SERVER_USE_TLS', xmpp_server_user_tls)
 374     replaceString(destination, 'XMPP_SERVER', xmppServer)
 375     replaceString(destination, 'DIRECTORY_BOT_JID',
 376                   os.environ.get('DIRECTORY_BOT_JID',
 377                                  'remoting@bot.talk.google.com'))
 378     replaceString(destination, 'THIRD_PARTY_AUTH_REDIRECT_URL',
 379                   thirdPartyAuthUrlJs)
 380
 381   # Set the correct API keys.
 382   # For overriding the client ID/secret via env vars, see google_api_keys.py.
 383   apiClientId = google_api_keys.GetClientID('REMOTING')
 384   apiClientSecret = google_api_keys.GetClientSecret('REMOTING')
 385   apiKey = google_api_keys.GetAPIKeyRemoting()
 386
 387   if is_app_remoting_webapp and buildtype != 'Dev':
 388     if not app_client_id:
 389       raise Exception('Invalid app_client_id passed in: "' +
 390           app_client_id + '"')
 391     apiClientIdV2 = app_client_id + '.apps.googleusercontent.com'
 392   else:
 393     apiClientIdV2 = os.environ.get(
 394         'REMOTING_IDENTITY_API_CLIENT_ID',
 395         google_api_keys.GetClientID('REMOTING_IDENTITY_API'))
 396
 397   if not is_app_remoting_webapp:
 398     replaceString(destination, 'API_CLIENT_ID', apiClientId)
 399     replaceString(destination, 'API_CLIENT_SECRET', apiClientSecret)
 400     replaceString(destination, 'API_KEY', apiKey)
 401
 402   # Write the application capabilities.
 403   if is_app_remoting_webapp:
 404     appCapabilities = ','.join(
 405         ['remoting.ClientSession.Capability.' + x for x in app_capabilities])
 406     findAndReplace(os.path.join(destination, 'arv_main.js'),
 407                    "'APPLICATION_CAPABILITIES'", appCapabilities)
 408
 409   # Official AppRemoting builds get the key from the gyp/gn build file. All
 410   # other builds use a fixed key. For dev builds, this ensures that the app
 411   # can be run directly from the output directory. For official CRD builds,
 412   # it allows QA to test the app without uploading it to Chrome Web Store.
 413   if is_app_remoting_webapp and buildtype != 'Dev':
 414     if not manifest_key:
 415       raise Exception('No manifest_key passed in')
 416   else:
 417       manifest_key = 'remotingdevbuild'
 418
 419   # Generate manifest.
 420   if manifest_template:
 421     context = {
 422         'webapp_type': webapp_type,
 423         'FULL_APP_VERSION': version,
 424         'MANIFEST_KEY': manifest_key,
 425         'OAUTH2_REDIRECT_URL': oauth2RedirectUrlJson,
 426         'TALK_GADGET_HOST': talkGadgetHostJson,
 427         'THIRD_PARTY_AUTH_REDIRECT_URL': thirdPartyAuthUrlJson,
 428         'REMOTING_IDENTITY_API_CLIENT_ID': apiClientIdV2,
 429         'OAUTH2_BASE_URL': oauth2BaseUrl,
 430         'OAUTH2_API_BASE_URL': oauth2ApiBaseUrl,
 431         'DIRECTORY_API_BASE_URL': directoryApiBaseUrl,
 432         'TELEMETRY_API_BASE_URL':telemetryApiBaseUrl ,
 433         'APP_REMOTING_API_BASE_URL': appRemotingApiBaseUrl,
 434         'CLOUD_PRINT_URL': '',
 435         'OAUTH2_ACCOUNTS_HOST': oauth2AccountsHost,
 436         'GOOGLE_API_HOSTS': googleApiHosts,
 437         'APP_NAME': app_name,
 438         'APP_DESCRIPTION': app_description,
 439         'OAUTH_CLOUD_PRINT_SCOPE': '',
 440         'OAUTH_GDRIVE_SCOPE': '',
 441         'USE_GCD': use_gcd,
 442         'XMPP_SERVER': xmppServer,
 443         # An URL match pattern that is added to the |permissions| section of the
 444         # manifest in case some URLs are redirected by corporate proxies.
 445         'PROXY_URL' : os.environ.get('PROXY_URL', ''),
 446     }
 447     if 'CLOUD_PRINT' in app_capabilities:
 448       context['OAUTH_CLOUD_PRINT_SCOPE'] = ('"https://www.googleapis.com/auth/cloudprint",')
 449       context['CLOUD_PRINT_URL'] = ('"https://www.google.com/cloudprint/*",')
 450     if 'GOOGLE_DRIVE' in app_capabilities:
 451       context['OAUTH_GDRIVE_SCOPE'] = ('"https://docs.google.com/feeds/", '
 452                                        '"https://www.googleapis.com/auth/drive",')
 453     processJinjaTemplate(manifest_template,
 454                          jinja_paths,
 455                          os.path.join(destination, 'manifest.json'),
 456                          context)
 457
 458   # Make the zipfile.
 459   createZip(zip_path, destination)
 460
 461   return 0
 462
 463
 464 def main():
 465   parser = argparse.ArgumentParser()
 466   parser.add_argument('buildtype')
 467   parser.add_argument('version')
 468   parser.add_argument('destination')
 469   parser.add_argument('zip_path')
 470   parser.add_argument('manifest_template')
 471   parser.add_argument('webapp_type')
 472   parser.add_argument('files', nargs='*', metavar='file', default=[])
 473   parser.add_argument('--app_name', metavar='NAME')
 474   parser.add_argument('--app_description', metavar='TEXT')
 475   parser.add_argument('--app_capabilities',
 476                       nargs='*', default=[], metavar='CAPABILITY')
 477   parser.add_argument('--appid')
 478   parser.add_argument('--app_client_id', default='')
 479   parser.add_argument('--manifest_key', default='')
 480   parser.add_argument('--files_listfile', default='', metavar='PATH')
 481   parser.add_argument('--locales_listfile', default='', metavar='PATH')
 482   parser.add_argument('--jinja_paths', nargs='*', default=[], metavar='PATH')
 483   parser.add_argument('--service_environment', default='', metavar='ENV')
 484   parser.add_argument('--use_gcd', choices=['0', '1'], default='0')
 485
 486   args = parser.parse_args()
 487   args.use_gcd = (args.use_gcd != '0')
 488   args.app_capabilities = set(args.app_capabilities)
 489   return buildWebApp(**vars(args))
 490
 491
 492 if __name__ == '__main__':
 493   sys.exit(main())