chrome/common/extensions/docs/server2/update_cache.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import cPickle
   6 import copy
   7 import getopt
   8 import json
   9 import logging
  10 import os.path
  11 import sys
  12 import traceback
  13
  14 from branch_utility import BranchUtility
  15 from commit_tracker import CommitTracker
  16 from compiled_file_system import CompiledFileSystem
  17 from data_source_registry import CreateDataSources
  18 from environment import GetAppVersion
  19 from environment_wrappers import CreateUrlFetcher, GetAccessToken
  20 from future import All
  21 from gcs_file_system_provider import CloudStorageFileSystemProvider
  22 from host_file_system_provider import HostFileSystemProvider
  23 from local_git_util import ParseRevision
  24 from object_store_creator import ObjectStoreCreator
  25 from persistent_object_store_fake import PersistentObjectStoreFake
  26 from render_refresher import RenderRefresher
  27 from server_instance import ServerInstance
  28 from timer import Timer
  29
  30
  31 # The path template to use for flushing the memcached. Should be formatted with
  32 # with the app version.
  33 _FLUSH_MEMCACHE_PATH = ('https://%s-dot-chrome-apps-doc.appspot.com/'
  34                         '_flush_memcache')
  35
  36
  37 def _UpdateCommitId(commit_name, commit_id):
  38   '''Sets the commit ID for a named commit. This is the final step performed
  39   during update. Once all the appropriate datastore entries have been populated
  40   for a new commit ID, the 'master' commit entry is updated to that ID and the
  41   frontend will begin serving the new data.
  42
  43   Note that this requires an access token identifying the main service account
  44   for the chrome-apps-doc project. VM instances will get this automatically
  45   from their environment, but if you want to do a local push to prod you will
  46   need to set the DOCSERVER_ACCESS_TOKEN environment variable appropriately.
  47   '''
  48   commit_tracker = CommitTracker(
  49       ObjectStoreCreator(store_type=PersistentObjectStoreFake,
  50                          start_empty=False))
  51   commit_tracker.Set(commit_name, commit_id).Get()
  52   logging.info('Commit "%s" updated to %s.' % (commit_name, commit_id))
  53
  54
  55 def _GetCachedCommitId(commit_name):
  56   '''Determines which commit ID was last cached.
  57   '''
  58   commit_tracker = CommitTracker(
  59       ObjectStoreCreator(store_type=PersistentObjectStoreFake,
  60                          start_empty=False))
  61   return commit_tracker.Get(commit_name).Get()
  62
  63
  64 def _FlushMemcache():
  65   '''Requests that the frontend flush its memcached to avoid serving stale data.
  66   '''
  67   flush_url = _FLUSH_MEMCACHE_PATH % GetAppVersion()
  68   headers = { 'Authorization': 'Bearer %s' % GetAccessToken() }
  69   response = CreateUrlFetcher().Fetch(flush_url, headers)
  70   if response.status_code != 200:
  71     logging.error('Unable to flush memcache: HTTP %s (%s)' %
  72                   (response.status_code, response.content))
  73   else:
  74     logging.info('Memcache flushed.')
  75
  76
  77 def _CreateServerInstance(commit):
  78   '''Creates a ServerInstance based on origin/master.
  79   '''
  80   object_store_creator = ObjectStoreCreator(
  81       start_empty=False, store_type=PersistentObjectStoreFake)
  82   branch_utility = BranchUtility.Create(object_store_creator)
  83   host_file_system_provider = HostFileSystemProvider(object_store_creator,
  84                                                      pinned_commit=commit)
  85   gcs_file_system_provider = CloudStorageFileSystemProvider(
  86       object_store_creator)
  87   return ServerInstance(object_store_creator,
  88                         CompiledFileSystem.Factory(object_store_creator),
  89                         branch_utility,
  90                         host_file_system_provider,
  91                         gcs_file_system_provider)
  92
  93
  94 def _UpdateDataSource(name, data_source):
  95   try:
  96     class_name = data_source.__class__.__name__
  97     timer = Timer()
  98     logging.info('Updating %s...' % name)
  99     data_source.Refresh().Get()
 100   except Exception as e:
 101     logging.error('%s: error %s' % (class_name, traceback.format_exc()))
 102     raise e
 103   finally:
 104     logging.info('Updating %s took %s' % (name, timer.Stop().FormatElapsed()))
 105
 106
 107 def UpdateCache(single_data_source=None, commit=None):
 108   '''Attempts to populate the datastore with a bunch of information derived from
 109   a given commit.
 110   '''
 111   server_instance = _CreateServerInstance(commit)
 112
 113   # This is the guy that would be responsible for refreshing the cache of
 114   # examples. Here for posterity, hopefully it will be added to the targets
 115   # below someday.
 116   # render_refresher = RenderRefresher(server_instance, self._request)
 117
 118   data_sources = CreateDataSources(server_instance)
 119   data_sources['content_providers'] = server_instance.content_providers
 120   data_sources['platform_bundle'] = server_instance.platform_bundle
 121   if single_data_source:
 122     _UpdateDataSource(single_data_source, data_sources[single_data_source])
 123   else:
 124     for name, source in data_sources.iteritems():
 125       _UpdateDataSource(name, source)
 126
 127
 128 def _Main(argv):
 129   try:
 130     opts = dict((name[2:], value) for name, value in
 131                 getopt.getopt(argv, '',
 132                               ['load-file=', 'data-source=', 'commit=',
 133                                'no-refresh', 'no-push', 'save-file=',
 134                                'no-master-update', 'push-all', 'force'])[0])
 135   except getopt.GetoptError as e:
 136     print '%s\n' % e
 137     print (
 138     'Usage: update_cache.py [options]\n\n'
 139     'Options:\n'
 140     '  --data-source=NAME        Limit update to a single data source.\n'
 141     '  --load-file=FILE          Load object store data from FILE before\n'
 142     '                            starting the update.\n'
 143     '  --save-file=FILE          Save object store data to FILE after running\n'
 144     '                            the update.\n'
 145     '  --no-refresh              Do not attempt to update any data sources.\n'
 146     '  --no-push                 Do not push to Datastore.\n'
 147     '  --commit=REV              Commit ID to use for master update.\n'
 148     '  --no-master-update        Do not update the master commit ID.\n'
 149     '  --push-all                Push all entities to the Datastore even if\n'
 150     '                            they do not differ from the loaded cache.\n\n'
 151     '  --force                   Force an update even if the latest commit is'
 152     '                            already cached.\n')
 153     exit(1)
 154
 155   logging.getLogger().setLevel(logging.INFO)
 156
 157   data_source = opts.get('data-source', None)
 158   load_file = opts.get('load-file', None)
 159   save_file = opts.get('save-file', None)
 160   do_refresh = 'no-refresh' not in opts
 161   do_push = 'no-push' not in opts
 162   do_master_update = 'no-master-update' not in opts
 163   push_all = do_push and ('push-all' in opts)
 164   commit = ParseRevision(opts.get('commit', 'origin/HEAD'))
 165   force_update = 'force' in opts
 166
 167   original_data = {}
 168   if load_file:
 169     logging.info('Loading cache...')
 170     PersistentObjectStoreFake.LoadFromFile(load_file)
 171     if not push_all:
 172       original_data = copy.deepcopy(PersistentObjectStoreFake.DATA)
 173
 174   last_commit = _GetCachedCommitId('master')
 175   if ParseRevision(commit) == last_commit and not force_update:
 176     logging.info('Latest cache (revision %s) is up to date. Bye.' % commit)
 177     exit(0)
 178
 179   timer = Timer()
 180   if do_refresh:
 181     logging.info('Starting refresh from commit %s...' % ParseRevision(commit))
 182     if data_source:
 183       UpdateCache(single_data_source=data_source,
 184                    commit=commit)
 185     else:
 186       UpdateCache(commit=commit)
 187
 188   if do_push:
 189     from datastore_util import PushData
 190     if do_master_update:
 191       _UpdateCommitId('master', commit)
 192     push_timer = Timer()
 193     logging.info('Pushing data into datastore...')
 194     PushData(PersistentObjectStoreFake.DATA, original_data=original_data)
 195     logging.info('Done. Datastore push took %s' %
 196                  push_timer.Stop().FormatElapsed())
 197     _FlushMemcache()
 198   if save_file:
 199     PersistentObjectStoreFake.SaveToFile(save_file)
 200
 201   logging.info('Update completed in %s' % timer.Stop().FormatElapsed())
 202
 203
 204 if __name__ == '__main__':
 205   _Main(sys.argv[1:])
 206