chrome/common/extensions/docs/server2/github_file_system.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import logging
   7 from StringIO import StringIO
   8
   9 from appengine_blobstore import AppEngineBlobstore, BLOBSTORE_GITHUB
  10 from appengine_url_fetcher import AppEngineUrlFetcher
  11 from appengine_wrappers import urlfetch, blobstore
  12 from docs_server_utils import StringIdentity
  13 from file_system import FileSystem, StatInfo
  14 from future import Future
  15 import url_constants
  16 from zipfile import ZipFile, BadZipfile
  17
  18 ZIP_KEY = 'zipball'
  19 USERNAME = None
  20 PASSWORD = None
  21
  22 def _MakeBlobstoreKey(version):
  23   return ZIP_KEY + '.' + str(version)
  24
  25 class _AsyncFetchFutureZip(object):
  26   def __init__(self,
  27                fetcher,
  28                username,
  29                password,
  30                blobstore,
  31                key_to_set,
  32                key_to_delete=None):
  33     self._fetcher = fetcher
  34     self._fetch = fetcher.FetchAsync(ZIP_KEY,
  35                                      username=username,
  36                                      password=password)
  37     self._blobstore = blobstore
  38     self._key_to_set = key_to_set
  39     self._key_to_delete = key_to_delete
  40
  41   def Get(self):
  42     try:
  43       result = self._fetch.Get()
  44       # Check if Github authentication failed.
  45       if result.status_code == 401:
  46         logging.error('Github authentication failed for %s, falling back to '
  47                       'unauthenticated.' % USERNAME)
  48         blob = self._fetcher.Fetch(ZIP_KEY).content
  49       else:
  50         blob = result.content
  51     except urlfetch.DownloadError as e:
  52       logging.error('Bad github zip file: %s' % e)
  53       return None
  54     if self._key_to_delete is not None:
  55       self._blobstore.Delete(_MakeBlobstoreKey(self._key_to_delete),
  56                              BLOBSTORE_GITHUB)
  57     try:
  58       return_zip = ZipFile(StringIO(blob))
  59     except BadZipfile as e:
  60       logging.error('Bad github zip file: %s' % e)
  61       return None
  62
  63     self._blobstore.Set(_MakeBlobstoreKey(self._key_to_set),
  64                         blob,
  65                         BLOBSTORE_GITHUB)
  66     return return_zip
  67
  68 class GithubFileSystem(FileSystem):
  69   @staticmethod
  70   def CreateChromeAppsSamples(object_store_creator):
  71     return GithubFileSystem(
  72         '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS,
  73         AppEngineBlobstore(),
  74         object_store_creator)
  75
  76   def __init__(self, url, blobstore, object_store_creator):
  77     # If we key the password store on the app version then the whole advantage
  78     # of having it in the first place is greatly lessened (likewise it should
  79     # always start populated).
  80     password_store = object_store_creator.Create(
  81         GithubFileSystem,
  82         app_version=None,
  83         category='password',
  84         start_empty=False)
  85     if USERNAME is None:
  86       password_data = password_store.GetMulti(('username', 'password')).Get()
  87       self._username, self._password = (password_data.get('username'),
  88                                         password_data.get('password'))
  89     else:
  90       password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
  91       self._username, self._password = (USERNAME, PASSWORD)
  92
  93     self._url = url
  94     self._fetcher = AppEngineUrlFetcher(url)
  95     self._blobstore = blobstore
  96     self._stat_object_store = object_store_creator.Create(GithubFileSystem)
  97     self._version = None
  98     self._GetZip(self.Stat(ZIP_KEY).version)
  99
 100   def _GetZip(self, version):
 101     try:
 102       blob = self._blobstore.Get(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
 103     except blobstore.BlobNotFoundError:
 104       self._zip_file = Future(value=None)
 105       return
 106     if blob is not None:
 107       try:
 108         self._zip_file = Future(value=ZipFile(StringIO(blob)))
 109       except BadZipfile as e:
 110         self._blobstore.Delete(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
 111         logging.error('Bad github zip file: %s' % e)
 112         self._zip_file = Future(value=None)
 113     else:
 114       self._zip_file = Future(
 115           delegate=_AsyncFetchFutureZip(self._fetcher,
 116                                         self._username,
 117                                         self._password,
 118                                         self._blobstore,
 119                                         version,
 120                                         key_to_delete=self._version))
 121     self._version = version
 122
 123   def _ReadFile(self, path):
 124     try:
 125       zip_file = self._zip_file.Get()
 126     except Exception as e:
 127       logging.error('Github ReadFile error: %s' % e)
 128       return ''
 129     if zip_file is None:
 130       logging.error('Bad github zip file.')
 131       return ''
 132     prefix = zip_file.namelist()[0][:-1]
 133     return zip_file.read(prefix + path)
 134
 135   def _ListDir(self, path):
 136     try:
 137       zip_file = self._zip_file.Get()
 138     except Exception as e:
 139       logging.error('Github ListDir error: %s' % e)
 140       return []
 141     if zip_file is None:
 142       logging.error('Bad github zip file.')
 143       return []
 144     filenames = zip_file.namelist()
 145     # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
 146     filenames = [f[len(filenames[0]) - 1:] for f in filenames]
 147     # Remove the path of the directory we're listing from the filenames.
 148     filenames = [f[len(path):] for f in filenames
 149                  if f != path and f.startswith(path)]
 150     # Remove all files not directly in this directory.
 151     return [f for f in filenames if f[:-1].count('/') == 0]
 152
 153   def Read(self, paths):
 154     version = self.Stat(ZIP_KEY).version
 155     if version != self._version:
 156       self._GetZip(version)
 157     result = {}
 158     for path in paths:
 159       if path.endswith('/'):
 160         result[path] = self._ListDir(path)
 161       else:
 162         result[path] = self._ReadFile(path)
 163     return Future(value=result)
 164
 165   def _DefaultStat(self, path):
 166     version = 0
 167     # TODO(kalman): we should replace all of this by wrapping the
 168     # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
 169     # CFS to be robust, and GFS is missing out.
 170     # For example: the following line is wrong, but it could be moot.
 171     self._stat_object_store.Set(path, version)
 172     return StatInfo(version)
 173
 174   def Stat(self, path):
 175     version = self._stat_object_store.Get(path).Get()
 176     if version is not None:
 177       return StatInfo(version)
 178     try:
 179       result = self._fetcher.Fetch('commits/HEAD',
 180                                    username=USERNAME,
 181                                    password=PASSWORD)
 182     except urlfetch.DownloadError as e:
 183       logging.warning('GithubFileSystem Stat: %s' % e)
 184       return self._DefaultStat(path)
 185
 186     # Check if Github authentication failed.
 187     if result.status_code == 401:
 188       logging.warning('Github authentication failed for %s, falling back to '
 189                       'unauthenticated.' % USERNAME)
 190       try:
 191         result = self._fetcher.Fetch('commits/HEAD')
 192       except urlfetch.DownloadError as e:
 193         logging.warning('GithubFileSystem Stat: %s' % e)
 194         return self._DefaultStat(path)
 195
 196     # Parse response JSON - but sometimes github gives us invalid JSON.
 197     try:
 198       version = json.loads(result.content)['sha']
 199       self._stat_object_store.Set(path, version)
 200       return StatInfo(version)
 201     except StandardError as e:
 202       logging.warning(
 203           ('%s: got invalid or unexpected JSON from github. Response status ' +
 204            'was %s, content %s') % (e, result.status_code, result.content))
 205       return self._DefaultStat(path)
 206
 207   def GetIdentity(self):
 208     return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))