chrome/common/extensions/docs/server2/github_file_system.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import logging
   7 from StringIO import StringIO
   8 import posixpath
   9
  10 from appengine_blobstore import AppEngineBlobstore, BLOBSTORE_GITHUB
  11 from appengine_url_fetcher import AppEngineUrlFetcher
  12 from appengine_wrappers import urlfetch, blobstore
  13 from docs_server_utils import StringIdentity
  14 from file_system import FileSystem, StatInfo
  15 from future import Future
  16 from path_util import IsDirectory
  17 import url_constants
  18 from zipfile import ZipFile, BadZipfile
  19
  20 ZIP_KEY = 'zipball'
  21 USERNAME = None
  22 PASSWORD = None
  23
  24
  25 def _MakeBlobstoreKey(version):
  26   return ZIP_KEY + '.' + str(version)
  27
  28
  29 def _GetAsyncFetchCallback(fetcher,
  30                            username,
  31                            password,
  32                            blobstore,
  33                            key_to_set,
  34                            key_to_delete=None):
  35   fetch = fetcher.FetchAsync(ZIP_KEY, username=username, password=password)
  36
  37   def resolve():
  38     try:
  39       result = fetch.Get()
  40       # Check if Github authentication failed.
  41       if result.status_code == 401:
  42         logging.error('Github authentication failed for %s, falling back to '
  43                       'unauthenticated.' % USERNAME)
  44         blob = fetcher.Fetch(ZIP_KEY).content
  45       else:
  46         blob = result.content
  47     except urlfetch.DownloadError as e:
  48       logging.error('Bad github zip file: %s' % e)
  49       return None
  50     if key_to_delete is not None:
  51       blobstore.Delete(_MakeBlobstoreKey(key_to_delete, BLOBSTORE_GITHUB))
  52     try:
  53       return_zip = ZipFile(StringIO(blob))
  54     except BadZipfile as e:
  55       logging.error('Bad github zip file: %s' % e)
  56       return None
  57
  58     blobstore.Set(_MakeBlobstoreKey(key_to_set), blob, BLOBSTORE_GITHUB)
  59     return return_zip
  60
  61   return resolve
  62
  63
  64 class GithubFileSystem(FileSystem):
  65   @staticmethod
  66   def CreateChromeAppsSamples(object_store_creator):
  67     return GithubFileSystem(
  68         '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS,
  69         AppEngineBlobstore(),
  70         object_store_creator)
  71
  72   def __init__(self, url, blobstore, object_store_creator):
  73     # If we key the password store on the app version then the whole advantage
  74     # of having it in the first place is greatly lessened (likewise it should
  75     # always start populated).
  76     password_store = object_store_creator.Create(
  77         GithubFileSystem,
  78         app_version=None,
  79         category='password',
  80         start_empty=False)
  81     if USERNAME is None:
  82       password_data = password_store.GetMulti(('username', 'password')).Get()
  83       self._username, self._password = (password_data.get('username'),
  84                                         password_data.get('password'))
  85     else:
  86       password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
  87       self._username, self._password = (USERNAME, PASSWORD)
  88
  89     self._url = url
  90     self._fetcher = AppEngineUrlFetcher(url)
  91     self._blobstore = blobstore
  92     self._stat_object_store = object_store_creator.Create(GithubFileSystem)
  93     self._version = None
  94     self._GetZip(self.Stat(ZIP_KEY).version)
  95
  96   def _GetZip(self, version):
  97     try:
  98       blob = self._blobstore.Get(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
  99     except blobstore.BlobNotFoundError:
 100       self._zip_file = Future(value=None)
 101       return
 102     if blob is not None:
 103       try:
 104         self._zip_file = Future(value=ZipFile(StringIO(blob)))
 105       except BadZipfile as e:
 106         self._blobstore.Delete(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
 107         logging.error('Bad github zip file: %s' % e)
 108         self._zip_file = Future(value=None)
 109     else:
 110       self._zip_file = Future(
 111           callback=_GetAsyncFetchCallback(self._fetcher,
 112                                           self._username,
 113                                           self._password,
 114                                           self._blobstore,
 115                                           version,
 116                                           key_to_delete=self._version))
 117     self._version = version
 118
 119   def _ReadFile(self, path):
 120     try:
 121       zip_file = self._zip_file.Get()
 122     except Exception as e:
 123       logging.error('Github ReadFile error: %s' % e)
 124       return ''
 125     if zip_file is None:
 126       logging.error('Bad github zip file.')
 127       return ''
 128     prefix = zip_file.namelist()[0]
 129     return zip_file.read(prefix + path)
 130
 131   def _ListDir(self, path):
 132     try:
 133       zip_file = self._zip_file.Get()
 134     except Exception as e:
 135       logging.error('Github ListDir error: %s' % e)
 136       return []
 137     if zip_file is None:
 138       logging.error('Bad github zip file.')
 139       return []
 140     filenames = zip_file.namelist()
 141     # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
 142     filenames = [f[len(filenames[0]):] for f in filenames]
 143     # Remove the path of the directory we're listing from the filenames.
 144     filenames = [f[len(path):] for f in filenames
 145                  if f != path and f.startswith(path)]
 146     # Remove all files not directly in this directory.
 147     return [f for f in filenames if f[:-1].count('/') == 0]
 148
 149   def Read(self, paths, skip_not_found=False):
 150     version = self.Stat(ZIP_KEY).version
 151     if version != self._version:
 152       self._GetZip(version)
 153     result = {}
 154     for path in paths:
 155       if IsDirectory(path):
 156         result[path] = self._ListDir(path)
 157       else:
 158         result[path] = self._ReadFile(path)
 159     return Future(value=result)
 160
 161   def _DefaultStat(self, path):
 162     version = 0
 163     # TODO(kalman): we should replace all of this by wrapping the
 164     # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
 165     # CFS to be robust, and GFS is missing out.
 166     # For example: the following line is wrong, but it could be moot.
 167     self._stat_object_store.Set(path, version)
 168     return StatInfo(version)
 169
 170   def Stat(self, path):
 171     version = self._stat_object_store.Get(path).Get()
 172     if version is not None:
 173       return StatInfo(version)
 174     try:
 175       result = self._fetcher.Fetch('commits/HEAD',
 176                                    username=USERNAME,
 177                                    password=PASSWORD)
 178     except urlfetch.DownloadError as e:
 179       logging.warning('GithubFileSystem Stat: %s' % e)
 180       return self._DefaultStat(path)
 181
 182     # Check if Github authentication failed.
 183     if result.status_code == 401:
 184       logging.warning('Github authentication failed for %s, falling back to '
 185                       'unauthenticated.' % USERNAME)
 186       try:
 187         result = self._fetcher.Fetch('commits/HEAD')
 188       except urlfetch.DownloadError as e:
 189         logging.warning('GithubFileSystem Stat: %s' % e)
 190         return self._DefaultStat(path)
 191
 192     # Parse response JSON - but sometimes github gives us invalid JSON.
 193     try:
 194       version = json.loads(result.content)['sha']
 195       self._stat_object_store.Set(path, version)
 196       return StatInfo(version)
 197     except StandardError as e:
 198       logging.warning(
 199           ('%s: got invalid or unexpected JSON from github. Response status ' +
 200            'was %s, content %s') % (e, result.status_code, result.content))
 201       return self._DefaultStat(path)
 202
 203   def GetIdentity(self):
 204     return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))