chrome/common/extensions/docs/server2/new_github_file_system_test.py

   1 #!/usr/bin/env python
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import json
   7 from copy import deepcopy
   8 from cStringIO import StringIO
   9 from functools import partial
  10 from hashlib import sha1
  11 from random import random
  12 import unittest
  13 from zipfile import ZipFile
  14
  15 from caching_file_system import CachingFileSystem
  16 from file_system import FileNotFoundError, StatInfo
  17 from fake_url_fetcher import FakeURLFSFetcher, MockURLFetcher
  18 from local_file_system import LocalFileSystem
  19 from new_github_file_system import GithubFileSystem
  20 from object_store_creator import ObjectStoreCreator
  21 from test_file_system import TestFileSystem
  22
  23
  24 class _TestBundle(object):
  25   '''Bundles test file data with a GithubFileSystem and test utilites. Create
  26   GithubFileSystems via |CreateGfs()|, the Fetcher it uses as |fetcher|,
  27   randomly mutate its contents via |Mutate()|, and access the underlying zip
  28   data via |files|.
  29   '''
  30
  31   def __init__(self):
  32     self.files = {
  33       'zipfile/': '',
  34       'zipfile/hello.txt': 'world',
  35       'zipfile/readme': 'test zip',
  36       'zipfile/dir/file1': 'contents',
  37       'zipfile/dir/file2': 'more contents'
  38     }
  39     self._test_files = {
  40       'test_owner': {
  41         'changing-repo': {
  42           'commits': {
  43             'HEAD': self._MakeShaJson(self._GenerateHash())
  44           },
  45           'zipball': self._ZipFromFiles(self.files)
  46         }
  47       }
  48     }
  49
  50     self._fake_fetcher = None
  51
  52
  53   def CreateGfsAndFetcher(self):
  54     fetchers = []
  55     def create_mock_url_fetcher(base_path):
  56       assert not fetchers
  57       # Save this reference so we can replace the TestFileSystem in Mutate.
  58       self._fake_fetcher = FakeURLFSFetcher(
  59           TestFileSystem(self._test_files), base_path)
  60       fetchers.append(MockURLFetcher(self._fake_fetcher))
  61       return fetchers[-1]
  62
  63     # Constructing |gfs| will create a fetcher.
  64     gfs = GithubFileSystem.ForTest(
  65         'changing-repo/', create_mock_url_fetcher, path='')
  66     assert len(fetchers) == 1
  67     return gfs, fetchers[0]
  68
  69   def Mutate(self):
  70     fake_version = self._GenerateHash()
  71     fake_data = self._GenerateHash()
  72     self.files['zipfile/hello.txt'] = fake_data
  73     self.files['zipfile/new-file'] = fake_data
  74     self.files['zipfile/dir/file1'] = fake_data
  75     self._test_files['test_owner']['changing-repo']['zipball'] = (
  76         self._ZipFromFiles(self.files))
  77     self._test_files['test_owner']['changing-repo']['commits']['HEAD'] = (
  78         self._MakeShaJson(fake_version))
  79
  80     # Update the file_system used by FakeURLFSFetcher so the above mutations
  81     # propagate.
  82     self._fake_fetcher.UpdateFS(TestFileSystem(self._test_files))
  83
  84     return fake_version, fake_data
  85
  86   def _GenerateHash(self):
  87     '''Generates an arbitrary SHA1 hash.
  88     '''
  89     return sha1(str(random())).hexdigest()
  90
  91   def _MakeShaJson(self, hash_value):
  92     commit_json = json.loads(deepcopy(LocalFileSystem('').ReadSingle(
  93         'test_data/github_file_system/test_owner/repo/commits/HEAD').Get()))
  94     commit_json['sha'] = hash_value
  95     return json.dumps(commit_json)
  96
  97   def _ZipFromFiles(self, file_dict):
  98     string = StringIO()
  99     zipfile = ZipFile(string, 'w')
 100     for filename, contents in file_dict.iteritems():
 101       zipfile.writestr(filename, contents)
 102     zipfile.close()
 103     return string.getvalue()
 104
 105
 106 class TestGithubFileSystem(unittest.TestCase):
 107   def setUp(self):
 108     self._gfs = GithubFileSystem.ForTest(
 109         'repo/', partial(FakeURLFSFetcher, LocalFileSystem('')))
 110     # Start and finish the repository load.
 111     self._cgfs = CachingFileSystem(self._gfs, ObjectStoreCreator.ForTest())
 112
 113   def testReadDirectory(self):
 114     self._gfs.Refresh().Get()
 115     self.assertEqual(
 116         sorted(['requirements.txt', '.gitignore', 'README.md', 'src/']),
 117         sorted(self._gfs.ReadSingle('').Get()))
 118     self.assertEqual(
 119         sorted(['__init__.notpy', 'hello.notpy']),
 120         sorted(self._gfs.ReadSingle('src/').Get()))
 121
 122   def testReadFile(self):
 123     self._gfs.Refresh().Get()
 124     expected = (
 125       '# Compiled Python files\n'
 126       '*.pyc\n'
 127     )
 128     self.assertEqual(expected, self._gfs.ReadSingle('.gitignore').Get())
 129
 130   def testMultipleReads(self):
 131     self._gfs.Refresh().Get()
 132     self.assertEqual(
 133         self._gfs.ReadSingle('requirements.txt').Get(),
 134         self._gfs.ReadSingle('requirements.txt').Get())
 135
 136   def testReads(self):
 137     self._gfs.Refresh().Get()
 138     expected = {
 139         'src/': sorted(['hello.notpy', '__init__.notpy']),
 140         '': sorted(['requirements.txt', '.gitignore', 'README.md', 'src/'])
 141     }
 142
 143     read = self._gfs.Read(['', 'src/']).Get()
 144     self.assertEqual(expected['src/'], sorted(read['src/']))
 145     self.assertEqual(expected[''], sorted(read['']))
 146
 147   def testStat(self):
 148     # This is the hash value from the zip on disk.
 149     real_hash = 'c36fc23688a9ec9e264d3182905dc0151bfff7d7'
 150
 151     self._gfs.Refresh().Get()
 152     dir_stat = StatInfo(real_hash, {
 153       'hello.notpy': StatInfo(real_hash),
 154       '__init__.notpy': StatInfo(real_hash)
 155     })
 156
 157     self.assertEqual(StatInfo(real_hash), self._gfs.Stat('README.md'))
 158     self.assertEqual(StatInfo(real_hash), self._gfs.Stat('src/hello.notpy'))
 159     self.assertEqual(dir_stat, self._gfs.Stat('src/'))
 160
 161   def testBadReads(self):
 162     self._gfs.Refresh().Get()
 163     self.assertRaises(FileNotFoundError, self._gfs.Stat, 'DONT_README.md')
 164     self.assertRaises(FileNotFoundError,
 165                       self._gfs.ReadSingle('DONT_README.md').Get)
 166
 167   def testCachingFileSystem(self):
 168     self._cgfs.Refresh().Get()
 169     initial_cgfs_read_one = self._cgfs.ReadSingle('src/hello.notpy').Get()
 170
 171     self.assertEqual(initial_cgfs_read_one,
 172                      self._gfs.ReadSingle('src/hello.notpy').Get())
 173     self.assertEqual(initial_cgfs_read_one,
 174                      self._cgfs.ReadSingle('src/hello.notpy').Get())
 175
 176     initial_cgfs_read_two = self._cgfs.Read(
 177         ['README.md', 'requirements.txt']).Get()
 178
 179     self.assertEqual(
 180         initial_cgfs_read_two,
 181         self._gfs.Read(['README.md', 'requirements.txt']).Get())
 182     self.assertEqual(
 183         initial_cgfs_read_two,
 184         self._cgfs.Read(['README.md', 'requirements.txt']).Get())
 185
 186   def testWithoutRefresh(self):
 187     # Without refreshing it will still read the content from blobstore, and it
 188     # does this via the magic of the FakeURLFSFetcher.
 189     self.assertEqual(['__init__.notpy', 'hello.notpy'],
 190                      sorted(self._gfs.ReadSingle('src/').Get()))
 191
 192   def testRefresh(self):
 193     test_bundle = _TestBundle()
 194     gfs, fetcher = test_bundle.CreateGfsAndFetcher()
 195
 196     # It shouldn't fetch until Refresh does so; then it will do 2, one for the
 197     # stat, and another for the read.
 198     self.assertTrue(*fetcher.CheckAndReset())
 199     gfs.Refresh().Get()
 200     self.assertTrue(*fetcher.CheckAndReset(fetch_count=1,
 201                                            fetch_async_count=1,
 202                                            fetch_resolve_count=1))
 203
 204     # Refresh is just an alias for Read('').
 205     gfs.Refresh().Get()
 206     self.assertTrue(*fetcher.CheckAndReset())
 207
 208     initial_dir_read = sorted(gfs.ReadSingle('').Get())
 209     initial_file_read = gfs.ReadSingle('dir/file1').Get()
 210
 211     version, data = test_bundle.Mutate()
 212
 213     # Check that changes have not effected the file system yet.
 214     self.assertEqual(initial_dir_read, sorted(gfs.ReadSingle('').Get()))
 215     self.assertEqual(initial_file_read, gfs.ReadSingle('dir/file1').Get())
 216     self.assertNotEqual(StatInfo(version), gfs.Stat(''))
 217
 218     gfs, fetcher = test_bundle.CreateGfsAndFetcher()
 219     gfs.Refresh().Get()
 220     self.assertTrue(*fetcher.CheckAndReset(fetch_count=1,
 221                                            fetch_async_count=1,
 222                                            fetch_resolve_count=1))
 223
 224     # Check that the changes have affected the file system.
 225     self.assertEqual(data, gfs.ReadSingle('new-file').Get())
 226     self.assertEqual(test_bundle.files['zipfile/dir/file1'],
 227                      gfs.ReadSingle('dir/file1').Get())
 228     self.assertEqual(StatInfo(version), gfs.Stat('new-file'))
 229
 230     # Regression test: ensure that reading the data after it's been mutated,
 231     # but before Refresh() has been realised, still returns the correct data.
 232     gfs, fetcher = test_bundle.CreateGfsAndFetcher()
 233     version, data = test_bundle.Mutate()
 234
 235     refresh_future = gfs.Refresh()
 236     self.assertTrue(*fetcher.CheckAndReset(fetch_count=1, fetch_async_count=1))
 237
 238     self.assertEqual(data, gfs.ReadSingle('new-file').Get())
 239     self.assertEqual(test_bundle.files['zipfile/dir/file1'],
 240                      gfs.ReadSingle('dir/file1').Get())
 241     self.assertEqual(StatInfo(version), gfs.Stat('new-file'))
 242
 243     refresh_future.Get()
 244     self.assertTrue(*fetcher.CheckAndReset(fetch_resolve_count=1))
 245
 246   def testGetThenRefreshOnStartup(self):
 247     # Regression test: Test that calling Get() but never resolving the future,
 248     # then Refresh()ing the data, causes the data to be refreshed.
 249     test_bundle = _TestBundle()
 250     gfs, fetcher = test_bundle.CreateGfsAndFetcher()
 251     self.assertTrue(*fetcher.CheckAndReset())
 252
 253     # Get a predictable version.
 254     version, data = test_bundle.Mutate()
 255
 256     read_future = gfs.ReadSingle('hello.txt')
 257     # Fetch for the Stat(), async-fetch for the Read().
 258     self.assertTrue(*fetcher.CheckAndReset(fetch_count=1, fetch_async_count=1))
 259
 260     refresh_future = gfs.Refresh()
 261     self.assertTrue(*fetcher.CheckAndReset())
 262
 263     self.assertEqual(data, read_future.Get())
 264     self.assertTrue(*fetcher.CheckAndReset(fetch_resolve_count=1))
 265     self.assertEqual(StatInfo(version), gfs.Stat('hello.txt'))
 266     self.assertTrue(*fetcher.CheckAndReset())
 267
 268     # The fetch will already have been resolved, so resolving the Refresh won't
 269     # affect anything.
 270     refresh_future.Get()
 271     self.assertTrue(*fetcher.CheckAndReset())
 272
 273     # Read data should not have changed.
 274     self.assertEqual(data, gfs.ReadSingle('hello.txt').Get())
 275     self.assertEqual(StatInfo(version), gfs.Stat('hello.txt'))
 276     self.assertTrue(*fetcher.CheckAndReset())
 277
 278
 279 if __name__ == '__main__':
 280   unittest.main()