chrome/common/extensions/docs/server2/caching_file_system_test.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import os
   7 import sys
   8 import unittest
   9
  10 from caching_file_system import CachingFileSystem
  11 from extensions_paths import SERVER2
  12 from file_system import FileNotFoundError, StatInfo
  13 from local_file_system import LocalFileSystem
  14 from mock_file_system import MockFileSystem
  15 from object_store_creator import ObjectStoreCreator
  16 from test_file_system import TestFileSystem
  17 from test_object_store import TestObjectStore
  18
  19
  20 def _CreateLocalFs():
  21   return LocalFileSystem.Create(SERVER2, 'test_data', 'file_system/')
  22
  23
  24 class CachingFileSystemTest(unittest.TestCase):
  25   def setUp(self):
  26     # Use this to make sure that every time _CreateCachingFileSystem is called
  27     # the underlying object store data is the same, within each test.
  28     self._object_store_dbs = {}
  29
  30   def _CreateCachingFileSystem(self, fs, start_empty=False):
  31     def store_type_constructor(namespace, start_empty=False):
  32       '''Returns an ObjectStore backed onto test-lifetime-persistent objects
  33       in |_object_store_dbs|.
  34       '''
  35       if namespace not in self._object_store_dbs:
  36         self._object_store_dbs[namespace] = {}
  37       db = self._object_store_dbs[namespace]
  38       if start_empty:
  39         db.clear()
  40       return TestObjectStore(namespace, init=db)
  41     object_store_creator = ObjectStoreCreator(start_empty=start_empty,
  42                                               store_type=store_type_constructor)
  43     return CachingFileSystem(fs, object_store_creator)
  44
  45   def testReadFiles(self):
  46     file_system = self._CreateCachingFileSystem(
  47         _CreateLocalFs(), start_empty=False)
  48     expected = {
  49       './test1.txt': 'test1\n',
  50       './test2.txt': 'test2\n',
  51       './test3.txt': 'test3\n',
  52     }
  53     self.assertEqual(
  54         expected,
  55         file_system.Read(['./test1.txt', './test2.txt', './test3.txt']).Get())
  56
  57   def testListDir(self):
  58     file_system = self._CreateCachingFileSystem(
  59         _CreateLocalFs(), start_empty=False)
  60     expected = ['dir/'] + ['file%d.html' % i for i in range(7)]
  61     file_system._read_cache.Set(
  62         'list/',
  63         (expected, file_system.Stat('list/').version))
  64     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
  65
  66     expected.remove('file0.html')
  67     file_system._read_cache.Set(
  68         'list/',
  69         (expected, file_system.Stat('list/').version))
  70     self.assertEqual(expected, sorted(file_system.ReadSingle('list/').Get()))
  71
  72   def testCaching(self):
  73     test_fs = TestFileSystem({
  74       'bob': {
  75         'bob0': 'bob/bob0 contents',
  76         'bob1': 'bob/bob1 contents',
  77         'bob2': 'bob/bob2 contents',
  78         'bob3': 'bob/bob3 contents',
  79       }
  80     })
  81     mock_fs = MockFileSystem(test_fs)
  82     def create_empty_caching_fs():
  83       return self._CreateCachingFileSystem(mock_fs, start_empty=True)
  84
  85     file_system = create_empty_caching_fs()
  86
  87     # The stat/read should happen before resolving the Future, and resolving
  88     # the future shouldn't do any additional work.
  89     get_future = file_system.ReadSingle('bob/bob0')
  90     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
  91     self.assertEqual('bob/bob0 contents', get_future.Get())
  92     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
  93
  94     # Resource has been cached, so test resource is not re-fetched.
  95     self.assertEqual('bob/bob0 contents',
  96                      file_system.ReadSingle('bob/bob0').Get())
  97     self.assertTrue(*mock_fs.CheckAndReset())
  98
  99     # Test if the Stat version is the same the resource is not re-fetched.
 100     file_system = create_empty_caching_fs()
 101     self.assertEqual('bob/bob0 contents',
 102                      file_system.ReadSingle('bob/bob0').Get())
 103     self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 104
 105     # Test if there is a newer version, the resource is re-fetched.
 106     file_system = create_empty_caching_fs()
 107     test_fs.IncrementStat();
 108     future = file_system.ReadSingle('bob/bob0')
 109     self.assertTrue(*mock_fs.CheckAndReset(read_count=1, stat_count=1))
 110     self.assertEqual('bob/bob0 contents', future.Get())
 111     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1))
 112
 113     # Test directory and subdirectory stats are cached.
 114     file_system = create_empty_caching_fs()
 115     file_system._stat_cache.Del('bob/bob0')
 116     file_system._read_cache.Del('bob/bob0')
 117     file_system._stat_cache.Del('bob/bob1')
 118     test_fs.IncrementStat();
 119     futures = (file_system.ReadSingle('bob/bob1'),
 120                file_system.ReadSingle('bob/bob0'))
 121     self.assertTrue(*mock_fs.CheckAndReset(read_count=2))
 122     self.assertEqual(('bob/bob1 contents', 'bob/bob0 contents'),
 123                      tuple(future.Get() for future in futures))
 124     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=2, stat_count=1))
 125     self.assertEqual('bob/bob1 contents',
 126                      file_system.ReadSingle('bob/bob1').Get())
 127     self.assertTrue(*mock_fs.CheckAndReset())
 128
 129     # Test a more recent parent directory doesn't force a refetch of children.
 130     file_system = create_empty_caching_fs()
 131     file_system._read_cache.Del('bob/bob0')
 132     file_system._read_cache.Del('bob/bob1')
 133     futures = (file_system.ReadSingle('bob/bob1'),
 134                file_system.ReadSingle('bob/bob2'),
 135                file_system.ReadSingle('bob/bob3'))
 136     self.assertTrue(*mock_fs.CheckAndReset(read_count=3))
 137     self.assertEqual(
 138         ('bob/bob1 contents', 'bob/bob2 contents', 'bob/bob3 contents'),
 139         tuple(future.Get() for future in futures))
 140     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=3, stat_count=1))
 141
 142     test_fs.IncrementStat(path='bob/bob0')
 143     file_system = create_empty_caching_fs()
 144     self.assertEqual('bob/bob1 contents',
 145                      file_system.ReadSingle('bob/bob1').Get())
 146     self.assertEqual('bob/bob2 contents',
 147                      file_system.ReadSingle('bob/bob2').Get())
 148     self.assertEqual('bob/bob3 contents',
 149                      file_system.ReadSingle('bob/bob3').Get())
 150     self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 151
 152     file_system = create_empty_caching_fs()
 153     file_system._stat_cache.Del('bob/bob0')
 154     future = file_system.ReadSingle('bob/bob0')
 155     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
 156     self.assertEqual('bob/bob0 contents', future.Get())
 157     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
 158     self.assertEqual('bob/bob0 contents',
 159                      file_system.ReadSingle('bob/bob0').Get())
 160     self.assertTrue(*mock_fs.CheckAndReset())
 161
 162     # Test skip_not_found caching behavior.
 163     file_system = create_empty_caching_fs()
 164     future = file_system.ReadSingle('bob/no_file', skip_not_found=True)
 165     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
 166     self.assertEqual(None, future.Get())
 167     self.assertTrue(*mock_fs.CheckAndReset(read_resolve_count=1, stat_count=1))
 168     future = file_system.ReadSingle('bob/no_file', skip_not_found=True)
 169     # There shouldn't be another read/stat from the file system;
 170     # we know the file is not there.
 171     self.assertTrue(*mock_fs.CheckAndReset())
 172     future = file_system.ReadSingle('bob/no_file')
 173     self.assertTrue(*mock_fs.CheckAndReset(read_count=1))
 174     # Even though we cached information about non-existent files,
 175     # trying to read one without specifiying skip_not_found should
 176     # still raise an error.
 177     self.assertRaises(FileNotFoundError, future.Get)
 178
 179   def testCachedStat(self):
 180     test_fs = TestFileSystem({
 181       'bob': {
 182         'bob0': 'bob/bob0 contents',
 183         'bob1': 'bob/bob1 contents'
 184       }
 185     })
 186     mock_fs = MockFileSystem(test_fs)
 187
 188     file_system = self._CreateCachingFileSystem(mock_fs, start_empty=False)
 189
 190     self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
 191     self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 192     self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
 193     self.assertTrue(*mock_fs.CheckAndReset())
 194
 195     # Caching happens on a directory basis, so reading other files from that
 196     # directory won't result in a stat.
 197     self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
 198     self.assertEqual(
 199         StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
 200         file_system.Stat('bob/'))
 201     self.assertTrue(*mock_fs.CheckAndReset())
 202
 203     # Even though the stat is bumped, the object store still has it cached so
 204     # this won't update.
 205     test_fs.IncrementStat()
 206     self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob0'))
 207     self.assertEqual(StatInfo('0'), file_system.Stat('bob/bob1'))
 208     self.assertEqual(
 209         StatInfo('0', child_versions={'bob0': '0', 'bob1': '0'}),
 210         file_system.Stat('bob/'))
 211     self.assertTrue(*mock_fs.CheckAndReset())
 212
 213   def testFreshStat(self):
 214     test_fs = TestFileSystem({
 215       'bob': {
 216         'bob0': 'bob/bob0 contents',
 217         'bob1': 'bob/bob1 contents'
 218       }
 219     })
 220     mock_fs = MockFileSystem(test_fs)
 221
 222     def run_expecting_stat(stat):
 223       def run():
 224         file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
 225         self.assertEqual(
 226             StatInfo(stat, child_versions={'bob0': stat, 'bob1': stat}),
 227             file_system.Stat('bob/'))
 228         self.assertTrue(*mock_fs.CheckAndReset(stat_count=1))
 229         self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
 230         self.assertEqual(StatInfo(stat), file_system.Stat('bob/bob0'))
 231         self.assertTrue(*mock_fs.CheckAndReset())
 232       run()
 233       run()
 234
 235     run_expecting_stat('0')
 236     test_fs.IncrementStat()
 237     run_expecting_stat('1')
 238
 239   def testSkipNotFound(self):
 240     caching_fs = self._CreateCachingFileSystem(TestFileSystem({
 241       'bob': {
 242         'bob0': 'bob/bob0 contents',
 243         'bob1': 'bob/bob1 contents'
 244       }
 245     }))
 246     def read_skip_not_found(paths):
 247       return caching_fs.Read(paths, skip_not_found=True).Get()
 248     self.assertEqual({}, read_skip_not_found(('grub',)))
 249     self.assertEqual({}, read_skip_not_found(('bob/bob2',)))
 250     self.assertEqual({
 251       'bob/bob0': 'bob/bob0 contents',
 252     }, read_skip_not_found(('bob/bob0', 'bob/bob2')))
 253
 254   def testWalkCaching(self):
 255     test_fs = TestFileSystem({
 256       'root': {
 257         'file1': 'file1',
 258         'file2': 'file2',
 259         'dir1': {
 260           'dir1_file1': 'dir1_file1',
 261           'dir2': {},
 262           'dir3': {
 263             'dir3_file1': 'dir3_file1',
 264             'dir3_file2': 'dir3_file2'
 265           }
 266         }
 267       }
 268     })
 269     mock_fs = MockFileSystem(test_fs)
 270     file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
 271     for walkinfo in file_system.Walk(''):
 272       pass
 273     self.assertTrue(*mock_fs.CheckAndReset(
 274         read_resolve_count=5, read_count=5, stat_count=5))
 275
 276     all_dirs, all_files = [], []
 277     for root, dirs, files in file_system.Walk(''):
 278       all_dirs.extend(dirs)
 279       all_files.extend(files)
 280     self.assertEqual(sorted(['root/', 'dir1/', 'dir2/', 'dir3/']),
 281                      sorted(all_dirs))
 282     self.assertEqual(
 283         sorted(['file1', 'file2', 'dir1_file1', 'dir3_file1', 'dir3_file2']),
 284         sorted(all_files))
 285     # All data should be cached.
 286     self.assertTrue(*mock_fs.CheckAndReset())
 287
 288     # Starting from a different root should still pull cached data.
 289     for walkinfo in file_system.Walk('root/dir1/'):
 290       pass
 291     self.assertTrue(*mock_fs.CheckAndReset())
 292     # TODO(ahernandez): Test with a new instance CachingFileSystem so a
 293     # different object store is utilized.
 294
 295   def testVersionedStat(self):
 296     test_fs = TestFileSystem({
 297       'bob': {
 298         'bob0': 'bob/bob0 contents',
 299         'bob1': 'bob/bob1 contents'
 300       }
 301     })
 302
 303     # Create a versioned FileSystem and verify that multiple CachingFileSystem
 304     # instances wrapping it will share the same stat cache.
 305     mock_fs = MockFileSystem(test_fs)
 306     mock_fs.SetVersion('abcdefg')
 307
 308     def run_and_expect_stat_count(paths, stat_count=0):
 309       file_system = self._CreateCachingFileSystem(mock_fs, start_empty=True)
 310       [file_system.Stat(path) for path in paths]
 311       self.assertTrue(*mock_fs.CheckAndReset(stat_count=stat_count))
 312
 313     run_and_expect_stat_count(['bob/', 'bob/bob0', 'bob/bob1'], stat_count=1)
 314     run_and_expect_stat_count(['bob/', 'bob/bob0', 'bob/bob1'], stat_count=0)
 315
 316 if __name__ == '__main__':
 317   unittest.main()