config/tests/unitMozZipFile.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import copy
   6 import os
   7 import random
   8 import shutil
   9 import sys
  10 import unittest
  11 from string import letters
  12
  13 """
  14 Test case infrastructure for MozZipFile.
  15
  16 This isn't really a unit test, but a test case generator and runner.
  17 For a given set of files, lengths, and number of writes, we create
  18 a testcase for every combination of the three. There are some
  19 symmetries used to reduce the number of test cases, the first file
  20 written is always the first file, the second is either the first or
  21 the second, the third is one of the first three. That is, if we
  22 had 4 files, but only three writes, the fourth file would never even
  23 get tried.
  24
  25 The content written to the jars is pseudorandom with a fixed seed.
  26 """
  27
  28 if not __file__:
  29     __file__ = sys.argv[0]
  30 sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
  31
  32 import zipfile
  33
  34 from MozZipFile import ZipFile
  35
  36 leafs = ("firstdir/oneleaf", "seconddir/twoleaf", "thirddir/with/sub/threeleaf")
  37 _lengths = map(lambda n: n * 64, [16, 64, 80])
  38 lengths = 3
  39 writes = 5
  40
  41
  42 def givenlength(i):
  43     """Return a length given in the _lengths array to allow manual
  44     tuning of which lengths of zip entries to use.
  45     """
  46     return _lengths[i]
  47
  48
  49 def prod(*iterables):
  50     """'Tensor product of a list of iterables.
  51
  52     This generator returns lists of items, one of each given
  53     iterable. It iterates over all possible combinations.
  54     """
  55     for item in iterables[0]:
  56         if len(iterables) == 1:
  57             yield [item]
  58         else:
  59             for others in prod(*iterables[1:]):
  60                 yield [item] + others
  61
  62
  63 def getid(descs):
  64     "Convert a list of ints to a string."
  65     return reduce(lambda x, y: x + "{0}{1}".format(*tuple(y)), descs, "")
  66
  67
  68 def getContent(length):
  69     "Get pseudo random content of given length."
  70     rv = [None] * length
  71     for i in xrange(length):
  72         rv[i] = random.choice(letters)
  73     return "".join(rv)
  74
  75
  76 def createWriter(sizer, *items):
  77     "Helper method to fill in tests, one set of writes, one for each item"
  78     locitems = copy.deepcopy(items)
  79     for item in locitems:
  80         item["length"] = sizer(item.pop("length", 0))
  81
  82     def helper(self):
  83         mode = "w"
  84         if os.path.isfile(self.f):
  85             mode = "a"
  86         zf = ZipFile(self.f, mode, self.compression)
  87         for item in locitems:
  88             self._write(zf, **item)
  89         zf = None
  90         pass
  91
  92     return helper
  93
  94
  95 def createTester(name, *writes):
  96     """Helper method to fill in tests, calls into a list of write
  97     helper methods.
  98     """
  99     _writes = copy.copy(writes)
 100
 101     def tester(self):
 102         for w in _writes:
 103             getattr(self, w)()
 104         self._verifyZip()
 105         pass
 106
 107     # unit tests get confused if the method name isn't test...
 108     tester.__name__ = name
 109     return tester
 110
 111
 112 class TestExtensiveStored(unittest.TestCase):
 113     """Unit tests for MozZipFile
 114
 115     The testcase are actually populated by code following the class
 116     definition.
 117     """
 118
 119     stage = "mozzipfilestage"
 120     compression = zipfile.ZIP_STORED
 121
 122     def leaf(self, *leafs):
 123         return os.path.join(self.stage, *leafs)
 124
 125     def setUp(self):
 126         if os.path.exists(self.stage):
 127             shutil.rmtree(self.stage)
 128         os.mkdir(self.stage)
 129         self.f = self.leaf("test.jar")
 130         self.ref = {}
 131         self.seed = 0
 132
 133     def tearDown(self):
 134         self.f = None
 135         self.ref = None
 136
 137     def _verifyZip(self):
 138         zf = zipfile.ZipFile(self.f)
 139         badEntry = zf.testzip()
 140         self.failIf(badEntry, badEntry)
 141         zlist = zf.namelist()
 142         zlist.sort()
 143         vlist = self.ref.keys()
 144         vlist.sort()
 145         self.assertEqual(zlist, vlist)
 146         for leaf, content in self.ref.iteritems():
 147             zcontent = zf.read(leaf)
 148             self.assertEqual(content, zcontent)
 149
 150     def _write(self, zf, seed=None, leaf=0, length=0):
 151         if seed is None:
 152             seed = self.seed
 153             self.seed += 1
 154         random.seed(seed)
 155         leaf = leafs[leaf]
 156         content = getContent(length)
 157         self.ref[leaf] = content
 158         zf.writestr(leaf, content)
 159         dir = os.path.dirname(self.leaf("stage", leaf))
 160         if not os.path.isdir(dir):
 161             os.makedirs(dir)
 162         open(self.leaf("stage", leaf), "w").write(content)
 163
 164
 165 # all leafs in all lengths
 166 atomics = list(prod(xrange(len(leafs)), xrange(lengths)))
 167
 168 # populate TestExtensiveStore with testcases
 169 for w in xrange(writes):
 170     # Don't iterate over all files for the the first n passes,
 171     # those are redundant as long as w < lengths.
 172     # There are symmetries in the trailing end, too, but I don't know
 173     # how to reduce those out right now.
 174     nonatomics = [
 175         list(prod(range(min(i, len(leafs))), xrange(lengths))) for i in xrange(1, w + 1)
 176     ] + [atomics]
 177     for descs in prod(*nonatomics):
 178         suffix = getid(descs)
 179         dicts = [dict(leaf=leaf, length=length) for leaf, length in descs]
 180         setattr(
 181             TestExtensiveStored, "_write" + suffix, createWriter(givenlength, *dicts)
 182         )
 183         setattr(
 184             TestExtensiveStored,
 185             "test" + suffix,
 186             createTester("test" + suffix, "_write" + suffix),
 187         )
 188
 189 # now create another round of tests, with two writing passes
 190 # first, write all file combinations into the jar, close it,
 191 # and then write all atomics again.
 192 # This should catch more or less all artifacts generated
 193 # by the final ordering step when closing the jar.
 194 files = [list(prod([i], xrange(lengths))) for i in xrange(len(leafs))]
 195 allfiles = reduce(
 196     lambda l, r: l + r, [list(prod(*files[: (i + 1)])) for i in xrange(len(leafs))]
 197 )
 198
 199 for first in allfiles:
 200     testbasename = "test{0}_".format(getid(first))
 201     test = [None, "_write" + getid(first), None]
 202     for second in atomics:
 203         test[0] = testbasename + getid([second])
 204         test[2] = "_write" + getid([second])
 205         setattr(TestExtensiveStored, test[0], createTester(*test))
 206
 207
 208 class TestExtensiveDeflated(TestExtensiveStored):
 209     "Test all that has been tested with ZIP_STORED with DEFLATED, too."
 210     compression = zipfile.ZIP_DEFLATED
 211
 212
 213 if __name__ == "__main__":
 214     unittest.main()