[SyncFS] Build indexes from FileTracker entries on disk.
[chromium-blink-merge.git] / native_client_sdk / src / tools / tests / create_nmf_test.py
blob86a89a2e272bdbb8dbc032fe7ef9b31d653de68c
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5 import json
6 import os
7 import posixpath
8 import shutil
9 import subprocess
10 import sys
11 import tempfile
12 import unittest
14 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
15 TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
16 DATA_DIR = os.path.join(TOOLS_DIR, 'lib', 'tests', 'data')
17 CHROME_SRC = os.path.dirname(os.path.dirname(os.path.dirname(TOOLS_DIR)))
18 MOCK_DIR = os.path.join(CHROME_SRC, 'third_party', 'pymock')
20 # For the mock library
21 sys.path.append(MOCK_DIR)
22 sys.path.append(TOOLS_DIR)
24 import build_paths
25 import create_nmf
26 import getos
27 import mock
29 TOOLCHAIN_OUT = os.path.join(build_paths.OUT_DIR, 'sdk_tests', 'toolchain')
30 NACL_X86_GLIBC_TOOLCHAIN = os.path.join(TOOLCHAIN_OUT,
31 '%s_x86' % getos.GetPlatform(),
32 'nacl_x86_glibc')
34 PosixRelPath = create_nmf.PosixRelPath
37 def StripSo(name):
38 """Strip trailing hexidecimal characters from the name of a shared object.
40 It strips everything after the last '.' in the name, and checks that the new
41 name ends with .so.
43 e.g.
45 libc.so.ad6acbfa => libc.so
46 foo.bar.baz => foo.bar.baz
47 """
48 stripped_name = '.'.join(name.split('.')[:-1])
49 if stripped_name.endswith('.so'):
50 return stripped_name
51 return name
54 class TestPosixRelPath(unittest.TestCase):
55 def testBasic(self):
56 # Note that PosixRelPath only converts from native path format to posix
57 # path format, that's why we have to use os.path.join here.
58 path = os.path.join(os.path.sep, 'foo', 'bar', 'baz.blah')
59 start = os.path.sep + 'foo'
60 self.assertEqual(PosixRelPath(path, start), 'bar/baz.blah')
63 class TestDefaultLibpath(unittest.TestCase):
64 def testWithoutNaClSDKRoot(self):
65 """GetDefaultLibPath wihtout NACL_SDK_ROOT set
67 In the absence of NACL_SDK_ROOT GetDefaultLibPath should
68 return the empty list."""
69 with mock.patch.dict('os.environ', clear=True):
70 paths = create_nmf.GetDefaultLibPath('Debug')
71 self.assertEqual(paths, [])
73 def testHonorNaClSDKRoot(self):
74 with mock.patch.dict('os.environ', {'NACL_SDK_ROOT': '/dummy/path'}):
75 paths = create_nmf.GetDefaultLibPath('Debug')
76 for path in paths:
77 self.assertTrue(path.startswith('/dummy/path'))
79 def testIncludesNaClPorts(self):
80 with mock.patch.dict('os.environ', {'NACL_SDK_ROOT': '/dummy/path'}):
81 paths = create_nmf.GetDefaultLibPath('Debug')
82 self.assertTrue(any(os.path.join('ports', 'lib') in p for p in paths),
83 "naclports libpath missing: %s" % str(paths))
86 class TestNmfUtils(unittest.TestCase):
87 """Tests for the main NmfUtils class in create_nmf."""
89 def setUp(self):
90 self.tempdir = None
91 self.toolchain = NACL_X86_GLIBC_TOOLCHAIN
92 self.objdump = os.path.join(self.toolchain, 'bin', 'i686-nacl-objdump')
93 if os.name == 'nt':
94 self.objdump += '.exe'
95 self._Mktemp()
97 def _CreateTestNexe(self, name, arch):
98 """Create an empty test .nexe file for use in create_nmf tests.
100 This is used rather than checking in test binaries since the
101 checked in binaries depend on .so files that only exist in the
102 certain SDK that build them.
104 compiler = os.path.join(self.toolchain, 'bin', '%s-nacl-g++' % arch)
105 if os.name == 'nt':
106 compiler += '.exe'
107 os.environ['CYGWIN'] = 'nodosfilewarning'
108 program = 'int main() { return 0; }'
109 name = os.path.join(self.tempdir, name)
110 dst_dir = os.path.dirname(name)
111 if not os.path.exists(dst_dir):
112 os.makedirs(dst_dir)
113 cmd = [compiler, '-pthread', '-x' , 'c', '-o', name, '-']
114 p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
115 p.communicate(input=program)
116 self.assertEqual(p.returncode, 0)
117 return name
119 def tearDown(self):
120 if self.tempdir:
121 shutil.rmtree(self.tempdir)
123 def _Mktemp(self):
124 self.tempdir = tempfile.mkdtemp()
126 def _CreateNmfUtils(self, nexes, **kwargs):
127 if not kwargs.get('lib_path'):
128 # Use lib instead of lib64 (lib64 is a symlink to lib).
129 kwargs['lib_path'] = [
130 os.path.join(self.toolchain, 'x86_64-nacl', 'lib'),
131 os.path.join(self.toolchain, 'x86_64-nacl', 'lib32')]
132 return create_nmf.NmfUtils(nexes,
133 objdump=self.objdump,
134 **kwargs)
136 def _CreateStatic(self, arch_path=None, **kwargs):
137 """Copy all static .nexe files from the DATA_DIR to a temporary directory.
139 Args:
140 arch_path: A dictionary mapping architecture to the directory to generate
141 the .nexe for the architecture in.
142 kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
143 constructor.
145 Returns:
146 A tuple with 2 elements:
147 * The generated NMF as a dictionary (i.e. parsed by json.loads)
148 * A list of the generated .nexe paths
150 arch_path = arch_path or {}
151 nexes = []
152 for arch in ('x86_64', 'x86_32', 'arm'):
153 nexe_name = 'test_static_%s.nexe' % arch
154 src_nexe = os.path.join(DATA_DIR, nexe_name)
155 dst_nexe = os.path.join(self.tempdir, arch_path.get(arch, ''), nexe_name)
156 dst_dir = os.path.dirname(dst_nexe)
157 if not os.path.exists(dst_dir):
158 os.makedirs(dst_dir)
159 shutil.copy(src_nexe, dst_nexe)
160 nexes.append(dst_nexe)
162 nexes.sort()
163 nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
164 nmf = json.loads(nmf_utils.GetJson())
165 return nmf, nexes
167 def _CreateDynamicAndStageDeps(self, arch_path=None, **kwargs):
168 """Create dynamic .nexe files and put them in a temporary directory, with
169 their dependencies staged in the same directory.
171 Args:
172 arch_path: A dictionary mapping architecture to the directory to generate
173 the .nexe for the architecture in.
174 kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
175 constructor.
177 Returns:
178 A tuple with 2 elements:
179 * The generated NMF as a dictionary (i.e. parsed by json.loads)
180 * A list of the generated .nexe paths
182 arch_path = arch_path or {}
183 nexes = []
184 for arch in ('x86_64', 'x86_32'):
185 nexe_name = 'test_dynamic_%s.nexe' % arch
186 rel_nexe = os.path.join(arch_path.get(arch, ''), nexe_name)
187 arch_alt = 'i686' if arch == 'x86_32' else arch
188 nexe = self._CreateTestNexe(rel_nexe, arch_alt)
189 nexes.append(nexe)
191 nexes.sort()
192 nmf_utils = self._CreateNmfUtils(nexes, **kwargs)
193 nmf = json.loads(nmf_utils.GetJson())
194 nmf_utils.StageDependencies(self.tempdir)
196 return nmf, nexes
198 def _CreatePexe(self, **kwargs):
199 """Copy test.pexe from the DATA_DIR to a temporary directory.
201 Args:
202 kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
203 constructor.
205 Returns:
206 A tuple with 2 elements:
207 * The generated NMF as a dictionary (i.e. parsed by json.loads)
208 * A list of the generated .pexe paths
210 pexe_name = 'test.pexe'
211 src_pexe = os.path.join(DATA_DIR, pexe_name)
212 dst_pexe = os.path.join(self.tempdir, pexe_name)
213 shutil.copy(src_pexe, dst_pexe)
215 pexes = [dst_pexe]
216 nmf_utils = self._CreateNmfUtils(pexes, **kwargs)
217 nmf = json.loads(nmf_utils.GetJson())
219 return nmf, pexes
221 def _CreateBitCode(self, **kwargs):
222 """Copy test.bc from the DATA_DIR to a temporary directory.
224 Args:
225 kwargs: Keyword arguments to pass through to create_nmf.NmfUtils
226 constructor.
228 Returns:
229 A tuple with 2 elements:
230 * The generated NMF as a dictionary (i.e. parsed by json.loads)
231 * A list of the generated .bc paths
233 bc_name = 'test.bc'
234 src_bc = os.path.join(DATA_DIR, bc_name)
235 dst_bc = os.path.join(self.tempdir, bc_name)
236 shutil.copy(src_bc, dst_bc)
238 bcs = [dst_bc]
239 nmf_utils = self._CreateNmfUtils(bcs, **kwargs)
240 nmf = json.loads(nmf_utils.GetJson())
242 return nmf, bcs
244 def assertManifestEquals(self, manifest, expected):
245 """Compare two manifest dictionaries.
247 The input manifest is regenerated with all string keys and values being
248 processed through StripSo, to remove the random hexidecimal characters at
249 the end of shared object names.
251 Args:
252 manifest: The generated manifest.
253 expected: The expected manifest.
255 def StripSoCopyDict(d):
256 new_d = {}
257 for k, v in d.iteritems():
258 new_k = StripSo(k)
259 if isinstance(v, (str, unicode)):
260 new_v = StripSo(v)
261 elif type(v) is list:
262 new_v = v[:]
263 elif type(v) is dict:
264 new_v = StripSoCopyDict(v)
265 else:
266 # Assume that anything else can be copied directly.
267 new_v = v
269 new_d[new_k] = new_v
270 return new_d
272 self.assertEqual(StripSoCopyDict(manifest), expected)
274 def assertStagingEquals(self, expected):
275 """Compare the contents of the temporary directory, to an expected
276 directory layout.
278 Args:
279 expected: The expected directory layout.
281 all_files = []
282 for root, _, files in os.walk(self.tempdir):
283 rel_root_posix = PosixRelPath(root, self.tempdir)
284 for f in files:
285 path = posixpath.join(rel_root_posix, StripSo(f))
286 if path.startswith('./'):
287 path = path[2:]
288 all_files.append(path)
289 self.assertEqual(set(expected), set(all_files))
292 def testStatic(self):
293 nmf, _ = self._CreateStatic()
294 expected_manifest = {
295 'files': {},
296 'program': {
297 'x86-64': {'url': 'test_static_x86_64.nexe'},
298 'x86-32': {'url': 'test_static_x86_32.nexe'},
299 'arm': {'url': 'test_static_arm.nexe'},
302 self.assertManifestEquals(nmf, expected_manifest)
304 def testStaticWithPath(self):
305 arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
306 nmf, _ = self._CreateStatic(arch_dir, nmf_root=self.tempdir)
307 expected_manifest = {
308 'files': {},
309 'program': {
310 'x86-32': {'url': 'x86_32/test_static_x86_32.nexe'},
311 'x86-64': {'url': 'x86_64/test_static_x86_64.nexe'},
312 'arm': {'url': 'arm/test_static_arm.nexe'},
315 self.assertManifestEquals(nmf, expected_manifest)
317 def testStaticWithPathNoNmfRoot(self):
318 # This case is not particularly useful, but it is similar to how create_nmf
319 # used to work. If there is no nmf_root given, all paths are relative to
320 # the first nexe passed on the commandline. I believe the assumption
321 # previously was that all .nexes would be in the same directory.
322 arch_dir = {'x86_32': 'x86_32', 'x86_64': 'x86_64', 'arm': 'arm'}
323 nmf, _ = self._CreateStatic(arch_dir)
324 expected_manifest = {
325 'files': {},
326 'program': {
327 'x86-32': {'url': '../x86_32/test_static_x86_32.nexe'},
328 'x86-64': {'url': '../x86_64/test_static_x86_64.nexe'},
329 'arm': {'url': 'test_static_arm.nexe'},
332 self.assertManifestEquals(nmf, expected_manifest)
334 def testStaticWithNexePrefix(self):
335 nmf, _ = self._CreateStatic(nexe_prefix='foo')
336 expected_manifest = {
337 'files': {},
338 'program': {
339 'x86-64': {'url': 'foo/test_static_x86_64.nexe'},
340 'x86-32': {'url': 'foo/test_static_x86_32.nexe'},
341 'arm': {'url': 'foo/test_static_arm.nexe'},
344 self.assertManifestEquals(nmf, expected_manifest)
346 def testDynamic(self):
347 nmf, nexes = self._CreateDynamicAndStageDeps()
348 expected_manifest = {
349 'files': {
350 'main.nexe': {
351 'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
352 'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
354 'libc.so': {
355 'x86-32': {'url': 'lib32/libc.so'},
356 'x86-64': {'url': 'lib64/libc.so'},
358 'libgcc_s.so': {
359 'x86-32': {'url': 'lib32/libgcc_s.so'},
360 'x86-64': {'url': 'lib64/libgcc_s.so'},
362 'libpthread.so': {
363 'x86-32': {'url': 'lib32/libpthread.so'},
364 'x86-64': {'url': 'lib64/libpthread.so'},
367 'program': {
368 'x86-32': {'url': 'lib32/runnable-ld.so'},
369 'x86-64': {'url': 'lib64/runnable-ld.so'},
373 expected_staging = [os.path.basename(f) for f in nexes]
374 expected_staging.extend([
375 'lib32/libc.so',
376 'lib32/libgcc_s.so',
377 'lib32/libpthread.so',
378 'lib32/runnable-ld.so',
379 'lib64/libc.so',
380 'lib64/libgcc_s.so',
381 'lib64/libpthread.so',
382 'lib64/runnable-ld.so'])
384 self.assertManifestEquals(nmf, expected_manifest)
385 self.assertStagingEquals(expected_staging)
387 def testDynamicWithPath(self):
388 arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
389 nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
390 nmf_root=self.tempdir)
391 expected_manifest = {
392 'files': {
393 'main.nexe': {
394 'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
395 'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
397 'libc.so': {
398 'x86-32': {'url': 'x86_32/lib32/libc.so'},
399 'x86-64': {'url': 'x86_64/lib64/libc.so'},
401 'libgcc_s.so': {
402 'x86-32': {'url': 'x86_32/lib32/libgcc_s.so'},
403 'x86-64': {'url': 'x86_64/lib64/libgcc_s.so'},
405 'libpthread.so': {
406 'x86-32': {'url': 'x86_32/lib32/libpthread.so'},
407 'x86-64': {'url': 'x86_64/lib64/libpthread.so'},
410 'program': {
411 'x86-32': {'url': 'x86_32/lib32/runnable-ld.so'},
412 'x86-64': {'url': 'x86_64/lib64/runnable-ld.so'},
416 expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
417 expected_staging.extend([
418 'x86_32/lib32/libc.so',
419 'x86_32/lib32/libgcc_s.so',
420 'x86_32/lib32/libpthread.so',
421 'x86_32/lib32/runnable-ld.so',
422 'x86_64/lib64/libc.so',
423 'x86_64/lib64/libgcc_s.so',
424 'x86_64/lib64/libpthread.so',
425 'x86_64/lib64/runnable-ld.so'])
427 self.assertManifestEquals(nmf, expected_manifest)
428 self.assertStagingEquals(expected_staging)
430 def testDynamicWithRelPath(self):
431 """Test that when the nmf root is a relative path that things work."""
432 arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
433 old_path = os.getcwd()
434 try:
435 os.chdir(self.tempdir)
436 nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir, nmf_root='')
437 expected_manifest = {
438 'files': {
439 'main.nexe': {
440 'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
441 'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
443 'libc.so': {
444 'x86-32': {'url': 'x86_32/lib32/libc.so'},
445 'x86-64': {'url': 'x86_64/lib64/libc.so'},
447 'libgcc_s.so': {
448 'x86-32': {'url': 'x86_32/lib32/libgcc_s.so'},
449 'x86-64': {'url': 'x86_64/lib64/libgcc_s.so'},
451 'libpthread.so': {
452 'x86-32': {'url': 'x86_32/lib32/libpthread.so'},
453 'x86-64': {'url': 'x86_64/lib64/libpthread.so'},
456 'program': {
457 'x86-32': {'url': 'x86_32/lib32/runnable-ld.so'},
458 'x86-64': {'url': 'x86_64/lib64/runnable-ld.so'},
462 expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
463 expected_staging.extend([
464 'x86_32/lib32/libc.so',
465 'x86_32/lib32/libgcc_s.so',
466 'x86_32/lib32/libpthread.so',
467 'x86_32/lib32/runnable-ld.so',
468 'x86_64/lib64/libc.so',
469 'x86_64/lib64/libgcc_s.so',
470 'x86_64/lib64/libpthread.so',
471 'x86_64/lib64/runnable-ld.so'])
473 self.assertManifestEquals(nmf, expected_manifest)
474 self.assertStagingEquals(expected_staging)
475 finally:
476 os.chdir(old_path)
478 def testDynamicWithPathNoArchPrefix(self):
479 arch_dir = {'x86_64': 'x86_64', 'x86_32': 'x86_32'}
480 nmf, nexes = self._CreateDynamicAndStageDeps(arch_dir,
481 nmf_root=self.tempdir,
482 no_arch_prefix=True)
483 expected_manifest = {
484 'files': {
485 'main.nexe': {
486 'x86-32': {'url': 'x86_32/test_dynamic_x86_32.nexe'},
487 'x86-64': {'url': 'x86_64/test_dynamic_x86_64.nexe'},
489 'libc.so': {
490 'x86-32': {'url': 'x86_32/libc.so'},
491 'x86-64': {'url': 'x86_64/libc.so'},
493 'libgcc_s.so': {
494 'x86-32': {'url': 'x86_32/libgcc_s.so'},
495 'x86-64': {'url': 'x86_64/libgcc_s.so'},
497 'libpthread.so': {
498 'x86-32': {'url': 'x86_32/libpthread.so'},
499 'x86-64': {'url': 'x86_64/libpthread.so'},
502 'program': {
503 'x86-32': {'url': 'x86_32/runnable-ld.so'},
504 'x86-64': {'url': 'x86_64/runnable-ld.so'},
508 expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
509 expected_staging.extend([
510 'x86_32/libc.so',
511 'x86_32/libgcc_s.so',
512 'x86_32/libpthread.so',
513 'x86_32/runnable-ld.so',
514 'x86_64/libc.so',
515 'x86_64/libgcc_s.so',
516 'x86_64/libpthread.so',
517 'x86_64/runnable-ld.so'])
519 self.assertManifestEquals(nmf, expected_manifest)
520 self.assertStagingEquals(expected_staging)
522 def testDynamicWithLibPrefix(self):
523 nmf, nexes = self._CreateDynamicAndStageDeps(lib_prefix='foo')
524 expected_manifest = {
525 'files': {
526 'main.nexe': {
527 'x86-32': {'url': 'test_dynamic_x86_32.nexe'},
528 'x86-64': {'url': 'test_dynamic_x86_64.nexe'},
530 'libc.so': {
531 'x86-32': {'url': 'foo/lib32/libc.so'},
532 'x86-64': {'url': 'foo/lib64/libc.so'},
534 'libgcc_s.so': {
535 'x86-32': {'url': 'foo/lib32/libgcc_s.so'},
536 'x86-64': {'url': 'foo/lib64/libgcc_s.so'},
538 'libpthread.so': {
539 'x86-32': {'url': 'foo/lib32/libpthread.so'},
540 'x86-64': {'url': 'foo/lib64/libpthread.so'},
543 'program': {
544 'x86-32': {'url': 'foo/lib32/runnable-ld.so'},
545 'x86-64': {'url': 'foo/lib64/runnable-ld.so'},
549 expected_staging = [PosixRelPath(f, self.tempdir) for f in nexes]
550 expected_staging.extend([
551 'foo/lib32/libc.so',
552 'foo/lib32/libgcc_s.so',
553 'foo/lib32/libpthread.so',
554 'foo/lib32/runnable-ld.so',
555 'foo/lib64/libc.so',
556 'foo/lib64/libgcc_s.so',
557 'foo/lib64/libpthread.so',
558 'foo/lib64/runnable-ld.so'])
560 self.assertManifestEquals(nmf, expected_manifest)
561 self.assertStagingEquals(expected_staging)
563 def testPexe(self):
564 nmf, _ = self._CreatePexe()
565 expected_manifest = {
566 'program': {
567 'portable': {
568 'pnacl-translate': {
569 'url': 'test.pexe'
574 self.assertManifestEquals(nmf, expected_manifest)
576 def testPexeOptLevel(self):
577 nmf, _ = self._CreatePexe(pnacl_optlevel=2)
578 expected_manifest = {
579 'program': {
580 'portable': {
581 'pnacl-translate': {
582 'url': 'test.pexe',
583 'optlevel': 2,
588 self.assertManifestEquals(nmf, expected_manifest)
590 def testBitCode(self):
591 nmf, _ = self._CreateBitCode(pnacl_debug_optlevel=0)
592 expected_manifest = {
593 'program': {
594 'portable': {
595 'pnacl-debug': {
596 'url': 'test.bc',
597 'optlevel': 0,
602 self.assertManifestEquals(nmf, expected_manifest)
605 if __name__ == '__main__':
606 unittest.main()