2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Tool for automatically creating .nmf files from .nexe/.pexe/.bc executables.
8 As well as creating the nmf file this tool can also find and stage
9 any shared libraries dependencies that the executables might have.
22 if sys
.version_info
< (2, 6, 0):
23 sys
.stderr
.write("python 2.6 or later is required run this script\n")
26 SCRIPT_DIR
= os
.path
.dirname(os
.path
.abspath(__file__
))
27 LIB_DIR
= os
.path
.join(SCRIPT_DIR
, 'lib')
29 sys
.path
.append(LIB_DIR
)
32 import get_shared_deps
43 # These constants are used within nmf files.
44 RUNNABLE_LD
= 'runnable-ld.so' # Name of the dynamic loader
45 MAIN_NEXE
= 'main.nexe' # Name of entry point for execution
46 PROGRAM_KEY
= 'program' # Key of the program section in an nmf file
47 URL_KEY
= 'url' # Key of the url field for a particular file in an nmf file
48 FILES_KEY
= 'files' # Key of the files section in an nmf file
49 PNACL_OPTLEVEL_KEY
= 'optlevel' # key for PNaCl optimization level
50 PORTABLE_KEY
= 'portable' # key for portable section of manifest
51 TRANSLATE_KEY
= 'pnacl-translate' # key for translatable objects
52 TRANSLATE_DEBUG_KEY
= 'pnacl-debug' # key for translatable debug objects
55 def DebugPrint(message
):
56 if DebugPrint
.debug_mode
:
57 sys
.stderr
.write('%s\n' % message
)
60 DebugPrint
.debug_mode
= False # Set to True to enable extra debug prints
64 """Returns all components of a path as a list.
67 'foo/bar/baz.blah' => ['foo', 'bar', 'baz.blah']
71 path
, part
= os
.path
.split(path
)
73 return result
[::-1] # Reverse.
76 def MakePosixPath(path
):
77 """Converts from the native format to posixpath format.
79 e.g. on Windows, "foo\\bar\\baz.blah" => "foo/bar/baz.blah"
80 on Mac/Linux this is a no-op.
82 if os
.path
== posixpath
:
84 return posixpath
.join(*SplitPath(path
))
87 def PosixRelPath(path
, start
):
88 """Takes two paths in native format, and produces a relative path in posix
92 For Windows: "foo\\bar\\baz.blah", "foo" => "bar/baz.blah"
93 For Mac/Linux: "foo/bar/baz.blah", "foo" => "bar/baz.blah"
95 NOTE: This function uses os.path.realpath to create a canonical path for
98 real_path
= os
.path
.realpath(path
)
99 real_start
= os
.path
.realpath(start
)
100 return MakePosixPath(os
.path
.relpath(real_path
, real_start
))
103 def DirectoryTreeContainsFile(dirname
, filename
):
104 """Returns True if a file is in a directory, or any of that directory's
105 subdirectories recursively.
108 DirectoryTreeContainsFile("foo", "foo/quux.txt") => True
109 DirectoryTreeContainsFile("foo", "foo/bar/baz/blah.txt") => True
110 DirectoryTreeContainsFile("foo", "bar/blah.txt") => False
112 real_dirname
= os
.path
.realpath(dirname
)
113 real_filename
= os
.path
.realpath(filename
)
114 return real_filename
.startswith(real_dirname
)
117 def MakeDir(dirname
):
118 """Just like os.makedirs but doesn't generate errors when dirname
121 if os
.path
.isdir(dirname
):
124 Trace("mkdir: %s" % dirname
)
127 except OSError as exception_info
:
128 if exception_info
.errno
!= errno
.EEXIST
:
132 def ParseElfHeader(path
):
133 """Wrap elf.ParseElfHeader to return raise this module's Error on failure."""
135 return elf
.ParseElfHeader(path
)
140 class Error(Exception):
141 """Local Error class for this file."""
145 class ArchFile(object):
146 """Simple structure containing information about an architecture-specific
150 name: Name of this file
151 path: Full path to this file on the build system
152 arch: Architecture of this file (e.g., x86-32)
153 url: Relative path to file in the staged web directory.
154 Used for specifying the "url" attribute in the nmf file."""
156 def __init__(self
, name
, path
, url
=None, arch
=None):
162 self
.arch
= ParseElfHeader(path
)[0]
165 return '<ArchFile %s>' % self
.path
168 """Return the file path when invoked with the str() function"""
172 class NmfUtils(object):
173 """Helper class for creating and managing nmf files"""
175 def __init__(self
, main_files
=None, objdump
=None,
176 lib_path
=None, extra_files
=None, lib_prefix
=None,
177 nexe_prefix
=None, no_arch_prefix
=None, remap
=None,
178 pnacl_optlevel
=None, pnacl_debug_optlevel
=None,
183 main_files: List of main entry program files. These will be named
184 files->main.nexe for dynamic nexes, and program for static nexes
185 objdump: path to x86_64-nacl-objdump tool (or Linux equivalent)
186 lib_path: List of paths to library directories
187 extra_files: List of extra files to include in the nmf
188 lib_prefix: A path prefix to prepend to the library paths, both for
189 staging the libraries and for inclusion into the nmf file.
190 Example: '../lib_dir'
191 nexe_prefix: Like lib_prefix, but is prepended to the nexes instead.
192 no_arch_prefix: Don't prefix shared libraries by lib32/lib64.
193 remap: Remaps the library name in the manifest.
194 pnacl_optlevel: Optimization level for PNaCl translation.
195 pnacl_debug_optlevel: Optimization level for debug PNaCl translation.
196 nmf_root: Directory of the NMF. All urls are relative to this directory.
198 assert len(main_files
) > 0
199 self
.objdump
= objdump
200 self
.main_files
= main_files
201 self
.extra_files
= extra_files
or []
202 self
.lib_path
= lib_path
or []
205 self
.lib_prefix
= lib_prefix
or ''
206 self
.nexe_prefix
= nexe_prefix
or ''
207 self
.no_arch_prefix
= no_arch_prefix
208 self
.remap
= remap
or {}
209 self
.pnacl
= main_files
[0].endswith(('.pexe', '.bc'))
210 self
.pnacl_optlevel
= pnacl_optlevel
211 self
.pnacl_debug_optlevel
= pnacl_debug_optlevel
212 if nmf_root
is not None:
213 self
.nmf_root
= nmf_root
215 # To match old behavior, if there is no nmf_root, use the directory of
216 # the first nexe found in main_files.
217 self
.nmf_root
= os
.path
.dirname(main_files
[0])
219 for filename
in self
.main_files
:
220 if not os
.path
.exists(filename
):
221 raise Error('Input file not found: %s' % filename
)
222 if not os
.path
.isfile(filename
):
223 raise Error('Input is not a file: %s' % filename
)
226 """Collect the list of dependencies for the main_files
229 A dict with key=filename and value=ArchFile of input files.
230 Includes the input files as well, with arch filled in if absent.
231 Example: { '/path/to/my.nexe': ArchFile(my.nexe),
232 '/path/to/libfoo.so': ArchFile(libfoo.so) }"""
237 DebugPrint('GetNeeded(%s)' % self
.main_files
)
240 self
.objdump
= FindObjdumpExecutable()
243 all_files
= get_shared_deps
.GetNeeded(self
.main_files
, self
.objdump
,
245 except get_shared_deps
.NoObjdumpError
:
246 raise Error('No objdump executable found (see --help for more info)')
247 except get_shared_deps
.Error
, e
:
252 # all_files is a dictionary mapping filename to architecture. self.needed
253 # should be a dictionary of filename to ArchFile.
254 for filename
, arch
in all_files
.iteritems():
255 name
= os
.path
.basename(filename
)
256 self
.needed
[filename
] = ArchFile(name
=name
, path
=filename
, arch
=arch
)
258 self
._SetArchFileUrls
()
262 def _SetArchFileUrls(self
):
263 """Fill in the url member of all ArchFiles in self.needed.
265 All urls are relative to the nmf_root. In addition, architecture-specific
266 files are relative to the .nexe with the matching architecture. This is
267 useful when making a multi-platform packaged app, so each architecture's
268 files are in a different directory.
270 # self.GetNeeded() should have already been called.
271 assert self
.needed
is not None
273 main_nexes
= [f
for f
in self
.main_files
if f
.endswith('.nexe')]
275 # map from each arch to its corresponding main nexe.
276 arch_to_main_dir
= {}
277 for main_file
in main_nexes
:
278 arch
, _
= ParseElfHeader(main_file
)
279 main_dir
= os
.path
.dirname(main_file
)
280 main_dir
= PosixRelPath(main_dir
, self
.nmf_root
)
283 arch_to_main_dir
[arch
] = main_dir
285 for arch_file
in self
.needed
.itervalues():
287 if DirectoryTreeContainsFile(self
.nmf_root
, arch_file
.path
):
288 # This file is already in the nmf_root tree, so it does not need to be
289 # staged. Just make the URL relative to the .nmf.
290 url
= PosixRelPath(arch_file
.path
, self
.nmf_root
)
292 # This file is outside of the nmf_root subtree, so it needs to be
293 # staged. Its path should be relative to the main .nexe with the same
295 prefix
= arch_to_main_dir
[arch_file
.arch
]
296 url
= os
.path
.basename(arch_file
.path
)
298 if arch_file
.name
.endswith('.nexe'):
299 prefix
= posixpath
.join(prefix
, self
.nexe_prefix
)
300 elif self
.no_arch_prefix
:
301 prefix
= posixpath
.join(prefix
, self
.lib_prefix
)
303 prefix
= posixpath
.join(
304 prefix
, self
.lib_prefix
, ARCH_LOCATION
[arch_file
.arch
])
305 arch_file
.url
= posixpath
.join(prefix
, url
)
307 def StageDependencies(self
, destination_dir
):
308 """Copies over the dependencies into a given destination directory
310 Each library will be put into a subdirectory that corresponds to the arch.
313 destination_dir: The destination directory for staging the dependencies
315 assert self
.needed
is not None
316 for arch_file
in self
.needed
.itervalues():
317 source
= arch_file
.path
318 destination
= os
.path
.join(destination_dir
, arch_file
.url
)
320 if (os
.path
.normcase(os
.path
.realpath(source
)) ==
321 os
.path
.normcase(os
.path
.realpath(destination
))):
324 # make sure target dir exists
325 MakeDir(os
.path
.dirname(destination
))
327 Trace('copy: %s -> %s' % (source
, destination
))
328 shutil
.copy2(source
, destination
)
330 def _GeneratePNaClManifest(self
):
332 manifest
[PROGRAM_KEY
] = {}
333 manifest
[PROGRAM_KEY
][PORTABLE_KEY
] = {}
334 portable
= manifest
[PROGRAM_KEY
][PORTABLE_KEY
]
335 for filename
in self
.main_files
:
337 'url': os
.path
.basename(filename
),
339 if filename
.endswith('.pexe'):
340 if self
.pnacl_optlevel
is not None:
341 translate_dict
[PNACL_OPTLEVEL_KEY
] = self
.pnacl_optlevel
342 if TRANSLATE_KEY
in portable
:
343 raise Error('Multiple .pexe files')
344 portable
[TRANSLATE_KEY
] = translate_dict
345 elif filename
.endswith('.bc'):
346 if self
.pnacl_debug_optlevel
is not None:
347 translate_dict
[PNACL_OPTLEVEL_KEY
] = self
.pnacl_debug_optlevel
348 if TRANSLATE_DEBUG_KEY
in portable
:
349 raise Error('Multiple .bc files')
350 portable
[TRANSLATE_DEBUG_KEY
] = translate_dict
352 raise Error('Unexpected executable type: %s' % filename
)
353 self
.manifest
= manifest
355 def _GenerateManifest(self
):
356 """Create a JSON formatted dict containing the files
358 NaCl will map url requests based on architecture. The startup NEXE
359 can always be found under the top key PROGRAM. Additional files are under
360 the FILES key further mapped by file name. In the case of 'runnable' the
361 PROGRAM key is populated with urls pointing the runnable-ld.so which acts
362 as the startup nexe. The application itself is then placed under the
363 FILES key mapped as 'main.exe' instead of the original name so that the
366 manifest
= { FILES_KEY
: {}, PROGRAM_KEY
: {} }
368 needed
= self
.GetNeeded()
370 runnable
= any(n
.endswith(RUNNABLE_LD
) for n
in needed
)
372 extra_files_kv
= [(key
, ArchFile(name
=key
,
376 for key
, arch
, url
in self
.extra_files
]
378 for need
, archinfo
in needed
.items() + extra_files_kv
:
379 urlinfo
= { URL_KEY
: archinfo
.url
}
382 # If starting with runnable-ld.so, make that the main executable.
384 if need
.endswith(RUNNABLE_LD
):
385 manifest
[PROGRAM_KEY
][archinfo
.arch
] = urlinfo
388 if need
in self
.main_files
:
389 if need
.endswith(".nexe"):
390 # Place it under program if we aren't using the runnable-ld.so.
392 manifest
[PROGRAM_KEY
][archinfo
.arch
] = urlinfo
394 # Otherwise, treat it like another another file named main.nexe.
397 name
= self
.remap
.get(name
, name
)
398 fileinfo
= manifest
[FILES_KEY
].get(name
, {})
399 fileinfo
[archinfo
.arch
] = urlinfo
400 manifest
[FILES_KEY
][name
] = fileinfo
401 self
.manifest
= manifest
403 def GetManifest(self
):
404 """Returns a JSON-formatted dict containing the NaCl dependencies"""
405 if not self
.manifest
:
407 self
._GeneratePNaClManifest
()
409 self
._GenerateManifest
()
413 """Returns the Manifest as a JSON-formatted string"""
414 pretty_string
= json
.dumps(self
.GetManifest(), indent
=2)
415 # json.dumps sometimes returns trailing whitespace and does not put
416 # a newline at the end. This code fixes these problems.
417 pretty_lines
= pretty_string
.split('\n')
418 return '\n'.join([line
.rstrip() for line
in pretty_lines
]) + '\n'
423 sys
.stderr
.write(str(msg
) + '\n')
425 Trace
.verbose
= False
428 def ParseExtraFiles(encoded_list
, err
):
429 """Parse the extra-files list and return a canonicalized list of
430 [key, arch, url] triples. The |encoded_list| should be a list of
431 strings of the form 'key:url' or 'key:arch:url', where an omitted
432 'arch' is taken to mean 'portable'.
434 All entries in |encoded_list| are checked for syntax errors before
435 returning. Error messages are written to |err| (typically
436 sys.stderr) so that the user has actionable feedback for fixing all
437 errors, rather than one at a time. If there are any errors, None is
438 returned instead of a list, since an empty list is a valid return
443 for ix
in range(len(encoded_list
)):
444 kv
= encoded_list
[ix
]
445 unquoted
= quote
.unquote(kv
, ':')
446 if len(unquoted
) == 3:
447 if unquoted
[1] != ':':
448 err
.write('Syntax error for key:value tuple ' +
449 'for --extra-files argument: ' + kv
+ '\n')
452 canonicalized
.append([unquoted
[0], 'portable', unquoted
[2]])
453 elif len(unquoted
) == 5:
454 if unquoted
[1] != ':' or unquoted
[3] != ':':
455 err
.write('Syntax error for key:arch:url tuple ' +
456 'for --extra-files argument: ' +
460 canonicalized
.append([unquoted
[0], unquoted
[2], unquoted
[4]])
462 err
.write('Bad key:arch:url tuple for --extra-files: ' + kv
+ '\n')
469 """Determine current NACL_SDK_ROOT, either via the environment variable
470 itself, or by attempting to derive it from the location of this script.
472 sdk_root
= os
.environ
.get('NACL_SDK_ROOT')
474 sdk_root
= os
.path
.dirname(SCRIPT_DIR
)
475 if not os
.path
.exists(os
.path
.join(sdk_root
, 'toolchain')):
481 def FindObjdumpExecutable():
482 """Derive path to objdump executable to use for determining shared
485 sdk_root
= GetSDKRoot()
489 osname
= getos
.GetPlatform()
490 toolchain
= os
.path
.join(sdk_root
, 'toolchain', '%s_x86_glibc' % osname
)
491 objdump
= os
.path
.join(toolchain
, 'bin', 'x86_64-nacl-objdump')
495 if not os
.path
.exists(objdump
):
496 sys
.stderr
.write('WARNING: failed to find objdump in default '
497 'location: %s' % objdump
)
503 def GetDefaultLibPath(config
):
504 """Derive default library path to use when searching for shared
505 objects. This currently include the toolchain library folders
506 as well as the top level SDK lib folder and the naclports lib
507 folder. We include both 32-bit and 64-bit library paths.
509 assert(config
in ('Debug', 'Release'))
510 sdk_root
= GetSDKRoot()
512 # TOOD(sbc): output a warning here? We would also need to suppress
513 # the warning when run from the chromium build.
516 osname
= getos
.GetPlatform()
518 # Core toolchain libraries
519 'toolchain/%s_x86_glibc/x86_64-nacl/lib' % osname
,
520 'toolchain/%s_x86_glibc/x86_64-nacl/lib32' % osname
,
521 # naclports installed libraries
522 'toolchain/%s_x86_glibc/x86_64-nacl/usr/lib' % osname
,
523 'toolchain/%s_x86_glibc/i686-nacl/usr/lib' % osname
,
524 # SDK bundle libraries
525 'lib/glibc_x86_32/%s' % config
,
526 'lib/glibc_x86_64/%s' % config
,
527 # naclports bundle libraries
528 'ports/lib/glibc_x86_32/%s' % config
,
529 'ports/lib/glibc_x86_64/%s' % config
,
532 bionic_dir
= 'toolchain/%s_arm_bionic' % osname
533 if os
.path
.isdir(os
.path
.join(sdk_root
, bionic_dir
)):
535 '%s/arm-nacl/lib' % bionic_dir
,
536 '%s/arm-nacl/usr/lib' % bionic_dir
,
537 'lib/bionic_arm/%s' % config
,
539 libpath
= [os
.path
.normpath(p
) for p
in libpath
]
540 libpath
= [os
.path
.join(sdk_root
, p
) for p
in libpath
]
545 parser
= optparse
.OptionParser(
546 usage
='Usage: %prog [options] nexe [extra_libs...]', description
=__doc__
)
547 parser
.add_option('-o', '--output', dest
='output',
548 help='Write manifest file to FILE (default is stdout)',
550 parser
.add_option('-D', '--objdump', dest
='objdump',
551 help='Override the default "objdump" tool used to find '
552 'shared object dependencies',
554 parser
.add_option('--no-default-libpath', action
='store_true',
555 help="Don't include the SDK default library paths")
556 parser
.add_option('--debug-libs', action
='store_true',
557 help='Use debug library paths when constructing default '
559 parser
.add_option('-L', '--library-path', dest
='lib_path',
560 action
='append', default
=[],
561 help='Add DIRECTORY to library search path',
563 parser
.add_option('-P', '--path-prefix', dest
='path_prefix', default
='',
564 help='Deprecated. An alias for --lib-prefix.',
566 parser
.add_option('-p', '--lib-prefix', dest
='lib_prefix', default
='',
567 help='A path to prepend to shared libraries in the .nmf',
569 parser
.add_option('-N', '--nexe-prefix', dest
='nexe_prefix', default
='',
570 help='A path to prepend to nexes in the .nmf',
572 parser
.add_option('-s', '--stage-dependencies', dest
='stage_dependencies',
573 help='Destination directory for staging libraries',
575 parser
.add_option('--no-arch-prefix', action
='store_true',
576 help='Don\'t put shared libraries in the lib32/lib64 '
577 'directories. Instead, they will be put in the same '
578 'directory as the .nexe that matches its architecture.')
579 parser
.add_option('-t', '--toolchain', help='Legacy option, do not use')
580 parser
.add_option('-n', '--name', dest
='name',
581 help='Rename FOO as BAR',
582 action
='append', default
=[], metavar
='FOO,BAR')
583 parser
.add_option('-x', '--extra-files',
584 help=('Add extra key:file tuple to the "files"' +
585 ' section of the .nmf'),
586 action
='append', default
=[], metavar
='FILE')
587 parser
.add_option('-O', '--pnacl-optlevel',
588 help='Set the optimization level to N in PNaCl manifests',
590 parser
.add_option('--pnacl-debug-optlevel',
591 help='Set the optimization level to N for debugging '
592 'sections in PNaCl manifests',
594 parser
.add_option('-v', '--verbose',
595 help='Verbose output', action
='store_true')
596 parser
.add_option('-d', '--debug-mode',
597 help='Debug mode', action
='store_true')
599 # To enable bash completion for this command first install optcomplete
600 # and then add this line to your .bashrc:
601 # complete -F _optcomplete create_nmf.py
604 optcomplete
.autocomplete(parser
)
608 options
, args
= parser
.parse_args(argv
)
611 if options
.debug_mode
:
612 DebugPrint
.debug_mode
= True
614 if options
.toolchain
is not None:
615 sys
.stderr
.write('warning: option -t/--toolchain is deprecated.\n')
618 parser
.error('No nexe files specified. See --help for more info')
620 canonicalized
= ParseExtraFiles(options
.extra_files
, sys
.stderr
)
621 if canonicalized
is None:
622 parser
.error('Bad --extra-files (-x) argument syntax')
625 for ren
in options
.name
:
626 parts
= ren
.split(',')
628 parser
.error('Expecting --name=<orig_arch.so>,<new_name.so>')
629 remap
[parts
[0]] = parts
[1]
631 if options
.path_prefix
:
632 options
.lib_prefix
= options
.path_prefix
634 for libpath
in options
.lib_path
:
635 if not os
.path
.exists(libpath
):
636 sys
.stderr
.write('Specified library path does not exist: %s\n' % libpath
)
637 elif not os
.path
.isdir(libpath
):
638 sys
.stderr
.write('Specified library is not a directory: %s\n' % libpath
)
640 if not options
.no_default_libpath
:
641 # Add default libraries paths to the end of the search path.
642 config
= options
.debug_libs
and 'Debug' or 'Release'
643 options
.lib_path
+= GetDefaultLibPath(config
)
644 for path
in options
.lib_path
:
645 Trace('libpath: %s' % path
)
647 pnacl_optlevel
= None
648 if options
.pnacl_optlevel
is not None:
649 pnacl_optlevel
= int(options
.pnacl_optlevel
)
650 if pnacl_optlevel
< 0 or pnacl_optlevel
> 3:
652 'warning: PNaCl optlevel %d is unsupported (< 0 or > 3)\n' %
654 if options
.pnacl_debug_optlevel
is not None:
655 pnacl_debug_optlevel
= int(options
.pnacl_debug_optlevel
)
657 pnacl_debug_optlevel
= pnacl_optlevel
661 nmf_root
= os
.path
.dirname(options
.output
)
663 nmf
= NmfUtils(objdump
=options
.objdump
,
665 lib_path
=options
.lib_path
,
666 extra_files
=canonicalized
,
667 lib_prefix
=options
.lib_prefix
,
668 nexe_prefix
=options
.nexe_prefix
,
669 no_arch_prefix
=options
.no_arch_prefix
,
671 pnacl_optlevel
=pnacl_optlevel
,
672 pnacl_debug_optlevel
=pnacl_debug_optlevel
,
675 if not options
.output
:
676 sys
.stdout
.write(nmf
.GetJson())
678 with
open(options
.output
, 'w') as output
:
679 output
.write(nmf
.GetJson())
681 if options
.stage_dependencies
and not nmf
.pnacl
:
682 Trace('Staging dependencies...')
683 nmf
.StageDependencies(options
.stage_dependencies
)
688 if __name__
== '__main__':
690 rtn
= main(sys
.argv
[1:])
692 sys
.stderr
.write('%s: %s\n' % (os
.path
.basename(__file__
), e
))
694 except KeyboardInterrupt:
695 sys
.stderr
.write('%s: interrupted\n' % os
.path
.basename(__file__
))