2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Tool for automatically creating .nmf files from .nexe/.pexe/.bc executables.
8 As well as creating the nmf file this tool can also find and stage
9 any shared libraries dependencies that the executables might have.
22 if sys
.version_info
< (2, 7, 0):
23 sys
.stderr
.write("python 2.7 or later is required run this script\n")
26 SCRIPT_DIR
= os
.path
.dirname(os
.path
.abspath(__file__
))
27 LIB_DIR
= os
.path
.join(SCRIPT_DIR
, 'lib')
29 sys
.path
.append(LIB_DIR
)
32 import get_shared_deps
43 # These constants are used within nmf files.
44 RUNNABLE_LD
= 'runnable-ld.so' # Name of the dynamic loader
45 MAIN_NEXE
= 'main.nexe' # Name of entry point for execution
46 PROGRAM_KEY
= 'program' # Key of the program section in an nmf file
47 URL_KEY
= 'url' # Key of the url field for a particular file in an nmf file
48 FILES_KEY
= 'files' # Key of the files section in an nmf file
49 PNACL_OPTLEVEL_KEY
= 'optlevel' # key for PNaCl optimization level
50 PORTABLE_KEY
= 'portable' # key for portable section of manifest
51 TRANSLATE_KEY
= 'pnacl-translate' # key for translatable objects
52 TRANSLATE_DEBUG_KEY
= 'pnacl-debug' # key for translatable debug objects
55 def DebugPrint(message
):
56 if DebugPrint
.debug_mode
:
57 sys
.stderr
.write('%s\n' % message
)
60 DebugPrint
.debug_mode
= False # Set to True to enable extra debug prints
64 """Returns all components of a path as a list.
67 'foo/bar/baz.blah' => ['foo', 'bar', 'baz.blah']
71 path
, part
= os
.path
.split(path
)
73 return result
[::-1] # Reverse.
76 def MakePosixPath(path
):
77 """Converts from the native format to posixpath format.
79 e.g. on Windows, "foo\\bar\\baz.blah" => "foo/bar/baz.blah"
80 on Mac/Linux this is a no-op.
82 if os
.path
== posixpath
:
84 return posixpath
.join(*SplitPath(path
))
87 def PosixRelPath(path
, start
):
88 """Takes two paths in native format, and produces a relative path in posix
92 For Windows: "foo\\bar\\baz.blah", "foo" => "bar/baz.blah"
93 For Mac/Linux: "foo/bar/baz.blah", "foo" => "bar/baz.blah"
95 NOTE: This function uses os.path.realpath to create a canonical path for
98 real_path
= os
.path
.realpath(path
)
99 real_start
= os
.path
.realpath(start
)
100 return MakePosixPath(os
.path
.relpath(real_path
, real_start
))
103 def DirectoryTreeContainsFile(dirname
, filename
):
104 """Returns True if a file is in a directory, or any of that directory's
105 subdirectories recursively.
108 DirectoryTreeContainsFile("foo", "foo/quux.txt") => True
109 DirectoryTreeContainsFile("foo", "foo/bar/baz/blah.txt") => True
110 DirectoryTreeContainsFile("foo", "bar/blah.txt") => False
112 real_dirname
= os
.path
.realpath(dirname
)
113 real_filename
= os
.path
.realpath(filename
)
114 return real_filename
.startswith(real_dirname
)
117 def MakeDir(dirname
):
118 """Just like os.makedirs but doesn't generate errors when dirname
121 if os
.path
.isdir(dirname
):
124 Trace("mkdir: %s" % dirname
)
127 except OSError as exception_info
:
128 if exception_info
.errno
!= errno
.EEXIST
:
132 def ParseElfHeader(path
):
133 """Wrap elf.ParseElfHeader to return raise this module's Error on failure."""
135 return elf
.ParseElfHeader(path
)
140 class Error(Exception):
141 """Local Error class for this file."""
145 class ArchFile(object):
146 """Simple structure containing information about an architecture-specific
150 name: Name of this file
151 path: Full path to this file on the build system
152 arch: Architecture of this file (e.g., x86-32)
153 url: Relative path to file in the staged web directory.
154 Used for specifying the "url" attribute in the nmf file."""
156 def __init__(self
, name
, path
, url
=None, arch
=None):
162 self
.arch
= ParseElfHeader(path
)[0]
165 return '<ArchFile %s>' % self
.path
168 """Return the file path when invoked with the str() function"""
172 class NmfUtils(object):
173 """Helper class for creating and managing nmf files"""
175 def __init__(self
, main_files
=None, objdump
=None,
176 lib_path
=None, extra_files
=None, lib_prefix
=None,
177 nexe_prefix
=None, no_arch_prefix
=None, remap
=None,
178 pnacl_optlevel
=None, pnacl_debug_optlevel
=None,
183 main_files: List of main entry program files. These will be named
184 files->main.nexe for dynamic nexes, and program for static nexes
185 objdump: path to x86_64-nacl-objdump tool (or Linux equivalent)
186 lib_path: List of paths to library directories
187 extra_files: List of extra files to include in the nmf
188 lib_prefix: A path prefix to prepend to the library paths, both for
189 staging the libraries and for inclusion into the nmf file.
190 Example: '../lib_dir'
191 nexe_prefix: Like lib_prefix, but is prepended to the nexes instead.
192 no_arch_prefix: Don't prefix shared libraries by lib32/lib64.
193 remap: Remaps the library name in the manifest.
194 pnacl_optlevel: Optimization level for PNaCl translation.
195 pnacl_debug_optlevel: Optimization level for debug PNaCl translation.
196 nmf_root: Directory of the NMF. All urls are relative to this directory.
198 assert len(main_files
) > 0
199 self
.objdump
= objdump
200 self
.main_files
= main_files
201 self
.extra_files
= extra_files
or []
202 self
.lib_path
= lib_path
or []
205 self
.lib_prefix
= lib_prefix
or ''
206 self
.nexe_prefix
= nexe_prefix
or ''
207 self
.no_arch_prefix
= no_arch_prefix
208 self
.remap
= remap
or {}
209 self
.pnacl
= main_files
[0].endswith(('.pexe', '.bc'))
210 self
.pnacl_optlevel
= pnacl_optlevel
211 self
.pnacl_debug_optlevel
= pnacl_debug_optlevel
212 if nmf_root
is not None:
213 self
.nmf_root
= nmf_root
215 # To match old behavior, if there is no nmf_root, use the directory of
216 # the first nexe found in main_files.
217 self
.nmf_root
= os
.path
.dirname(main_files
[0])
219 for filename
in self
.main_files
:
220 if not os
.path
.exists(filename
):
221 raise Error('Input file not found: %s' % filename
)
222 if not os
.path
.isfile(filename
):
223 raise Error('Input is not a file: %s' % filename
)
226 """Collect the list of dependencies for the main_files
229 A dict with key=filename and value=ArchFile of input files.
230 Includes the input files as well, with arch filled in if absent.
231 Example: { '/path/to/my.nexe': ArchFile(my.nexe),
232 '/path/to/libfoo.so': ArchFile(libfoo.so) }"""
237 DebugPrint('GetNeeded(%s)' % self
.main_files
)
240 self
.objdump
= FindObjdumpExecutable()
243 all_files
= get_shared_deps
.GetNeeded(self
.main_files
, self
.objdump
,
245 except get_shared_deps
.NoObjdumpError
:
246 raise Error('No objdump executable found (see --help for more info)')
247 except get_shared_deps
.Error
, e
:
252 # all_files is a dictionary mapping filename to architecture. self.needed
253 # should be a dictionary of filename to ArchFile.
254 for filename
, arch
in all_files
.iteritems():
255 name
= os
.path
.basename(filename
)
256 self
.needed
[filename
] = ArchFile(name
=name
, path
=filename
, arch
=arch
)
258 self
._SetArchFileUrls
()
262 def _SetArchFileUrls(self
):
263 """Fill in the url member of all ArchFiles in self.needed.
265 All urls are relative to the nmf_root. In addition, architecture-specific
266 files are relative to the .nexe with the matching architecture. This is
267 useful when making a multi-platform packaged app, so each architecture's
268 files are in a different directory.
270 # self.GetNeeded() should have already been called.
271 assert self
.needed
is not None
273 main_nexes
= [f
for f
in self
.main_files
if f
.endswith('.nexe')]
275 # map from each arch to its corresponding main nexe.
276 arch_to_main_dir
= {}
277 for main_file
in main_nexes
:
278 arch
, _
= ParseElfHeader(main_file
)
279 main_dir
= os
.path
.dirname(main_file
)
280 main_dir
= PosixRelPath(main_dir
, self
.nmf_root
)
283 arch_to_main_dir
[arch
] = main_dir
285 for arch_file
in self
.needed
.itervalues():
287 if DirectoryTreeContainsFile(self
.nmf_root
, arch_file
.path
):
288 # This file is already in the nmf_root tree, so it does not need to be
289 # staged. Just make the URL relative to the .nmf.
290 url
= PosixRelPath(arch_file
.path
, self
.nmf_root
)
292 # This file is outside of the nmf_root subtree, so it needs to be
293 # staged. Its path should be relative to the main .nexe with the same
295 prefix
= arch_to_main_dir
[arch_file
.arch
]
296 url
= os
.path
.basename(arch_file
.path
)
298 if arch_file
.name
.endswith('.nexe'):
299 prefix
= posixpath
.join(prefix
, self
.nexe_prefix
)
300 elif self
.no_arch_prefix
:
301 prefix
= posixpath
.join(prefix
, self
.lib_prefix
)
303 prefix
= posixpath
.join(
304 prefix
, self
.lib_prefix
, ARCH_LOCATION
[arch_file
.arch
])
305 arch_file
.url
= posixpath
.join(prefix
, url
)
307 def StageDependencies(self
, destination_dir
):
308 """Copies over the dependencies into a given destination directory
310 Each library will be put into a subdirectory that corresponds to the arch.
313 destination_dir: The destination directory for staging the dependencies
315 assert self
.needed
is not None
316 for arch_file
in self
.needed
.itervalues():
317 source
= arch_file
.path
318 destination
= os
.path
.join(destination_dir
, arch_file
.url
)
320 if (os
.path
.normcase(os
.path
.realpath(source
)) ==
321 os
.path
.normcase(os
.path
.realpath(destination
))):
324 # make sure target dir exists
325 MakeDir(os
.path
.dirname(destination
))
327 Trace('copy: %s -> %s' % (source
, destination
))
328 shutil
.copy2(source
, destination
)
330 def _GeneratePNaClManifest(self
):
332 manifest
[PROGRAM_KEY
] = {}
333 manifest
[PROGRAM_KEY
][PORTABLE_KEY
] = {}
334 portable
= manifest
[PROGRAM_KEY
][PORTABLE_KEY
]
335 for filename
in self
.main_files
:
337 'url': os
.path
.basename(filename
),
339 if filename
.endswith('.pexe'):
340 if self
.pnacl_optlevel
is not None:
341 translate_dict
[PNACL_OPTLEVEL_KEY
] = self
.pnacl_optlevel
342 if TRANSLATE_KEY
in portable
:
343 raise Error('Multiple .pexe files')
344 portable
[TRANSLATE_KEY
] = translate_dict
345 elif filename
.endswith('.bc'):
346 if self
.pnacl_debug_optlevel
is not None:
347 translate_dict
[PNACL_OPTLEVEL_KEY
] = self
.pnacl_debug_optlevel
348 if TRANSLATE_DEBUG_KEY
in portable
:
349 raise Error('Multiple .bc files')
350 portable
[TRANSLATE_DEBUG_KEY
] = translate_dict
352 raise Error('Unexpected executable type: %s' % filename
)
353 self
.manifest
= manifest
355 def _GenerateManifest(self
):
356 """Create a JSON formatted dict containing the files
358 NaCl will map url requests based on architecture. The startup NEXE
359 can always be found under the top key PROGRAM. Additional files are under
360 the FILES key further mapped by file name. In the case of 'runnable' the
361 PROGRAM key is populated with urls pointing the runnable-ld.so which acts
362 as the startup nexe. The application itself is then placed under the
363 FILES key mapped as 'main.exe' instead of the original name so that the
366 manifest
= { FILES_KEY
: {}, PROGRAM_KEY
: {} }
368 needed
= self
.GetNeeded()
370 runnable
= any(n
.endswith(RUNNABLE_LD
) for n
in needed
)
372 extra_files_kv
= [(key
, ArchFile(name
=key
,
376 for key
, arch
, url
in self
.extra_files
]
378 for need
, archinfo
in needed
.items() + extra_files_kv
:
379 urlinfo
= { URL_KEY
: archinfo
.url
}
382 # If starting with runnable-ld.so, make that the main executable.
384 if need
.endswith(RUNNABLE_LD
):
385 manifest
[PROGRAM_KEY
][archinfo
.arch
] = urlinfo
388 if need
in self
.main_files
:
389 if need
.endswith(".nexe"):
390 # Place it under program if we aren't using the runnable-ld.so.
392 manifest
[PROGRAM_KEY
][archinfo
.arch
] = urlinfo
394 # Otherwise, treat it like another another file named main.nexe.
397 name
= self
.remap
.get(name
, name
)
398 fileinfo
= manifest
[FILES_KEY
].get(name
, {})
399 fileinfo
[archinfo
.arch
] = urlinfo
400 manifest
[FILES_KEY
][name
] = fileinfo
401 self
.manifest
= manifest
403 def GetManifest(self
):
404 """Returns a JSON-formatted dict containing the NaCl dependencies"""
405 if self
.manifest
is None:
407 self
._GeneratePNaClManifest
()
409 self
._GenerateManifest
()
413 """Returns the Manifest as a JSON-formatted string"""
414 pretty_string
= json
.dumps(self
.GetManifest(), indent
=2)
415 # json.dumps sometimes returns trailing whitespace and does not put
416 # a newline at the end. This code fixes these problems.
417 pretty_lines
= pretty_string
.split('\n')
418 return '\n'.join([line
.rstrip() for line
in pretty_lines
]) + '\n'
423 sys
.stderr
.write(str(msg
) + '\n')
425 Trace
.verbose
= False
428 def ParseExtraFiles(encoded_list
, err
):
429 """Parse the extra-files list and return a canonicalized list of
430 [key, arch, url] triples. The |encoded_list| should be a list of
431 strings of the form 'key:url' or 'key:arch:url', where an omitted
432 'arch' is taken to mean 'portable'.
434 All entries in |encoded_list| are checked for syntax errors before
435 returning. Error messages are written to |err| (typically
436 sys.stderr) so that the user has actionable feedback for fixing all
437 errors, rather than one at a time. If there are any errors, None is
438 returned instead of a list, since an empty list is a valid return
443 for ix
in range(len(encoded_list
)):
444 kv
= encoded_list
[ix
]
445 unquoted
= quote
.unquote(kv
, ':')
446 if len(unquoted
) == 3:
447 if unquoted
[1] != ':':
448 err
.write('Syntax error for key:value tuple ' +
449 'for --extra-files argument: ' + kv
+ '\n')
452 canonicalized
.append([unquoted
[0], 'portable', unquoted
[2]])
453 elif len(unquoted
) == 5:
454 if unquoted
[1] != ':' or unquoted
[3] != ':':
455 err
.write('Syntax error for key:arch:url tuple ' +
456 'for --extra-files argument: ' +
460 canonicalized
.append([unquoted
[0], unquoted
[2], unquoted
[4]])
462 err
.write('Bad key:arch:url tuple for --extra-files: ' + kv
+ '\n')
469 """Determine current NACL_SDK_ROOT, either via the environment variable
470 itself, or by attempting to derive it from the location of this script.
472 sdk_root
= os
.environ
.get('NACL_SDK_ROOT')
474 sdk_root
= os
.path
.dirname(SCRIPT_DIR
)
475 if not os
.path
.exists(os
.path
.join(sdk_root
, 'toolchain')):
481 def FindObjdumpExecutable():
482 """Derive path to objdump executable to use for determining shared
485 sdk_root
= GetSDKRoot()
489 osname
= getos
.GetPlatform()
490 toolchain
= os
.path
.join(sdk_root
, 'toolchain', '%s_x86_glibc' % osname
)
491 objdump
= os
.path
.join(toolchain
, 'bin', 'x86_64-nacl-objdump')
495 if not os
.path
.exists(objdump
):
496 sys
.stderr
.write('WARNING: failed to find objdump in default '
497 'location: %s' % objdump
)
503 def GetDefaultLibPath(config
):
504 """Derive default library path to use when searching for shared
505 objects. This currently include the toolchain library folders
506 as well as the top level SDK lib folder and the naclports lib
507 folder. We include both 32-bit and 64-bit library paths.
509 assert(config
in ('Debug', 'Release'))
510 sdk_root
= GetSDKRoot()
512 # TOOD(sbc): output a warning here? We would also need to suppress
513 # the warning when run from the chromium build.
516 osname
= getos
.GetPlatform()
518 # Core toolchain libraries
519 'toolchain/%s_x86_glibc/x86_64-nacl/lib' % osname
,
520 'toolchain/%s_x86_glibc/x86_64-nacl/lib32' % osname
,
521 # naclports installed libraries
522 'toolchain/%s_x86_glibc/x86_64-nacl/usr/lib' % osname
,
523 'toolchain/%s_x86_glibc/i686-nacl/usr/lib' % osname
,
524 # SDK bundle libraries
525 'lib/glibc_x86_32/%s' % config
,
526 'lib/glibc_x86_64/%s' % config
,
527 # naclports bundle libraries
528 'ports/lib/glibc_x86_32/%s' % config
,
529 'ports/lib/glibc_x86_64/%s' % config
,
532 bionic_dir
= 'toolchain/%s_arm_bionic' % osname
533 if os
.path
.isdir(os
.path
.join(sdk_root
, bionic_dir
)):
535 '%s/arm-nacl/lib' % bionic_dir
,
536 '%s/arm-nacl/usr/lib' % bionic_dir
,
537 'lib/bionic_arm/%s' % config
,
539 libpath
= [os
.path
.normpath(p
) for p
in libpath
]
540 libpath
= [os
.path
.join(sdk_root
, p
) for p
in libpath
]
545 parser
= argparse
.ArgumentParser(description
=__doc__
)
546 parser
.add_argument('-o', '--output', dest
='output',
547 help='Write manifest file to FILE (default is stdout)',
549 parser
.add_argument('-D', '--objdump', dest
='objdump',
550 help='Override the default "objdump" tool used to find '
551 'shared object dependencies',
553 parser
.add_argument('--no-default-libpath', action
='store_true',
554 help="Don't include the SDK default library paths")
555 parser
.add_argument('--debug-libs', action
='store_true',
556 help='Use debug library paths when constructing default '
558 parser
.add_argument('-L', '--library-path', dest
='lib_path',
559 action
='append', default
=[],
560 help='Add DIRECTORY to library search path',
562 parser
.add_argument('-P', '--path-prefix', dest
='path_prefix', default
='',
563 help='Deprecated. An alias for --lib-prefix.',
565 parser
.add_argument('-p', '--lib-prefix', dest
='lib_prefix', default
='',
566 help='A path to prepend to shared libraries in the .nmf',
568 parser
.add_argument('-N', '--nexe-prefix', dest
='nexe_prefix', default
='',
569 help='A path to prepend to nexes in the .nmf',
571 parser
.add_argument('-s', '--stage-dependencies', dest
='stage_dependencies',
572 help='Destination directory for staging libraries',
574 parser
.add_argument('--no-arch-prefix', action
='store_true',
575 help='Don\'t put shared libraries in the lib32/lib64 '
576 'directories. Instead, they will be put in the same '
577 'directory as the .nexe that matches its architecture.')
578 parser
.add_argument('-t', '--toolchain', help='Legacy option, do not use')
579 parser
.add_argument('-n', '--name', dest
='name',
580 help='Rename FOO as BAR',
581 action
='append', default
=[], metavar
='FOO,BAR')
582 parser
.add_argument('-x', '--extra-files',
583 help=('Add extra key:file tuple to the "files"' +
584 ' section of the .nmf'),
585 action
='append', default
=[], metavar
='FILE')
586 parser
.add_argument('-O', '--pnacl-optlevel',
587 help='Set the optimization level to N in PNaCl manifests',
589 parser
.add_argument('--pnacl-debug-optlevel',
590 help='Set the optimization level to N for debugging '
591 'sections in PNaCl manifests',
593 parser
.add_argument('-v', '--verbose',
594 help='Verbose output', action
='store_true')
595 parser
.add_argument('-d', '--debug-mode',
596 help='Debug mode', action
='store_true')
597 parser
.add_argument('executables', metavar
='EXECUTABLE', nargs
='+')
599 # To enable bash completion for this command first install optcomplete
600 # and then add this line to your .bashrc:
601 # complete -F _optcomplete create_nmf.py
604 optcomplete
.autocomplete(parser
)
608 options
= parser
.parse_args(args
)
611 if options
.debug_mode
:
612 DebugPrint
.debug_mode
= True
614 if options
.toolchain
is not None:
615 sys
.stderr
.write('warning: option -t/--toolchain is deprecated.\n')
617 canonicalized
= ParseExtraFiles(options
.extra_files
, sys
.stderr
)
618 if canonicalized
is None:
619 parser
.error('Bad --extra-files (-x) argument syntax')
622 for ren
in options
.name
:
623 parts
= ren
.split(',')
625 parser
.error('Expecting --name=<orig_arch.so>,<new_name.so>')
626 remap
[parts
[0]] = parts
[1]
628 if options
.path_prefix
:
629 options
.lib_prefix
= options
.path_prefix
631 for libpath
in options
.lib_path
:
632 if not os
.path
.exists(libpath
):
633 sys
.stderr
.write('Specified library path does not exist: %s\n' % libpath
)
634 elif not os
.path
.isdir(libpath
):
635 sys
.stderr
.write('Specified library is not a directory: %s\n' % libpath
)
637 if not options
.no_default_libpath
:
638 # Add default libraries paths to the end of the search path.
639 config
= options
.debug_libs
and 'Debug' or 'Release'
640 options
.lib_path
+= GetDefaultLibPath(config
)
641 for path
in options
.lib_path
:
642 Trace('libpath: %s' % path
)
644 pnacl_optlevel
= None
645 if options
.pnacl_optlevel
is not None:
646 pnacl_optlevel
= int(options
.pnacl_optlevel
)
647 if pnacl_optlevel
< 0 or pnacl_optlevel
> 3:
649 'warning: PNaCl optlevel %d is unsupported (< 0 or > 3)\n' %
651 if options
.pnacl_debug_optlevel
is not None:
652 pnacl_debug_optlevel
= int(options
.pnacl_debug_optlevel
)
654 pnacl_debug_optlevel
= pnacl_optlevel
658 nmf_root
= os
.path
.dirname(options
.output
)
660 nmf
= NmfUtils(objdump
=options
.objdump
,
661 main_files
=options
.executables
,
662 lib_path
=options
.lib_path
,
663 extra_files
=canonicalized
,
664 lib_prefix
=options
.lib_prefix
,
665 nexe_prefix
=options
.nexe_prefix
,
666 no_arch_prefix
=options
.no_arch_prefix
,
668 pnacl_optlevel
=pnacl_optlevel
,
669 pnacl_debug_optlevel
=pnacl_debug_optlevel
,
672 if not options
.output
:
673 sys
.stdout
.write(nmf
.GetJson())
675 with
open(options
.output
, 'w') as output
:
676 output
.write(nmf
.GetJson())
678 if options
.stage_dependencies
and not nmf
.pnacl
:
679 Trace('Staging dependencies...')
680 nmf
.StageDependencies(options
.stage_dependencies
)
685 if __name__
== '__main__':
687 rtn
= main(sys
.argv
[1:])
689 sys
.stderr
.write('%s: %s\n' % (os
.path
.basename(__file__
), e
))
691 except KeyboardInterrupt:
692 sys
.stderr
.write('%s: interrupted\n' % os
.path
.basename(__file__
))