naclbuild.py

   1
   2 import optparse
   3 import os
   4 import posixpath
   5 import re
   6 import signal
   7 import subprocess
   8 import sys
   9 import tempfile
  10
  11 import action_tree
  12 import cmd_env
  13 import deps_update
  14
  15
  16 # Work out tar/Python bug.
  17 # See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=532570
  18 # and http://bugs.python.org/issue1652
  19 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
  20
  21
  22 urls = {
  23     "python": "http://www.python.org/ftp/python/2.6.1/Python-2.6.1.tar.bz2",
  24     }
  25
  26 svn_repos = [
  27     ("http://nativeclient.googlecode.com/svn/trunk/src/third_party@1108",
  28      "third_party"),
  29     ("http://nativeclient.googlecode.com/svn/trunk/src/native_client/src/third_party/npapi@1424",
  30      "native_client/src/third_party/npapi"),
  31     ]
  32
  33
  34 def stamp_file(filename):
  35     fh = open(filename, "w")
  36     fh.close()
  37
  38
  39 def is_elf_file(filename):
  40     fh = open(filename)
  41     header = fh.read(4)
  42     fh.close()
  43     return header == "\x7fELF"
  44
  45
  46 def assert_equals(x, y):
  47     if x != y:
  48         if type(x) == str and type(y) == str:
  49             raise AssertionError("ACTUAL:\n%s\nEXPECTED:\n%s" % (x, y))
  50         raise AssertionError("%r != %r" % (x, y))
  51
  52
  53 # This is an alternative to subprocess.PIPE, which can cause deadlocks
  54 # if the pipe buffer gets filled up.
  55 def make_fh_pair():
  56     # Make read/write file handle pair.  This is like creating a pipe
  57     # FD pair, but without a pipe buffer limit.
  58     fd, filename = tempfile.mkstemp(prefix="nacl_test_")
  59     try:
  60         write_fh = os.fdopen(fd, "w", 0)
  61         read_fh = open(filename, "r")
  62     finally:
  63         os.unlink(filename)
  64     return write_fh, read_fh
  65
  66
  67 # Workaround for Python's variable binding semantics.
  68 def make_action(func, *args):
  69     return lambda log: func(*args)
  70
  71
  72 class NaClBuild(object):
  73
  74     def __init__(self, env, dir_path):
  75         self._env = env
  76         self._dir_path = dir_path
  77         self._glibc_src = "glibc"
  78         # This is the directory where scons looks for the toolchain by
  79         # default.
  80         self._sdkloc = os.path.join(
  81             self._dir_path, "native_client/src/third_party/nacl_sdk/linux/sdk")
  82
  83         def add_path(subdirs):
  84             path = ":".join(
  85                 [os.path.join(self._dir_path, subdir) for subdir in subdirs] +
  86                 [os.environ["PATH"]])
  87             return ["env", "PATH=%s" % path]
  88
  89         subdirs = ["native_client/scons-out/dbg-linux-x86-32/staging",
  90                    os.path.join(self._sdkloc, "nacl-sdk/bin"),
  91                    "bin"]
  92         self._path_env = cmd_env.PrefixCmdEnv(add_path(subdirs), env)
  93         self._gcc_override_env = cmd_env.PrefixCmdEnv(
  94             add_path(subdirs + ["override-bin"]), env)
  95         self.path_env = self._path_env
  96
  97     def checkout_thirdparty(self, log):
  98         for url, dest in svn_repos:
  99             self._env.cmd(["svn", "checkout", url, dest])
 100
 101     def svn_checkout(self, log):
 102         self._env.cmd(["python", "mini_gclient.py"])
 103
 104     def _git_fetch_repo(self, dir_path, url):
 105         # This is like "git clone --no-checkout", but it is more incremental.
 106         git_dir = os.path.join(dir_path, ".git")
 107         if not os.path.exists(git_dir):
 108             self._env.cmd(["mkdir", "-p", dir_path])
 109             self._env.cmd(cmd_env.in_dir(dir_path) +
 110                           ["git", "init"])
 111             self._env.cmd(cmd_env.in_dir(dir_path) +
 112                           ["git", "remote", "add", "origin", url])
 113         refspec = "+refs/heads/*:refs/remotes/origin/*"
 114         self._env.cmd(cmd_env.in_dir(dir_path) +
 115                       ["git", "fetch", url, refspec])
 116
 117     def _git_checkout_repo(self, dir_path, commit):
 118         # This will leave HEAD set to a commit ID, not a branch.
 119         # Be careful.  One could lose commits that are not saved
 120         # in branches.
 121         self._env.cmd(cmd_env.in_dir(dir_path) +
 122                       ["git", "checkout", commit])
 123
 124     @action_tree.action_node
 125     def git_fetch(self):
 126         for mod in deps_update.get_modules():
 127             yield mod["name"], make_action(
 128                 self._git_fetch_repo, mod["path"], mod["url"])
 129
 130     @action_tree.action_node
 131     def git_checkout(self):
 132         for mod in deps_update.get_modules():
 133             yield mod["name"], make_action(
 134                 self._git_checkout_repo, mod["path"], mod["commit"])
 135
 136     def make_stamps(self, log):
 137         # Stops tools/Makefile deleting our source and Git repos.
 138         assert os.path.exists("native_client/tools/BUILD/binutils-2.20")
 139         stamp_file("native_client/tools/BUILD/stamp-binutils-2.20")
 140
 141     def fetch_sqlite(self, log):
 142         self._env.cmd(["mkdir", "-p", "debs"])
 143         url = "http://archive.ubuntu.com/ubuntu/pool/main/s/sqlite3/sqlite3_3.6.10-1.dsc"
 144         # Disable signature checking because it doesn't work under
 145         # karmic where the public key is apparently not available.
 146         # Our SVN downloads are not secure so this is not a big deal.
 147         # We'd really want to include a hash here anyway, not a public
 148         # key check.
 149         # TODO: do a secure download.
 150         self._env.cmd(cmd_env.in_dir("debs") +
 151                       ["dget", "--allow-unauthenticated", "-x", url])
 152
 153     def fetch_python(self, log):
 154         self._env.cmd(["wget", "-c", urls["python"],
 155                        "-O", posixpath.basename(urls["python"])])
 156
 157     def unpack_python(self, log):
 158         self._env.cmd(["tar", "-xjf", posixpath.basename(urls["python"])])
 159
 160     @action_tree.action_node
 161     def checkout(self):
 162         return [self.checkout_thirdparty,
 163                 self.git_fetch,
 164                 self.make_stamps,
 165                 self.git_checkout,
 166                 self.svn_checkout,
 167                 self.fetch_sqlite,
 168                 self.fetch_python,
 169                 self.unpack_python]
 170
 171     def build_toolchain(self, log):
 172         self._env.cmd(cmd_env.in_dir("native_client/tools")
 173                       + ["make", "SDKLOC=%s" % self._sdkloc])
 174
 175     def build_nacl(self, log):
 176         self._env.cmd(cmd_env.in_dir("native_client")
 177                       + ["./scons", "--mode=dbg-linux"])
 178
 179     def test_nacl(self, log):
 180         # TODO: use --mode=dbg-linux,nacl
 181         self._env.cmd(
 182             cmd_env.in_dir("native_client")
 183             + ["./scons", "--mode=dbg-linux", "run_all_tests"])
 184
 185     def configure_glibc(self, log):
 186         self._path_env.cmd(cmd_env.in_dir(self._glibc_src)
 187                            + ["./myconfig.sh"])
 188
 189     def build_glibc(self, log):
 190         self._env.cmd(cmd_env.in_dir("glibc/build")
 191                       + ["make"])
 192
 193     def test_glibc_static(self, log):
 194         self._path_env.cmd(cmd_env.in_dir(self._glibc_src)
 195                            + ["./make-example.sh"])
 196         self._path_env.cmd(
 197             ["ncval_stubout", "glibc/hellow"])
 198         write_fh, read_fh = make_fh_pair()
 199         proc = self._path_env.cmd(["sel_ldr", "-d", "glibc/hellow"],
 200                                   stdout=write_fh)
 201         assert_equals(read_fh.read(),
 202                       "Hello (via write())\n"
 203                       "Hello world (via printf())\n")
 204
 205     def _run_dynamic(self, args, **kwargs):
 206         return self._path_env.cmd(
 207             ["sel_ldr",
 208              "-d", "install-stubout/lib/ld-linux.so.2",
 209              "--", "--library-path", "install-stubout/lib"]
 210             + args, **kwargs)
 211
 212     def test_glibc_dynamic(self, log):
 213         self._path_env.cmd(cmd_env.in_dir(self._glibc_src)
 214                            + ["./make-example-dynamic.sh"])
 215         write_fh, read_fh = make_fh_pair()
 216         proc = self._run_dynamic(["glibc/hellow-dyn"], stdout=write_fh)
 217         assert_equals(read_fh.read(),
 218                       "Hello (via write())\n"
 219                       "Hello world (via printf())\n")
 220
 221     def test_running_ldso(self, log):
 222         # TODO: expect a non-zero exit code.
 223         write_fh, read_fh = make_fh_pair()
 224         try:
 225             self._run_dynamic([], stderr=write_fh)
 226         except cmd_env.CommandFailedError, exn:
 227             assert_equals(exn.rc, 127)
 228         else:
 229             raise AssertionError("Expected non-zero exit code")
 230         output = read_fh.read()
 231         print output,
 232         # Check for ld.so's help message.
 233         assert ("You have invoked `ld.so', the helper program "
 234                 "for shared library executables." in output)
 235
 236     def install_glibc(self, log):
 237         self._env.cmd(cmd_env.in_dir("glibc/build")
 238                       + ["make", "install"])
 239
 240     def stub_out(self, log):
 241         self._env.cmd(["rm", "-rf",
 242                        "install-stubout"])
 243         self._env.cmd(["cp", "-a",
 244                        "install",
 245                        "install-stubout"])
 246         self._path_env.cmd(
 247             ["sh", "-c",
 248              "ncval_stubout install-stubout/lib/*.so.*"])
 249
 250     def build_sqlite(self, log):
 251         self._gcc_override_env.cmd(
 252             cmd_env.in_dir("debs/sqlite3-3.6.10")
 253             + ["dpkg-buildpackage", "-b", "-us", "-uc", "-rfakeroot"])
 254
 255     def configure_python(self, log):
 256         self._gcc_override_env.cmd(cmd_env.in_dir("Python-2.6.1")
 257                                    + ["i386", "./configure"])
 258
 259     def build_python(self, log):
 260         self._gcc_override_env.cmd(
 261             cmd_env.in_dir("Python-2.6.1") + ["make"])
 262
 263     def _get_install_dir(self, subdir_name):
 264         install_dir = os.path.join(self._dir_path, "install-trees", subdir_name)
 265         self._env.cmd(["rm", "-rf", install_dir])
 266         self._env.cmd(["mkdir", "-p", install_dir])
 267         return install_dir
 268
 269     def _rewrite_for_nacl(self, install_dir):
 270         for dir_path, dirnames, filenames in os.walk(install_dir):
 271             for filename in filenames:
 272                 pathname = os.path.join(dir_path, filename)
 273                 if is_elf_file(pathname):
 274                     self._path_env.cmd(["ncrewrite", pathname])
 275
 276     def install_python(self, log):
 277         install_dir = self._get_install_dir("python")
 278         self._gcc_override_env.cmd(
 279             cmd_env.in_dir("Python-2.6.1")
 280             + ["make", "install", "DESTDIR=%s" % install_dir])
 281         self._rewrite_for_nacl(install_dir)
 282
 283     def install_sqlite(self, log):
 284         install_dir = self._get_install_dir("sqlite")
 285         self._env.cmd(["dpkg-deb", "-x", "debs/libsqlite3-0_3.6.10-1_i386.deb",
 286                        install_dir])
 287         self._rewrite_for_nacl(install_dir)
 288
 289     def build_python_extension(self, log):
 290         self._env.cmd(
 291             ["rm", "-rf", "googleclient/native_client/python_extension/build"])
 292         self._gcc_override_env.cmd(
 293             cmd_env.in_dir("googleclient/native_client/python_extension") +
 294             ["python", "setup.py", "install", "--install-lib=../imcplugin"])
 295         self._path_env.cmd(
 296             ["ncrewrite", "googleclient/native_client/imcplugin/nacl.so"])
 297
 298     def test_libraries(self, log):
 299         # Minimal library test.  Tests that we can load more libraries
 300         # than just libc.so, and that libraries can call via the PLT.
 301         gcc = "nacl-glibc-gcc"
 302         self._path_env.cmd([gcc, "-shared", "-fPIC", "tests/libhello.c",
 303                             "-o", "tests/libhello.so"])
 304         self._path_env.cmd([gcc, "tests/hello.c", "tests/libhello.so",
 305                             "-o", "tests/hello"])
 306         # Check that the executable runs natively as well as under sel_ldr.
 307         subprocess.check_call(["./tests/hello"])
 308         self._path_env.cmd(["ncrewrite", "tests/libhello.so", "tests/hello"])
 309         write_fh, read_fh = make_fh_pair()
 310         self._run_dynamic(["./tests/hello"], stdout=write_fh)
 311         assert_equals(read_fh.read(), "Hello world, in libhello\n")
 312
 313     def test_dlopen(self, log):
 314         self._path_env.cmd(["nacl-glibc-gcc", "tests/dlopen.c", "-ldl",
 315                             "-o", "tests/dlopen"])
 316         self._path_env.cmd(["ncrewrite", "tests/dlopen"])
 317         write_fh, read_fh = make_fh_pair()
 318         self._run_dynamic(["./tests/dlopen"], stdout=write_fh)
 319         assert_equals(read_fh.read(),
 320                       "Trying dlopen\nHello world, in libhello\nDone\n")
 321
 322     def test_static_dlopen(self, log):
 323         self._path_env.cmd([
 324                 "nacl-glibc-gcc", "-static",
 325                 "-Wl,-T,glibc/elf_i386.x",
 326                 "tests/dlopen.c", "-ldl", "-o", "tests/dlopen-static"])
 327         self._path_env.cmd(["ncrewrite", "tests/dlopen-static"])
 328         self._path_env.cmd(["ncval_stubout", "tests/dlopen-static"])
 329         write_fh, read_fh = make_fh_pair()
 330         self._path_env.cmd(["sel_ldr", "-d",
 331                             "-E", "LD_LIBRARY_PATH=install-stubout/lib",
 332                             "./tests/dlopen-static"], stdout=write_fh)
 333         # The output comes out in a different order from before
 334         # because the executable and library use different copies of
 335         # stdio, and thus different buffers for stdout.
 336         assert_equals(read_fh.read(),
 337                       "Hello world, in libhello\nTrying dlopen\nDone\n")
 338
 339     def test_toolchain(self, log):
 340         self._path_env.cmd(["python", "toolchain_test.py"])
 341
 342     def test_gcc_wrapper(self, log):
 343         self._path_env.cmd(["python", "gccwrapper_test.py"])
 344
 345     def test_python(self, log):
 346         self._path_env.cmd(["ncrewrite", "Python-2.6.1/python"])
 347         write_fh, read_fh = make_fh_pair()
 348         proc = self._run_dynamic(
 349             ["Python-2.6.1/python", "-c", 'print "Hello world!"'],
 350             stdout=write_fh)
 351         assert_equals(read_fh.read(), "Hello world!\n")
 352
 353     def test_debugger(self, log):
 354         self._path_env.cmd(["nacl-glibc-gcc", "-g", "tests/faulty.c",
 355                             "-o", "tests/faulty"])
 356         self._path_env.cmd(["ncrewrite", "tests/faulty"])
 357         write_fh, read_fh = make_fh_pair()
 358         self._path_env.cmd(["python", "debugger.py", "tests/faulty"],
 359                            stdout=write_fh)
 360         stdout = read_fh.read()
 361         expected = ("Caught signal\n",
 362                     "tests/faulty.c:5\n",
 363                     "csu/libc-start.c",
 364                     "Signal handler done, exiting\n")
 365         print stdout,
 366         for line in expected:
 367             if line not in stdout:
 368                 raise AssertionError("Text %r not found in output" % line)
 369         print "** Debug output OK"
 370
 371     @action_tree.action_node
 372     def build(self):
 373         return [self.build_toolchain,
 374                 self.build_nacl,
 375                 self.configure_glibc,
 376                 self.build_glibc,
 377                 self.install_glibc,
 378                 self.stub_out,
 379                 self.build_sqlite,
 380                 self.configure_python,
 381                 self.build_python,
 382                 self.install_python,
 383                 self.install_sqlite,
 384                 # self.build_python_extension, # TODO: re-enable
 385                 ]
 386
 387     @action_tree.action_node
 388     def test(self):
 389         return [self.test_gcc_wrapper,
 390                 self.test_nacl,
 391                 self.test_toolchain,
 392                 self.test_glibc_static,
 393                 self.test_running_ldso,
 394                 self.test_glibc_dynamic,
 395                 self.test_libraries,
 396                 self.test_dlopen,
 397                 self.test_static_dlopen,
 398                 self.test_python,
 399                 self.test_debugger]
 400
 401     @action_tree.action_node
 402     def all(self):
 403         return [self.checkout, self.build, self.test]
 404
 405
 406 class DryRunEnv(object):
 407
 408     def cmd(self, args, **kwargs):
 409         print " ".join(args)
 410
 411
 412 def main(args):
 413     parser = optparse.OptionParser()
 414     parser.add_option("--dry-run", action="store_true", dest="dry_run",
 415                       help="Print commands instead of running them")
 416     options, args = parser.parse_args(args)
 417     if options.dry_run:
 418         env = DryRunEnv()
 419     else:
 420         env = cmd_env.BasicEnv()
 421     tree = NaClBuild(env, os.getcwd())
 422     action_tree.action_main(tree.all, args)
 423
 424
 425 if __name__ == "__main__":
 426     main(sys.argv[1:])