libcxx/utils/adb_run.py

   1 #!/usr/bin/env python3
   2 # ===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 # ===----------------------------------------------------------------------===##
   9
  10 """adb_run.py is a utility for running a libc++ test program via adb.
  11 """
  12
  13 import argparse
  14 import hashlib
  15 import os
  16 import re
  17 import shlex
  18 import socket
  19 import subprocess
  20 import sys
  21 from typing import List, Tuple
  22
  23
  24 # Sync a host file /path/to/dir/file to ${REMOTE_BASE_DIR}/run-${HASH}/dir/file.
  25 REMOTE_BASE_DIR = "/data/local/tmp/adb_run"
  26
  27 g_job_limit_socket = None
  28 g_verbose = False
  29
  30
  31 def run_adb_sync_command(command: List[str]) -> None:
  32     """Run an adb command and discard the output, unless the command fails. If
  33     the command fails, dump the output instead, and exit the script with
  34     failure.
  35     """
  36     if g_verbose:
  37         sys.stderr.write(f"running: {shlex.join(command)}\n")
  38     proc = subprocess.run(command, universal_newlines=True,
  39                           stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
  40                           stderr=subprocess.STDOUT, encoding="utf-8")
  41     if proc.returncode != 0:
  42         # adb's stdout (e.g. for adb push) should normally be discarded, but
  43         # on failure, it should be shown. Print it to stderr because it's
  44         # unrelated to the test program's stdout output. A common error caught
  45         # here is "No space left on device".
  46         sys.stderr.write(f"{proc.stdout}\n"
  47                          f"error: adb command exited with {proc.returncode}: "
  48                          f"{shlex.join(command)}\n")
  49         sys.exit(proc.returncode)
  50
  51
  52 def sync_test_dir(local_dir: str, remote_dir: str) -> None:
  53     """Sync the libc++ test directory on the host to the remote device."""
  54
  55     # Optimization: The typical libc++ test directory has only a single
  56     # *.tmp.exe file in it. In that case, skip the `mkdir` command, which is
  57     # normally necessary because we don't know if the target directory already
  58     # exists on the device.
  59     local_files = os.listdir(local_dir)
  60     if len(local_files) == 1:
  61         local_file = os.path.join(local_dir, local_files[0])
  62         remote_file = os.path.join(remote_dir, local_files[0])
  63         if not os.path.islink(local_file) and os.path.isfile(local_file):
  64             run_adb_sync_command(["adb", "push", "--sync", local_file,
  65                                   remote_file])
  66             return
  67
  68     assert os.path.basename(local_dir) == os.path.basename(remote_dir)
  69     run_adb_sync_command(["adb", "shell", "mkdir", "-p", remote_dir])
  70     run_adb_sync_command(["adb", "push", "--sync", local_dir,
  71                           os.path.dirname(remote_dir)])
  72
  73
  74 def build_env_arg(env_args: List[str], prepend_path_args: List[Tuple[str, str]]) -> str:
  75     components = []
  76     for arg in env_args:
  77         k, v = arg.split("=", 1)
  78         components.append(f"export {k}={shlex.quote(v)}; ")
  79     for k, v in prepend_path_args:
  80         components.append(f"export {k}={shlex.quote(v)}${{{k}:+:${k}}}; ")
  81     return "".join(components)
  82
  83
  84 def run_command(args: argparse.Namespace) -> int:
  85     local_dir = args.execdir
  86     assert local_dir.startswith("/")
  87     assert not local_dir.endswith("/")
  88
  89     # Copy each execdir to a subdir of REMOTE_BASE_DIR. Name the directory using
  90     # a hash of local_dir so that concurrent adb_run invocations don't create
  91     # the same intermediate parent directory. At least `adb push` has trouble
  92     # with concurrent mkdir syscalls on common parent directories. (Somehow
  93     # mkdir fails with EAGAIN/EWOULDBLOCK, see internal Google bug,
  94     # b/289311228.)
  95     local_dir_hash = hashlib.sha1(local_dir.encode()).hexdigest()
  96     remote_dir = f"{REMOTE_BASE_DIR}/run-{local_dir_hash}/{os.path.basename(local_dir)}"
  97     sync_test_dir(local_dir, remote_dir)
  98
  99     adb_shell_command = (
 100         # Set the environment early so that PATH can be overridden. Overriding
 101         # PATH is useful for:
 102         #  - Replacing older shell utilities with toybox (e.g. on old devices).
 103         #  - Adding a `bash` command that delegates to `sh` (mksh).
 104         f"{build_env_arg(args.env, args.prepend_path_env)}"
 105
 106         # Set a high oom_score_adj so that, if the test program uses too much
 107         # memory, it is killed before anything else on the device. The default
 108         # oom_score_adj is -1000, so a test using too much memory typically
 109         # crashes the device.
 110         "echo 1000 >/proc/self/oom_score_adj; "
 111
 112         # If we're running as root, switch to the shell user. The libc++
 113         # filesystem tests require running without root permissions. Some x86
 114         # emulator devices (before Android N) do not have a working `adb unroot`
 115         # and always run as root. Non-debug builds typically lack `su` and only
 116         # run as the shell user.
 117         #
 118         # Some libc++ tests create temporary files in the working directory,
 119         # which might be owned by root. Before switching to shell, make the
 120         # cwd writable (and readable+executable) to every user.
 121         #
 122         # N.B.:
 123         #  - Avoid "id -u" because it wasn't supported until Android M.
 124         #  - The `env` and `which` commands were also added in Android M.
 125         #  - Starting in Android M, su from root->shell resets PATH, so we need
 126         #    to modify it again in the new environment.
 127         #  - Avoid chmod's "a+rwx" syntax because it's not supported until
 128         #    Android N.
 129         #  - Defining this function allows specifying the arguments to the test
 130         #    program (i.e. "$@") only once.
 131         "run_without_root() {"
 132         "  chmod 777 .;"
 133         "  case \"$(id)\" in"
 134         "    *\"uid=0(root)\"*)"
 135         "    if command -v env >/dev/null; then"
 136         "      su shell \"$(command -v env)\" PATH=\"$PATH\" \"$@\";"
 137         "    else"
 138         "      su shell \"$@\";"
 139         "    fi;;"
 140         "    *) \"$@\";;"
 141         "  esac;"
 142         "}; "
 143     )
 144
 145     # Older versions of Bionic limit the length of argv[0] to 127 bytes
 146     # (SOINFO_NAME_LEN-1), and the path to libc++ tests tend to exceed this
 147     # limit. Changing the working directory works around this limit. The limit
 148     # is increased to 4095 (PATH_MAX-1) in Android M (API 23).
 149     command_line = [arg.replace(local_dir + "/", "./") for arg in args.command]
 150
 151     # Prior to the adb feature "shell_v2" (added in Android N), `adb shell`
 152     # always created a pty:
 153     #  - This merged stdout and stderr together.
 154     #  - The pty converts LF to CRLF.
 155     #  - The exit code of the shell command wasn't propagated.
 156     # Work around all three limitations, unless "shell_v2" is present.
 157     proc = subprocess.run(["adb", "features"], check=True,
 158                           stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
 159                           encoding="utf-8")
 160     adb_features = set(proc.stdout.strip().split())
 161     has_shell_v2 = "shell_v2" in adb_features
 162     if has_shell_v2:
 163         adb_shell_command += (
 164             f"cd {remote_dir} && run_without_root {shlex.join(command_line)}"
 165         )
 166     else:
 167         adb_shell_command += (
 168             f"{{"
 169             f"  stdout=$("
 170             f"    cd {remote_dir} && run_without_root {shlex.join(command_line)};"
 171             f"    echo -n __libcxx_adb_exit__=$?"
 172             f"  ); "
 173             f"}} 2>&1; "
 174             f"echo -n __libcxx_adb_stdout__\"$stdout\""
 175         )
 176
 177     adb_command_line = ["adb", "shell", adb_shell_command]
 178     if g_verbose:
 179         sys.stderr.write(f"running: {shlex.join(adb_command_line)}\n")
 180
 181     if has_shell_v2:
 182         proc = subprocess.run(adb_command_line, shell=False, check=False,
 183                               encoding="utf-8")
 184         return proc.returncode
 185     else:
 186         proc = subprocess.run(adb_command_line, shell=False, check=False,
 187                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
 188                               encoding="utf-8")
 189         # The old `adb shell` mode used a pty, which converted LF to CRLF.
 190         # Convert it back.
 191         output = proc.stdout.replace("\r\n", "\n")
 192
 193         if proc.returncode:
 194             sys.stderr.write(f"error: adb failed:\n"
 195                              f"  command: {shlex.join(adb_command_line)}\n"
 196                              f"  output: {output}\n")
 197             return proc.returncode
 198
 199         match = re.match(r"(.*)__libcxx_adb_stdout__(.*)__libcxx_adb_exit__=(\d+)$",
 200                      output, re.DOTALL)
 201         if not match:
 202             sys.stderr.write(f"error: could not parse adb output:\n"
 203                              f"  command: {shlex.join(adb_command_line)}\n"
 204                              f"  output: {output}\n")
 205             return 1
 206
 207         sys.stderr.write(match.group(1))
 208         sys.stdout.write(match.group(2))
 209         return int(match.group(3))
 210
 211
 212 def connect_to_job_limiter_server(sock_addr: str) -> None:
 213     sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 214
 215     try:
 216         sock.connect(sock_addr)
 217     except (FileNotFoundError, ConnectionRefusedError) as e:
 218         # Copying-and-pasting an adb_run.py command-line from a lit test failure
 219         # is likely to fail because the socket no longer exists (or is
 220         # inactive), so just give a warning.
 221         sys.stderr.write(f"warning: could not connect to {sock_addr}: {e}\n")
 222         return
 223
 224     # The connect call can succeed before the server has called accept, because
 225     # of the listen backlog, so wait for the server to send a byte.
 226     sock.recv(1)
 227
 228     # Keep the socket open until this process ends, then let the OS close the
 229     # connection automatically.
 230     global g_job_limit_socket
 231     g_job_limit_socket = sock
 232
 233
 234 def main() -> int:
 235     """Main function (pylint wants this docstring)."""
 236     parser = argparse.ArgumentParser()
 237     parser.add_argument("--execdir", type=str, required=True)
 238     parser.add_argument("--env", type=str, required=False, action="append",
 239                         default=[], metavar="NAME=VALUE")
 240     parser.add_argument("--prepend-path-env", type=str, nargs=2, required=False,
 241                         action="append", default=[],
 242                         metavar=("NAME", "PATH"))
 243     parser.add_argument("--job-limit-socket")
 244     parser.add_argument("--verbose", "-v", default=False, action="store_true")
 245     parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
 246     args = parser.parse_args()
 247
 248     global g_verbose
 249     g_verbose = args.verbose
 250     if args.job_limit_socket is not None:
 251         connect_to_job_limiter_server(args.job_limit_socket)
 252     return run_command(args)
 253
 254
 255 if __name__ == '__main__':
 256     sys.exit(main())