llvm/utils/remote-exec.py

   1 #!/usr/bin/env python
   2 #===----------------------------------------------------------------------===##
   3 #
   4 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   5 # See https://llvm.org/LICENSE.txt for license information.
   6 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   7 #
   8 #===----------------------------------------------------------------------===##
   9
  10 """
  11 Runs an executable on a remote host.
  12
  13 This is meant to be used as an executor when running the LLVM and the Libraries
  14 tests on a target.
  15 """
  16
  17 import argparse
  18 import os
  19 import posixpath
  20 import shlex
  21 import subprocess
  22 import sys
  23 import tarfile
  24 import tempfile
  25 import re
  26
  27 def ssh(args, command):
  28     cmd = ['ssh', '-oBatchMode=yes']
  29     if args.extra_ssh_args is not None:
  30         cmd.extend(shlex.split(args.extra_ssh_args))
  31     return cmd + [args.host, command]
  32
  33
  34 def scp(args, src, dst):
  35     cmd = ['scp', '-q', '-oBatchMode=yes']
  36     if args.extra_scp_args is not None:
  37         cmd.extend(shlex.split(args.extra_scp_args))
  38     return cmd + [src, '{}:{}'.format(args.host, dst)]
  39
  40
  41 def main():
  42     parser = argparse.ArgumentParser()
  43     parser.add_argument('--host', type=str, required=True)
  44     parser.add_argument('--execdir', type=str, required=True)
  45     parser.add_argument('--extra-ssh-args', type=str, required=False)
  46     parser.add_argument('--extra-scp-args', type=str, required=False)
  47     parser.add_argument('--codesign_identity', type=str, required=False, default=None)
  48     parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
  49
  50     # Note: The default value is for the backward compatibility with a hack in
  51     # libcxx test suite.
  52     # If an argument is a file that ends in `.tmp.exe`, assume it is the name
  53     # of an executable generated by a test file. We call these test-executables
  54     # below. This allows us to do custom processing like codesigning test-executables
  55     # and changing their path when running on the remote host. It's also possible
  56     # for there to be no such executable, for example in the case of a .sh.cpp
  57     # test.
  58     parser.add_argument('--exec-pattern', type=str, required=False, default='.*\.tmp\.exe',
  59                         help='The name regex pattern of the executables generated by \
  60                               a test file. Specifying it allows us to do custom \
  61                               processing like codesigning test-executables \
  62                               and changing their path when running on \
  63                               the remote host. It\'s also possible for there \
  64                               to be no such executable, for example in \
  65                               the case of a .sh.cpp test.')
  66
  67     parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
  68     args = parser.parse_args()
  69     commandLine = args.command
  70
  71     # Create a temporary directory where the test will be run.
  72     # That is effectively the value of %T on the remote host.
  73     tmp = subprocess.check_output(
  74                   ssh(args, 'mktemp -d /tmp/llvm.XXXXXXXXXX'),
  75                   universal_newlines=True
  76               ).strip()
  77
  78     isExecutable = lambda exe: re.match(args.exec_pattern, exe) and os.path.exists(exe)
  79     pathOnRemote = lambda file: posixpath.join(tmp, os.path.basename(file))
  80
  81     try:
  82         # Do any necessary codesigning of test-executables found in the command line.
  83         if args.codesign_identity:
  84             for exe in filter(isExecutable, commandLine):
  85                 subprocess.check_call(
  86                     ['xcrun', 'codesign', '-f', '-s', args.codesign_identity, exe],
  87                     env={})
  88
  89         # tar up the execution directory (which contains everything that's needed
  90         # to run the test), and copy the tarball over to the remote host.
  91         try:
  92             tmpTar = tempfile.NamedTemporaryFile(suffix='.tar', delete=False)
  93             with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
  94                 tarball.add(args.execdir, arcname=os.path.basename(args.execdir))
  95
  96             # Make sure we close the file before we scp it, because accessing
  97             # the temporary file while still open doesn't work on Windows.
  98             tmpTar.close()
  99             remoteTarball = pathOnRemote(tmpTar.name)
 100             subprocess.check_call(scp(args, tmpTar.name, remoteTarball))
 101         finally:
 102             # Make sure we close the file in case an exception happens before
 103             # we've closed it above -- otherwise close() is idempotent.
 104             tmpTar.close()
 105             os.remove(tmpTar.name)
 106
 107         # Untar the dependencies in the temporary directory and remove the tarball.
 108         remoteCommands = [
 109             'tar -xf {} -C {} --strip-components 1'.format(remoteTarball, tmp),
 110             'rm {}'.format(remoteTarball)
 111         ]
 112
 113         # Make sure all executables in the remote command line have 'execute'
 114         # permissions on the remote host. The host that compiled the test-executable
 115         # might not have a notion of 'executable' permissions.
 116         for exe in filter(isExecutable, commandLine):
 117             remoteCommands.append('chmod +x {}'.format(pathOnRemote(exe)))
 118
 119         # Execute the command through SSH in the temporary directory, with the
 120         # correct environment. We tweak the command line to run it on the remote
 121         # host by transforming the path of test-executables to their path in the
 122         # temporary directory on the remote host.
 123         for i, x in enumerate(commandLine):
 124             if isExecutable(x):
 125                 commandLine[i] = pathOnRemote(x)
 126         remoteCommands.append('cd {}'.format(tmp))
 127         if args.env:
 128             remoteCommands.append('export {}'.format(' '.join(args.env)))
 129         remoteCommands.append(subprocess.list2cmdline(commandLine))
 130
 131         # Finally, SSH to the remote host and execute all the commands.
 132         rc = subprocess.call(ssh(args, ' && '.join(remoteCommands)))
 133         return rc
 134
 135     finally:
 136         # Make sure the temporary directory is removed when we're done.
 137         subprocess.check_call(ssh(args, 'rm -r {}'.format(tmp)))
 138
 139
 140 if __name__ == '__main__':
 141     exit(main())