mojo/services/upload_service.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import imp
   8 import os
   9 import subprocess
  10 import sys
  11 import tempfile
  12 import time
  13 import zipfile
  14
  15 SERVICES = ["html_viewer", "network_service", "network_service_apptests"]
  16
  17 # A service does not need to expose interfaces. Those that do expose interfaces
  18 # have their mojoms located in the directories listed below, in paths relative
  19 # to the directory of this script.
  20 MOJOMS_IN_DIR = {
  21     "network_service": os.path.join("network", "public", "interfaces")
  22 }
  23
  24 # The network service is downloaded out-of-band rather than dynamically by the
  25 # shell and thus can be stored zipped in the cloud. Other services are intended
  26 # to be downloaded dynamically by the shell, which doesn't currently understand
  27 # zipped binaries.
  28 SERVICES_WITH_ZIPPED_BINARIES = ["network_service", "network_service_apptests"]
  29
  30 if not sys.platform.startswith("linux"):
  31   print "Only support linux for now"
  32   sys.exit(1)
  33
  34 root_path = os.path.realpath(
  35     os.path.join(
  36         os.path.dirname(
  37             os.path.realpath(__file__)),
  38         os.pardir,
  39         os.pardir))
  40
  41 find_depot_tools_path = os.path.join(root_path, "tools", "find_depot_tools.py")
  42 find_depot_tools = imp.load_source("find_depot_tools", find_depot_tools_path)
  43
  44 depot_tools_path = find_depot_tools.add_depot_tools_to_path()
  45 gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
  46
  47 def get_version_name(custom_build):
  48   if custom_build:
  49     branch = subprocess.check_output(
  50         ["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=root_path).strip()
  51     try:
  52       base = subprocess.check_output(
  53           ["git", "config", "--get", "branch." + branch + ".base"],
  54           cwd=root_path).strip()
  55       issue = subprocess.check_output(
  56           ["git", "config", "--get", "branch." + branch + ".rietveldissue"],
  57           cwd=root_path).strip()
  58       patchset = subprocess.check_output(
  59           ["git", "config", "--get", "branch." + branch + ".rietveldpatchset"],
  60           cwd=root_path).strip()
  61     except subprocess.CalledProcessError:
  62       return None
  63
  64     if not base or not issue or not patchset:
  65       return None
  66     else:
  67       return "custom_build_base_%s_issue_%s_patchset_%s" % (base, issue,
  68                                                             patchset)
  69   else:
  70     return subprocess.check_output(["git", "rev-parse", "HEAD"],
  71                                    cwd=root_path).strip()
  72
  73 def gsutil_cp(source, dest, dry_run):
  74   if dry_run:
  75     print "gsutil cp %s %s" % (source, dest)
  76   else:
  77     subprocess.check_call([gsutil_exe, "cp", source, dest])
  78
  79
  80 def upload_mojoms(version_name, service, absolute_mojom_directory_path,
  81                   dry_run):
  82   dest = "gs://mojo/" + service + "/" + version_name + "/" + "mojoms.zip"
  83
  84   with tempfile.NamedTemporaryFile() as mojom_zip_file:
  85     with zipfile.ZipFile(mojom_zip_file, 'w') as z:
  86       for root, _, files in os.walk(absolute_mojom_directory_path):
  87         for filename in files:
  88           absolute_file_path = os.path.join(root, filename)
  89           relative_file_path = os.path.relpath(absolute_file_path, root)
  90           z.write(absolute_file_path, relative_file_path)
  91     gsutil_cp(mojom_zip_file.name, dest, dry_run)
  92
  93
  94 def upload_binary(version_name, service, binary_dir, platform, dry_run):
  95   dest_dir = "gs://mojo/" + service + "/" + version_name + "/" + platform + "/"
  96   should_zip = service in SERVICES_WITH_ZIPPED_BINARIES
  97   binary_name = service + ".mojo"
  98   absolute_binary_path = os.path.join(root_path, binary_dir, binary_name)
  99
 100   if not should_zip:
 101     # Upload the binary.
 102     dest = dest_dir + binary_name
 103     gsutil_cp(absolute_binary_path, dest, dry_run)
 104
 105     # Update the pointer to the service's location to point to the
 106     # newly-uploaded binary.
 107     service_location = dest.replace("gs://", "https://storage.googleapis.com/")
 108     location_file = ("gs://mojo/services/" + platform + "/" + service +
 109                      "_location")
 110     with tempfile.NamedTemporaryFile() as tmp:
 111       tmp.write(service_location)
 112       tmp.flush()
 113       gsutil_cp(tmp.name, location_file, dry_run)
 114     return
 115
 116   # Zip the binary before uploading it to the cloud.
 117   dest = dest_dir + binary_name + ".zip"
 118   with tempfile.NamedTemporaryFile() as binary_zip_file:
 119     with zipfile.ZipFile(binary_zip_file, 'w') as z:
 120       with open(absolute_binary_path) as service_binary:
 121         zipinfo = zipfile.ZipInfo(binary_name)
 122         zipinfo.external_attr = 0o777 << 16
 123         zipinfo.compress_type = zipfile.ZIP_DEFLATED
 124         zipinfo.date_time = time.gmtime(os.path.getmtime(absolute_binary_path))
 125         z.writestr(zipinfo, service_binary.read())
 126     gsutil_cp(binary_zip_file.name, dest, dry_run)
 127
 128
 129 def main():
 130   parser = argparse.ArgumentParser(
 131       description="Upload service mojoms and binaries to Google storage")
 132   parser.add_argument("-n", "--dry-run", action="store_true", help="Dry run")
 133   parser.add_argument(
 134       "--linux-x64-binary-dir",
 135       help="Path to the dir containing the linux-x64 service binary relative "
 136            "to the repo root, e.g. out/Release")
 137   parser.add_argument(
 138       "--android-arm-binary-dir",
 139       help="Path to the dir containing the android-arm service binary relative "
 140            "to the repo root, e.g. out/android_Release")
 141   parser.add_argument("service",
 142                       help="The service to be uploaded (one of %s)" % SERVICES)
 143   parser.add_argument(
 144       "--custom-build", action="store_true",
 145       help="Indicates that this is a build with change that is not committed. "
 146            "The change must be uploaded to Rietveld. The script needs to be "
 147            "run from the branch associated with the change.")
 148   args = parser.parse_args()
 149
 150   if args.service not in SERVICES:
 151     print args.service + " is not one of the recognized services:"
 152     print SERVICES
 153     return 1
 154
 155   version_name = get_version_name(args.custom_build)
 156   if args.custom_build and not version_name:
 157     print ("When uploading a custom build, the corresponding change to source "
 158            "code must be uploaded to Rietveld. Besides, this script needs to "
 159            "be run from the branch associated with the change.")
 160     return 1
 161
 162   if args.service in MOJOMS_IN_DIR:
 163     script_dir = os.path.dirname(os.path.realpath(__file__))
 164     absolute_mojom_directory_path = os.path.join(script_dir,
 165                                                  MOJOMS_IN_DIR[args.service])
 166     upload_mojoms(version_name, args.service, absolute_mojom_directory_path,
 167                   args.dry_run)
 168
 169   if args.linux_x64_binary_dir:
 170     upload_binary(version_name, args.service, args.linux_x64_binary_dir,
 171                   "linux-x64", args.dry_run)
 172
 173   if args.android_arm_binary_dir:
 174     upload_binary(version_name, args.service, args.android_arm_binary_dir,
 175                   "android-arm", args.dry_run)
 176
 177   if not args.dry_run:
 178     print "Uploaded artifacts for version %s" % (version_name, )
 179   else:
 180     print "No artifacts uploaded (dry run)"
 181   return 0
 182
 183 if __name__ == '__main__':
 184   sys.exit(main())