tools/perf/page_sets/PRESUBMIT.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import sys
   8
   9
  10 def LoadSupport(input_api):
  11   if 'cloud_storage' not in globals():
  12     # Avoid leaking changes to global sys.path.
  13     _old_sys_path = sys.path
  14     try:
  15       telemetry_path = os.path.join(os.path.dirname(os.path.dirname(
  16           input_api.PresubmitLocalPath())), 'telemetry')
  17       sys.path = [telemetry_path] + sys.path
  18       from telemetry.util import cloud_storage
  19       globals()['cloud_storage'] = cloud_storage
  20     finally:
  21       sys.path = _old_sys_path
  22
  23   return globals()['cloud_storage']
  24
  25
  26 def _GetFilesNotInCloud(input_api):
  27   """Searches for .sha1 files and uploads them to Cloud Storage.
  28
  29   It validates all the hashes and skips upload if not necessary.
  30   """
  31   hash_paths = []
  32   for affected_file in input_api.AffectedFiles(include_deletes=False):
  33     hash_path = affected_file.AbsoluteLocalPath()
  34     _, extension = os.path.splitext(hash_path)
  35     if extension == '.sha1':
  36       hash_paths.append(hash_path)
  37   if not hash_paths:
  38     return []
  39
  40   cloud_storage = LoadSupport(input_api)
  41
  42   # Look in both buckets, in case the user uploaded the file manually. But this
  43   # script focuses on WPR archives, so it only uploads to the internal bucket.
  44   hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
  45   try:
  46     hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
  47   except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
  48     pass
  49
  50   files = []
  51   for hash_path in hash_paths:
  52     file_hash = cloud_storage.ReadHash(hash_path)
  53     if file_hash not in hashes_in_cloud_storage:
  54       files.append((hash_path, file_hash))
  55
  56   return files
  57
  58
  59 def _SyncFilesToCloud(input_api, output_api):
  60   """Searches for .sha1 files and uploads them to Cloud Storage.
  61
  62   It validates all the hashes and skips upload if not necessary.
  63   """
  64
  65   cloud_storage = LoadSupport(input_api)
  66
  67   results = []
  68   for hash_path, file_hash in _GetFilesNotInCloud(input_api):
  69     file_path, _ = os.path.splitext(hash_path)
  70
  71     if not re.match('^([A-Za-z0-9]{40})$', file_hash):
  72       results.append(output_api.PresubmitError(
  73           'Hash file does not contain a valid SHA-1 hash: %s' % hash_path))
  74       continue
  75     if not os.path.exists(file_path):
  76       results.append(output_api.PresubmitError(
  77           'Hash file exists, but file not found: %s' % hash_path))
  78       continue
  79     if cloud_storage.CalculateHash(file_path) != file_hash:
  80       results.append(output_api.PresubmitError(
  81           'Hash file does not match file\'s actual hash: %s' % hash_path))
  82       continue
  83
  84     try:
  85       bucket_input = raw_input('Uploading to Cloud Storage: %s\n'
  86                                'Is this file [P]ublic or Google-[i]nternal?'
  87                                % file_path).lower()
  88       if 'public'.startswith(bucket_input):
  89         bucket = cloud_storage.PUBLIC_BUCKET
  90       elif ('internal'.startswith(bucket_input) or
  91             'google-internal'.startswith(bucket_input)):
  92         bucket = cloud_storage.INTERNAL_BUCKET
  93       else:
  94         results.append(output_api.PresubmitError(
  95             'Response was neither "public" nor "internal": %s' % bucket_input))
  96         return results
  97
  98       cloud_storage.Insert(bucket, file_hash, file_path)
  99       results.append(output_api.PresubmitNotifyResult(
 100           'Uploaded file to Cloud Storage: %s' % file_path))
 101     except cloud_storage.CloudStorageError, e:
 102       results.append(output_api.PresubmitError(
 103           'Unable to upload to Cloud Storage: %s\n\n%s' % (file_path, e)))
 104
 105   return results
 106
 107
 108 def _VerifyFilesInCloud(input_api, output_api):
 109   """Searches for .sha1 files and uploads them to Cloud Storage.
 110
 111   It validates all the hashes and skips upload if not necessary.
 112   """
 113   results = []
 114   for hash_path, _ in _GetFilesNotInCloud(input_api):
 115     results.append(output_api.PresubmitError(
 116         'Attemping to commit hash file, but corresponding '
 117         'data file is not in Cloud Storage: %s' % hash_path))
 118   return results
 119
 120
 121 def _IsNewJsonPageSet(affected_file):
 122   return (affected_file.Action() == 'A' and
 123           'page_sets/data/' not in affected_file.AbsoluteLocalPath()
 124           and affected_file.AbsoluteLocalPath().endswith('.json'))
 125
 126
 127 def _GetNewJsonPageSets(input_api):
 128   return input_api.AffectedFiles(file_filter=_IsNewJsonPageSet)
 129
 130 def CheckChangeOnUpload(input_api, output_api):
 131   results = _SyncFilesToCloud(input_api, output_api)
 132   return results
 133
 134
 135 def CheckChangeOnCommit(input_api, output_api):
 136   results = _VerifyFilesInCloud(input_api, output_api)
 137   return results