tools/perf/page_sets/PRESUBMIT.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import re
   7 import sys
   8
   9
  10 def LoadSupport(input_api):
  11   if 'cloud_storage' not in globals():
  12     # Avoid leaking changes to global sys.path.
  13     _old_sys_path = sys.path
  14     try:
  15       telemetry_path = os.path.join(os.path.dirname(os.path.dirname(
  16           input_api.PresubmitLocalPath())), 'telemetry')
  17       sys.path = [telemetry_path] + sys.path
  18       from telemetry.util import cloud_storage
  19       globals()['cloud_storage'] = cloud_storage
  20     finally:
  21       sys.path = _old_sys_path
  22
  23   return globals()['cloud_storage']
  24
  25
  26 def _GetFilesNotInCloud(input_api):
  27   """Searches for .sha1 files and uploads them to Cloud Storage.
  28
  29   It validates all the hashes and skips upload if not necessary.
  30   """
  31   hash_paths = []
  32   for affected_file in input_api.AffectedFiles(include_deletes=False):
  33     hash_path = affected_file.AbsoluteLocalPath()
  34     _, extension = os.path.splitext(hash_path)
  35     if extension == '.sha1':
  36       hash_paths.append(hash_path)
  37   if not hash_paths:
  38     return []
  39
  40   cloud_storage = LoadSupport(input_api)
  41
  42   # Look in both buckets, in case the user uploaded the file manually. But this
  43   # script focuses on WPR archives, so it only uploads to the internal bucket.
  44   hashes_in_cloud_storage = cloud_storage.List(cloud_storage.PUBLIC_BUCKET)
  45   try:
  46     hashes_in_cloud_storage += cloud_storage.List(cloud_storage.INTERNAL_BUCKET)
  47   except (cloud_storage.PermissionError, cloud_storage.CredentialsError):
  48     pass
  49
  50   files = []
  51   for hash_path in hash_paths:
  52     file_hash = cloud_storage.ReadHash(hash_path)
  53     if file_hash not in hashes_in_cloud_storage:
  54       files.append((hash_path, file_hash))
  55
  56   return files
  57
  58
  59 def _SyncFilesToCloud(input_api, output_api):
  60   """Searches for .sha1 files and uploads them to Cloud Storage.
  61
  62   It validates all the hashes and skips upload if not necessary.
  63   """
  64
  65   cloud_storage = LoadSupport(input_api)
  66
  67   results = []
  68   for hash_path, file_hash in _GetFilesNotInCloud(input_api):
  69     file_path, _ = os.path.splitext(hash_path)
  70
  71     if not re.match('^([A-Za-z0-9]{40})$', file_hash):
  72       results.append(output_api.PresubmitError(
  73           'Hash file does not contain a valid SHA-1 hash: %s' % hash_path))
  74       continue
  75     if not os.path.exists(file_path):
  76       results.append(output_api.PresubmitError(
  77           'Hash file exists, but file not found: %s' % hash_path))
  78       continue
  79     if cloud_storage.CalculateHash(file_path) != file_hash:
  80       results.append(output_api.PresubmitError(
  81           'Hash file does not match file\'s actual hash: %s' % hash_path))
  82       continue
  83
  84     try:
  85       bucket_aliases_string = ', '.join(cloud_storage.BUCKET_ALIASES)
  86       bucket_input = raw_input(
  87           'Uploading to Cloud Storage: %s\n'
  88           'Which bucket should this go in? (%s) '
  89           % (file_path, bucket_aliases_string)).lower()
  90       bucket = cloud_storage.BUCKET_ALIASES.get(bucket_input, None)
  91       if not bucket:
  92         results.append(output_api.PresubmitError(
  93             '"%s" was not one of %s' % (bucket_input, bucket_aliases_string)))
  94         return results
  95
  96       cloud_storage.Insert(bucket, file_hash, file_path)
  97       results.append(output_api.PresubmitNotifyResult(
  98           'Uploaded file to Cloud Storage: %s' % file_path))
  99     except cloud_storage.CloudStorageError, e:
 100       results.append(output_api.PresubmitError(
 101           'Unable to upload to Cloud Storage: %s\n\n%s' % (file_path, e)))
 102
 103   return results
 104
 105
 106 def _VerifyFilesInCloud(input_api, output_api):
 107   """Searches for .sha1 files and uploads them to Cloud Storage.
 108
 109   It validates all the hashes and skips upload if not necessary.
 110   """
 111   results = []
 112   for hash_path, _ in _GetFilesNotInCloud(input_api):
 113     results.append(output_api.PresubmitError(
 114         'Attemping to commit hash file, but corresponding '
 115         'data file is not in Cloud Storage: %s' % hash_path))
 116   return results
 117
 118
 119 def _IsNewJsonPageSet(affected_file):
 120   return (affected_file.Action() == 'A' and
 121           'page_sets/data/' not in affected_file.AbsoluteLocalPath()
 122           and affected_file.AbsoluteLocalPath().endswith('.json'))
 123
 124
 125 def _GetNewJsonPageSets(input_api):
 126   return input_api.AffectedFiles(file_filter=_IsNewJsonPageSet)
 127
 128 def CheckChangeOnUpload(input_api, output_api):
 129   results = _SyncFilesToCloud(input_api, output_api)
 130   return results
 131
 132
 133 def CheckChangeOnCommit(input_api, output_api):
 134   results = _VerifyFilesInCloud(input_api, output_api)
 135   return results