tools/telemetry/refactor

   1 #! /usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import cStringIO
   7 import imp
   8 import inspect
   9 import os
  10 import re
  11 import sys
  12
  13 from telemetry.internal.util import command_line
  14 from telemetry.internal.util import path
  15
  16
  17 # All folders dependent on Telemetry, found using a code search.
  18 BASE_DIRS = (
  19   path.GetTelemetryDir(),
  20   os.path.join(path.GetChromiumSrcDir(), 'chrome', 'test', 'telemetry'),
  21   os.path.join(path.GetChromiumSrcDir(), 'content', 'test', 'gpu'),
  22   os.path.join(path.GetChromiumSrcDir(), 'tools', 'bisect-manual-test.py'),
  23   os.path.join(path.GetChromiumSrcDir(), 'tools', 'chrome_proxy'),
  24   os.path.join(path.GetChromiumSrcDir(), 'tools', 'perf'),
  25   os.path.join(path.GetChromiumSrcDir(),
  26                'tools', 'profile_chrome', 'perf_controller.py'),
  27   os.path.join(path.GetChromiumSrcDir(), 'tools', 'run-bisect-manual-test.py'),
  28   os.path.join(path.GetChromiumSrcDir(),
  29                'third_party', 'skia', 'tools', 'skp', 'page_sets'),
  30   os.path.join(path.GetChromiumSrcDir(), 'third_party', 'trace-viewer'),
  31 )
  32
  33
  34 def SortImportGroups(module_path):
  35   """Sort each group of imports in the given Python module.
  36
  37   A group is a collection of adjacent import statements, with no non-import
  38   lines in between. Groups are sorted according to the Google Python Style
  39   Guide: "lexicographically, ignoring case, according to each module's full
  40   package path."
  41   """
  42   _TransformImportGroups(module_path, _SortImportGroup)
  43
  44
  45 def _SortImportGroup(import_group):
  46   def _ImportComparator(import1, import2):
  47     _, root1, module1, _, _ = import1
  48     _, root2, module2, _, _ = import2
  49     full_module1 = (root1 + '.' + module1 if root1 else module1).lower()
  50     full_module2 = (root2 + '.' + module2 if root2 else module2).lower()
  51     return cmp(full_module1, full_module2)
  52   return sorted(import_group, cmp=_ImportComparator)
  53
  54
  55 def _TransformImportGroups(module_path, transformation):
  56   """Apply a transformation to each group of imports in the given module.
  57
  58   An import is a tuple of (indent, root, module, alias, suffix),
  59   serialized as <indent>from <root> import <module> as <alias><suffix>.
  60
  61   Args:
  62     module_path: The module to apply transformations on.
  63     transformation: A function that takes in an import group and returns a
  64         modified import group. An import group is a list of import tuples.
  65
  66   Returns:
  67     True iff the module was modified, and False otherwise.
  68   """
  69   def _WriteImports(output_stream, import_group):
  70     for indent, root, module, alias, suffix in transformation(import_group):
  71       output_stream.write(indent)
  72       if root:
  73         output_stream.write('from ')
  74         output_stream.write(root)
  75         output_stream.write(' ')
  76       output_stream.write('import ')
  77       output_stream.write(module)
  78       if alias:
  79         output_stream.write(' as ')
  80         output_stream.write(alias)
  81       output_stream.write(suffix)
  82       output_stream.write('\n')
  83
  84   # Read the file so we can diff it later to determine if we made any changes.
  85   with open(module_path, 'r') as module_file:
  86     original_file = module_file.read()
  87
  88   # Locate imports using regex, group them, and transform each one.
  89   # This regex produces a tuple of (indent, root, module, alias, suffix).
  90   regex = (r'(\s*)(?:from ((?:[a-z0-9_]+\.)*[a-z0-9_]+) )?'
  91       r'import ((?:[a-z0-9_]+\.)*[A-Za-z0-9_]+)(?: as ([A-Za-z0-9_]+))?(.*)')
  92   pattern = re.compile(regex)
  93
  94   updated_file = cStringIO.StringIO()
  95   with open(module_path, 'r') as module_file:
  96     import_group = []
  97     for line in module_file:
  98       import_match = pattern.match(line)
  99       if import_match:
 100         import_group.append(list(import_match.groups()))
 101         continue
 102
 103       if not import_group:
 104         updated_file.write(line)
 105         continue
 106
 107       _WriteImports(updated_file, import_group)
 108       import_group = []
 109
 110       updated_file.write(line)
 111
 112   if import_group:
 113     _WriteImports(updated_file, import_group)
 114     import_group = []
 115
 116   if original_file == updated_file.getvalue():
 117     return False
 118
 119   with open(module_path, 'w') as module_file:
 120     module_file.write(updated_file.getvalue())
 121   return True
 122
 123
 124 def _ListFiles(base_directory, should_include_dir, should_include_file):
 125   matching_files = []
 126   for root, dirs, files in os.walk(base_directory):
 127     dirs[:] = [dir_name for dir_name in dirs if should_include_dir(dir_name)]
 128     matching_files += [os.path.join(root, file_name)
 129                        for file_name in files if should_include_file(file_name)]
 130   return sorted(matching_files)
 131
 132
 133 def _IsSourceDir(dir_name):
 134   return dir_name[0] != '.' and dir_name != 'third_party'
 135
 136
 137 def _IsPythonModule(file_name):
 138   _, ext = os.path.splitext(file_name)
 139   return ext == '.py'
 140
 141
 142 class Count(command_line.Command):
 143   """Print the number of public modules."""
 144
 145   def Run(self, args):
 146     modules = _ListFiles(path.GetTelemetryDir(),
 147                          self._IsPublicApiDir, self._IsPublicApiFile)
 148     print len(modules)
 149     return 0
 150
 151   @staticmethod
 152   def _IsPublicApiDir(dir_name):
 153     return (dir_name[0] != '.' and dir_name[0] != '_' and
 154         not dir_name.startswith('internal') and not dir_name == 'third_party')
 155
 156   @staticmethod
 157   def _IsPublicApiFile(file_name):
 158     root, ext = os.path.splitext(file_name)
 159     return (file_name[0] != '.' and
 160         not root.endswith('_unittest') and ext == '.py')
 161
 162
 163 class Mv(command_line.Command):
 164   """Move modules or packages."""
 165
 166   @classmethod
 167   def AddCommandLineArgs(cls, parser):
 168     parser.add_argument('source', nargs='+')
 169     parser.add_argument('destination')
 170
 171   @classmethod
 172   def ProcessCommandLineArgs(cls, parser, args):
 173     for source in args.source:
 174       # Ensure source path exists.
 175       if not os.path.exists(source):
 176         parser.error('"%s" not found.' % source)
 177
 178       # Ensure source path is in one of the BASE_DIRS.
 179       for base_dir in BASE_DIRS:
 180         if path.IsSubpath(source, base_dir):
 181           break
 182       else:
 183         parser.error('Source path "%s" is not in any of the base dirs.')
 184
 185     # Ensure destination path exists.
 186     if not os.path.exists(args.destination):
 187       parser.error('"%s" not found.' % args.destination)
 188
 189     # Ensure destination path is in one of the BASE_DIRS.
 190     for base_dir in BASE_DIRS:
 191       if path.IsSubpath(args.destination, base_dir):
 192         break
 193     else:
 194       parser.error('Destination path "%s" is not in any of the base dirs.')
 195
 196     # If there are multiple source paths, ensure destination is a directory.
 197     if len(args.source) > 1 and not os.path.isdir(args.destination):
 198       parser.error('Target "%s" is not a directory.' % args.destination)
 199
 200     # Ensure destination is not in any of the source paths.
 201     for source in args.source:
 202       if path.IsSubpath(args.destination, source):
 203         parser.error('Cannot move "%s" to a subdirectory of itself, "%s".' %
 204             (source, args.destination))
 205
 206   def Run(self, args):
 207     for dest_base_dir in BASE_DIRS:
 208       if path.IsSubpath(args.destination, dest_base_dir):
 209         break
 210
 211     # Get a list of old and new module names for renaming imports.
 212     moved_modules = {}
 213     for source in args.source:
 214       for source_base_dir in BASE_DIRS:
 215         if path.IsSubpath(source, source_base_dir):
 216           break
 217
 218       source_dir = os.path.dirname(os.path.normpath(source))
 219
 220       if os.path.isdir(source):
 221         source_files = _ListFiles(source, _IsSourceDir, _IsPythonModule)
 222       else:
 223         source_files = (source,)
 224
 225       for source_file_path in source_files:
 226         source_rel_path = os.path.relpath(source_file_path, source_base_dir)
 227         source_module_name = os.path.splitext(
 228             source_rel_path)[0].replace(os.sep, '.')
 229
 230         source_tree = os.path.relpath(source_file_path, source_dir)
 231         dest_path = os.path.join(args.destination, source_tree)
 232         dest_rel_path = os.path.relpath(dest_path, dest_base_dir)
 233         dest_module_name = os.path.splitext(
 234             dest_rel_path)[0].replace(os.sep, '.')
 235
 236         moved_modules[source_module_name] = dest_module_name
 237
 238     # Move things!
 239     if os.path.isdir(args.destination):
 240       for source in args.source:
 241         destination_path = os.path.join(
 242             args.destination, os.path.split(os.path.normpath(source))[1])
 243         os.rename(source, destination_path)
 244     else:
 245       assert len(args.source) == 1
 246       os.rename(args.source.pop(), args.destination)
 247
 248     # Update imports!
 249     def _UpdateImportGroup(import_group):
 250       modified = False
 251       for import_line in import_group:
 252         _, root, module, _, _ = import_line
 253         full_module = root + '.' + module if root else module
 254
 255         if full_module not in moved_modules:
 256           continue
 257
 258         modified = True
 259
 260         # Update import line.
 261         new_root, _, new_module = moved_modules[full_module].rpartition('.')
 262         import_line[1] = new_root
 263         import_line[2] = new_module
 264
 265       if modified:
 266         return _SortImportGroup(import_group)
 267       else:
 268         return import_group
 269
 270     for base_dir in BASE_DIRS:
 271       for module_path in _ListFiles(base_dir, _IsSourceDir, _IsPythonModule):
 272         if not _TransformImportGroups(module_path, _UpdateImportGroup):
 273           continue
 274
 275         # TODO(dtu): Update occurrences.
 276
 277     print moved_modules
 278
 279     return 0
 280
 281
 282 class Sort(command_line.Command):
 283   """Sort imports."""
 284
 285   @classmethod
 286   def AddCommandLineArgs(cls, parser):
 287     parser.add_argument('target', nargs='*')
 288
 289   @classmethod
 290   def ProcessCommandLineArgs(cls, parser, args):
 291     for target in args.target:
 292       if not os.path.exists(target):
 293         parser.error('"%s" not found.' % target)
 294
 295   def Run(self, args):
 296     if args.target:
 297       targets = args.target
 298     else:
 299       targets = BASE_DIRS
 300
 301     for base_dir in targets:
 302       for module_path in _ListFiles(base_dir, _IsSourceDir, _IsPythonModule):
 303         SortImportGroups(module_path)
 304     return 0
 305
 306
 307 class RefactorCommand(command_line.SubcommandCommand):
 308   commands = (Count, Mv, Sort,)
 309
 310
 311 if __name__ == '__main__':
 312   sys.exit(RefactorCommand.main())