tools/telemetry/refactor

   1 #! /usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import cStringIO
   7 import imp
   8 import inspect
   9 import os
  10 import re
  11 import sys
  12
  13 from telemetry.core import command_line
  14 from telemetry.util import path
  15
  16
  17 # All folders dependent on Telemetry, found using a code search.
  18 BASE_DIRS = (
  19   path.GetTelemetryDir(),
  20   os.path.join(path.GetChromiumSrcDir(), 'chrome', 'test', 'telemetry'),
  21   os.path.join(path.GetChromiumSrcDir(), 'content', 'test', 'gpu'),
  22   os.path.join(path.GetChromiumSrcDir(), 'tools', 'bisect-manual-test.py'),
  23   os.path.join(path.GetChromiumSrcDir(), 'tools', 'chrome_proxy'),
  24   os.path.join(path.GetChromiumSrcDir(), 'tools', 'perf'),
  25   os.path.join(path.GetChromiumSrcDir(),
  26                'tools', 'profile_chrome', 'perf_controller.py'),
  27   os.path.join(path.GetChromiumSrcDir(), 'tools', 'run-bisect-manual-test.py'),
  28   os.path.join(path.GetChromiumSrcDir(),
  29                'third_party', 'skia', 'tools', 'skp', 'page_sets'),
  30   os.path.join(path.GetChromiumSrcDir(), 'third_party', 'trace-viewer'),
  31 )
  32
  33
  34 def SortImportGroups(module_path):
  35   """Sort each group of imports in the given Python module.
  36
  37   A group is a collection of adjacent import statements, with no non-import
  38   lines in between. Groups are sorted according to the Google Python Style
  39   Guide: "lexicographically, ignoring case, according to each module's full
  40   package path."
  41   """
  42   _TransformImportGroups(module_path, _SortImportGroup)
  43
  44
  45 def _SortImportGroup(import_group):
  46   def _ImportComparator(import1, import2):
  47     _, root1, module1, _, _ = import1
  48     _, root2, module2, _, _ = import2
  49     full_module1 = (root1 + '.' + module1 if root1 else module1).lower()
  50     full_module2 = (root2 + '.' + module2 if root2 else module2).lower()
  51     return cmp(full_module1, full_module2)
  52   return sorted(import_group, cmp=_ImportComparator)
  53
  54
  55 def _TransformImportGroups(module_path, transformation):
  56   """Apply a transformation to each group of imports in the given module.
  57
  58   An import is a tuple of (indent, root, module, alias, suffix),
  59   serialized as <indent>from <root> import <module> as <alias><suffix>.
  60
  61   Args:
  62     module_path: The module to apply transformations on.
  63     transformation: A function that takes in an import group and returns a
  64         modified import group. An import group is a list of import tuples.
  65
  66   Returns:
  67     True iff the module was modified, and False otherwise.
  68   """
  69   def _WriteImports(output_stream, import_group):
  70     for indent, root, module, alias, suffix in transformation(import_group):
  71       output_stream.write(indent)
  72       if root:
  73         output_stream.write('from ')
  74         output_stream.write(root)
  75         output_stream.write(' ')
  76       output_stream.write('import ')
  77       output_stream.write(module)
  78       if alias:
  79         output_stream.write(' as ')
  80         output_stream.write(alias)
  81       output_stream.write(suffix)
  82       output_stream.write('\n')
  83
  84   # Read the file so we can diff it later to determine if we made any changes.
  85   with open(module_path, 'r') as module_file:
  86     original_file = module_file.read()
  87
  88   # Locate imports using regex, group them, and transform each one.
  89   # This regex produces a tuple of (indent, root, module, alias, suffix).
  90   regex = (r'(\s*)(?:from ((?:[a-z0-9_]+\.)*[a-z0-9_]+) )?'
  91       r'import ((?:[a-z0-9_]+\.)*[A-Za-z0-9_]+)(?: as ([A-Za-z0-9_]+))?(.*)')
  92   pattern = re.compile(regex)
  93
  94   updated_file = cStringIO.StringIO()
  95   with open(module_path, 'r') as module_file:
  96     import_group = []
  97     for line in module_file:
  98       import_match = pattern.match(line)
  99       if import_match:
 100         import_group.append(list(import_match.groups()))
 101         continue
 102
 103       if not import_group:
 104         updated_file.write(line)
 105         continue
 106
 107       _WriteImports(updated_file, import_group)
 108       import_group = []
 109
 110       updated_file.write(line)
 111
 112   if import_group:
 113     _WriteImports(updated_file, import_group)
 114     import_group = []
 115
 116   if original_file == updated_file.getvalue():
 117     return False
 118
 119   with open(module_path, 'w') as module_file:
 120     module_file.write(updated_file.getvalue())
 121   return True
 122
 123
 124 def _ListFiles(base_directory, should_include_dir, should_include_file):
 125   matching_files = []
 126   for root, dirs, files in os.walk(base_directory):
 127     dirs[:] = [dir_name for dir_name in dirs if should_include_dir(dir_name)]
 128     matching_files += [os.path.join(root, file_name)
 129                        for file_name in files if should_include_file(file_name)]
 130   return sorted(matching_files)
 131
 132
 133 class Count(command_line.Command):
 134   """Print the number of public modules."""
 135
 136   def Run(self, args):
 137     modules = _ListFiles(path.GetTelemetryDir(),
 138                          self._IsPublicApiDir, self._IsPublicApiFile)
 139     print len(modules)
 140     return 0
 141
 142   @staticmethod
 143   def _IsPublicApiDir(dir_name):
 144     return (dir_name[0] != '.' and dir_name[0] != '_' and
 145         not dir_name.startswith('internal') and not dir_name == 'third_party')
 146
 147   @staticmethod
 148   def _IsPublicApiFile(file_name):
 149     root, ext = os.path.splitext(file_name)
 150     return (file_name[0] != '.' and
 151         not root.endswith('_unittest') and ext == '.py')
 152
 153
 154 class Mv(command_line.Command):
 155   """Move modules or packages."""
 156
 157   @classmethod
 158   def AddCommandLineArgs(cls, parser):
 159     parser.add_argument('source', nargs='+')
 160     parser.add_argument('destination')
 161
 162   @classmethod
 163   def ProcessCommandLineArgs(cls, parser, args):
 164     for source in args.source:
 165       # Ensure source path exists.
 166       if not os.path.exists(source):
 167         parser.error('"%s" not found.' % source)
 168
 169       # Ensure source path is in one of the BASE_DIRS.
 170       for base_dir in BASE_DIRS:
 171         if path.IsSubpath(source, base_dir):
 172           break
 173       else:
 174         parser.error('Source path "%s" is not in any of the base dirs.')
 175
 176     # Ensure destination path exists.
 177     if not os.path.exists(args.destination):
 178       parser.error('"%s" not found.' % args.destination)
 179
 180     # Ensure destination path is in one of the BASE_DIRS.
 181     for base_dir in BASE_DIRS:
 182       if path.IsSubpath(args.destination, base_dir):
 183         break
 184     else:
 185       parser.error('Destination path "%s" is not in any of the base dirs.')
 186
 187     # If there are multiple source paths, ensure destination is a directory.
 188     if len(args.source) > 1 and not os.path.isdir(args.destination):
 189       parser.error('Target "%s" is not a directory.' % args.destination)
 190
 191     # Ensure destination is not in any of the source paths.
 192     for source in args.source:
 193       if path.IsSubpath(args.destination, source):
 194         parser.error('Cannot move "%s" to a subdirectory of itself, "%s".' %
 195             (source, args.destination))
 196
 197   def Run(self, args):
 198     for dest_base_dir in BASE_DIRS:
 199       if path.IsSubpath(args.destination, dest_base_dir):
 200         break
 201
 202     # Get a list of old and new module names for renaming imports.
 203     moved_modules = {}
 204     for source in args.source:
 205       for source_base_dir in BASE_DIRS:
 206         if path.IsSubpath(source, source_base_dir):
 207           break
 208
 209       source_dir = os.path.dirname(os.path.normpath(source))
 210
 211       if os.path.isdir(source):
 212         source_files = _ListFiles(source,
 213                                   self._IsSourceDir, self._IsPythonModule)
 214       else:
 215         source_files = (source,)
 216
 217       for source_file_path in source_files:
 218         source_rel_path = os.path.relpath(source_file_path, source_base_dir)
 219         source_module_name = os.path.splitext(
 220             source_rel_path)[0].replace(os.sep, '.')
 221
 222         source_tree = os.path.relpath(source_file_path, source_dir)
 223         dest_path = os.path.join(args.destination, source_tree)
 224         dest_rel_path = os.path.relpath(dest_path, dest_base_dir)
 225         dest_module_name = os.path.splitext(
 226             dest_rel_path)[0].replace(os.sep, '.')
 227
 228         moved_modules[source_module_name] = dest_module_name
 229
 230     # Move things!
 231     if os.path.isdir(args.destination):
 232       for source in args.source:
 233         destination_path = os.path.join(
 234             args.destination, os.path.split(os.path.normpath(source))[1])
 235         os.rename(source, destination_path)
 236     else:
 237       assert len(args.source) == 1
 238       os.rename(args.source.pop(), args.destination)
 239
 240     # Update imports!
 241     def _UpdateImportGroup(import_group):
 242       modified = False
 243       for import_line in import_group:
 244         _, root, module, _, _ = import_line
 245         full_module = root + '.' + module if root else module
 246
 247         if full_module not in moved_modules:
 248           continue
 249
 250         modified = True
 251
 252         # Update import line.
 253         new_root, _, new_module = moved_modules[full_module].rpartition('.')
 254         import_line[1] = new_root
 255         import_line[2] = new_module
 256
 257       if modified:
 258         return _SortImportGroup(import_group)
 259       else:
 260         return import_group
 261
 262     for base_dir in BASE_DIRS:
 263       for module_path in _ListFiles(base_dir,
 264                                     self._IsSourceDir, self._IsPythonModule):
 265         if not _TransformImportGroups(module_path, _UpdateImportGroup):
 266           continue
 267
 268         # TODO(dtu): Update occurrences.
 269
 270     print moved_modules
 271
 272     return 0
 273
 274   @staticmethod
 275   def _IsSourceDir(dir_name):
 276     return dir_name[0] != '.' and dir_name != 'third_party'
 277
 278   @staticmethod
 279   def _IsPythonModule(file_name):
 280     _, ext = os.path.splitext(file_name)
 281     return ext == '.py'
 282
 283
 284 class RefactorCommand(command_line.SubcommandCommand):
 285   commands = (Count, Mv,)
 286
 287
 288 if __name__ == '__main__':
 289   sys.exit(RefactorCommand.main())