tools/telemetry/refactor

   1 #! /usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import cStringIO
   7 import functools
   8 import imp
   9 import inspect
  10 import itertools
  11 import operator
  12 import os
  13 import re
  14 import sys
  15
  16 from catapult_base import refactor
  17 from catapult_base.refactor_util import move
  18
  19 from telemetry.internal.util import command_line
  20 from telemetry.internal.util import path
  21
  22
  23 _RELATIVE_BASE_DIRS = (
  24   ('chrome', 'test', 'telemetry'),
  25   ('content', 'test', 'gpu'),
  26   ('tools', 'bisect-manual-test.py'),
  27   ('tools', 'chrome_proxy'),
  28   ('tools', 'perf'),
  29   ('tools', 'profile_chrome', 'perf_controller.py'),
  30   ('tools', 'run-bisect-manual-test.py'),
  31   ('third_party', 'skia', 'tools', 'skp', 'page_sets'),
  32   ('third_party', 'trace-viewer'),
  33 )
  34 # All folders dependent on Telemetry, found using a code search.
  35 # Note that this is not the same as the directory that imports are relative to.
  36 BASE_DIRS = [path.GetTelemetryDir()] + [
  37     os.path.join(path.GetChromiumSrcDir(), *dir_path)
  38     for dir_path in _RELATIVE_BASE_DIRS]
  39
  40
  41 def SortImportGroups(module_path):
  42   """Sort each group of imports in the given Python module.
  43
  44   A group is a collection of adjacent import statements, with no non-import
  45   lines in between. Groups are sorted according to the Google Python Style
  46   Guide: "lexicographically, ignoring case, according to each module's full
  47   package path."
  48   """
  49   _TransformImportGroups(module_path, _SortImportGroup)
  50
  51
  52 def _SortImportGroup(import_group):
  53   def _ImportComparator(import1, import2):
  54     _, root1, module1, _, _ = import1
  55     _, root2, module2, _, _ = import2
  56     full_module1 = (root1 + '.' + module1 if root1 else module1).lower()
  57     full_module2 = (root2 + '.' + module2 if root2 else module2).lower()
  58     return cmp(full_module1, full_module2)
  59   return sorted(import_group, cmp=_ImportComparator)
  60
  61
  62 def _TransformImportGroups(module_path, transformation):
  63   """Apply a transformation to each group of imports in the given module.
  64
  65   An import is a tuple of (indent, root, module, alias, suffix),
  66   serialized as <indent>from <root> import <module> as <alias><suffix>.
  67
  68   Args:
  69     module_path: The module to apply transformations on.
  70     transformation: A function that takes in an import group and returns a
  71         modified import group. An import group is a list of import tuples.
  72
  73   Returns:
  74     True iff the module was modified, and False otherwise.
  75   """
  76   def _WriteImports(output_stream, import_group):
  77     for indent, root, module, alias, suffix in transformation(import_group):
  78       output_stream.write(indent)
  79       if root:
  80         output_stream.write('from ')
  81         output_stream.write(root)
  82         output_stream.write(' ')
  83       output_stream.write('import ')
  84       output_stream.write(module)
  85       if alias:
  86         output_stream.write(' as ')
  87         output_stream.write(alias)
  88       output_stream.write(suffix)
  89       output_stream.write('\n')
  90
  91   # Read the file so we can diff it later to determine if we made any changes.
  92   with open(module_path, 'r') as module_file:
  93     original_file = module_file.read()
  94
  95   # Locate imports using regex, group them, and transform each one.
  96   # This regex produces a tuple of (indent, root, module, alias, suffix).
  97   regex = (r'(\s*)(?:from ((?:[a-z0-9_]+\.)*[a-z0-9_]+) )?'
  98       r'import ((?:[a-z0-9_]+\.)*[A-Za-z0-9_]+)(?: as ([A-Za-z0-9_]+))?(.*)')
  99   pattern = re.compile(regex)
 100
 101   updated_file = cStringIO.StringIO()
 102   with open(module_path, 'r') as module_file:
 103     import_group = []
 104     for line in module_file:
 105       import_match = pattern.match(line)
 106       if import_match:
 107         import_group.append(list(import_match.groups()))
 108         continue
 109
 110       if not import_group:
 111         updated_file.write(line)
 112         continue
 113
 114       _WriteImports(updated_file, import_group)
 115       import_group = []
 116
 117       updated_file.write(line)
 118
 119   if import_group:
 120     _WriteImports(updated_file, import_group)
 121     import_group = []
 122
 123   if original_file == updated_file.getvalue():
 124     return False
 125
 126   with open(module_path, 'w') as module_file:
 127     module_file.write(updated_file.getvalue())
 128   return True
 129
 130
 131 def _CountInterfaces(module):
 132   return (len(list(module.FindChildren(refactor.Class))) +
 133       len(list(module.FindChildren(refactor.Function))))
 134
 135
 136 def _IsSourceDir(dir_name):
 137   return dir_name[0] != '.' and dir_name != 'third_party'
 138
 139
 140 def _IsPythonModule(file_name):
 141   _, ext = os.path.splitext(file_name)
 142   return ext == '.py'
 143
 144
 145 class Count(command_line.Command):
 146   """Print the number of public modules."""
 147
 148   @classmethod
 149   def AddCommandLineArgs(cls, parser):
 150     parser.add_argument('type', nargs='?', choices=('interfaces', 'modules'),
 151                         default='modules')
 152
 153   def Run(self, args):
 154     module_paths = path.ListFiles(
 155         path.GetTelemetryDir(), self._IsPublicApiDir, self._IsPublicApiFile)
 156
 157     if args.type == 'modules':
 158       print len(module_paths)
 159     elif args.type == 'interfaces':
 160       print reduce(operator.add, refactor.Transform(_CountInterfaces, module_paths))
 161
 162     return 0
 163
 164   @staticmethod
 165   def _IsPublicApiDir(dir_name):
 166     return (dir_name[0] != '.' and dir_name[0] != '_' and
 167         dir_name != 'internal' and dir_name != 'third_party')
 168
 169   @staticmethod
 170   def _IsPublicApiFile(file_name):
 171     root, ext = os.path.splitext(file_name)
 172     return (file_name[0] != '.' and
 173         not root.endswith('_unittest') and ext == '.py')
 174
 175
 176 def _TelemetryFiles():
 177   list_files = functools.partial(path.ListFiles,
 178                                  should_include_dir=_IsSourceDir,
 179                                  should_include_file=_IsPythonModule)
 180   return sorted(itertools.chain(*map(list_files, BASE_DIRS)))
 181
 182
 183 class Mv(command_line.Command):
 184   """Move modules or packages."""
 185
 186   @classmethod
 187   def AddCommandLineArgs(cls, parser):
 188     parser.add_argument('source', nargs='+')
 189     parser.add_argument('target')
 190
 191   @classmethod
 192   def ProcessCommandLineArgs(cls, parser, args):
 193     # Check source file paths.
 194     for source_path in args.source:
 195       # Ensure source path exists.
 196       if not os.path.exists(source_path):
 197         parser.error('"%s" not found.' % source_path)
 198
 199       # Ensure source path is in one of the BASE_DIRS.
 200       for base_dir in BASE_DIRS:
 201         if path.IsSubpath(source_path, base_dir):
 202           break
 203       else:
 204         parser.error('"%s" is not in any of the base dirs.')
 205
 206     # Ensure target directory exists.
 207     if not (os.path.exists(args.target) or
 208         os.path.exists(os.path.dirname(args.target))):
 209       parser.error('"%s" not found.' % args.target)
 210
 211     # Ensure target path is in one of the BASE_DIRS.
 212     for base_dir in BASE_DIRS:
 213       if path.IsSubpath(args.target, base_dir):
 214         break
 215     else:
 216       parser.error('"%s" is not in any of the base dirs.')
 217
 218     # If there are multiple source paths, ensure target is a directory.
 219     if len(args.source) > 1 and not os.path.isdir(args.target):
 220       parser.error('Target "%s" is not a directory.' % args.target)
 221
 222     # Ensure target is not in any of the source paths.
 223     for source_path in args.source:
 224       if path.IsSubpath(args.target, source_path):
 225         parser.error('Cannot move "%s" to a subdirectory of itself, "%s".' %
 226             (source_path, args.target))
 227
 228   def Run(self, args):
 229     move.Run(args.source, args.target, _TelemetryFiles())
 230     for module_path in _TelemetryFiles():
 231       SortImportGroups(module_path)
 232     return 0
 233
 234
 235 class Sort(command_line.Command):
 236   """Sort imports."""
 237
 238   @classmethod
 239   def AddCommandLineArgs(cls, parser):
 240     parser.add_argument('target', nargs='*')
 241
 242   @classmethod
 243   def ProcessCommandLineArgs(cls, parser, args):
 244     for target in args.target:
 245       if not os.path.exists(target):
 246         parser.error('"%s" not found.' % target)
 247
 248   def Run(self, args):
 249     if args.target:
 250       targets = args.target
 251     else:
 252       targets = BASE_DIRS
 253
 254     for base_dir in targets:
 255       for module_path in path.ListFiles(base_dir, _IsSourceDir, _IsPythonModule):
 256         SortImportGroups(module_path)
 257     return 0
 258
 259
 260 class RefactorCommand(command_line.SubcommandCommand):
 261   commands = (Count, Mv, Sort,)
 262
 263
 264 if __name__ == '__main__':
 265   sys.exit(RefactorCommand.main())