flight/libraries/PyMite/tools/pycscope.py

   1 #!/usr/bin/env python
   2
   3 # This file is Copyright 2003, 2006, 2007, 2009, 2010 Dean Hall.
   4 #
   5 # This file is part of the Python-on-a-Chip program.
   6 # Python-on-a-Chip is free software: you can redistribute it and/or modify
   7 # it under the terms of the GNU LESSER GENERAL PUBLIC LICENSE Version 2.1.
   8 #
   9 # Python-on-a-Chip is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  12 # A copy of the GNU LESSER GENERAL PUBLIC LICENSE Version 2.1
  13 # is seen in the file COPYING up one directory from this.
  14
  15 """
  16 PyCscope
  17
  18 PyCscope creates a Cscope-like index file for a tree of Python source.
  19 """
  20
  21 ## @file
  22 #  @copybrief pycscope
  23
  24 ## @package pycscope
  25 #  @brief PyCscope creates a Cscope-like index file for a tree of Python source.
  26 #
  27 # 2007/12/25:
  28 #   Improvements contributed by K. Rader of Google:
  29 #   - Added the `-i` argument to specify a file-list file
  30 #   - Fixups to the header and footer to make a valid file that cscope can read
  31 #
  32
  33
  34 __author__ = "Dean Hall"
  35 __copyright__ = "Copyright 2003, 2006, 2007, 2009, 2010 Dean Hall.  See LICENSE for details."
  36 __date__ = "2007/12/25"
  37 __version__ = "0.3"
  38 __usage__ = """Usage: pycscope.py [-R] [-f reffile] [-i srclistfile] [files ...]
  39
  40 -R              Recurse directories for files.
  41 -f reffile      Use reffile as cross-ref file name instead of cscope.out.
  42 -i srclistfile  Use a file that contains a list of source files to scan."""
  43
  44
  45 import getopt, sys, os, os.path, string, types
  46 import keyword, parser, symbol, token
  47
  48 # Marks as defined by Cscope
  49 MARK_FILE = "\t@"
  50 MARK_FUNC_DEF = "\t$"
  51 MARK_FUNC_CALL = "\t`"
  52 MARK_FUNC_END = "\t}"
  53 MARK_INCLUDE = "\t~<"   # TODO: assume all includes are global for now
  54 MARK_ASGN = "\t="
  55 MARK_CLASS = "\tc"
  56 MARK_GLOBAL = "\tg"
  57 MARK_FUNC_PARM = "\tp"
  58
  59 # Reverse the key,value pairs in the token dict
  60 tok_name_lookup = dict((v,k) for k,v in token.tok_name.iteritems())
  61 TOK_NEWLINE = tok_name_lookup["NEWLINE"]
  62 TOK_NAME = tok_name_lookup["NAME"]
  63 TOK_LPAR = tok_name_lookup["LPAR"]
  64 TOK_ENDMARKER = tok_name_lookup["ENDMARKER"]
  65 TOK_INDENT = tok_name_lookup["INDENT"]
  66 TOK_DEDENT = tok_name_lookup["DEDENT"]
  67
  68 # Reverse the key,value pairs in the symbol dict
  69 sym_name_lookup = dict((v,k) for k,v in symbol.sym_name.iteritems())
  70 SYM_TRAILER = sym_name_lookup["trailer"]
  71 SYM_VARARGSLIST = sym_name_lookup["varargslist"]
  72
  73 # Get the list of Python keywords and add a few common builtins
  74 kwlist = keyword.kwlist
  75 kwlist.extend(("True", "False", "None", "object"))
  76
  77 # Globals for the recursive walkAst function
  78 latestnewline = 1
  79 latestsymbol = ""
  80 latesttoken = ""
  81 prevtoken = ""
  82 mark = ""
  83 infuncdef = False
  84 indentcount = 0
  85
  86
  87 def main():
  88     """Parse command line args and act accordingly.
  89     """
  90     # Parse the command line arguments
  91     try:
  92         opts, args = getopt.getopt(sys.argv[1:], "Rf:i:")
  93     except getopt.GetoptError:
  94         print __usage__
  95         sys.exit(2)
  96     recurse = False
  97     indexfn = "cscope.out"
  98     for o, a in opts:
  99         if o == "-R":
 100             recurse = True
 101         if o == "-f":
 102             indexfn = a
 103         if o == "-i":
 104             args.extend(map(string.rstrip, open(a, 'r').readlines()))
 105
 106     # Create the buffer to store the output (list of strings)
 107     indexbuff = []
 108     fnamesbuff = []
 109
 110     # Search current dir by default
 111     if len(args) == 0:
 112         args = "."
 113
 114     # Parse the given list of files/dirs
 115     basepath = os.getcwd()
 116     for name in args:
 117         if os.path.isdir(os.path.join(basepath, name)):
 118             parseDir(basepath, name, indexbuff, recurse, fnamesbuff)
 119         else:
 120             try:
 121                 parseFile(basepath, name, indexbuff, fnamesbuff)
 122             except SyntaxError:
 123                 pass
 124
 125     # Symbol data for the last file ends with a file mark
 126     indexbuff.append("\n" + MARK_FILE)
 127     writeIndex(basepath, indexfn, indexbuff, fnamesbuff)
 128
 129
 130 def parseDir(basepath, relpath, indexbuff, recurse, fnamesbuff):
 131     """Parses all files in the directory and
 132     recurses into subdirectories if requested.
 133     """
 134     dirpath = os.path.join(basepath, relpath)
 135     for name in os.listdir(dirpath):
 136         fullpath = os.path.join(dirpath, name)
 137         if os.path.isdir(fullpath) and recurse:
 138             parseDir(basepath, os.path.join(relpath, name), indexbuff, recurse,
 139                      fnamesbuff)
 140         else:
 141             try:
 142                 parseFile(basepath, os.path.join(relpath, name), indexbuff,
 143                           fnamesbuff)
 144             except SyntaxError:
 145                 pass
 146
 147
 148 def parseFile(basepath, relpath, indexbuff, fnamesbuff):
 149     """Parses a source file and puts the resulting index into the buffer.
 150     """
 151     # Don't parse if it's not python source
 152     if relpath[-3:] != ".py":
 153         return
 154
 155     # Open the file and get the contents
 156     fullpath = os.path.join(basepath, relpath)
 157     f = open(fullpath, 'r')
 158     filecontents = f.read()
 159     f.close()
 160
 161     # Add the file mark to the index
 162     fnamesbuff.append(relpath)
 163     indexbuff.append("\n%s%s" % (MARK_FILE, relpath))
 164     global latestnewline
 165     latestnewline = len(indexbuff)
 166
 167     # Add path info to any syntax errors in the source files
 168     try:
 169         parseSource(filecontents, indexbuff)
 170     except SyntaxError, se:
 171         se.filename = fullpath
 172         raise se
 173
 174
 175 def parseSource(sourcecode, indexbuff):
 176     """Parses python source code and puts the resulting index into the buffer.
 177     """
 178     # Parse the source to an Abstract Syntax Tree
 179     ast = parser.suite(sourcecode)
 180     astlist = parser.ast2list(ast, True)
 181
 182     # Set these globals before each file's AST is walked
 183     global sourcelinehassymbol
 184     sourcelinehassymbol = False
 185     global currentlinenum
 186     currentlinenum = 0
 187
 188     # Walk the AST to index the rest of the file
 189     walkAst(astlist, indexbuff)
 190
 191
 192 def walkAst(astlist, indexbuff):
 193     """Scan the AST for tokens, write out index lines.
 194     """
 195     global latestnewline
 196     global latestsymbol
 197     global latesttoken
 198     global prevtoken
 199     global mark
 200     global sourcelinehassymbol
 201     global infuncdef
 202     global indentcount
 203     global currentlinenum
 204
 205     # Remember the latest symbol
 206     if astlist[0] > 256:
 207         latestsymbol = astlist[0]
 208
 209     # Handle the tokens
 210     else:
 211         # Save the previous token and get the latest one
 212         prevtoken = latesttoken
 213         latesttoken = astlist[0]
 214
 215         # If this code is on a new line number
 216         if astlist[2] != currentlinenum:
 217             currentlinenum = astlist[2]
 218
 219             # If there was a symbol of interest,
 220             # remember this location in the index
 221             if sourcelinehassymbol:
 222                 latestnewline = len(indexbuff)
 223                 sourcelinehassymbol = False
 224
 225             # If there was no symbol of interest between this and the previous
 226             # newline, remove all entries added since the previous newline
 227             else:
 228                 del indexbuff[latestnewline:]
 229
 230             # Write the new line number
 231             indexbuff.append("\n\n%d " % astlist[2])
 232
 233             # Clear an include mark when a newline token is reached
 234             # This is what ends a comma-separated list of modules after import
 235             if mark == MARK_INCLUDE:
 236                 mark = ""
 237
 238         if latesttoken == TOK_NAME:
 239             # If a name is not a python keyword, it is a symbol of interest
 240             if astlist[1] not in kwlist:
 241
 242                 # Remember that there is a symbol of interest
 243                 sourcelinehassymbol = True
 244
 245                 # Write the mark and the symbol
 246                 indexbuff.append("\n%s%s\n" % (mark, astlist[1]))
 247
 248                 # Clear the mark unless it's an include mark
 249                 # This is what allows a comma-separated list of modules after import
 250                 if mark != MARK_INCLUDE:
 251                     mark = ""
 252
 253             # If the name is a python keyword
 254             else:
 255                 # Some keywords determine what mark should prefix the next name
 256                 kw = astlist[1]
 257                 if kw == "def":
 258                     mark = MARK_FUNC_DEF
 259
 260                     # Remember that we're in a function definition
 261                     infuncdef = True
 262                     indentcount = 0
 263                 elif kw == "import":
 264                     mark = MARK_INCLUDE
 265                 elif kw == "class":
 266                     mark = MARK_CLASS
 267
 268                 # Write out the keyword
 269                 indexbuff.append("%s " % kw)
 270
 271         # This set of tokens and symbols indicates a function call (not perfect)
 272         elif (latesttoken == TOK_LPAR) and (prevtoken == TOK_NAME) and (
 273             (latestsymbol == SYM_TRAILER) or (latestsymbol == SYM_VARARGSLIST)):
 274
 275             # Insert a function-call mark before the previous name
 276             indexbuff[-1] = "\n%s%s( " % (MARK_FUNC_CALL, indexbuff[-1][1:])
 277
 278         # Count the number of indents; to be used by dedent
 279         elif latesttoken == TOK_INDENT:
 280             if infuncdef:
 281                 indentcount += 1
 282
 283         # When dedent reaches the level of the function def,
 284         # write the function-end mark
 285         elif latesttoken == TOK_DEDENT:
 286             if infuncdef:
 287                 indentcount -= 1
 288                 if indentcount == 0:
 289                     indexbuff.insert(-1, "\n\n%d \n%s\n" % (astlist[2], MARK_FUNC_END))
 290                     latestnewline += 1
 291                     infuncdef = False
 292
 293         # Replace the last line number placeholder with a newline
 294         # when at the end of a file
 295         elif latesttoken == TOK_ENDMARKER:
 296             if len(indexbuff) > 0:
 297                 indexbuff[-1] = "\n"
 298
 299         # For uninteresting tokens, just write the accompanying string
 300         else:
 301             if len(astlist[1]) > 0:
 302                 nonsymboltext = astlist[1].replace("\n","\\n") + ' '
 303             else:
 304                 nonsymboltext = ''
 305             indexbuff.append(nonsymboltext)
 306
 307     # Recurse into all nodes
 308     for i in range(1, len(astlist)):
 309         if type(astlist[i]) == types.ListType:
 310             walkAst(astlist[i], indexbuff)
 311
 312
 313 def writeIndex(basepath, indexfn, indexbuff, fnamesbuff):
 314     """Write the index buffer to the output file.
 315     """
 316     fout = open(os.path.join(basepath, indexfn), 'w')
 317
 318     # Write the header and index
 319     index = ''.join(indexbuff)
 320     index_len = len(index)
 321     hdr_len = len(basepath) + 25
 322     fout.write("cscope 15 %s -c %010d" % (basepath, hdr_len + index_len))
 323     fout.write(index)
 324
 325     # Write trailer info
 326     fnames = '\n'.join(fnamesbuff) + '\n'
 327     fout.write("\n1\n.\n0\n")
 328     fout.write("%d\n" % len(fnamesbuff))
 329     fout.write("%d\n" % len(fnames))
 330     fout.write(fnames)
 331     fout.close()
 332
 333
 334 if __name__ == "__main__":
 335     main()