pythonparser-exp/cdmbriefparser.py

   1 #
   2 # -*- coding: utf-8 -*-
   3 #
   4 # codimension - graphics python two-way code editor and analyzer
   5 # Copyright (C) 2010  Sergey Satskiy <sergey.satskiy@gmail.com>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19 #
  20 # $Id$
  21 #
  22
  23 """ The file holds types and a glue code between python and C python
  24     code parser """
  25
  26 import _cdmpyparser
  27 from sys import maxint
  28
  29
  30
  31 def trim_docstring( docstring ):
  32     " Taken from http://www.python.org/dev/peps/pep-0257/ "
  33
  34     if not docstring:
  35         return ''
  36
  37     # Convert tabs to spaces (following the normal Python rules)
  38     # and split into a list of lines:
  39     lines = docstring.expandtabs().splitlines()
  40
  41     # Determine minimum indentation (first line doesn't count):
  42     indent = maxint
  43     for line in lines[ 1: ]:
  44         stripped = line.lstrip()
  45         if stripped:
  46             indent = min( indent, len( line ) - len( stripped ) )
  47
  48     # Remove indentation (first line is special):
  49     lines[ 0 ] = lines[ 0 ].strip()
  50     if indent < maxint:
  51         index = 1
  52         for line in lines[ 1: ]:
  53             lines[ index ] = line[ indent: ].rstrip()
  54             index += 1
  55
  56     # Strip off trailing and leading blank lines:
  57     while lines and not lines[ -1 ]:
  58         del lines[ -1 ]
  59     while lines and not lines[ 0 ]:
  60         del lines[ 0 ]
  61
  62     # Return a single string:
  63     return '\n'.join( lines )
  64
  65
  66
  67 class ModuleInfoBase( object ):
  68     " Common part for the module information "
  69
  70     __slots__ = [ "name", "line", "pos", "absPosition" ]
  71
  72     def __init__( self, name, line, pos, absPosition ):
  73         self.name = name        # Item identifier
  74         self.line = line        # Line number where item is found (1-based)
  75         self.pos  = pos         # Item position in the line (1-based)
  76
  77         self.absPosition = absPosition    # Absolute position of the first item
  78                                           # identifier character starting from
  79                                           # the beginning of the input stream
  80                                           # (0-based)
  81         return
  82
  83     def isPrivate( self ):
  84         " True if it is private "
  85         return self.name.startswith( '__' )
  86
  87     def isProtected( self ):
  88         " True if it is protected "
  89         return not self.isPrivate() and self.name.startswith( '_' )
  90
  91     def _getLPA( self ):
  92         " Provides line, pos and absPosition line as string "
  93         return str( self.line ) + ":" + \
  94                str( self.pos ) + ":" + \
  95                str( self.absPosition )
  96
  97     def getDisplayName( self ):
  98         """ Provides a name for display purpose.
  99             It is going to be overriden in deriving classes """
 100         return self.name
 101
 102
 103 class Encoding( ModuleInfoBase ):
 104     " Holds information about encoding string "
 105
 106     __slots__ = []
 107
 108     def __init__( self, encName, line, pos, absPosition ):
 109         ModuleInfoBase.__init__( self, encName, line, pos, absPosition )
 110         return
 111
 112     def __str__( self ):
 113         return "Encoding[" + self._getLPA() + "]: '" + self.name + "'"
 114
 115
 116 class ImportWhat( ModuleInfoBase ):
 117     " Holds information about a single imported item "
 118
 119     __slots__ = [ "alias" ]
 120
 121     def __init__( self, whatName, line, pos, absPosition ):
 122         ModuleInfoBase.__init__( self, whatName, line, pos, absPosition )
 123         self.alias = ""
 124         return
 125
 126     def __str__( self ):
 127         if self.alias == "":
 128             return self.name + "[" + self._getLPA() + "]"
 129         return self.name + "[" + self._getLPA() + "] as " + self.alias
 130
 131     def getDisplayName( self ):
 132         " Provides a name for display purpose respecting the alias "
 133         if self.alias == "":
 134             return self.name
 135         return self.name + " as " + self.alias
 136
 137
 138 class Import( ModuleInfoBase ):
 139     " Holds information about a single import name "
 140
 141     __slots__ = [ "alias", "what" ]
 142
 143     def __init__( self, importName, line, pos, absPosition ):
 144         ModuleInfoBase.__init__( self, importName, line, pos, absPosition )
 145         self.alias = ""
 146         self.what = []
 147         return
 148
 149     def __str__( self ):
 150         out = "Import[" + self._getLPA() + "]: '" + self.name + "'"
 151         if self.alias != "":
 152             out += " as '" + self.alias + "'"
 153         whatPart = ""
 154         for item in self.what:
 155             whatPart += "\n    " + str( item )
 156         return out + whatPart
 157
 158     def getDisplayName( self ):
 159         " Provides a name for display purpose respecting the alias "
 160         if self.alias == "":
 161             return self.name
 162         return self.name + " as " + self.alias
 163
 164
 165 class Global( ModuleInfoBase ):
 166     " Holds information about a single global variable "
 167
 168     __slots__ = []
 169
 170     def __init__( self, globalName, line, pos, absPosition ):
 171         ModuleInfoBase.__init__( self, globalName, line, pos, absPosition )
 172         return
 173
 174     def __str__( self ):
 175         return "Global[" + self._getLPA() + "]: '" + self.name + "'"
 176
 177
 178 class ClassAttribute( ModuleInfoBase ):
 179     " Holds information about a class attribute "
 180
 181     __slots__ = []
 182
 183     def __init__( self, attrName, line, pos, absPosition ):
 184         ModuleInfoBase.__init__( self, attrName, line, pos, absPosition )
 185         return
 186
 187     def __str__( self ):
 188         return "Class attribute[" + self._getLPA() + "]: '" + self.name + "'"
 189
 190
 191 class InstanceAttribute( ModuleInfoBase ):
 192     " Holds information about a class instance attribute "
 193
 194     __slots__ = []
 195
 196     def __init__( self, attrName, line, pos, absPosition ):
 197         ModuleInfoBase.__init__( self, attrName, line, pos, absPosition )
 198         return
 199
 200     def __str__( self ):
 201         return "Instance attribute[" + self._getLPA() + "]: '" + self.name + "'"
 202
 203
 204 class Decorator( ModuleInfoBase ):
 205     " Holds information about a class/function decorator "
 206
 207     __slots__ = [ "arguments" ]
 208
 209     def __init__( self, decorName, line, pos, absPosition ):
 210         ModuleInfoBase.__init__( self, decorName, line, pos, absPosition )
 211         self.arguments = []
 212         return
 213
 214     def __str__( self ):
 215         val = "Decorator[" + self._getLPA() + "]: '" + self.name
 216         if self.arguments:
 217             val += "( "
 218             first = True
 219             for item in self.arguments:
 220                 if first:
 221                     val += item
 222                     first = False
 223                 else:
 224                     val += ", " + item
 225             val += " )'"
 226         return val
 227
 228     def getDisplayName( self ):
 229         " Provides a name for display purpose "
 230         displayName = self.name
 231         if self.arguments:
 232             displayName += "( " + ", ".join( self.arguments ) + " )"
 233         return displayName
 234
 235
 236 class Docstring( object ):
 237     " Holds a docstring information "
 238
 239     __slots__ = [ "line", "text" ]
 240
 241     def __init__( self, text, line ):
 242         self.line = line
 243         self.text = text
 244         return
 245
 246     def __str__( self ):
 247         return "Docstring[" + str( self.line ) + "]: '" + self.text + "'"
 248
 249
 250 class Function( ModuleInfoBase ):
 251     " Holds information about a single function"
 252
 253     __slots__ = [ "keywordLine", "keywordPos", "colonLine", "colonPos",
 254                   "docstring", "arguments", "decorators", "functions",
 255                   "classes" ]
 256
 257     def __init__( self, funcName, line, pos, absPosition,
 258                         keywordLine, keywordPos,
 259                         colonLine, colonPos ):
 260         ModuleInfoBase.__init__( self, funcName, line, pos, absPosition )
 261
 262         self.keywordLine = keywordLine  # line where 'def' keyword
 263                                         # starts (1-based).
 264         self.keywordPos = keywordPos    # pos where 'def' keyword
 265                                         # starts (1-based).
 266         self.colonLine = colonLine      # line where ':' char starts (1-based)
 267         self.colonPos = colonPos        # pos where ':' char starts (1-based)
 268
 269         self.docstring = None
 270         self.arguments = []
 271         self.decorators = []
 272         self.functions = []     # nested functions
 273         self.classes = []       # nested classes
 274         return
 275
 276     def isStaticMethod( self ):
 277         " Returns True if it is a static method "
 278         for item in self.decorators:
 279             if item.name == 'staticmethod':
 280                 return True
 281         return False
 282
 283     def niceStringify( self, level ):
 284         " Returns a string representation with new lines and shifts "
 285
 286         out = level * "    " + "Function[" + str(self.keywordLine) + \
 287                                        ":" + str(self.keywordPos) + \
 288                                        ":" + self._getLPA() + \
 289                                        ":" + str(self.colonLine) + \
 290                                        ":" + str(self.colonPos) + \
 291                                        "]: '" + self.name + "'"
 292         for item in self.arguments:
 293             out += '\n' + level * "    " + "Argument: '" + item + "'"
 294         for item in self.decorators:
 295             out += '\n' + level * "    " + str( item )
 296         if self.docstring is not None:
 297             out += '\n' + level * "    " + str( self.docstring )
 298         for item in self.functions:
 299             out += '\n' + item.niceStringify( level + 1 )
 300         for item in self.classes:
 301             out += '\n' + item.niceStringify( level + 1 )
 302         return out
 303
 304     def getDisplayName( self ):
 305         " Provides a name for display purpose "
 306         displayName = self.name + "("
 307         if self.arguments:
 308             displayName += " " + ", ".join( self.arguments ) + " "
 309         displayName += ")"
 310         return displayName
 311
 312
 313 class Class( ModuleInfoBase ):
 314     " Holds information about a single class"
 315
 316     __slots__ = [ "keywordLine", "keywordPos", "colonLine", "colonPos",
 317                   "docstring", "base", "decorators", "classAttributes",
 318                   "instanceAttributes", "functions", "classes" ]
 319
 320     def __init__( self, className, line, pos, absPosition,
 321                         keywordLine, keywordPos,
 322                         colonLine, colonPos ):
 323         ModuleInfoBase.__init__( self, className, line, pos, absPosition )
 324
 325         self.keywordLine = keywordLine  # line where 'def' keyword
 326                                         # starts (1-based).
 327         self.keywordPos = keywordPos    # pos where 'def' keyword
 328                                         # starts (1-based).
 329         self.colonLine = colonLine      # line where ':' char starts (1-based)
 330         self.colonPos = colonPos        # pos where ':' char starts (1-based)
 331
 332         self.docstring = None
 333         self.base = []
 334         self.decorators = []
 335         self.classAttributes = []
 336         self.instanceAttributes = []
 337         self.functions = []             # member functions
 338         self.classes = []               # nested classes
 339         return
 340
 341     def niceStringify( self, level ):
 342         " Returns a string representation with new lines and shifts "
 343
 344         out = level * "    " + "Class[" + str(self.keywordLine) + \
 345                                     ":" + str(self.keywordPos) + \
 346                                     ":" + self._getLPA() + \
 347                                     ":" + str(self.colonLine) + \
 348                                     ":" + str(self.colonPos) + \
 349                                     "]: '" + self.name + "'"
 350         for item in self.base:
 351             out += '\n' + level * "    " + "Base class: '" + item + "'"
 352         for item in self.decorators:
 353             out += '\n' + level * "    " + str( item )
 354         if self.docstring is not None:
 355             out += '\n' + level * "    " + str( self.docstring )
 356         for item in self.classAttributes:
 357             out += '\n' + level * "    " + str(item)
 358         for item in self.instanceAttributes:
 359             out += '\n' + level * "    " + str(item)
 360         for item in self.functions:
 361             out += '\n' + item.niceStringify( level + 1 )
 362         for item in self.classes:
 363             out += '\n' + item.niceStringify( level + 1 )
 364         return out
 365
 366     def getDisplayName( self ):
 367         " Provides a name for display purpose "
 368         displayName = self.name
 369         if self.base:
 370             displayName += "( " + ", ".join( self.base ) + " )"
 371         return displayName
 372
 373
 374 class BriefModuleInfo( object ):
 375     " Holds a single module content information "
 376
 377     __slots__ = [ "isOK", "docstring", "encoding", "imports", "globals",
 378                   "functions", "classes", "errors", "lexerErrors",
 379                   "objectsStack", "__lastImport", "__lastDecorators" ]
 380
 381     def __init__( self ):
 382         self.isOK = True
 383
 384         self.docstring = None
 385         self.encoding = None
 386         self.imports = []
 387         self.globals = []
 388         self.functions = []
 389         self.classes = []
 390         self.errors = []
 391         self.lexerErrors = []
 392
 393         self.objectsStack = []
 394         self.__lastImport = None
 395         self.__lastDecorators = None
 396         return
 397
 398     def niceStringify( self ):
 399         " Returns a string representation with new lines and shifts "
 400
 401         out = ""
 402         if self.docstring is not None:
 403             out += str( self.docstring )
 404         if not self.encoding is None:
 405             if out != "":
 406                 out += '\n'
 407             out += str( self.encoding )
 408         for item in self.imports:
 409             if out != "":
 410                 out += '\n'
 411             out += str( item )
 412         for item in self.globals:
 413             if out != "":
 414                 out += '\n'
 415             out += str( item )
 416         for item in self.functions:
 417             if out != "":
 418                 out += '\n'
 419             out += item.niceStringify( 0 )
 420         for item in self.classes:
 421             if out != "":
 422                 out += '\n'
 423             out += item.niceStringify( 0 )
 424         return out
 425
 426
 427     def flush( self ):
 428         " Flushes the collected information "
 429         self.__flushLevel( 0 )
 430         if self.__lastImport is not None:
 431             self.imports.append( self.__lastImport )
 432         return
 433
 434     def __flushLevel( self, level ):
 435         " Merge the found objects to the required level "
 436
 437         objectsCount = len( self.objectsStack )
 438
 439         while objectsCount > level:
 440             lastIndex = objectsCount - 1
 441             if lastIndex == 0:
 442                 # We have exactly one element in the stack
 443                 if self.objectsStack[ 0 ].__class__.__name__ == "Class":
 444                     self.classes.append( self.objectsStack[ 0 ] )
 445                 else:
 446                     self.functions.append( self.objectsStack[ 0 ] )
 447                 self.objectsStack = []
 448                 break
 449
 450             # Append to the previous level
 451             if self.objectsStack[ lastIndex ].__class__.__name__ == "Class":
 452                 self.objectsStack[ lastIndex - 1 ].classes. \
 453                         append( self.objectsStack[ lastIndex ] )
 454             else:
 455                 self.objectsStack[ lastIndex - 1 ].functions. \
 456                         append( self.objectsStack[ lastIndex ] )
 457
 458             del self.objectsStack[ lastIndex ]
 459             objectsCount -= 1
 460
 461         return
 462
 463     def _onEncoding( self, encString, line, pos, absPosition ):
 464         " Memorizes module encoding "
 465         self.encoding = Encoding( encString, line, pos, absPosition )
 466         return
 467
 468     def _onGlobal( self, name, line, pos, absPosition, level ):
 469         " Memorizes a global variable "
 470
 471         # level is ignored
 472         for item in self.globals:
 473             if item.name == name:
 474                 return
 475         self.globals.append( Global( name, line, pos, absPosition ) )
 476         return
 477
 478     def _onClass( self, name, line, pos, absPosition,
 479                         keywordLine, keywordPos,
 480                         colonLine, colonPos, level ):
 481         " Memorizes a class "
 482         self.__flushLevel( level )
 483         c = Class( name, line, pos, absPosition, keywordLine, keywordPos,
 484                    colonLine, colonPos )
 485         if self.__lastDecorators is not None:
 486             c.decorators = self.__lastDecorators
 487             self.__lastDecorators = None
 488         self.objectsStack.append( c )
 489         return
 490
 491     def _onFunction( self, name, line, pos, absPosition,
 492                            keywordLine, keywordPos,
 493                            colonLine, colonPos, level ):
 494         " Memorizes a function "
 495         self.__flushLevel( level )
 496         f = Function( name, line, pos, absPosition, keywordLine, keywordPos,
 497                       colonLine, colonPos )
 498         if self.__lastDecorators is not None:
 499             f.decorators = self.__lastDecorators
 500             self.__lastDecorators = None
 501         self.objectsStack.append( f )
 502         return
 503
 504     def _onImport( self, name, line, pos, absPosition ):
 505         " Memorizes an import "
 506         if self.__lastImport is not None:
 507             self.imports.append( self.__lastImport )
 508         self.__lastImport = Import( name, line, pos, absPosition )
 509         return
 510
 511     def _onAs( self, name ):
 512         " Memorizes an alias for an import or an imported item "
 513         if self.__lastImport.what:
 514             self.__lastImport.what[ -1 ].alias = name
 515         else:
 516             self.__lastImport.alias = name
 517         return
 518
 519     def _onWhat( self, name, line, pos, absPosition ):
 520         " Memorizes an imported item "
 521         self.__lastImport.what.append( ImportWhat( name, line, pos,
 522                                                          absPosition ) )
 523         return
 524
 525     def _onClassAttribute( self, name, line, pos, absPosition, level ):
 526         " Memorizes a class attribute "
 527         # A class must be on the top of the stack
 528         attributes = self.objectsStack[ level ].classAttributes
 529         for item in attributes:
 530             if item.name == name:
 531                 return
 532         attributes.append( ClassAttribute( name, line, pos, absPosition ) )
 533         return
 534
 535     def _onInstanceAttribute( self, name, line, pos, absPosition, level ):
 536         " Memorizes a class instance attribute "
 537         # Instance attributes may appear in member functions only so we already
 538         # have a function on the stack of objects. To get the class object one
 539         # more step is required so we -1 here.
 540         attributes = self.objectsStack[ level - 1 ].instanceAttributes
 541         for item in attributes:
 542             if item.name == name:
 543                 return
 544         attributes.append( InstanceAttribute( name, line, pos, absPosition ) )
 545         return
 546
 547     def _onDecorator( self, name, line, pos, absPosition ):
 548         " Memorizes a function or a class decorator "
 549         # A class or a function must be on the top of the stack
 550         d = Decorator( name, line, pos, absPosition )
 551         if self.__lastDecorators is None:
 552             self.__lastDecorators = [ d ]
 553         else:
 554             self.__lastDecorators.append( d )
 555         return
 556
 557     def _onDecoratorArgument( self, name ):
 558         " Memorizes a decorator argument "
 559         self.__lastDecorators[ -1 ].arguments.append( name )
 560         return
 561
 562     def _onDocstring( self, docstr, line ):
 563         " Memorizes a function/class/module docstring "
 564         if self.objectsStack:
 565             self.objectsStack[ -1 ].docstring = \
 566                                  Docstring( trim_docstring( docstr ), line )
 567             return
 568
 569         self.docstring = Docstring( trim_docstring( docstr ), line )
 570         return
 571
 572     def _onArgument( self, name ):
 573         " Memorizes a function argument "
 574         self.objectsStack[ -1 ].arguments.append( name )
 575         return
 576
 577     def _onArgumentValue( self, value ):
 578         " Memorizes a function argument value "
 579         self.objectsStack[ -1 ].arguments[ -1 ] += " = " + value
 580         return
 581
 582     def _onBaseClass( self, name ):
 583         " Memorizes a class base class "
 584         # A class must be on the top of the stack
 585         self.objectsStack[ -1 ].base.append( name )
 586         return
 587
 588     def _onError( self, message ):
 589         " Memorizies a parser error message "
 590         self.isOK = False
 591         if message.strip() != "":
 592             self.errors.append( message )
 593         return
 594
 595     def _onLexerError( self, message ):
 596         " Memorizes a lexer error message "
 597         self.isOK = False
 598         if message.strip() != "":
 599             self.lexerErrors.append( message )
 600         return
 601
 602
 603 def getBriefModuleInfoFromFile( fileName ):
 604     " Builds the brief module info from file "
 605
 606     modInfo = BriefModuleInfo()
 607     _cdmpyparser.getBriefModuleInfoFromFile( modInfo, fileName )
 608     modInfo.flush()
 609     return modInfo
 610
 611
 612 def getBriefModuleInfoFromMemory( content ):
 613     " Builds the brief module info from memory "
 614
 615     modInfo = BriefModuleInfo()
 616     _cdmpyparser.getBriefModuleInfoFromMemory( modInfo, content )
 617     modInfo.flush()
 618     return modInfo
 619
 620 def getVersion():
 621     " Provides the parser version "
 622     return _cdmpyparser.version
 623