Lib/distutils/filelist.py

   1 """distutils.filelist
   2
   3 Provides the FileList class, used for poking about the filesystem
   4 and building lists of files.
   5 """
   6
   7 # created 2000/07/17, Rene Liebscher (as template.py)
   8 # most parts taken from commands/sdist.py
   9 # renamed 2000/07/29 (to filelist.py) and officially added to
  10 #  the Distutils source, Greg Ward
  11
  12 __revision__ = "$Id$"
  13
  14 import os, string, re
  15 import fnmatch
  16 from types import *
  17 from glob import glob
  18 from distutils.util import convert_path
  19 from distutils.errors import DistutilsTemplateError, DistutilsInternalError
  20 from distutils import log
  21
  22 class FileList:
  23
  24     """A list of files built by on exploring the filesystem and filtered by
  25     applying various patterns to what we find there.
  26
  27     Instance attributes:
  28       dir
  29         directory from which files will be taken -- only used if
  30         'allfiles' not supplied to constructor
  31       files
  32         list of filenames currently being built/filtered/manipulated
  33       allfiles
  34         complete list of files under consideration (ie. without any
  35         filtering applied)
  36     """
  37
  38     def __init__(self,
  39                  warn=None,
  40                  debug_print=None):
  41         # ignore argument to FileList, but keep them for backwards
  42         # compatibility
  43
  44         self.allfiles = None
  45         self.files = []
  46
  47     def set_allfiles (self, allfiles):
  48         self.allfiles = allfiles
  49
  50     def findall (self, dir=os.curdir):
  51         self.allfiles = findall(dir)
  52
  53     def debug_print (self, msg):
  54         """Print 'msg' to stdout if the global DEBUG (taken from the
  55         DISTUTILS_DEBUG environment variable) flag is true.
  56         """
  57         from distutils.debug import DEBUG
  58         if DEBUG:
  59             print msg
  60
  61     # -- List-like methods ---------------------------------------------
  62
  63     def append (self, item):
  64         self.files.append(item)
  65
  66     def extend (self, items):
  67         self.files.extend(items)
  68
  69     def sort (self):
  70         # Not a strict lexical sort!
  71         sortable_files = map(os.path.split, self.files)
  72         sortable_files.sort()
  73         self.files = []
  74         for sort_tuple in sortable_files:
  75             self.files.append(apply(os.path.join, sort_tuple))
  76
  77
  78     # -- Other miscellaneous utility methods ---------------------------
  79
  80     def remove_duplicates (self):
  81         # Assumes list has been sorted!
  82         for i in range(len(self.files) - 1, 0, -1):
  83             if self.files[i] == self.files[i - 1]:
  84                 del self.files[i]
  85
  86
  87     # -- "File template" methods ---------------------------------------
  88
  89     def _parse_template_line (self, line):
  90         words = string.split(line)
  91         action = words[0]
  92
  93         patterns = dir = dir_pattern = None
  94
  95         if action in ('include', 'exclude',
  96                       'global-include', 'global-exclude'):
  97             if len(words) < 2:
  98                 raise DistutilsTemplateError, \
  99                       "'%s' expects <pattern1> <pattern2> ..." % action
 100
 101             patterns = map(convert_path, words[1:])
 102
 103         elif action in ('recursive-include', 'recursive-exclude'):
 104             if len(words) < 3:
 105                 raise DistutilsTemplateError, \
 106                       "'%s' expects <dir> <pattern1> <pattern2> ..." % action
 107
 108             dir = convert_path(words[1])
 109             patterns = map(convert_path, words[2:])
 110
 111         elif action in ('graft', 'prune'):
 112             if len(words) != 2:
 113                 raise DistutilsTemplateError, \
 114                      "'%s' expects a single <dir_pattern>" % action
 115
 116             dir_pattern = convert_path(words[1])
 117
 118         else:
 119             raise DistutilsTemplateError, "unknown action '%s'" % action
 120
 121         return (action, patterns, dir, dir_pattern)
 122
 123     # _parse_template_line ()
 124
 125
 126     def process_template_line (self, line):
 127
 128         # Parse the line: split it up, make sure the right number of words
 129         # is there, and return the relevant words.  'action' is always
 130         # defined: it's the first word of the line.  Which of the other
 131         # three are defined depends on the action; it'll be either
 132         # patterns, (dir and patterns), or (dir_pattern).
 133         (action, patterns, dir, dir_pattern) = self._parse_template_line(line)
 134
 135         # OK, now we know that the action is valid and we have the
 136         # right number of words on the line for that action -- so we
 137         # can proceed with minimal error-checking.
 138         if action == 'include':
 139             self.debug_print("include " + string.join(patterns))
 140             for pattern in patterns:
 141                 if not self.include_pattern(pattern, anchor=1):
 142                     log.warn("warning: no files found matching '%s'",
 143                              pattern)
 144
 145         elif action == 'exclude':
 146             self.debug_print("exclude " + string.join(patterns))
 147             for pattern in patterns:
 148                 if not self.exclude_pattern(pattern, anchor=1):
 149                     log.warn(("warning: no previously-included files "
 150                               "found matching '%s'"), pattern)
 151
 152         elif action == 'global-include':
 153             self.debug_print("global-include " + string.join(patterns))
 154             for pattern in patterns:
 155                 if not self.include_pattern(pattern, anchor=0):
 156                     log.warn(("warning: no files found matching '%s' " +
 157                               "anywhere in distribution"), pattern)
 158
 159         elif action == 'global-exclude':
 160             self.debug_print("global-exclude " + string.join(patterns))
 161             for pattern in patterns:
 162                 if not self.exclude_pattern(pattern, anchor=0):
 163                     log.warn(("warning: no previously-included files matching "
 164                               "'%s' found anywhere in distribution"),
 165                              pattern)
 166
 167         elif action == 'recursive-include':
 168             self.debug_print("recursive-include %s %s" %
 169                              (dir, string.join(patterns)))
 170             for pattern in patterns:
 171                 if not self.include_pattern(pattern, prefix=dir):
 172                     log.warn(("warngin: no files found matching '%s' " +
 173                                 "under directory '%s'"),
 174                              pattern, dir)
 175
 176         elif action == 'recursive-exclude':
 177             self.debug_print("recursive-exclude %s %s" %
 178                              (dir, string.join(patterns)))
 179             for pattern in patterns:
 180                 if not self.exclude_pattern(pattern, prefix=dir):
 181                     log.warn(("warning: no previously-included files matching "
 182                               "'%s' found under directory '%s'"),
 183                              pattern, dir)
 184
 185         elif action == 'graft':
 186             self.debug_print("graft " + dir_pattern)
 187             if not self.include_pattern(None, prefix=dir_pattern):
 188                 log.warn("warning: no directories found matching '%s'",
 189                          dir_pattern)
 190
 191         elif action == 'prune':
 192             self.debug_print("prune " + dir_pattern)
 193             if not self.exclude_pattern(None, prefix=dir_pattern):
 194                 log.warn(("no previously-included directories found " +
 195                           "matching '%s'"), dir_pattern)
 196         else:
 197             raise DistutilsInternalError, \
 198                   "this cannot happen: invalid action '%s'" % action
 199
 200     # process_template_line ()
 201
 202
 203     # -- Filtering/selection methods -----------------------------------
 204
 205     def include_pattern (self, pattern,
 206                          anchor=1, prefix=None, is_regex=0):
 207         """Select strings (presumably filenames) from 'self.files' that
 208         match 'pattern', a Unix-style wildcard (glob) pattern.  Patterns
 209         are not quite the same as implemented by the 'fnmatch' module: '*'
 210         and '?'  match non-special characters, where "special" is platform-
 211         dependent: slash on Unix; colon, slash, and backslash on
 212         DOS/Windows; and colon on Mac OS.
 213
 214         If 'anchor' is true (the default), then the pattern match is more
 215         stringent: "*.py" will match "foo.py" but not "foo/bar.py".  If
 216         'anchor' is false, both of these will match.
 217
 218         If 'prefix' is supplied, then only filenames starting with 'prefix'
 219         (itself a pattern) and ending with 'pattern', with anything in between
 220         them, will match.  'anchor' is ignored in this case.
 221
 222         If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
 223         'pattern' is assumed to be either a string containing a regex or a
 224         regex object -- no translation is done, the regex is just compiled
 225         and used as-is.
 226
 227         Selected strings will be added to self.files.
 228
 229         Return 1 if files are found.
 230         """
 231         files_found = 0
 232         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
 233         self.debug_print("include_pattern: applying regex r'%s'" %
 234                          pattern_re.pattern)
 235
 236         # delayed loading of allfiles list
 237         if self.allfiles is None:
 238             self.findall()
 239
 240         for name in self.allfiles:
 241             if pattern_re.search(name):
 242                 self.debug_print(" adding " + name)
 243                 self.files.append(name)
 244                 files_found = 1
 245
 246         return files_found
 247
 248     # include_pattern ()
 249
 250
 251     def exclude_pattern (self, pattern,
 252                          anchor=1, prefix=None, is_regex=0):
 253         """Remove strings (presumably filenames) from 'files' that match
 254         'pattern'.  Other parameters are the same as for
 255         'include_pattern()', above.
 256         The list 'self.files' is modified in place.
 257         Return 1 if files are found.
 258         """
 259         files_found = 0
 260         pattern_re = translate_pattern(pattern, anchor, prefix, is_regex)
 261         self.debug_print("exclude_pattern: applying regex r'%s'" %
 262                          pattern_re.pattern)
 263         for i in range(len(self.files)-1, -1, -1):
 264             if pattern_re.search(self.files[i]):
 265                 self.debug_print(" removing " + self.files[i])
 266                 del self.files[i]
 267                 files_found = 1
 268
 269         return files_found
 270
 271     # exclude_pattern ()
 272
 273 # class FileList
 274
 275
 276 # ----------------------------------------------------------------------
 277 # Utility functions
 278
 279 def findall (dir = os.curdir):
 280     """Find all files under 'dir' and return the list of full filenames
 281     (relative to 'dir').
 282     """
 283     from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK
 284
 285     list = []
 286     stack = [dir]
 287     pop = stack.pop
 288     push = stack.append
 289
 290     while stack:
 291         dir = pop()
 292         names = os.listdir(dir)
 293
 294         for name in names:
 295             if dir != os.curdir:        # avoid the dreaded "./" syndrome
 296                 fullname = os.path.join(dir, name)
 297             else:
 298                 fullname = name
 299
 300             # Avoid excess stat calls -- just one will do, thank you!
 301             stat = os.stat(fullname)
 302             mode = stat[ST_MODE]
 303             if S_ISREG(mode):
 304                 list.append(fullname)
 305             elif S_ISDIR(mode) and not S_ISLNK(mode):
 306                 push(fullname)
 307
 308     return list
 309
 310
 311 def glob_to_re (pattern):
 312     """Translate a shell-like glob pattern to a regular expression; return
 313     a string containing the regex.  Differs from 'fnmatch.translate()' in
 314     that '*' does not match "special characters" (which are
 315     platform-specific).
 316     """
 317     pattern_re = fnmatch.translate(pattern)
 318
 319     # '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
 320     # IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
 321     # and by extension they shouldn't match such "special characters" under
 322     # any OS.  So change all non-escaped dots in the RE to match any
 323     # character except the special characters.
 324     # XXX currently the "special characters" are just slash -- i.e. this is
 325     # Unix-only.
 326     pattern_re = re.sub(r'(^|[^\\])\.', r'\1[^/]', pattern_re)
 327     return pattern_re
 328
 329 # glob_to_re ()
 330
 331
 332 def translate_pattern (pattern, anchor=1, prefix=None, is_regex=0):
 333     """Translate a shell-like wildcard pattern to a compiled regular
 334     expression.  Return the compiled regex.  If 'is_regex' true,
 335     then 'pattern' is directly compiled to a regex (if it's a string)
 336     or just returned as-is (assumes it's a regex object).
 337     """
 338     if is_regex:
 339         if type(pattern) is StringType:
 340             return re.compile(pattern)
 341         else:
 342             return pattern
 343
 344     if pattern:
 345         pattern_re = glob_to_re(pattern)
 346     else:
 347         pattern_re = ''
 348
 349     if prefix is not None:
 350         prefix_re = (glob_to_re(prefix))[0:-1] # ditch trailing $
 351         pattern_re = "^" + os.path.join(prefix_re, ".*" + pattern_re)
 352     else:                               # no prefix -- respect anchor flag
 353         if anchor:
 354             pattern_re = "^" + pattern_re
 355
 356     return re.compile(pattern_re)
 357
 358 # translate_pattern ()