3 General-purpose utility functions used throughout the Distutils
4 (especially in command classes). Mostly filesystem manipulation, but
5 not limited to that. The functions in this module generally raise
6 DistutilsFileError when they have problems with the filesystem, because
7 os.error in pre-1.5.2 Python only gives the error message and not the
10 # created 1999/03/08, Greg Ward
14 import sys
, os
, string
, re
, shutil
15 from distutils
.errors
import *
16 from distutils
.spawn
import spawn
18 # cache for by mkpath() -- in addition to cheapening redundant calls,
19 # eliminates redundant "creating /foo/bar/baz" messages in dry-run mode
22 # for generating verbose output in 'copy_file()'
23 _copy_action
= { None: 'copying',
24 'hard': 'hard linking',
25 'sym': 'symbolically linking' }
27 # I don't use os.makedirs because a) it's new to Python 1.5.2, and
28 # b) it blows up if the directory already exists (I want to silently
29 # succeed in that case).
30 def mkpath (name
, mode
=0777, verbose
=0, dry_run
=0):
31 """Create a directory and any missing ancestor directories. If the
32 directory already exists (or if 'name' is the empty string, which
33 means the current directory, which of course exists), then do
34 nothing. Raise DistutilsFileError if unable to create some
35 directory along the way (eg. some sub-path exists, but is a file
36 rather than a directory). If 'verbose' is true, print a one-line
37 summary of each mkdir to stdout. Return the list of directories
42 # XXX what's the better way to handle verbosity? print as we create
43 # each directory in the path (the current behaviour), or only announce
44 # the creation of the whole path? (quite easy to do the latter since
45 # we're not using a recursive algorithm)
47 name
= os
.path
.normpath (name
)
49 if os
.path
.isdir (name
) or name
== '':
51 if PATH_CREATED
.get (name
):
54 (head
, tail
) = os
.path
.split (name
)
55 tails
= [tail
] # stack of lone dirs to create
57 while head
and tail
and not os
.path
.isdir (head
):
58 #print "splitting '%s': " % head,
59 (head
, tail
) = os
.path
.split (head
)
60 #print "to ('%s','%s')" % (head, tail)
61 tails
.insert (0, tail
) # push next higher dir onto stack
63 #print "stack of tails:", tails
65 # now 'head' contains the deepest directory that already exists
66 # (that is, the child of 'head' in 'name' is the highest directory
67 # that does *not* exist)
69 #print "head = %s, d = %s: " % (head, d),
70 head
= os
.path
.join (head
, d
)
71 if PATH_CREATED
.get (head
):
75 print "creating", head
80 created_dirs
.append(head
)
82 raise DistutilsFileError
, \
83 "could not create '%s': %s" % (head
, exc
[-1])
85 PATH_CREATED
[head
] = 1
91 def create_tree (base_dir
, files
, mode
=0777, verbose
=0, dry_run
=0):
93 """Create all the empty directories under 'base_dir' needed to
94 put 'files' there. 'base_dir' is just the a name of a directory
95 which doesn't necessarily exist yet; 'files' is a list of filenames
96 to be interpreted relative to 'base_dir'. 'base_dir' + the
97 directory portion of every file in 'files' will be created if it
98 doesn't already exist. 'mode', 'verbose' and 'dry_run' flags are as
101 # First get the list of directories to create
104 need_dir
[os
.path
.join (base_dir
, os
.path
.dirname (file))] = 1
105 need_dirs
= need_dir
.keys()
109 for dir in need_dirs
:
110 mkpath (dir, mode
, verbose
, dry_run
)
115 def newer (source
, target
):
116 """Return true if 'source' exists and is more recently modified than
117 'target', or if 'source' exists and 'target' doesn't. Return
118 false if both exist and 'target' is the same age or younger than
119 'source'. Raise DistutilsFileError if 'source' does not
122 if not os
.path
.exists (source
):
123 raise DistutilsFileError
, "file '%s' does not exist" % source
124 if not os
.path
.exists (target
):
127 from stat
import ST_MTIME
128 mtime1
= os
.stat(source
)[ST_MTIME
]
129 mtime2
= os
.stat(target
)[ST_MTIME
]
131 return mtime1
> mtime2
136 def newer_pairwise (sources
, targets
):
137 """Walk two filename lists in parallel, testing if each source is newer
138 than its corresponding target. Return a pair of lists (sources,
139 targets) where source is newer than target, according to the
140 semantics of 'newer()'."""
142 if len (sources
) != len (targets
):
143 raise ValueError, "'sources' and 'targets' must be same length"
145 # build a pair of lists (sources, targets) where source is newer
148 for i
in range (len (sources
)):
149 if newer (sources
[i
], targets
[i
]):
150 n_sources
.append (sources
[i
])
151 n_targets
.append (targets
[i
])
153 return (n_sources
, n_targets
)
158 def newer_group (sources
, target
, missing
='error'):
159 """Return true if 'target' is out-of-date with respect to any
160 file listed in 'sources'. In other words, if 'target' exists and
161 is newer than every file in 'sources', return false; otherwise
162 return true. 'missing' controls what we do when a source file is
163 missing; the default ("error") is to blow up with an OSError from
164 inside 'stat()'; if it is "ignore", we silently drop any missing
165 source files; if it is "newer", any missing source files make us
166 assume that 'target' is out-of-date (this is handy in "dry-run"
167 mode: it'll make you pretend to carry out commands that wouldn't
168 work because inputs are missing, but that doesn't matter because
169 you're not actually going to run the commands)."""
171 # If the target doesn't even exist, then it's definitely out-of-date.
172 if not os
.path
.exists (target
):
175 # Otherwise we have to find out the hard way: if *any* source file
176 # is more recent than 'target', then 'target' is out-of-date and
177 # we can immediately return true. If we fall through to the end
178 # of the loop, then 'target' is up-to-date and we return false.
179 from stat
import ST_MTIME
180 target_mtime
= os
.stat (target
)[ST_MTIME
]
181 for source
in sources
:
182 if not os
.path
.exists (source
):
183 if missing
== 'error': # blow up when we stat() the file
185 elif missing
== 'ignore': # missing source dropped from
186 continue # target's dependency list
187 elif missing
== 'newer': # missing source means target is
188 return 1 # out-of-date
190 source_mtime
= os
.stat(source
)[ST_MTIME
]
191 if source_mtime
> target_mtime
:
199 # XXX this isn't used anywhere, and worse, it has the same name as a method
200 # in Command with subtly different semantics. (This one just has one
201 # source -> one dest; that one has many sources -> one dest.) Nuke it?
202 def make_file (src
, dst
, func
, args
,
203 verbose
=0, update_message
=None, noupdate_message
=None):
204 """Makes 'dst' from 'src' (both filenames) by calling 'func' with
205 'args', but only if it needs to: i.e. if 'dst' does not exist or
206 'src' is newer than 'dst'."""
209 if verbose
and update_message
:
213 if verbose
and noupdate_message
:
214 print noupdate_message
219 def _copy_file_contents (src
, dst
, buffer_size
=16*1024):
220 """Copy the file 'src' to 'dst'; both must be filenames. Any error
221 opening either file, reading from 'src', or writing to 'dst',
222 raises DistutilsFileError. Data is read/written in chunks of
223 'buffer_size' bytes (default 16k). No attempt is made to handle
224 anything apart from regular files."""
226 # Stolen from shutil module in the standard library, but with
227 # custom error-handling added.
233 fsrc
= open(src
, 'rb')
234 except os
.error
, (errno
, errstr
):
235 raise DistutilsFileError
, \
236 "could not open '%s': %s" % (src
, errstr
)
239 fdst
= open(dst
, 'wb')
240 except os
.error
, (errno
, errstr
):
241 raise DistutilsFileError
, \
242 "could not create '%s': %s" % (dst
, errstr
)
246 buf
= fsrc
.read (buffer_size
)
247 except os
.error
, (errno
, errstr
):
248 raise DistutilsFileError
, \
249 "could not read from '%s': %s" % (src
, errstr
)
256 except os
.error
, (errno
, errstr
):
257 raise DistutilsFileError
, \
258 "could not write to '%s': %s" % (dst
, errstr
)
266 # _copy_file_contents()
269 def copy_file (src
, dst
,
277 """Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src'
278 is copied there with the same name; otherwise, it must be a
279 filename. (If the file exists, it will be ruthlessly clobbered.)
280 If 'preserve_mode' is true (the default), the file's mode (type
281 and permission bits, or whatever is analogous on the current
282 platform) is copied. If 'preserve_times' is true (the default),
283 the last-modified and last-access times are copied as well. If
284 'update' is true, 'src' will only be copied if 'dst' does not
285 exist, or if 'dst' does exist but is older than 'src'. If
286 'verbose' is true, then a one-line summary of the copy will be
289 'link' allows you to make hard links (os.link) or symbolic links
290 (os.symlink) instead of copying: set it to "hard" or "sym"; if it
291 is None (the default), files are copied. Don't set 'link' on
292 systems that don't support it: 'copy_file()' doesn't check if
293 hard or symbolic linking is availalble.
295 Under Mac OS, uses the native file copy function in macostools;
296 on other systems, uses '_copy_file_contents()' to copy file
299 Return true if the file was copied (or would have been copied),
300 false otherwise (ie. 'update' was true and the destination is
303 # XXX if the destination file already exists, we clobber it if
304 # copying, but blow up if linking. Hmmm. And I don't know what
305 # macostools.copyfile() does. Should definitely be consistent, and
306 # should probably blow up if destination exists and we would be
307 # changing it (ie. it's not already a hard/soft link to src OR
308 # (not update) and (src newer than dst).
312 if not os
.path
.isfile (src
):
313 raise DistutilsFileError
, \
314 "can't copy '%s': doesn't exist or not a regular file" % src
316 if os
.path
.isdir (dst
):
318 dst
= os
.path
.join (dst
, os
.path
.basename (src
))
320 dir = os
.path
.dirname (dst
)
322 if update
and not newer (src
, dst
):
324 print "not copying %s (output up-to-date)" % src
328 action
= _copy_action
[link
]
331 "invalid value '%s' for 'link' argument" % link
333 print "%s %s -> %s" % (action
, src
, dir)
338 # On a Mac, use the native file copy routine
342 macostools
.copy (src
, dst
, 0, preserve_times
)
344 raise DistutilsFileError
, \
345 "could not copy '%s' to '%s': %s" % (src
, dst
, exc
[-1])
347 # If linking (hard or symbolic), use the appropriate system call
348 # (Unix only, of course, but that's the caller's responsibility)
350 if not (os
.path
.exists (dst
) and os
.path
.samefile (src
, dst
)):
353 if not (os
.path
.exists (dst
) and os
.path
.samefile (src
, dst
)):
354 os
.symlink (src
, dst
)
356 # Otherwise (non-Mac, not linking), copy the file contents and
357 # (optionally) copy the times and mode.
359 _copy_file_contents (src
, dst
)
360 if preserve_mode
or preserve_times
:
363 # According to David Ascher <da@ski.org>, utime() should be done
364 # before chmod() (at least under NT).
366 os
.utime (dst
, (st
[ST_ATIME
], st
[ST_MTIME
]))
368 os
.chmod (dst
, S_IMODE (st
[ST_MODE
]))
375 def copy_tree (src
, dst
,
383 """Copy an entire directory tree 'src' to a new location 'dst'. Both
384 'src' and 'dst' must be directory names. If 'src' is not a
385 directory, raise DistutilsFileError. If 'dst' does not exist, it is
386 created with 'mkpath()'. The end result of the copy is that every
387 file in 'src' is copied to 'dst', and directories under 'src' are
388 recursively copied to 'dst'. Return the list of files that were
389 copied or might have been copied, using their output name. The
390 return value is unaffected by 'update' or 'dry_run': it is simply
391 the list of all files under 'src', with the names changed to be
394 'preserve_mode' and 'preserve_times' are the same as for
395 'copy_file'; note that they only apply to regular files, not to
396 directories. If 'preserve_symlinks' is true, symlinks will be
397 copied as symlinks (on platforms that support them!); otherwise
398 (the default), the destination of the symlink will be copied.
399 'update' and 'verbose' are the same as for 'copy_file'."""
401 if not dry_run
and not os
.path
.isdir (src
):
402 raise DistutilsFileError
, \
403 "cannot copy tree '%s': not a directory" % src
405 names
= os
.listdir (src
)
406 except os
.error
, (errno
, errstr
):
410 raise DistutilsFileError
, \
411 "error listing files in '%s': %s" % (src
, errstr
)
414 mkpath (dst
, verbose
=verbose
)
419 src_name
= os
.path
.join (src
, n
)
420 dst_name
= os
.path
.join (dst
, n
)
422 if preserve_symlinks
and os
.path
.islink (src_name
):
423 link_dest
= os
.readlink (src_name
)
425 print "linking %s -> %s" % (dst_name
, link_dest
)
427 os
.symlink (link_dest
, dst_name
)
428 outputs
.append (dst_name
)
430 elif os
.path
.isdir (src_name
):
432 copy_tree (src_name
, dst_name
,
433 preserve_mode
, preserve_times
, preserve_symlinks
,
434 update
, verbose
, dry_run
))
436 copy_file (src_name
, dst_name
,
437 preserve_mode
, preserve_times
,
438 update
, None, verbose
, dry_run
)
439 outputs
.append (dst_name
)
446 def remove_tree (directory
, verbose
=0, dry_run
=0):
447 """Recursively remove an entire directory tree. Any errors are ignored
448 (apart from being reported to stdout if 'verbose' is true)."""
451 print "removing '%s' (and everything under it)" % directory
455 shutil
.rmtree(directory
,1)
456 except (IOError, OSError), exc
:
459 print "error removing %s: %s (%s)" % \
460 (directory
, exc
.strerror
, exc
.filename
)
462 print "error removing %s: %s" % (directory
, exc
.strerror
)
465 # XXX I suspect this is Unix-specific -- need porting help!
466 def move_file (src
, dst
,
470 """Move a file 'src' to 'dst'. If 'dst' is a directory, the file
471 will be moved into it with the same name; otherwise, 'src' is
472 just renamed to 'dst'. Return the new full name of the file.
474 Handles cross-device moves on Unix using
475 'copy_file()'. What about other systems???"""
477 from os
.path
import exists
, isfile
, isdir
, basename
, dirname
480 print "moving %s -> %s" % (src
, dst
)
486 raise DistutilsFileError
, \
487 "can't move '%s': not a regular file" % src
490 dst
= os
.path
.join (dst
, basename (src
))
492 raise DistutilsFileError
, \
493 "can't move '%s': destination '%s' already exists" % \
496 if not isdir (dirname (dst
)):
497 raise DistutilsFileError
, \
498 "can't move '%s': destination '%s' not a valid path" % \
504 except os
.error
, (num
, msg
):
505 if num
== errno
.EXDEV
:
508 raise DistutilsFileError
, \
509 "couldn't move '%s' to '%s': %s" % (src
, dst
, msg
)
515 except os
.error
, (num
, msg
):
520 raise DistutilsFileError
, \
521 ("couldn't move '%s' to '%s' by copy/delete: " +
522 "delete '%s' failed: %s") % \
530 def write_file (filename
, contents
):
531 """Create a file with the specified name and write 'contents' (a
532 sequence of strings without line terminators) to it."""
534 f
= open (filename
, "w")
535 for line
in contents
:
536 f
.write (line
+ "\n")
541 """Return a string (suitable for tacking onto directory names) that
542 identifies the current platform. Under Unix, identifies both the OS
543 and hardware architecture, e.g. "linux-i586", "solaris-sparc",
544 "irix-mips". For Windows and Mac OS, just returns 'sys.platform' --
545 i.e. "???" or "???"."""
547 if os
.name
== 'posix':
548 (OS
, _
, rel
, _
, arch
) = os
.uname()
549 return "%s%c-%s" % (string
.lower (OS
), rel
[0], string
.lower (arch
))
556 def native_path (pathname
):
557 """Return 'pathname' as a name that will work on the native
558 filesystem, i.e. split it on '/' and put it back together again
559 using the current directory separator. Needed because filenames in
560 the setup script are always supplied in Unix style, and have to be
561 converted to the local convention before we can actually use them in
562 the filesystem. Raises DistutilsValueError if 'pathname' is
563 absolute (starts with '/') or contains local directory separators
564 (unless the local separator is '/', of course)."""
566 if pathname
[0] == '/':
567 raise DistutilsValueError
, "path '%s' cannot be absolute" % pathname
568 if pathname
[-1] == '/':
569 raise DistutilsValueError
, "path '%s' cannot end with '/'" % pathname
570 if os
.sep
!= '/' and os
.sep
in pathname
:
571 raise DistutilsValueError
, \
572 "path '%s' cannot contain '%c' character" % \
575 paths
= string
.split (pathname
, '/')
576 return apply (os
.path
.join
, paths
)
583 def _check_environ ():
584 """Ensure that 'os.environ' has all the environment variables we
585 guarantee that users can use in config files, command-line
586 options, etc. Currently this includes:
587 HOME - user's home directory (Unix only)
588 PLAT - desription of the current platform, including hardware
589 and OS (see 'get_platform()')
592 if os
.name
== 'posix' and not os
.environ
.has_key('HOME'):
594 os
.environ
['HOME'] = pwd
.getpwuid (os
.getuid())[5]
596 if not os
.environ
.has_key('PLAT'):
597 os
.environ
['PLAT'] = get_platform ()
600 def subst_vars (str, local_vars
):
601 """Perform shell/Perl-style variable substitution on 'string'.
602 Every occurence of '$' followed by a name, or a name enclosed in
603 braces, is considered a variable. Every variable is substituted by
604 the value found in the 'local_vars' dictionary, or in 'os.environ'
605 if it's not in 'local_vars'. 'os.environ' is first checked/
606 augmented to guarantee that it contains certain values: see
607 '_check_environ()'. Raise ValueError for any variables not found in
608 either 'local_vars' or 'os.environ'."""
611 def _subst (match
, local_vars
=local_vars
):
612 var_name
= match
.group(1)
613 if local_vars
.has_key (var_name
):
614 return str (local_vars
[var_name
])
616 return os
.environ
[var_name
]
618 return re
.sub (r
'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst
, str)
623 def make_tarball (base_name
, base_dir
, compress
="gzip",
624 verbose
=0, dry_run
=0):
625 """Create a (possibly compressed) tar file from all the files under
626 'base_dir'. 'compress' must be "gzip" (the default), "compress", or
627 None. Both "tar" and the compression utility named by 'compress'
628 must be on the default program search path, so this is probably
629 Unix-specific. The output tar file will be named 'base_dir' +
630 ".tar", possibly plus the appropriate compression extension
631 (".gz" or ".Z"). Return the output filename."""
633 # XXX GNU tar 1.13 has a nifty option to add a prefix directory.
634 # It's pretty new, though, so we certainly can't require it --
635 # but it would be nice to take advantage of it to skip the
636 # "create a tree of hardlinks" step! (Would also be nice to
637 # detect GNU tar to use its 'z' option and save a step.)
639 compress_ext
= { 'gzip': ".gz",
642 if compress
is not None and compress
not in ('gzip', 'compress'):
644 "bad value for 'compress': must be None, 'gzip', or 'compress'"
646 archive_name
= base_name
+ ".tar"
647 cmd
= ["tar", "-cf", archive_name
, base_dir
]
648 spawn (cmd
, verbose
=verbose
, dry_run
=dry_run
)
651 spawn ([compress
, archive_name
], verbose
=verbose
, dry_run
=dry_run
)
652 return archive_name
+ compress_ext
[compress
]
659 def make_zipfile (base_name
, base_dir
, verbose
=0, dry_run
=0):
660 """Create a zip file from all the files under 'base_dir'. The
661 output zip file will be named 'base_dir' + ".zip". Uses either the
662 InfoZIP "zip" utility (if installed and found on the default search
663 path) or the "zipfile" Python module (if available). If neither
664 tool is available, raises DistutilsExecError. Returns the name
665 of the output zip file."""
667 # This initially assumed the Unix 'zip' utility -- but
668 # apparently InfoZIP's zip.exe works the same under Windows, so
671 zip_filename
= base_name
+ ".zip"
673 spawn (["zip", "-rq", zip_filename
, base_dir
],
674 verbose
=verbose
, dry_run
=dry_run
)
675 except DistutilsExecError
:
677 # XXX really should distinguish between "couldn't find
678 # external 'zip' command" and "zip failed" -- shouldn't try
679 # again in the latter case. (I think fixing this will
680 # require some cooperation from the spawn module -- perhaps
681 # a utility function to search the path, so we can fallback
682 # on zipfile.py without the failed spawn.)
686 raise DistutilsExecError
, \
687 ("unable to create zip file '%s': " +
688 "could neither find a standalone zip utility nor " +
689 "import the 'zipfile' module") % zip_filename
692 print "creating '%s' and adding '%s' to it" % \
693 (zip_filename
, base_dir
)
695 def visit (z
, dirname
, names
):
697 path
= os
.path
.join (dirname
, name
)
698 if os
.path
.isfile (path
):
702 z
= zipfile
.ZipFile (zip_filename
, "wb",
703 compression
=zipfile
.ZIP_DEFLATED
)
705 os
.path
.walk (base_dir
, visit
, z
)
713 def make_archive (base_name
, format
,
714 root_dir
=None, base_dir
=None,
715 verbose
=0, dry_run
=0):
717 """Create an archive file (eg. zip or tar). 'base_name' is the name
718 of the file to create, minus any format-specific extension; 'format'
719 is the archive format: one of "zip", "tar", "ztar", or "gztar".
720 'root_dir' is a directory that will be the root directory of the
721 archive; ie. we typically chdir into 'root_dir' before creating the
722 archive. 'base_dir' is the directory where we start archiving from;
723 ie. 'base_dir' will be the common prefix of all files and
724 directories in the archive. 'root_dir' and 'base_dir' both default
725 to the current directory."""
727 save_cwd
= os
.getcwd()
728 if root_dir
is not None:
730 print "changing into '%s'" % root_dir
731 base_name
= os
.path
.abspath (base_name
)
738 kwargs
= { 'verbose': verbose
,
741 if format
== 'gztar':
743 kwargs
['compress'] = 'gzip'
744 elif format
== 'ztar':
746 kwargs
['compress'] = 'compress'
747 elif format
== 'tar':
749 kwargs
['compress'] = None
750 elif format
== 'zip':
753 apply (func
, (base_name
, base_dir
), kwargs
)
755 if root_dir
is not None:
757 print "changing back to '%s'" % save_cwd