cpython/Lib/distutils/util.py

761 lines
27 KiB
Python

"""distutils.util
General-purpose utility functions used throughout the Distutils
(especially in command classes). Mostly filesystem manipulation, but
not limited to that. The functions in this module generally raise
DistutilsFileError when they have problems with the filesystem, because
os.error in pre-1.5.2 Python only gives the error message and not the
file causing it."""
# created 1999/03/08, Greg Ward
__revision__ = "$Id$"
import sys, os, string, re, shutil
from distutils.errors import *
from distutils.spawn import spawn
# cache for by mkpath() -- in addition to cheapening redundant calls,
# eliminates redundant "creating /foo/bar/baz" messages in dry-run mode
PATH_CREATED = {}
# for generating verbose output in 'copy_file()'
_copy_action = { None: 'copying',
'hard': 'hard linking',
'sym': 'symbolically linking' }
# I don't use os.makedirs because a) it's new to Python 1.5.2, and
# b) it blows up if the directory already exists (I want to silently
# succeed in that case).
def mkpath (name, mode=0777, verbose=0, dry_run=0):
"""Create a directory and any missing ancestor directories. If the
directory already exists (or if 'name' is the empty string, which
means the current directory, which of course exists), then do
nothing. Raise DistutilsFileError if unable to create some
directory along the way (eg. some sub-path exists, but is a file
rather than a directory). If 'verbose' is true, print a one-line
summary of each mkdir to stdout. Return the list of directories
actually created."""
global PATH_CREATED
# XXX what's the better way to handle verbosity? print as we create
# each directory in the path (the current behaviour), or only announce
# the creation of the whole path? (quite easy to do the latter since
# we're not using a recursive algorithm)
name = os.path.normpath (name)
created_dirs = []
if os.path.isdir (name) or name == '':
return created_dirs
if PATH_CREATED.get (name):
return created_dirs
(head, tail) = os.path.split (name)
tails = [tail] # stack of lone dirs to create
while head and tail and not os.path.isdir (head):
#print "splitting '%s': " % head,
(head, tail) = os.path.split (head)
#print "to ('%s','%s')" % (head, tail)
tails.insert (0, tail) # push next higher dir onto stack
#print "stack of tails:", tails
# now 'head' contains the deepest directory that already exists
# (that is, the child of 'head' in 'name' is the highest directory
# that does *not* exist)
for d in tails:
#print "head = %s, d = %s: " % (head, d),
head = os.path.join (head, d)
if PATH_CREATED.get (head):
continue
if verbose:
print "creating", head
if not dry_run:
try:
os.mkdir (head)
created_dirs.append(head)
except OSError, exc:
raise DistutilsFileError, \
"could not create '%s': %s" % (head, exc[-1])
PATH_CREATED[head] = 1
return created_dirs
# mkpath ()
def create_tree (base_dir, files, mode=0777, verbose=0, dry_run=0):
"""Create all the empty directories under 'base_dir' needed to
put 'files' there. 'base_dir' is just the a name of a directory
which doesn't necessarily exist yet; 'files' is a list of filenames
to be interpreted relative to 'base_dir'. 'base_dir' + the
directory portion of every file in 'files' will be created if it
doesn't already exist. 'mode', 'verbose' and 'dry_run' flags are as
for 'mkpath()'."""
# First get the list of directories to create
need_dir = {}
for file in files:
need_dir[os.path.join (base_dir, os.path.dirname (file))] = 1
need_dirs = need_dir.keys()
need_dirs.sort()
# Now create them
for dir in need_dirs:
mkpath (dir, mode, verbose, dry_run)
# create_tree ()
def newer (source, target):
"""Return true if 'source' exists and is more recently modified than
'target', or if 'source' exists and 'target' doesn't. Return
false if both exist and 'target' is the same age or younger than
'source'. Raise DistutilsFileError if 'source' does not
exist."""
if not os.path.exists (source):
raise DistutilsFileError, "file '%s' does not exist" % source
if not os.path.exists (target):
return 1
from stat import ST_MTIME
mtime1 = os.stat(source)[ST_MTIME]
mtime2 = os.stat(target)[ST_MTIME]
return mtime1 > mtime2
# newer ()
def newer_pairwise (sources, targets):
"""Walk two filename lists in parallel, testing if each source is newer
than its corresponding target. Return a pair of lists (sources,
targets) where source is newer than target, according to the
semantics of 'newer()'."""
if len (sources) != len (targets):
raise ValueError, "'sources' and 'targets' must be same length"
# build a pair of lists (sources, targets) where source is newer
n_sources = []
n_targets = []
for i in range (len (sources)):
if newer (sources[i], targets[i]):
n_sources.append (sources[i])
n_targets.append (targets[i])
return (n_sources, n_targets)
# newer_pairwise ()
def newer_group (sources, target, missing='error'):
"""Return true if 'target' is out-of-date with respect to any
file listed in 'sources'. In other words, if 'target' exists and
is newer than every file in 'sources', return false; otherwise
return true. 'missing' controls what we do when a source file is
missing; the default ("error") is to blow up with an OSError from
inside 'stat()'; if it is "ignore", we silently drop any missing
source files; if it is "newer", any missing source files make us
assume that 'target' is out-of-date (this is handy in "dry-run"
mode: it'll make you pretend to carry out commands that wouldn't
work because inputs are missing, but that doesn't matter because
you're not actually going to run the commands)."""
# If the target doesn't even exist, then it's definitely out-of-date.
if not os.path.exists (target):
return 1
# Otherwise we have to find out the hard way: if *any* source file
# is more recent than 'target', then 'target' is out-of-date and
# we can immediately return true. If we fall through to the end
# of the loop, then 'target' is up-to-date and we return false.
from stat import ST_MTIME
target_mtime = os.stat (target)[ST_MTIME]
for source in sources:
if not os.path.exists (source):
if missing == 'error': # blow up when we stat() the file
pass
elif missing == 'ignore': # missing source dropped from
continue # target's dependency list
elif missing == 'newer': # missing source means target is
return 1 # out-of-date
source_mtime = os.stat(source)[ST_MTIME]
if source_mtime > target_mtime:
return 1
else:
return 0
# newer_group ()
# XXX this isn't used anywhere, and worse, it has the same name as a method
# in Command with subtly different semantics. (This one just has one
# source -> one dest; that one has many sources -> one dest.) Nuke it?
def make_file (src, dst, func, args,
verbose=0, update_message=None, noupdate_message=None):
"""Makes 'dst' from 'src' (both filenames) by calling 'func' with
'args', but only if it needs to: i.e. if 'dst' does not exist or
'src' is newer than 'dst'."""
if newer (src, dst):
if verbose and update_message:
print update_message
apply (func, args)
else:
if verbose and noupdate_message:
print noupdate_message
# make_file ()
def _copy_file_contents (src, dst, buffer_size=16*1024):
"""Copy the file 'src' to 'dst'; both must be filenames. Any error
opening either file, reading from 'src', or writing to 'dst',
raises DistutilsFileError. Data is read/written in chunks of
'buffer_size' bytes (default 16k). No attempt is made to handle
anything apart from regular files."""
# Stolen from shutil module in the standard library, but with
# custom error-handling added.
fsrc = None
fdst = None
try:
try:
fsrc = open(src, 'rb')
except os.error, (errno, errstr):
raise DistutilsFileError, \
"could not open '%s': %s" % (src, errstr)
try:
fdst = open(dst, 'wb')
except os.error, (errno, errstr):
raise DistutilsFileError, \
"could not create '%s': %s" % (dst, errstr)
while 1:
try:
buf = fsrc.read (buffer_size)
except os.error, (errno, errstr):
raise DistutilsFileError, \
"could not read from '%s': %s" % (src, errstr)
if not buf:
break
try:
fdst.write(buf)
except os.error, (errno, errstr):
raise DistutilsFileError, \
"could not write to '%s': %s" % (dst, errstr)
finally:
if fdst:
fdst.close()
if fsrc:
fsrc.close()
# _copy_file_contents()
def copy_file (src, dst,
preserve_mode=1,
preserve_times=1,
update=0,
link=None,
verbose=0,
dry_run=0):
"""Copy a file 'src' to 'dst'. If 'dst' is a directory, then 'src'
is copied there with the same name; otherwise, it must be a
filename. (If the file exists, it will be ruthlessly clobbered.)
If 'preserve_mode' is true (the default), the file's mode (type
and permission bits, or whatever is analogous on the current
platform) is copied. If 'preserve_times' is true (the default),
the last-modified and last-access times are copied as well. If
'update' is true, 'src' will only be copied if 'dst' does not
exist, or if 'dst' does exist but is older than 'src'. If
'verbose' is true, then a one-line summary of the copy will be
printed to stdout.
'link' allows you to make hard links (os.link) or symbolic links
(os.symlink) instead of copying: set it to "hard" or "sym"; if it
is None (the default), files are copied. Don't set 'link' on
systems that don't support it: 'copy_file()' doesn't check if
hard or symbolic linking is availalble.
Under Mac OS, uses the native file copy function in macostools;
on other systems, uses '_copy_file_contents()' to copy file
contents.
Return true if the file was copied (or would have been copied),
false otherwise (ie. 'update' was true and the destination is
up-to-date)."""
# XXX if the destination file already exists, we clobber it if
# copying, but blow up if linking. Hmmm. And I don't know what
# macostools.copyfile() does. Should definitely be consistent, and
# should probably blow up if destination exists and we would be
# changing it (ie. it's not already a hard/soft link to src OR
# (not update) and (src newer than dst).
from stat import *
if not os.path.isfile (src):
raise DistutilsFileError, \
"can't copy '%s': doesn't exist or not a regular file" % src
if os.path.isdir (dst):
dir = dst
dst = os.path.join (dst, os.path.basename (src))
else:
dir = os.path.dirname (dst)
if update and not newer (src, dst):
if verbose:
print "not copying %s (output up-to-date)" % src
return 0
try:
action = _copy_action[link]
except KeyError:
raise ValueError, \
"invalid value '%s' for 'link' argument" % link
if verbose:
print "%s %s -> %s" % (action, src, dir)
if dry_run:
return 1
# On a Mac, use the native file copy routine
if os.name == 'mac':
import macostools
try:
macostools.copy (src, dst, 0, preserve_times)
except OSError, exc:
raise DistutilsFileError, \
"could not copy '%s' to '%s': %s" % (src, dst, exc[-1])
# If linking (hard or symbolic), use the appropriate system call
# (Unix only, of course, but that's the caller's responsibility)
elif link == 'hard':
if not (os.path.exists (dst) and os.path.samefile (src, dst)):
os.link (src, dst)
elif link == 'sym':
if not (os.path.exists (dst) and os.path.samefile (src, dst)):
os.symlink (src, dst)
# Otherwise (non-Mac, not linking), copy the file contents and
# (optionally) copy the times and mode.
else:
_copy_file_contents (src, dst)
if preserve_mode or preserve_times:
st = os.stat (src)
# According to David Ascher <da@ski.org>, utime() should be done
# before chmod() (at least under NT).
if preserve_times:
os.utime (dst, (st[ST_ATIME], st[ST_MTIME]))
if preserve_mode:
os.chmod (dst, S_IMODE (st[ST_MODE]))
return 1
# copy_file ()
def copy_tree (src, dst,
preserve_mode=1,
preserve_times=1,
preserve_symlinks=0,
update=0,
verbose=0,
dry_run=0):
"""Copy an entire directory tree 'src' to a new location 'dst'. Both
'src' and 'dst' must be directory names. If 'src' is not a
directory, raise DistutilsFileError. If 'dst' does not exist, it is
created with 'mkpath()'. The end result of the copy is that every
file in 'src' is copied to 'dst', and directories under 'src' are
recursively copied to 'dst'. Return the list of files that were
copied or might have been copied, using their output name. The
return value is unaffected by 'update' or 'dry_run': it is simply
the list of all files under 'src', with the names changed to be
under 'dst'.
'preserve_mode' and 'preserve_times' are the same as for
'copy_file'; note that they only apply to regular files, not to
directories. If 'preserve_symlinks' is true, symlinks will be
copied as symlinks (on platforms that support them!); otherwise
(the default), the destination of the symlink will be copied.
'update' and 'verbose' are the same as for 'copy_file'."""
if not dry_run and not os.path.isdir (src):
raise DistutilsFileError, \
"cannot copy tree '%s': not a directory" % src
try:
names = os.listdir (src)
except os.error, (errno, errstr):
if dry_run:
names = []
else:
raise DistutilsFileError, \
"error listing files in '%s': %s" % (src, errstr)
if not dry_run:
mkpath (dst, verbose=verbose)
outputs = []
for n in names:
src_name = os.path.join (src, n)
dst_name = os.path.join (dst, n)
if preserve_symlinks and os.path.islink (src_name):
link_dest = os.readlink (src_name)
if verbose:
print "linking %s -> %s" % (dst_name, link_dest)
if not dry_run:
os.symlink (link_dest, dst_name)
outputs.append (dst_name)
elif os.path.isdir (src_name):
outputs.extend (
copy_tree (src_name, dst_name,
preserve_mode, preserve_times, preserve_symlinks,
update, verbose, dry_run))
else:
copy_file (src_name, dst_name,
preserve_mode, preserve_times,
update, None, verbose, dry_run)
outputs.append (dst_name)
return outputs
# copy_tree ()
def remove_tree (directory, verbose=0, dry_run=0):
"""Recursively remove an entire directory tree. Any errors are ignored
(apart from being reported to stdout if 'verbose' is true)."""
if verbose:
print "removing '%s' (and everything under it)" % directory
if dry_run:
return
try:
shutil.rmtree(directory,1)
except (IOError, OSError), exc:
if verbose:
if exc.filename:
print "error removing %s: %s (%s)" % \
(directory, exc.strerror, exc.filename)
else:
print "error removing %s: %s" % (directory, exc.strerror)
# XXX I suspect this is Unix-specific -- need porting help!
def move_file (src, dst,
verbose=0,
dry_run=0):
"""Move a file 'src' to 'dst'. If 'dst' is a directory, the file
will be moved into it with the same name; otherwise, 'src' is
just renamed to 'dst'. Return the new full name of the file.
Handles cross-device moves on Unix using
'copy_file()'. What about other systems???"""
from os.path import exists, isfile, isdir, basename, dirname
if verbose:
print "moving %s -> %s" % (src, dst)
if dry_run:
return dst
if not isfile (src):
raise DistutilsFileError, \
"can't move '%s': not a regular file" % src
if isdir (dst):
dst = os.path.join (dst, basename (src))
elif exists (dst):
raise DistutilsFileError, \
"can't move '%s': destination '%s' already exists" % \
(src, dst)
if not isdir (dirname (dst)):
raise DistutilsFileError, \
"can't move '%s': destination '%s' not a valid path" % \
(src, dst)
copy_it = 0
try:
os.rename (src, dst)
except os.error, (num, msg):
if num == errno.EXDEV:
copy_it = 1
else:
raise DistutilsFileError, \
"couldn't move '%s' to '%s': %s" % (src, dst, msg)
if copy_it:
copy_file (src, dst)
try:
os.unlink (src)
except os.error, (num, msg):
try:
os.unlink (dst)
except os.error:
pass
raise DistutilsFileError, \
("couldn't move '%s' to '%s' by copy/delete: " +
"delete '%s' failed: %s") % \
(src, dst, src, msg)
return dst
# move_file ()
def write_file (filename, contents):
"""Create a file with the specified name and write 'contents' (a
sequence of strings without line terminators) to it."""
f = open (filename, "w")
for line in contents:
f.write (line + "\n")
f.close ()
def get_platform ():
"""Return a string (suitable for tacking onto directory names) that
identifies the current platform. Under Unix, identifies both the OS
and hardware architecture, e.g. "linux-i586", "solaris-sparc",
"irix-mips". For Windows and Mac OS, just returns 'sys.platform' --
i.e. "???" or "???"."""
if os.name == 'posix':
(OS, _, rel, _, arch) = os.uname()
return "%s%c-%s" % (string.lower (OS), rel[0], string.lower (arch))
else:
return sys.platform
# get_platform()
def native_path (pathname):
"""Return 'pathname' as a name that will work on the native
filesystem, i.e. split it on '/' and put it back together again
using the current directory separator. Needed because filenames in
the setup script are always supplied in Unix style, and have to be
converted to the local convention before we can actually use them in
the filesystem. Raises DistutilsValueError if 'pathname' is
absolute (starts with '/') or contains local directory separators
(unless the local separator is '/', of course)."""
if pathname[0] == '/':
raise DistutilsValueError, "path '%s' cannot be absolute" % pathname
if pathname[-1] == '/':
raise DistutilsValueError, "path '%s' cannot end with '/'" % pathname
if os.sep != '/' and os.sep in pathname:
raise DistutilsValueError, \
"path '%s' cannot contain '%c' character" % \
(pathname, os.sep)
paths = string.split (pathname, '/')
return apply (os.path.join, paths)
else:
return pathname
# native_path ()
def _check_environ ():
"""Ensure that 'os.environ' has all the environment variables we
guarantee that users can use in config files, command-line
options, etc. Currently this includes:
HOME - user's home directory (Unix only)
PLAT - desription of the current platform, including hardware
and OS (see 'get_platform()')
"""
if os.name == 'posix' and not os.environ.has_key('HOME'):
import pwd
os.environ['HOME'] = pwd.getpwuid (os.getuid())[5]
if not os.environ.has_key('PLAT'):
os.environ['PLAT'] = get_platform ()
def subst_vars (str, local_vars):
"""Perform shell/Perl-style variable substitution on 'string'.
Every occurence of '$' followed by a name, or a name enclosed in
braces, is considered a variable. Every variable is substituted by
the value found in the 'local_vars' dictionary, or in 'os.environ'
if it's not in 'local_vars'. 'os.environ' is first checked/
augmented to guarantee that it contains certain values: see
'_check_environ()'. Raise ValueError for any variables not found in
either 'local_vars' or 'os.environ'."""
_check_environ ()
def _subst (match, local_vars=local_vars):
var_name = match.group(1)
if local_vars.has_key (var_name):
return str (local_vars[var_name])
else:
return os.environ[var_name]
return re.sub (r'\$([a-zA-Z_][a-zA-Z_0-9]*)', _subst, str)
# subst_vars ()
def make_tarball (base_name, base_dir, compress="gzip",
verbose=0, dry_run=0):
"""Create a (possibly compressed) tar file from all the files under
'base_dir'. 'compress' must be "gzip" (the default), "compress", or
None. Both "tar" and the compression utility named by 'compress'
must be on the default program search path, so this is probably
Unix-specific. The output tar file will be named 'base_dir' +
".tar", possibly plus the appropriate compression extension
(".gz" or ".Z"). Return the output filename."""
# XXX GNU tar 1.13 has a nifty option to add a prefix directory.
# It's pretty new, though, so we certainly can't require it --
# but it would be nice to take advantage of it to skip the
# "create a tree of hardlinks" step! (Would also be nice to
# detect GNU tar to use its 'z' option and save a step.)
compress_ext = { 'gzip': ".gz",
'compress': ".Z" }
if compress is not None and compress not in ('gzip', 'compress'):
raise ValueError, \
"bad value for 'compress': must be None, 'gzip', or 'compress'"
archive_name = base_name + ".tar"
cmd = ["tar", "-cf", archive_name, base_dir]
spawn (cmd, verbose=verbose, dry_run=dry_run)
if compress:
spawn ([compress, archive_name], verbose=verbose, dry_run=dry_run)
return archive_name + compress_ext[compress]
else:
return archive_name
# make_tarball ()
def make_zipfile (base_name, base_dir, verbose=0, dry_run=0):
"""Create a zip file from all the files under 'base_dir'. The
output zip file will be named 'base_dir' + ".zip". Uses either the
InfoZIP "zip" utility (if installed and found on the default search
path) or the "zipfile" Python module (if available). If neither
tool is available, raises DistutilsExecError. Returns the name
of the output zip file."""
# This initially assumed the Unix 'zip' utility -- but
# apparently InfoZIP's zip.exe works the same under Windows, so
# no changes needed!
zip_filename = base_name + ".zip"
try:
spawn (["zip", "-rq", zip_filename, base_dir],
verbose=verbose, dry_run=dry_run)
except DistutilsExecError:
# XXX really should distinguish between "couldn't find
# external 'zip' command" and "zip failed" -- shouldn't try
# again in the latter case. (I think fixing this will
# require some cooperation from the spawn module -- perhaps
# a utility function to search the path, so we can fallback
# on zipfile.py without the failed spawn.)
try:
import zipfile
except ImportError:
raise DistutilsExecError, \
("unable to create zip file '%s': " +
"could neither find a standalone zip utility nor " +
"import the 'zipfile' module") % zip_filename
if verbose:
print "creating '%s' and adding '%s' to it" % \
(zip_filename, base_dir)
def visit (z, dirname, names):
for name in names:
path = os.path.join (dirname, name)
if os.path.isfile (path):
z.write (path, path)
if not dry_run:
z = zipfile.ZipFile (zip_filename, "wb",
compression=zipfile.ZIP_DEFLATED)
os.path.walk (base_dir, visit, z)
z.close()
return zip_filename
# make_zipfile ()
def make_archive (base_name, format,
root_dir=None, base_dir=None,
verbose=0, dry_run=0):
"""Create an archive file (eg. zip or tar). 'base_name' is the name
of the file to create, minus any format-specific extension; 'format'
is the archive format: one of "zip", "tar", "ztar", or "gztar".
'root_dir' is a directory that will be the root directory of the
archive; ie. we typically chdir into 'root_dir' before creating the
archive. 'base_dir' is the directory where we start archiving from;
ie. 'base_dir' will be the common prefix of all files and
directories in the archive. 'root_dir' and 'base_dir' both default
to the current directory."""
save_cwd = os.getcwd()
if root_dir is not None:
if verbose:
print "changing into '%s'" % root_dir
base_name = os.path.abspath (base_name)
if not dry_run:
os.chdir (root_dir)
if base_dir is None:
base_dir = os.curdir
kwargs = { 'verbose': verbose,
'dry_run': dry_run }
if format == 'gztar':
func = make_tarball
kwargs['compress'] = 'gzip'
elif format == 'ztar':
func = make_tarball
kwargs['compress'] = 'compress'
elif format == 'tar':
func = make_tarball
kwargs['compress'] = None
elif format == 'zip':
func = make_zipfile
apply (func, (base_name, base_dir), kwargs)
if root_dir is not None:
if verbose:
print "changing back to '%s'" % save_cwd
os.chdir (save_cwd)
# make_archive ()