cpython/Lib/distutils/filelist.py

360 lines
13 KiB
Python
Raw Normal View History

"""distutils.filelist
Provides the FileList class, used for poking about the filesystem
and building lists of files.
"""
# created 2000/07/17, Rene Liebscher (as template.py)
# most parts taken from commands/sdist.py
# renamed 2000/07/29 (to filelist.py) and officially added to
# the Distutils source, Greg Ward
__revision__ = "$Id$"
import sys, os, string, re
import fnmatch
from types import *
from glob import glob
from distutils.util import convert_path
from distutils.errors import DistutilsTemplateError, DistutilsInternalError
class FileList:
"""A list of files built by on exploring the filesystem and filtered by
applying various patterns to what we find there.
Instance attributes:
dir
directory from which files will be taken -- only used if
'allfiles' not supplied to constructor
files
list of filenames currently being built/filtered/manipulated
allfiles
complete list of files under consideration (ie. without any
filtering applied)
"""
def __init__(self,
files=[],
dir=os.curdir,
allfiles=None,
warn=None,
debug_print=None):
# use standard warning and debug functions, if no other given
if warn is None: warn = self.__warn
if debug_print is None: debug_print = self.__debug_print
self.warn = warn
self.debug_print = debug_print
self.files = files
self.dir = dir
# if None, 'allfiles' will be filled when used for first time
self.allfiles = allfiles
# standard warning and debug functions, if no other given
def __warn (self, msg):
sys.stderr.write ("warning: %s\n" % msg)
def __debug_print (self, msg):
"""Print 'msg' to stdout if the global DEBUG (taken from the
DISTUTILS_DEBUG environment variable) flag is true.
"""
from distutils.core import DEBUG
if DEBUG:
print msg
def _parse_template_line (self, line):
words = string.split (line)
action = words[0]
patterns = dir = dir_pattern = None
if action in ('include', 'exclude',
'global-include', 'global-exclude'):
if len (words) < 2:
raise DistutilsTemplateError, \
"'%s' expects <pattern1> <pattern2> ..." % action
patterns = map(convert_path, words[1:])
elif action in ('recursive-include', 'recursive-exclude'):
if len (words) < 3:
raise DistutilsTemplateError, \
"'%s' expects <dir> <pattern1> <pattern2> ..." % action
dir = convert_path(words[1])
patterns = map(convert_path, words[2:])
elif action in ('graft', 'prune'):
if len (words) != 2:
raise DistutilsTemplateError, \
"'%s' expects a single <dir_pattern>" % action
dir_pattern = convert_path(words[1])
else:
raise DistutilsTemplateError, "unknown action '%s'" % action
return (action, pattern, dir, dir_pattern)
# _parse_template_line ()
def process_template_line (self, line):
# Parse the line: split it up, make sure the right number of words
# are there, and return the relevant words. 'action' is always
# defined: it's the first word of the line. Which of the other
# three are defined depends on the action; it'll be either
# patterns, (dir and patterns), or (dir_pattern).
(action, patterns, dir, dir_pattern) = self._parse_template_line(line)
# OK, now we know that the action is valid and we have the
# right number of words on the line for that action -- so we
# can proceed with minimal error-checking.
if action == 'include':
self.debug_print("include " + string.join(patterns))
for pattern in patterns:
if not self.select_pattern (pattern, anchor=1):
self.warn("no files found matching '%s'" % pattern)
elif action == 'exclude':
self.debug_print("exclude " + string.join(patterns))
for pattern in patterns:
if not self.exclude_pattern (pattern, anchor=1):
self.warn(
"no previously-included files found matching '%s'"%
pattern)
elif action == 'global-include':
self.debug_print("global-include " + string.join(patterns))
for pattern in patterns:
if not self.select_pattern (pattern, anchor=0):
self.warn (("no files found matching '%s' " +
"anywhere in distribution") %
pattern)
elif action == 'global-exclude':
self.debug_print("global-exclude " + string.join(patterns))
for pattern in patterns:
if not self.exclude_pattern (pattern, anchor=0):
self.warn(("no previously-included files matching '%s' " +
"found anywhere in distribution") %
pattern)
elif action == 'recursive-include':
self.debug_print("recursive-include %s %s" %
(dir, string.join(patterns)))
for pattern in patterns:
if not self.select_pattern (pattern, prefix=dir):
self.warn (("no files found matching '%s' " +
"under directory '%s'") %
(pattern, dir))
elif action == 'recursive-exclude':
self.debug_print("recursive-exclude %s %s" %
(dir, string.join(patterns)))
for pattern in patterns:
if not self.exclude_pattern(pattern, prefix=dir):
self.warn(("no previously-included files matching '%s' " +
"found under directory '%s'") %
(pattern, dir))
elif action == 'graft':
self.debug_print("graft " + dir_pattern)
if not self.select_pattern(None, prefix=dir_pattern):
self.warn ("no directories found matching '%s'" % dir_pattern)
elif action == 'prune':
self.debug_print("prune " + dir_pattern)
if not self.exclude_pattern(None, prefix=dir_pattern):
self.warn(("no previously-included directories found " +
"matching '%s'") %
dir_pattern)
else:
raise DistutilsInternalError, \
"this cannot happen: invalid action '%s'" % action
# process_template_line ()
def select_pattern (self, pattern,
anchor=1, prefix=None, is_regex=0):
"""Select strings (presumably filenames) from 'files' that match
'pattern', a Unix-style wildcard (glob) pattern. Patterns are not
quite the same as implemented by the 'fnmatch' module: '*' and '?'
match non-special characters, where "special" is platform-dependent:
slash on Unix, colon, slash, and backslash on DOS/Windows, and colon on
Mac OS.
If 'anchor' is true (the default), then the pattern match is more
stringent: "*.py" will match "foo.py" but not "foo/bar.py". If
'anchor' is false, both of these will match.
If 'prefix' is supplied, then only filenames starting with 'prefix'
(itself a pattern) and ending with 'pattern', with anything in between
them, will match. 'anchor' is ignored in this case.
If 'is_regex' is true, 'anchor' and 'prefix' are ignored, and
'pattern' is assumed to be either a string containing a regex or a
regex object -- no translation is done, the regex is just compiled
and used as-is.
Selected strings will be added to self.files.
Return 1 if files are found.
"""
files_found = 0
pattern_re = translate_pattern (pattern, anchor, prefix, is_regex)
self.debug_print("select_pattern: applying regex r'%s'" %
pattern_re.pattern)
# delayed loading of allfiles list
if self.allfiles is None: self.allfiles = findall (self.dir)
for name in self.allfiles:
if pattern_re.search (name):
self.debug_print(" adding " + name)
self.files.append (name)
files_found = 1
return files_found
# select_pattern ()
def exclude_pattern (self, pattern,
anchor=1, prefix=None, is_regex=0):
"""Remove strings (presumably filenames) from 'files' that match
'pattern'. Other parameters are the same as for
'select_pattern()', above.
The list 'self.files' is modified in place.
Return 1 if files are found.
"""
files_found = 0
pattern_re = translate_pattern (pattern, anchor, prefix, is_regex)
self.debug_print("exclude_pattern: applying regex r'%s'" %
pattern_re.pattern)
for i in range (len(self.files)-1, -1, -1):
if pattern_re.search (self.files[i]):
self.debug_print(" removing " + self.files[i])
del self.files[i]
files_found = 1
return files_found
# exclude_pattern ()
def recursive_exclude_pattern (self, dir, pattern=None):
"""Remove filenames from 'self.files' that are under 'dir' and
whose basenames match 'pattern'.
Return 1 if files are found.
"""
files_found = 0
self.debug_print("recursive_exclude_pattern: dir=%s, pattern=%s" %
(dir, pattern))
if pattern is None:
pattern_re = None
else:
pattern_re = translate_pattern (pattern)
for i in range (len (self.files)-1, -1, -1):
(cur_dir, cur_base) = os.path.split (self.files[i])
if (cur_dir == dir and
(pattern_re is None or pattern_re.match (cur_base))):
self.debug_print("removing %s" % self.files[i])
del self.files[i]
files_found = 1
return files_found
# class FileList
# ----------------------------------------------------------------------
# Utility functions
def findall (dir = os.curdir):
"""Find all files under 'dir' and return the list of full filenames
(relative to 'dir').
"""
from stat import ST_MODE, S_ISREG, S_ISDIR, S_ISLNK
list = []
stack = [dir]
pop = stack.pop
push = stack.append
while stack:
dir = pop()
names = os.listdir (dir)
for name in names:
if dir != os.curdir: # avoid the dreaded "./" syndrome
fullname = os.path.join (dir, name)
else:
fullname = name
# Avoid excess stat calls -- just one will do, thank you!
stat = os.stat(fullname)
mode = stat[ST_MODE]
if S_ISREG(mode):
list.append (fullname)
elif S_ISDIR(mode) and not S_ISLNK(mode):
push (fullname)
return list
def glob_to_re (pattern):
"""Translate a shell-like glob pattern to a regular expression; return
a string containing the regex. Differs from 'fnmatch.translate()' in
that '*' does not match "special characters" (which are
platform-specific).
"""
pattern_re = fnmatch.translate (pattern)
# '?' and '*' in the glob pattern become '.' and '.*' in the RE, which
# IMHO is wrong -- '?' and '*' aren't supposed to match slash in Unix,
# and by extension they shouldn't match such "special characters" under
# any OS. So change all non-escaped dots in the RE to match any
# character except the special characters.
# XXX currently the "special characters" are just slash -- i.e. this is
# Unix-only.
pattern_re = re.sub (r'(^|[^\\])\.', r'\1[^/]', pattern_re)
return pattern_re
# glob_to_re ()
def translate_pattern (pattern, anchor=1, prefix=None, is_regex=0):
"""Translate a shell-like wildcard pattern to a compiled regular
expression. Return the compiled regex. If 'is_regex' true,
then 'pattern' is directly compiled to a regex (if it's a string)
or just returned as-is (assumes it's a regex object).
"""
if is_regex:
if type(pattern) is StringType:
return re.compile(pattern)
else:
return pattern
if pattern:
pattern_re = glob_to_re (pattern)
else:
pattern_re = ''
if prefix is not None:
prefix_re = (glob_to_re (prefix))[0:-1] # ditch trailing $
pattern_re = "^" + os.path.join (prefix_re, ".*" + pattern_re)
else: # no prefix -- respect anchor flag
if anchor:
pattern_re = "^" + pattern_re
return re.compile (pattern_re)
# translate_pattern ()