mirror of https://github.com/python/cpython
Totally new "freeze" program.
This commit is contained in:
Guido van Rossum
parent
dbaf332107
commit
00ff4336a9
|
|
@ -0,0 +1,128 @@
|
|||
# Determine the names and filenames of the modules imported by a
|
||||
# script, recursively. This is done by scanning for lines containing
|
||||
# import statements. (The scanning has only superficial knowledge of
|
||||
# Python syntax and no knowledge of semantics, so in theory the result
|
||||
# may be incorrect -- however this is quite unlikely if you don't
|
||||
# intentionally obscure your Python code.)
|
||||
|
||||
import os
|
||||
import regex
|
||||
import string
|
||||
import sys
|
||||
|
||||
|
||||
# Top-level interface.
|
||||
# First argument is the main program (script).
|
||||
# Second optional argument is list of modules to be searched as well.
|
||||
|
||||
def findmodules(scriptfile, modules = [], path = sys.path):
|
||||
todo = {}
|
||||
todo['__main__'] = scriptfile
|
||||
for name in modules:
|
||||
mod = os.path.basename(name)
|
||||
if mod[-3:] == '.py': mod = mod[:-3]
|
||||
todo[mod] = name
|
||||
done = closure(todo)
|
||||
return done
|
||||
|
||||
|
||||
# Compute the closure of scanfile() and findmodule().
|
||||
# Return a dictionary mapping module names to filenames.
|
||||
# Writes to stderr if a file can't be or read.
|
||||
|
||||
def closure(todo):
|
||||
done = {}
|
||||
while todo:
|
||||
newtodo = {}
|
||||
for modname in todo.keys():
|
||||
if not done.has_key(modname):
|
||||
filename = todo[modname]
|
||||
if filename is None:
|
||||
filename = findmodule(modname)
|
||||
done[modname] = filename
|
||||
if filename in ('<builtin>', '<unknown>'):
|
||||
continue
|
||||
try:
|
||||
modules = scanfile(filename)
|
||||
except IOError, msg:
|
||||
sys.stderr.write("%s: %s\n" %
|
||||
(filename, str(msg)))
|
||||
continue
|
||||
for m in modules:
|
||||
if not done.has_key(m):
|
||||
newtodo[m] = None
|
||||
todo = newtodo
|
||||
return done
|
||||
|
||||
|
||||
# Scan a file looking for import statements.
|
||||
# Return list of module names.
|
||||
# Can raise IOError.
|
||||
|
||||
importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
|
||||
fromstr = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
|
||||
isimport = regex.compile(importstr)
|
||||
isfrom = regex.compile(fromstr)
|
||||
|
||||
def scanfile(filename):
|
||||
allmodules = {}
|
||||
f = open(filename, 'r')
|
||||
try:
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line: break # EOF
|
||||
while line[-2:] == '\\\n': # Continuation line
|
||||
line = line[:-2] + ' '
|
||||
line = line + f.readline()
|
||||
if isimport.search(line) >= 0:
|
||||
rawmodules = isimport.group(2)
|
||||
modules = string.splitfields(rawmodules, ',')
|
||||
for i in range(len(modules)):
|
||||
modules[i] = string.strip(modules[i])
|
||||
elif isfrom.search(line) >= 0:
|
||||
modules = [isfrom.group(2)]
|
||||
else:
|
||||
continue
|
||||
for mod in modules:
|
||||
allmodules[mod] = None
|
||||
finally:
|
||||
f.close()
|
||||
return allmodules.keys()
|
||||
|
||||
|
||||
# Find the file containing a module, given its name.
|
||||
# Return filename, or '<builtin>', or '<unknown>'.
|
||||
|
||||
builtins = sys.builtin_module_names
|
||||
if 'sys' not in builtins: builtins.append('sys')
|
||||
# XXX this table may have to be changed depending on your platform:
|
||||
tails = ['.so', 'module.so', '.py', '.pyc']
|
||||
|
||||
def findmodule(modname, path = sys.path):
|
||||
if modname in builtins: return '<builtin>'
|
||||
for dirname in path:
|
||||
for tail in tails:
|
||||
fullname = os.path.join(dirname, modname + tail)
|
||||
try:
|
||||
f = open(fullname, 'r')
|
||||
except IOError:
|
||||
continue
|
||||
f.close()
|
||||
return fullname
|
||||
return '<unknown>'
|
||||
|
||||
|
||||
# Test the above functions.
|
||||
|
||||
def test():
|
||||
if not sys.argv[1:]:
|
||||
print 'usage: python findmodules.py scriptfile [morefiles ...]'
|
||||
sys.exit(2)
|
||||
done = findmodules(sys.argv[1], sys.argv[2:])
|
||||
items = done.items()
|
||||
items.sort()
|
||||
for mod, file in [('Module', 'File')] + items:
|
||||
print "%-15s %s" % (mod, file)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
|
@ -1,576 +1,176 @@
|
|||
#! /usr/local/bin/python
|
||||
|
||||
# Given a Python script, create a binary that runs the script.
|
||||
# The binary is 100% independent of Python libraries and binaries.
|
||||
# It will not contain any Python source code -- only "compiled" Python
|
||||
# (as initialized static variables containing marshalled code objects).
|
||||
# It even does the right thing for dynamically loaded modules!
|
||||
# The module search path of the binary is set to the current directory.
|
||||
#
|
||||
# Some problems remain:
|
||||
# - You need to have the Python source tree lying around as well as
|
||||
# the various libraries used to generate the Python binary.
|
||||
# - For scripts that use many modules it generates absurdly large
|
||||
# files (frozen.c and config.o as well as the final binary),
|
||||
# and is consequently rather slow.
|
||||
#
|
||||
# Caveats:
|
||||
# - The search for modules sometimes finds modules that are never
|
||||
# actually imported since the code importing them is never executed.
|
||||
# - If an imported module isn't found, you get a warning but the
|
||||
# process of freezing continues. The binary will fail if it
|
||||
# actually tries to import one of these modules.
|
||||
# - This often happens with the module 'mac', which module 'os' tries
|
||||
# to import (to determine whether it is running on a Macintosh).
|
||||
# You can ignore the warning about this.
|
||||
# - If the program dynamically reads or generates Python code and
|
||||
# executes it, this code may reference built-in or library modules
|
||||
# that aren't present in the frozen binary, and this will fail.
|
||||
# - Your program may be using external data files, e.g. compiled
|
||||
# forms definitions (*.fd). These aren't incorporated. By default,
|
||||
# the sys.path in the resulting binary is only '.' (but you can override
|
||||
# that with the -P option).
|
||||
#
|
||||
# Usage hints:
|
||||
# - If you have a bunch of scripts that you want to freeze, instead
|
||||
# of freezing each of them separately, you might consider writing
|
||||
# a tiny main script that looks at sys.argv[0] and then imports
|
||||
# the corresponding module. You can then make links to the
|
||||
# frozen binary named after the various scripts you support.
|
||||
# Pass the additional scripts as arguments after the main script.
|
||||
# A minimal script to do this is the following.
|
||||
# import sys, posixpath
|
||||
# exec('import ' + posixpath.basename(sys.argv[0]) + '\n')
|
||||
#
|
||||
# Mods by Jack, August 94:
|
||||
# - Removed all static configuration stuff. Now, Setup and Makefile files
|
||||
# are parsed to obtain the linking info for the libraries. You have to
|
||||
# supply the -B option, though.
|
||||
# - Added -P (set sys.path) and -I/-D/-L/-l options (passed on to cc and
|
||||
# ld).
|
||||
# "Freeze" a Python script into a binary.
|
||||
# Usage: see first function below (before the imports!)
|
||||
|
||||
# HINTS:
|
||||
# - Edit the line at XXX below before running!
|
||||
# - You must have done "make inclinstall libainstall" in the Python
|
||||
# build directory.
|
||||
# - The script should not use dynamically loaded modules
|
||||
# (*.so on most systems).
|
||||
|
||||
|
||||
# XXX Change the following line to point to your Demo/freeze directory!
|
||||
pack = '/ufs/guido/src/python/Demo/freeze'
|
||||
|
||||
|
||||
# Print usage message and exit
|
||||
|
||||
def usage(msg = None):
|
||||
if msg:
|
||||
sys.stderr.write(str(msg) + '\n')
|
||||
sys.stderr.write('usage: freeze [-p prefix] script [module] ...\n')
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
# Import standard modules
|
||||
|
||||
import os
|
||||
import sys
|
||||
import regex
|
||||
import getopt
|
||||
import regsub
|
||||
import os
|
||||
import string
|
||||
import marshal
|
||||
import sys
|
||||
import addpack
|
||||
|
||||
# Exception used when scanfile fails
|
||||
NoSuchFile = 'NoSuchFile'
|
||||
|
||||
# Global options
|
||||
builddir = '' # -B dir
|
||||
quiet = 0 # -q
|
||||
verbose = 0 # -v
|
||||
noexec = 0 # -n
|
||||
nowrite = 0 # -N
|
||||
ofile = 'a.out' # -o file
|
||||
path = '\'"."\'' # -P path
|
||||
# Set the directory to look for the freeze-private modules
|
||||
|
||||
cc_options = [] # Collects cc options
|
||||
ld_options = [] # Collects ld options
|
||||
module_libraries = {} # ld options for each module
|
||||
global_libraries = [] # Libraries we always need
|
||||
include_path = '' # Include path, from Makefile
|
||||
lib_path = '' # and lib path, ditto
|
||||
compiler = 'cc' # and compiler
|
||||
dir = os.path.dirname(sys.argv[0])
|
||||
if dir:
|
||||
pack = dir
|
||||
addpack.addpack(pack)
|
||||
|
||||
|
||||
# Import the freeze-private modules
|
||||
|
||||
import findmodules
|
||||
import makeconfig
|
||||
import makefreeze
|
||||
import makemakefile
|
||||
import parsesetup
|
||||
|
||||
hint = """
|
||||
Use the '-p prefix' command line option to specify the prefix used
|
||||
when you ran 'Make inclinstall libainstall' in the Python build directory.
|
||||
(Please specify an absolute path.)
|
||||
"""
|
||||
|
||||
|
||||
# Main program
|
||||
|
||||
# Main program -- argument parsing etc.
|
||||
def main():
|
||||
global quiet, verbose, noexec, nowrite, ofile, builddir, path
|
||||
# overridable context
|
||||
prefix = '/usr/local' # settable with -p option
|
||||
path = sys.path
|
||||
|
||||
# output files
|
||||
frozen_c = 'frozen.c'
|
||||
config_c = 'config.c'
|
||||
target = 'a.out' # normally derived from script name
|
||||
makefile = 'Makefile'
|
||||
|
||||
# parse command line
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'B:nNo:P:qvI:D:L:l:')
|
||||
opts, args = getopt.getopt(sys.argv[1:], 'p:')
|
||||
if not args:
|
||||
raise getopt.error, 'not enough arguments'
|
||||
except getopt.error, msg:
|
||||
usage(str(msg))
|
||||
sys.exit(2)
|
||||
usage('getopt error: ' + str(msg))
|
||||
|
||||
# proces option arguments
|
||||
for o, a in opts:
|
||||
if o == '-B': builddir = a
|
||||
if o == '-n': noexec = 1
|
||||
if o == '-N': nowrite = 1
|
||||
if o == '-o': ofile = a
|
||||
if o == '-P':
|
||||
if '"' in a:
|
||||
usage('sorry, cannot have " in -P option')
|
||||
sys.exit(2)
|
||||
path = `'"' + a + '"'`
|
||||
if o == '-q': verbose = 0; quiet = 1
|
||||
if o == '-v': verbose = verbose + 1; quiet = 0
|
||||
if o in ('-I', '-D'): cc_options.append(o+a)
|
||||
if o in ('-L', '-l'): ld_options.append(o+a)
|
||||
if not builddir:
|
||||
usage('sorry, you have to pass a -B option')
|
||||
sys.exit(2)
|
||||
if len(args) < 1:
|
||||
usage('please pass at least one file argument')
|
||||
sys.exit(2)
|
||||
process(args[0], args[1:])
|
||||
if o == '-p':
|
||||
prefix = a
|
||||
|
||||
# Print usage message to stderr
|
||||
def usage(*msgs):
|
||||
sys.stdout = sys.stderr
|
||||
for msg in msgs: print msg
|
||||
print 'Usage: freeze [options] scriptfile [modulefile ...]'
|
||||
print '-B dir : name of python build dir (no default)'
|
||||
print '-n : generate the files but don\'t compile and link'
|
||||
print '-N : don\'t write frozen.c (do compile unless -n given)'
|
||||
print '-o file : binary output file (default a.out)'
|
||||
print '-P path : set sys.path for program (default ".")'
|
||||
print '-q : quiet (no messages at all except errors)'
|
||||
print '-v : verbose (lots of extra messages)'
|
||||
print '-D and -I options are passed to cc, -L and -l to ld'
|
||||
# locations derived from options
|
||||
binlib = os.path.join(prefix, 'lib/python/lib')
|
||||
incldir = os.path.join(prefix, 'include/Py')
|
||||
config_c_in = os.path.join(binlib, 'config.c.in')
|
||||
frozenmain_c = os.path.join(binlib, 'frozenmain.c')
|
||||
makefile_in = os.path.join(binlib, 'Makefile')
|
||||
defines = ['-DHAVE_CONFIG_H', '-DUSE_FROZEN', '-DNO_MAIN',
|
||||
'-DPTHONPATH=\\"$(PYTHONPATH)\\"']
|
||||
includes = ['-I' + incldir, '-I' + binlib]
|
||||
|
||||
# Process the script file
|
||||
def process(filename, addmodules):
|
||||
global noexec
|
||||
#
|
||||
if not quiet: print 'Computing needed modules ...'
|
||||
todo = {}
|
||||
todo['__main__'] = filename
|
||||
for name in addmodules:
|
||||
mod = os.path.basename(name)
|
||||
if mod[-3:] == '.py': mod = mod[:-3]
|
||||
todo[mod] = name
|
||||
try:
|
||||
dict = closure(todo)
|
||||
except NoSuchFile, filename:
|
||||
sys.stderr.write('Can\'t open file %s\n' % filename)
|
||||
sys.exit(1)
|
||||
#
|
||||
mods = dict.keys()
|
||||
mods.sort()
|
||||
#
|
||||
if verbose:
|
||||
print '%-15s %s' % ('Module', 'Filename')
|
||||
for mod in mods:
|
||||
print '%-15s %s' % (`mod`, dict[mod])
|
||||
#
|
||||
if not quiet: print 'Looking for dynamically linked modules ...'
|
||||
dlmodules = []
|
||||
objs = []
|
||||
libs = []
|
||||
for mod in mods:
|
||||
if dict[mod][-2:] == '.o':
|
||||
if verbose: print 'Found', mod, dict[mod]
|
||||
dlmodules.append(mod)
|
||||
objs.append(dict[mod])
|
||||
libsname = dict[mod][:-2] + '.libs'
|
||||
try:
|
||||
f = open(libsname, 'r')
|
||||
except IOError:
|
||||
f = None
|
||||
if f:
|
||||
libtext = f.read()
|
||||
f.close()
|
||||
for lib in string.split(libtext):
|
||||
if lib in libs: libs.remove(lib)
|
||||
libs.append(lib)
|
||||
#
|
||||
if not nowrite:
|
||||
if not quiet: print 'Writing frozen.c ...'
|
||||
writefrozen('frozen.c', dict)
|
||||
else:
|
||||
if not quiet: print 'NOT writing frozen.c ...'
|
||||
#
|
||||
if not quiet:
|
||||
print 'Deducing compile/link options from', builddir
|
||||
#
|
||||
# Parse the config info
|
||||
#
|
||||
parse(builddir)
|
||||
CONFIG_IN = lib_path + '/config.c.in'
|
||||
FMAIN = lib_path + '/frozenmain.c'
|
||||
CC = compiler
|
||||
#
|
||||
## if not dlmodules:
|
||||
if 0:
|
||||
config = CONFIG
|
||||
if not quiet: print 'Using existing', config, '...'
|
||||
else:
|
||||
config = 'tmpconfig.c'
|
||||
if nowrite:
|
||||
if not quiet: print 'NOT writing config.c ...'
|
||||
# sanity check of locations
|
||||
for dir in prefix, binlib, incldir:
|
||||
if not os.path.exists(dir):
|
||||
usage('needed directory %s not found' % dir + hint)
|
||||
if not os.path.isdir(dir):
|
||||
usage('%s: not a directory' % dir)
|
||||
for file in config_c_in, makefile_in, frozenmain_c:
|
||||
if not os.path.exists(file):
|
||||
usage('needed file %s not found' % file)
|
||||
if not os.path.isfile(file):
|
||||
usage('%s: not a plain file' % file)
|
||||
|
||||
# check that file arguments exist
|
||||
for arg in args:
|
||||
if not os.path.exists(arg):
|
||||
usage('argument %s not found' % arg)
|
||||
if not os.path.isfile(arg):
|
||||
usage('%s: not a plain file' % arg)
|
||||
|
||||
# process non-option arguments
|
||||
scriptfile = args[0]
|
||||
modules = args[1:]
|
||||
|
||||
# derive target name from script name
|
||||
base = os.path.basename(scriptfile)
|
||||
base, ext = os.path.splitext(base)
|
||||
if base:
|
||||
if base != scriptfile:
|
||||
target = base
|
||||
else:
|
||||
if not quiet:
|
||||
print 'Writing config.c with dl modules ...'
|
||||
f = open(CONFIG_IN, 'r')
|
||||
g = open(config, 'w')
|
||||
m1 = regex.compile('-- ADDMODULE MARKER 1 --')
|
||||
m2 = regex.compile('-- ADDMODULE MARKER 2 --')
|
||||
builtinmodules = []
|
||||
stdmodules = ('sys', '__main__', '__builtin__',
|
||||
'marshal')
|
||||
for mod in dict.keys():
|
||||
if dict[mod] == '<builtin>' and \
|
||||
mod not in stdmodules:
|
||||
builtinmodules.append(mod)
|
||||
todomodules = builtinmodules + dlmodules
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line: break
|
||||
g.write(line)
|
||||
if m1.search(line) >= 0:
|
||||
if verbose: print 'Marker 1 ...'
|
||||
for mod in todomodules:
|
||||
g.write('extern void init' + \
|
||||
mod + '();\n')
|
||||
if m2.search(line) >= 0:
|
||||
if verbose: print 'Marker 2 ...'
|
||||
for mod in todomodules:
|
||||
g.write('{"' + mod + \
|
||||
'", init' + mod + '},\n')
|
||||
g.close()
|
||||
#
|
||||
if not quiet:
|
||||
if noexec: print 'Generating compilation commands ...'
|
||||
else: print 'Starting compilation ...'
|
||||
defs = ['-DNO_MAIN', '-DUSE_FROZEN']
|
||||
defs.append('-DPYTHONPATH='+path)
|
||||
#
|
||||
incs = ['-I.', '-I' + include_path]
|
||||
# if dict.has_key('stdwin'):
|
||||
# incs.append('-I' + j(STDWIN, 'H'))
|
||||
#
|
||||
srcs = [config, FMAIN]
|
||||
#
|
||||
modlibs = module_libraries
|
||||
|
||||
target = base + '.bin'
|
||||
|
||||
# Actual work starts here...
|
||||
|
||||
dict = findmodules.findmodules(scriptfile, modules, path)
|
||||
|
||||
builtins = []
|
||||
for mod in dict.keys():
|
||||
if modlibs.has_key(mod):
|
||||
libs = libs + modlibs[mod]
|
||||
if dict[mod] == '<builtin>':
|
||||
builtins.append(mod)
|
||||
|
||||
libs = libs + global_libraries
|
||||
#
|
||||
# remove dups:
|
||||
# XXXX Not well tested...
|
||||
nskip = 0
|
||||
newlibs = []
|
||||
while libs:
|
||||
l = libs[0]
|
||||
del libs[0]
|
||||
if l[:2] == '-L' and l in newlibs:
|
||||
nskip = nskip + 1
|
||||
continue
|
||||
if (l[:2] == '-l' or l[-2:] == '.a') and l in libs:
|
||||
nskip = nskip + 1
|
||||
continue
|
||||
newlibs.append(l)
|
||||
libs = newlibs
|
||||
if nskip and not quiet:
|
||||
print 'Removed %d duplicate libraries'%nskip
|
||||
#
|
||||
sts = 0
|
||||
#
|
||||
cmd = CC + ' -c'
|
||||
if cc_options:
|
||||
cmd = cmd + ' ' + string.join(cc_options)
|
||||
cmd = cmd + ' ' + string.join(defs)
|
||||
cmd = cmd + ' ' + string.join(incs)
|
||||
cmd = cmd + ' ' + string.join(srcs)
|
||||
print cmd
|
||||
#
|
||||
if not noexec:
|
||||
sts = os.system(cmd)
|
||||
if sts:
|
||||
print 'Exit status', sts, '-- turning on -n'
|
||||
noexec = 1
|
||||
#
|
||||
for s in srcs:
|
||||
s = os.path.basename(s)
|
||||
if s[-2:] == '.c': s = s[:-2]
|
||||
o = s + '.o'
|
||||
objs.insert(0, o)
|
||||
#
|
||||
cmd = CC
|
||||
cmd = cmd + ' ' + string.join(objs)
|
||||
cmd = cmd + ' ' + string.join(libs)
|
||||
if ld_options:
|
||||
cmd = cmd + ' ' + string.join(ld_options)
|
||||
cmd = cmd + ' -o ' + ofile
|
||||
print cmd
|
||||
#
|
||||
if not noexec:
|
||||
sts = os.system(cmd)
|
||||
if sts:
|
||||
print 'Exit status', sts
|
||||
else:
|
||||
print 'Done.'
|
||||
#
|
||||
if not quiet and not noexec and sts == 0:
|
||||
print 'Note: consider this:'; print '\tstrip', ofile
|
||||
#
|
||||
sys.exit(sts)
|
||||
|
||||
|
||||
# Generate code for a given module
|
||||
def makecode(filename):
|
||||
if filename[-2:] == '.o':
|
||||
return None
|
||||
outfp = open(frozen_c, 'w')
|
||||
try:
|
||||
f = open(filename, 'r')
|
||||
except IOError:
|
||||
return None
|
||||
if verbose: print 'Making code from', filename, '...'
|
||||
text = f.read()
|
||||
code = compile(text, filename, 'exec')
|
||||
f.close()
|
||||
return marshal.dumps(code)
|
||||
|
||||
|
||||
# Write the C source file containing the frozen Python code
|
||||
def writefrozen(filename, dict):
|
||||
f = open(filename, 'w')
|
||||
codelist = []
|
||||
for mod in dict.keys():
|
||||
codestring = makecode(dict[mod])
|
||||
if codestring is not None:
|
||||
codelist.append((mod, codestring))
|
||||
write = sys.stdout.write
|
||||
save_stdout = sys.stdout
|
||||
try:
|
||||
sys.stdout = f
|
||||
for mod, codestring in codelist:
|
||||
if verbose:
|
||||
write('Writing initializer for %s\n'%mod)
|
||||
print 'static unsigned char M_' + mod + '[' + \
|
||||
str(len(codestring)) + '+1] = {'
|
||||
for i in range(0, len(codestring), 16):
|
||||
for c in codestring[i:i+16]:
|
||||
print str(ord(c)) + ',',
|
||||
print
|
||||
print '};'
|
||||
print 'struct frozen {'
|
||||
print ' char *name;'
|
||||
print ' unsigned char *code;'
|
||||
print ' int size;'
|
||||
print '} frozen_modules[] = {'
|
||||
for mod, codestring in codelist:
|
||||
print ' {"' + mod + '",',
|
||||
print 'M_' + mod + ',',
|
||||
print str(len(codestring)) + '},'
|
||||
print ' {0, 0, 0} /* sentinel */'
|
||||
print '};'
|
||||
makefreeze.makefreeze(outfp, dict)
|
||||
finally:
|
||||
sys.stdout = save_stdout
|
||||
f.close()
|
||||
outfp.close()
|
||||
|
||||
|
||||
# Determine the names and filenames of the modules imported by the
|
||||
# script, recursively. This is done by scanning for lines containing
|
||||
# import statements. (The scanning has only superficial knowledge of
|
||||
# Python syntax and no knowledge of semantics, so in theory the result
|
||||
# may be incorrect -- however this is quite unlikely if you don't
|
||||
# intentionally obscure your Python code.)
|
||||
|
||||
# Compute the closure of scanfile() -- special first file because of script
|
||||
def closure(todo):
|
||||
done = {}
|
||||
while todo:
|
||||
newtodo = {}
|
||||
for modname in todo.keys():
|
||||
if not done.has_key(modname):
|
||||
filename = todo[modname]
|
||||
if filename is None:
|
||||
filename = findmodule(modname)
|
||||
done[modname] = filename
|
||||
if filename in ('<builtin>', '<unknown>'):
|
||||
continue
|
||||
modules = scanfile(filename)
|
||||
for m in modules:
|
||||
if not done.has_key(m):
|
||||
newtodo[m] = None
|
||||
todo = newtodo
|
||||
return done
|
||||
|
||||
# Scan a file looking for import statements
|
||||
importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
|
||||
fromstr = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
|
||||
isimport = regex.compile(importstr)
|
||||
isfrom = regex.compile(fromstr)
|
||||
def scanfile(filename):
|
||||
allmodules = {}
|
||||
infp = open(config_c_in)
|
||||
outfp = open(config_c, 'w')
|
||||
try:
|
||||
f = open(filename, 'r')
|
||||
except IOError, msg:
|
||||
raise NoSuchFile, filename
|
||||
while 1:
|
||||
line = f.readline()
|
||||
if not line: break # EOF
|
||||
while line[-2:] == '\\\n': # Continuation line
|
||||
line = line[:-2] + ' '
|
||||
line = line + f.readline()
|
||||
if isimport.search(line) >= 0:
|
||||
rawmodules = isimport.group(2)
|
||||
modules = string.splitfields(rawmodules, ',')
|
||||
for i in range(len(modules)):
|
||||
modules[i] = string.strip(modules[i])
|
||||
elif isfrom.search(line) >= 0:
|
||||
modules = [isfrom.group(2)]
|
||||
else:
|
||||
continue
|
||||
for mod in modules:
|
||||
allmodules[mod] = None
|
||||
f.close()
|
||||
return allmodules.keys()
|
||||
makeconfig.makeconfig(infp, outfp, builtins)
|
||||
finally:
|
||||
outfp.close()
|
||||
infp.close()
|
||||
|
||||
# Find the file containing a module, given its name; None if not found
|
||||
builtins = sys.builtin_module_names + ['sys']
|
||||
def findmodule(modname):
|
||||
if modname in builtins: return '<builtin>'
|
||||
for dirname in sys.path:
|
||||
dlfullname = os.path.join(dirname, modname + 'module.o')
|
||||
try:
|
||||
f = open(dlfullname, 'r')
|
||||
except IOError:
|
||||
f = None
|
||||
if f:
|
||||
f.close()
|
||||
return dlfullname
|
||||
fullname = os.path.join(dirname, modname + '.py')
|
||||
try:
|
||||
f = open(fullname, 'r')
|
||||
except IOError:
|
||||
continue
|
||||
f.close()
|
||||
return fullname
|
||||
if not quiet:
|
||||
sys.stderr.write('Warning: module %s not found\n' % modname)
|
||||
return '<unknown>'
|
||||
#
|
||||
# Parse a setup file. Returns two dictionaries, one containing variables
|
||||
# defined with their values and one containing module definitions
|
||||
#
|
||||
def parse_setup(fp):
|
||||
modules = {}
|
||||
variables = {}
|
||||
for line in fp.readlines():
|
||||
if '#' in line: # Strip comments
|
||||
line = string.splitfields(line, '#')[0]
|
||||
line = string.strip(line[:-1]) # Strip whitespace
|
||||
if not line:
|
||||
continue
|
||||
words = string.split(line)
|
||||
if '=' in words[0]:
|
||||
#
|
||||
# equal sign before first space. Definition
|
||||
#
|
||||
pos = string.index(line, '=')
|
||||
name = line[:pos]
|
||||
value = string.strip(line[pos+1:])
|
||||
variables[name] = value
|
||||
else:
|
||||
modules[words[0]] = words[1:]
|
||||
return modules, variables
|
||||
#
|
||||
# Parse a makefile. Returns a list of the variables defined.
|
||||
#
|
||||
def parse_makefile(fp):
|
||||
variables = {}
|
||||
for line in fp.readlines():
|
||||
if '#' in line: # Strip comments
|
||||
line = string.splitfields(line, '#')[0]
|
||||
if not line:
|
||||
continue
|
||||
if line[0] in string.whitespace:
|
||||
continue
|
||||
line = string.strip(line[:-1]) # Strip whitespace
|
||||
if not line:
|
||||
continue
|
||||
if '=' in string.splitfields(line, ':')[0]:
|
||||
#
|
||||
# equal sign before first colon. Definition
|
||||
#
|
||||
pos = string.index(line, '=')
|
||||
name = line[:pos]
|
||||
value = string.strip(line[pos+1:])
|
||||
variables[name] = value
|
||||
return variables
|
||||
cflags = defines + includes + ['$(OPT)']
|
||||
libs = []
|
||||
for n in 'Modules', 'Python', 'Objects', 'Parser':
|
||||
n = 'lib%s.a' % n
|
||||
n = os.path.join(binlib, n)
|
||||
libs.append(n)
|
||||
|
||||
#
|
||||
# Recursively add loader options from Setup files in extension
|
||||
# directories.
|
||||
#
|
||||
def add_extension_directory(name, isinstalldir):
|
||||
if verbose:
|
||||
print 'Adding extension directory', name
|
||||
fp = open(name + '/Setup', 'r')
|
||||
modules, variables = parse_setup(fp)
|
||||
#
|
||||
# Locate all new modules and remember the ld flags needed for them
|
||||
#
|
||||
for m in modules.keys():
|
||||
if module_libraries.has_key(m):
|
||||
continue
|
||||
options = modules[m]
|
||||
if isinstalldir:
|
||||
ld_options = []
|
||||
else:
|
||||
ld_options = [name + '/lib.a']
|
||||
for o in options:
|
||||
# ld options are all capital except DUIC and l
|
||||
if o[:-2] == '.a':
|
||||
ld_options.append(o)
|
||||
elif o[0] == '-':
|
||||
if o[1] == 'l':
|
||||
ld_options.append(o)
|
||||
elif o[1] in string.uppercase and not o[1] in 'DUIC':
|
||||
ld_options.append(o)
|
||||
module_libraries[m] = ld_options
|
||||
#
|
||||
# See if we have to bother with base setups
|
||||
#
|
||||
if variables.has_key('BASESETUP'):
|
||||
if isinstalldir:
|
||||
raise 'installdir has base setup'
|
||||
setupfiles = string.split(variables['BASESETUP'])
|
||||
for s in setupfiles:
|
||||
if s[-6:] <> '/Setup':
|
||||
raise 'Incorrect BASESETUP', s
|
||||
s = s[:-6]
|
||||
if s[0] <> '/':
|
||||
s = name + '/' + s
|
||||
s = os.path.normpath(s)
|
||||
add_extension_directory(s, 0)
|
||||
#
|
||||
# Main routine for this module: given a build directory, get all
|
||||
# information needed for the linker.
|
||||
#
|
||||
def parse(dir):
|
||||
global include_path
|
||||
global lib_path
|
||||
global compiler
|
||||
|
||||
fp = open(dir + '/Makefile', 'r')
|
||||
#
|
||||
# First find the global libraries and the base python
|
||||
#
|
||||
vars = parse_makefile(fp)
|
||||
if vars.has_key('CC'):
|
||||
compiler = vars['CC']
|
||||
if not vars.has_key('installdir'):
|
||||
raise 'No $installdir in Makefile'
|
||||
include_path = vars['installdir'] + '/include/Py'
|
||||
lib_path = vars['installdir'] + '/lib/python/lib'
|
||||
global_libraries.append('-L' + lib_path)
|
||||
global_libraries.append('-lPython')
|
||||
global_libraries.append('-lParser')
|
||||
global_libraries.append('-lObjects')
|
||||
global_libraries.append('-lModules')
|
||||
for name in ('LIBS', 'LIBM', 'LIBC'):
|
||||
if not vars.has_key(name):
|
||||
raise 'Missing required def in Makefile', name
|
||||
for lib in string.split(vars[name]):
|
||||
global_libraries.append(lib)
|
||||
#
|
||||
# Next, parse the modules from the base python
|
||||
#
|
||||
add_extension_directory(lib_path, 1)
|
||||
#
|
||||
# Finally, parse the modules from the extension python
|
||||
#
|
||||
if dir <> lib_path:
|
||||
add_extension_directory(dir, 0)
|
||||
makevars = parsesetup.getmakevars(makefile_in)
|
||||
somevars = {}
|
||||
for key in makevars.keys():
|
||||
somevars[key] = makevars[key]
|
||||
|
||||
somevars['CFLAGS'] = string.join(cflags) # override
|
||||
files = ['$(OPT)', config_c, frozenmain_c] + libs + \
|
||||
['$(MODLIBS)', '$(LIBS)', '$(SYSLIBS)']
|
||||
|
||||
outfp = open(makefile, 'w')
|
||||
try:
|
||||
makemakefile.makemakefile(outfp, somevars, files, target)
|
||||
finally:
|
||||
outfp.close()
|
||||
|
||||
# Done!
|
||||
|
||||
print 'Now run make to build the target:', target
|
||||
|
||||
# Call the main program
|
||||
main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
import regex
|
||||
|
||||
|
||||
# Write the config.c file
|
||||
|
||||
never = ['marshal', '__main__', '__builtin__', 'sys']
|
||||
|
||||
def makeconfig(infp, outfp, modules):
|
||||
m1 = regex.compile('-- ADDMODULE MARKER 1 --')
|
||||
m2 = regex.compile('-- ADDMODULE MARKER 2 --')
|
||||
while 1:
|
||||
line = infp.readline()
|
||||
if not line: break
|
||||
outfp.write(line)
|
||||
if m1 and m1.search(line) >= 0:
|
||||
m1 = None
|
||||
for mod in modules:
|
||||
if mod in never:
|
||||
continue
|
||||
outfp.write('extern void init%s();\n' % mod)
|
||||
elif m2 and m2.search(line) >= 0:
|
||||
m2 = None
|
||||
for mod in modules:
|
||||
if mod in never:
|
||||
continue
|
||||
outfp.write('\t{"%s", init%s},\n' %
|
||||
(mod, mod))
|
||||
if m1:
|
||||
sys.stderr.write('MARKER 1 never found\n')
|
||||
elif m2:
|
||||
sys.stderr.write('MARKER 2 never found\n')
|
||||
|
||||
|
||||
# Test program.
|
||||
|
||||
def test():
|
||||
import sys
|
||||
if not sys.argv[3:]:
|
||||
print 'usage: python makeconfig.py config.c.in outputfile',
|
||||
print 'modulename ...'
|
||||
sys.exit(2)
|
||||
if sys.argv[1] == '-':
|
||||
infp = sys.stdin
|
||||
else:
|
||||
infp = open(sys.argv[1])
|
||||
if sys.argv[2] == '-':
|
||||
outfp = sys.stdout
|
||||
else:
|
||||
outfp = open(sys.argv[2], 'w')
|
||||
makeconfig(infp, outfp, sys.argv[3:])
|
||||
if outfp != sys.stdout:
|
||||
outfp.close()
|
||||
if infp != sys.stdin:
|
||||
infp.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
import marshal
|
||||
|
||||
|
||||
# Write a file containing frozen code for the modules in the dictionary.
|
||||
|
||||
header = """
|
||||
struct frozen {
|
||||
char *name;
|
||||
unsigned char *code;
|
||||
int size;
|
||||
} frozen_modules[] = {
|
||||
"""
|
||||
trailer = """\
|
||||
{0, 0, 0} /* sentinel */
|
||||
};
|
||||
"""
|
||||
|
||||
def makefreeze(outfp, dict):
|
||||
done = []
|
||||
mods = dict.keys()
|
||||
mods.sort()
|
||||
for mod in mods:
|
||||
modfn = dict[mod]
|
||||
try:
|
||||
str = makecode(modfn)
|
||||
except IOError, msg:
|
||||
sys.stderr.write("%s: %s\n" % (modfn, str(msg)))
|
||||
continue
|
||||
if str:
|
||||
done.append(mod, len(str))
|
||||
writecode(outfp, mod, str)
|
||||
outfp.write(header)
|
||||
for mod, size in done:
|
||||
outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mod, size))
|
||||
outfp.write(trailer)
|
||||
|
||||
|
||||
# Return code string for a given module -- either a .py or a .pyc
|
||||
# file. Return either a string or None (if it's not Python code).
|
||||
# May raise IOError.
|
||||
|
||||
def makecode(filename):
|
||||
if filename[-3:] == '.py':
|
||||
f = open(filename, 'r')
|
||||
try:
|
||||
text = f.read()
|
||||
code = compile(text, filename, 'exec')
|
||||
finally:
|
||||
f.close()
|
||||
return marshal.dumps(code)
|
||||
if filename[-4:] == '.pyc':
|
||||
f = open(filename, 'rb')
|
||||
try:
|
||||
f.seek(8)
|
||||
str = f.read()
|
||||
finally:
|
||||
f.close()
|
||||
return str
|
||||
# Can't generate code for this extension
|
||||
return None
|
||||
|
||||
|
||||
# Write a C initializer for a module containing the frozen python code.
|
||||
# The array is called M_<mod>.
|
||||
|
||||
def writecode(outfp, mod, str):
|
||||
outfp.write('static unsigned char M_%s[] = {' % mod)
|
||||
for i in range(0, len(str), 16):
|
||||
outfp.write('\n\t')
|
||||
for c in str[i:i+16]:
|
||||
outfp.write('%d,' % ord(c))
|
||||
outfp.write('\n};\n')
|
||||
|
||||
|
||||
# Test for the above functions.
|
||||
|
||||
def test():
|
||||
import os
|
||||
import sys
|
||||
if not sys.argv[1:]:
|
||||
print 'usage: python freezepython.py file.py(c) ...'
|
||||
sys.exit(2)
|
||||
dict = {}
|
||||
for arg in sys.argv[1:]:
|
||||
base = os.path.basename(arg)
|
||||
mod, ext = os.path.splitext(base)
|
||||
dict[mod] = arg
|
||||
makefreeze(sys.stdout, dict)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
# Write the actual Makefile.
|
||||
|
||||
import os
|
||||
import string
|
||||
|
||||
def makemakefile(outfp, makevars, files, target):
|
||||
outfp.write("# Makefile generated by freeze.py script\n\n")
|
||||
|
||||
keys = makevars.keys()
|
||||
keys.sort()
|
||||
for key in keys:
|
||||
outfp.write("%s=%s\n" % (key, makevars[key]))
|
||||
outfp.write("\nall: %s\n" % target)
|
||||
|
||||
deps = []
|
||||
for i in range(len(files)):
|
||||
file = files[i]
|
||||
if file[-2:] == '.c':
|
||||
base = os.path.basename(file)
|
||||
dest = base[:-2] + '.o'
|
||||
outfp.write("%s: %s\n" % (dest, file))
|
||||
outfp.write("\t$(CC) $(CFLAGS) -c %s\n" % file)
|
||||
files[i] = dest
|
||||
deps.append(dest)
|
||||
|
||||
outfp.write("\n%s: %s\n" % (target, string.join(deps)))
|
||||
outfp.write("\t$(CC) %s -o %s\n" % (string.join(files), target))
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
# Parse Makefiles and Python Setup(.in) files.
|
||||
|
||||
import regex
|
||||
import string
|
||||
|
||||
|
||||
# Extract variable definitions from a Makefile.
|
||||
# Return a dictionary mapping names to values.
|
||||
# May raise IOError.
|
||||
|
||||
makevardef = regex.compile('^\([a-zA-Z0-9_]+\)[ \t]*=\(.*\)')
|
||||
|
||||
def getmakevars(filename):
|
||||
variables = {}
|
||||
fp = open(filename)
|
||||
try:
|
||||
while 1:
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
break
|
||||
if makevardef.match(line) < 0:
|
||||
continue
|
||||
name, value = makevardef.group(1, 2)
|
||||
# Strip trailing comment
|
||||
i = string.find(value, '#')
|
||||
if i >= 0:
|
||||
value = value[:i]
|
||||
value = string.strip(value)
|
||||
variables[name] = value
|
||||
finally:
|
||||
fp.close()
|
||||
return variables
|
||||
|
||||
|
||||
# Parse a Python Setup(.in) file.
|
||||
# Return two dictionaries, the first mapping modules to their
|
||||
# definitions, the second mapping variable names to their values.
|
||||
# May raise IOError.
|
||||
|
||||
setupvardef = regex.compile('^\([a-zA-Z0-9_]+\)=\(.*\)')
|
||||
|
||||
def getsetupinfo(filename):
|
||||
modules = {}
|
||||
variables = {}
|
||||
fp = open(filename)
|
||||
try:
|
||||
while 1:
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
break
|
||||
# Strip comments
|
||||
i = string.find(line, '#')
|
||||
if i >= 0:
|
||||
line = line[:i]
|
||||
if setupvardef.match(line) >= 0:
|
||||
name, value = setupvardef.group(1, 2)
|
||||
variables[name] = string.strip(value)
|
||||
else:
|
||||
words = string.split(line)
|
||||
if words:
|
||||
modules[words[0]] = words[1:]
|
||||
finally:
|
||||
fp.close()
|
||||
return modules, variables
|
||||
|
||||
|
||||
# Test the above functions.
|
||||
|
||||
def test():
|
||||
import sys
|
||||
import os
|
||||
if not sys.argv[1:]:
|
||||
print 'usage: python parsesetup.py Makefile*|Setup* ...'
|
||||
sys.exit(2)
|
||||
for arg in sys.argv[1:]:
|
||||
base = os.path.basename(arg)
|
||||
if base[:8] == 'Makefile':
|
||||
print 'Make style parsing:', arg
|
||||
v = getmakevars(arg)
|
||||
prdict(v)
|
||||
elif base[:5] == 'Setup':
|
||||
print 'Setup style parsing:', arg
|
||||
m, v = getsetupinfo(arg)
|
||||
prdict(m)
|
||||
prdict(v)
|
||||
else:
|
||||
print arg, 'is neither a Makefile nor a Setup file'
|
||||
print '(name must begin with "Makefile" or "Setup")'
|
||||
|
||||
def prdict(d):
|
||||
keys = d.keys()
|
||||
keys.sort()
|
||||
for key in keys:
|
||||
value = d[key]
|
||||
print "%-15s" % key, str(value)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
Loading…
Reference in New Issue