From baf0603493f6f1eee073a479497291785b067f61 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 25 Aug 1998 14:06:55 +0000 Subject: [PATCH] New version, with contributions from Sjoerd Mullender and Mark Hammond. Sjoerd writes: This version of freeze creates one file per Python module, instead of one humongous file for all Python modules. bkfile: new module to used to write files with backups. No new file is produced if the new contents is identical to the old. New option "-x excluded-module" for modulefinder test program. New option "-i filename" for freeze main program to include a list of options in place of the -i option. --- Tools/freeze/README | 10 +-- Tools/freeze/bkfile.py | 50 ++++++++++++++ Tools/freeze/checkextensions_win32.py | 41 +++++++++--- Tools/freeze/extensions_win32.ini | 2 +- Tools/freeze/freeze.py | 95 +++++++++++---------------- Tools/freeze/makefreeze.py | 19 +++++- Tools/freeze/modulefinder.py | 11 +++- Tools/freeze/winmakemakefile.py | 50 ++++++++++---- 8 files changed, 189 insertions(+), 89 deletions(-) create mode 100644 Tools/freeze/bkfile.py diff --git a/Tools/freeze/README b/Tools/freeze/README index 09a05d749aa..5617eeec164 100644 --- a/Tools/freeze/README +++ b/Tools/freeze/README @@ -77,10 +77,12 @@ such as /usr/joe/python/Tools/freeze/freeze.py). What do I do next? ------------------ -Freeze creates three files: frozen.c, config.c and Makefile. To -produce the frozen version of your program, you can simply type -"make". This should produce a binary file. If the filename argument -to Freeze was "hello.py", the binary will be called "hello". +Freeze creates a number of files: frozen.c, config.c and Makefile, +plus one file for each Python module that gets included named +M_.c. To produce the frozen version of your program, you can +simply type "make". This should produce a binary file. If the +filename argument to Freeze was "hello.py", the binary will be called +"hello". Note: you can use the -o option to freeze to specify an alternative directory where these files are created. This makes it easier to diff --git a/Tools/freeze/bkfile.py b/Tools/freeze/bkfile.py new file mode 100644 index 00000000000..6d0ccdeac78 --- /dev/null +++ b/Tools/freeze/bkfile.py @@ -0,0 +1,50 @@ +_orig_open = open + +class _BkFile: + def __init__(self, file, mode, bufsize): + import os + self.__filename = file + self.__backup = file + '~' + try: + os.unlink(self.__backup) + except os.error: + pass + try: + os.rename(file, self.__backup) + except os.error: + self.__backup = None + self.__file = _orig_open(file, mode, bufsize) + self.closed = self.__file.closed + self.fileno = self.__file.fileno + self.flush = self.__file.flush + self.isatty = self.__file.isatty + self.mode = self.__file.mode + self.name = self.__file.name + self.read = self.__file.read + self.readinto = self.__file.readinto + self.readline = self.__file.readline + self.readlines = self.__file.readlines + self.seek = self.__file.seek + self.softspace = self.__file.softspace + self.tell = self.__file.tell + self.truncate = self.__file.truncate + self.write = self.__file.write + self.writelines = self.__file.writelines + + def close(self): + self.__file.close() + if self.__backup is None: + return + import cmp + # don't use cmp.cmp because of NFS bugs :-( and + # anyway, the stat mtime values differ so do_cmp will + # most likely be called anyway + if cmp.do_cmp(self.__backup, self.__filename): + import os + os.unlink(self.__filename) + os.rename(self.__backup, self.__filename) + +def open(file, mode = 'r', bufsize = -1): + if 'w' not in mode: + return _orig_open(file, mode, bufsize) + return _BkFile(file, mode, bufsize) diff --git a/Tools/freeze/checkextensions_win32.py b/Tools/freeze/checkextensions_win32.py index 69643b3fa38..ff86ab0f4ad 100644 --- a/Tools/freeze/checkextensions_win32.py +++ b/Tools/freeze/checkextensions_win32.py @@ -9,11 +9,13 @@ options anyway (eg, to enable or disable specific functionality) So my basic stragtegy is: -* Have a Windows INI file which "describes" an extension module. +* Have some Windows INI files which "describe" one or more extension modules. + (Freeze comes with a default one for all known modules - but you can specify + your own). * This description can include: - The MSVC .dsp file for the extension. The .c source file names are extraced from there. - - Specific compiler options + - Specific compiler/linker options - Flag to indicate if Unicode compilation is expected. At the moment the name and location of this INI file is hardcoded, @@ -52,31 +54,52 @@ class CExtension: def GetLinkerLibs(self): return self.linkerLibs -def checkextensions(unknown, ignored): +def checkextensions(unknown, extra_inis): # Create a table of frozen extensions - mapFileName = os.path.join( os.path.split(sys.argv[0])[0], "extensions_win32.ini") + defaultMapName = os.path.join( os.path.split(sys.argv[0])[0], "extensions_win32.ini") + if not os.path.isfile(defaultMapName): + sys.stderr.write("WARNING: %s can not be found - standard extensions may not be found" % mapFileName) + else: + # must go on end, so other inis can override. + extra_inis.append(defaultMapName) + ret = [] for mod in unknown: - defn = get_extension_defn( mod, mapFileName ) - if defn is not None: - ret.append( defn ) + for ini in extra_inis: +# print "Looking for", mod, "in", win32api.GetFullPathName(ini),"...", + defn = get_extension_defn( mod, ini ) + if defn is not None: +# print "Yay - found it!" + ret.append( defn ) + break +# print "Nope!" + else: # For not broken! + sys.stderr.write("No definition of module %s in any specified map file.\n" % (mod)) + return ret def get_extension_defn(moduleName, mapFileName): if win32api is None: return None dsp = win32api.GetProfileVal(moduleName, "dsp", "", mapFileName) if dsp=="": - sys.stderr.write("No definition of module %s in map file '%s'\n" % (moduleName, mapFileName)) return None # We allow environment variables in the file name dsp = win32api.ExpandEnvironmentStrings(dsp) + # If the path to the .DSP file is not absolute, assume it is relative + # to the description file. + if not os.path.isabs(dsp): + dsp = os.path.join( os.path.split(mapFileName)[0], dsp) + # Parse it to extract the source files. sourceFiles = parse_dsp(dsp) if sourceFiles is None: return None module = CExtension(moduleName, sourceFiles) + # Put the path to the DSP into the environment so entries can reference it. + os.environ['dsp_path'] = os.path.split(dsp)[0] + os.environ['ini_path'] = os.path.split(mapFileName)[0] cl_options = win32api.GetProfileVal(moduleName, "cl", "", mapFileName) if cl_options: @@ -90,7 +113,7 @@ def get_extension_defn(moduleName, mapFileName): libs = string.split(win32api.GetProfileVal(moduleName, "libs", "", mapFileName)) for lib in libs: - module.AddLinkerLib(lib) + module.AddLinkerLib(win32api.ExpandEnvironmentStrings(lib)) for exc in exclude: if exc in module.sourceFiles: diff --git a/Tools/freeze/extensions_win32.ini b/Tools/freeze/extensions_win32.ini index 49da9d63408..3b6d08e4976 100644 --- a/Tools/freeze/extensions_win32.ini +++ b/Tools/freeze/extensions_win32.ini @@ -112,6 +112,6 @@ Unicode = 1 ; Pythonwin [win32ui] dsp=%PYTHONEX%\Pythonwin\win32ui.dsp -cl=/I %PYTHONEX%\win32\src +cl=/D _AFXDLL /D FREEZE_WIN32UI /GX /I %PYTHONEX%\win32\src libs=mfc42.lib diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index dbca9639a1c..2911a35ee93 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -5,7 +5,6 @@ usage: freeze [options...] script [module]... Options: - -p prefix: This is the prefix used when you ran ``make install'' in the Python build directory. (If you never ran this, freeze won't work.) @@ -22,6 +21,8 @@ Options: -e extension: A directory containing additional .o files that may be used to resolve modules. This directory should also have a Setup file describing the .o files. + On Windows, the name of a .INI file describing one + or more extensions is passed. More than one -e option may be given. -o dir: Directory where the output files are created; default '.'. @@ -41,15 +42,20 @@ Options: -h: Print this help message. --w: Toggle Windows (NT or 95) behavior. - (For debugging only -- on a win32 platform, win32 behaviour - is automatic.) - -x module Exclude the specified module. +-i filename: Include a file with additional command line options. Used + to prevent command lines growing beyond the capabilities of + the shell/OS. All arguments specified in filename + are read and the -i option replaced with the parsed + params (note - quoting args in this file is NOT supported) + -s subsystem: Specify the subsystem (For Windows only.); 'console' (default), 'windows', 'service' or 'com_dll' +-w: Toggle Windows (NT or 95) behavior. + (For debugging only -- on a win32 platform, win32 behaviour + is automatic.) Arguments: @@ -87,6 +93,7 @@ import makeconfig import makefreeze import makemakefile import parsesetup +import bkfile # Main program @@ -105,8 +112,8 @@ def main(): win = sys.platform[:3] == 'win' # default the exclude list for each platform -## if win: exclude = exclude + [ -## 'dos', 'dospath', 'mac', 'macpath', 'MACFS', 'posix', 'os2'] + if win: exclude = exclude + [ + 'dos', 'dospath', 'mac', 'macpath', 'macfs', 'MACFS', 'posix', 'os2'] # modules that are imported by the Python runtime implicits = ["site", "exceptions"] @@ -118,7 +125,20 @@ def main(): makefile = 'Makefile' subsystem = 'console' - # parse command line + # parse command line by first replacing any "-i" options with the file contents. + pos = 1 + while pos < len(sys.argv)-1: # last option can not be "-i", so this ensures "pos+1" is in range! + if sys.argv[pos] == '-i': + try: + options = string.split(open(sys.argv[pos+1]).read()) + except IOError, why: + usage("File name '%s' specified with the -i option can not be read - %s" % (sys.argv[pos+1], why) ) + # Replace the '-i' and the filename with the read params. + sys.argv[pos:pos+2] = options + pos = pos + len(options) - 1 # Skip the name and the included args. + pos = pos + 1 + + # Now parse the command line with the extras inserted. try: opts, args = getopt.getopt(sys.argv[1:], 'a:de:hmo:p:P:qs:wx:l:') except getopt.error, msg: @@ -197,13 +217,15 @@ def main(): includes = ['-I' + incldir, '-I' + config_h_dir] # sanity check of directories and files - for dir in [prefix, exec_prefix, binlib, incldir] + extensions: + check_dirs = [prefix, exec_prefix, binlib, incldir] + if not win: check_dirs = check_dirs + extensions # These are not directories on Windows. + for dir in check_dirs: if not os.path.exists(dir): usage('needed directory %s not found' % dir) if not os.path.isdir(dir): usage('%s: not a directory' % dir) if win: - files = supp_sources + files = supp_sources + extensions # extensions are files on Windows. else: files = [config_c_in, makefile_in] + supp_sources for file in supp_sources: @@ -260,7 +282,9 @@ def main(): print "Created output directory", odir except os.error, msg: usage('%s: mkdir failed (%s)' % (odir, str(msg))) + base = '' if odir: + base = os.path.join(odir, '') frozen_c = os.path.join(odir, frozen_c) config_c = os.path.join(odir, config_c) target = os.path.join(odir, target) @@ -323,21 +347,7 @@ def main(): dict = mf.modules # generate output for frozen modules - backup = frozen_c + '~' - try: - os.rename(frozen_c, backup) - except os.error: - backup = None - outfp = open(frozen_c, 'w') - try: - makefreeze.makefreeze(outfp, dict, debug, custom_entry_point) - finally: - outfp.close() - if backup: - if cmp.cmp(backup, frozen_c): - sys.stderr.write('%s not changed, not written\n' % frozen_c) - os.unlink(frozen_c) - os.rename(backup, frozen_c) + files = makefreeze.makefreeze(base, dict, debug, custom_entry_point) # look for unfrozen modules (builtin and of unknown origin) builtins = [] @@ -387,7 +397,7 @@ def main(): frozen_extensions) # Create a module definition for the bootstrap C code. xtras = [frozenmain_c, os.path.basename(frozen_c), - frozendllmain_c, extensions_c] + frozendllmain_c, extensions_c] + files maindefn = checkextensions_win32.CExtension( '__main__', xtras ) frozen_extensions.append( maindefn ) outfp = open(makefile, 'w') @@ -403,22 +413,12 @@ def main(): # generate config.c and Makefile builtins.sort() infp = open(config_c_in) - backup = config_c + '~' - try: - os.rename(config_c, backup) - except os.error: - backup = None - outfp = open(config_c, 'w') + outfp = bkfile.open(config_c, 'w') try: makeconfig.makeconfig(infp, outfp, builtins) finally: outfp.close() infp.close() - if backup: - if cmp.cmp(backup, config_c): - sys.stderr.write('%s not changed, not written\n' % config_c) - os.unlink(config_c) - os.rename(backup, config_c) cflags = defines + includes + ['$(OPT)'] libs = [os.path.join(binlib, 'libpython$(VERSION).a')] @@ -431,31 +431,14 @@ def main(): somevars['CFLAGS'] = string.join(cflags) # override files = ['$(OPT)', '$(LDFLAGS)', base_config_c, base_frozen_c] + \ - supp_sources + addfiles + libs + \ + files + supp_sources + addfiles + libs + \ ['$(MODLIBS)', '$(LIBS)', '$(SYSLIBS)'] - backup = makefile + '~' - if os.path.exists(makefile): - try: - os.unlink(backup) - except os.error: - pass - try: - os.rename(makefile, backup) - except os.error: - backup = None - outfp = open(makefile, 'w') + outfp = bkfile.open(makefile, 'w') try: makemakefile.makemakefile(outfp, somevars, files, base_target) finally: outfp.close() - if backup: - if not cmp.cmp(backup, makefile): - print 'previous Makefile saved as', backup - else: - sys.stderr.write('%s not changed, not written\n' % makefile) - os.unlink(makefile) - os.rename(backup, makefile) # Done! diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py index f11c59b1104..4ea1905b102 100644 --- a/Tools/freeze/makefreeze.py +++ b/Tools/freeze/makefreeze.py @@ -1,5 +1,6 @@ import marshal import string +import bkfile # Write a file containing frozen code for the modules in the dictionary. @@ -30,15 +31,19 @@ main(argc, argv) """ -def makefreeze(outfp, dict, debug=0, entry_point = None): +def makefreeze(base, dict, debug=0, entry_point = None): if entry_point is None: entry_point = default_entry_point done = [] + files = [] mods = dict.keys() mods.sort() for mod in mods: m = dict[mod] mangled = string.join(string.split(mod, "."), "__") if m.__code__: + file = 'M_' + mangled + '.c' + outfp = bkfile.open(base + file, 'w') + files.append(file) if debug: print "freezing", mod, "..." str = marshal.dumps(m.__code__) @@ -48,13 +53,19 @@ def makefreeze(outfp, dict, debug=0, entry_point = None): size = -size done.append((mod, mangled, size)) writecode(outfp, mangled, str) + outfp.close() if debug: print "generating table of frozen modules" + outfp = bkfile.open(base + 'frozen.c', 'w') + for mod, mangled, size in done: + outfp.write('extern unsigned char M_%s[];\n' % mangled) outfp.write(header) for mod, mangled, size in done: outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size)) outfp.write(trailer) outfp.write(entry_point) + outfp.close() + return files @@ -62,9 +73,13 @@ def makefreeze(outfp, dict, debug=0, entry_point = None): # The array is called M_. def writecode(outfp, mod, str): - outfp.write('static unsigned char M_%s[] = {' % mod) + outfp.write('unsigned char M_%s[] = {' % mod) for i in range(0, len(str), 16): outfp.write('\n\t') for c in str[i:i+16]: outfp.write('%d,' % ord(c)) outfp.write('\n};\n') + +## def writecode(outfp, mod, str): +## outfp.write('unsigned char M_%s[%d] = "%s";\n' % (mod, len(str), +## string.join(map(lambda s: `s`[1:-1], string.split(str, '"')), '\\"'))) diff --git a/Tools/freeze/modulefinder.py b/Tools/freeze/modulefinder.py index 4c54ebd8ea0..4985d52f2b9 100644 --- a/Tools/freeze/modulefinder.py +++ b/Tools/freeze/modulefinder.py @@ -359,14 +359,16 @@ class ModuleFinder: keys = self.badmodules.keys() keys.sort() for key in keys: - print "?", key + # ... but not if they were explicitely excluded. + if key not in self.excludes: + print "?", key def test(): # Parse command line import getopt try: - opts, args = getopt.getopt(sys.argv[1:], "dmp:q") + opts, args = getopt.getopt(sys.argv[1:], "dmp:qx:") except getopt.error, msg: print msg return @@ -375,6 +377,7 @@ def test(): debug = 1 domods = 0 addpath = [] + exclude = [] for o, a in opts: if o == '-d': debug = debug + 1 @@ -384,6 +387,8 @@ def test(): addpath = addpath + string.split(a, os.pathsep) if o == '-q': debug = 0 + if o == '-x': + exclude.append(a) # Provide default arguments if not args: @@ -401,7 +406,7 @@ def test(): print " ", `item` # Create the module finder and turn its crank - mf = ModuleFinder(path, debug) + mf = ModuleFinder(path, debug, exclude) for arg in args[1:]: if arg == '-m': domods = 1 diff --git a/Tools/freeze/winmakemakefile.py b/Tools/freeze/winmakemakefile.py index 3f37e771a21..351e9cad95b 100644 --- a/Tools/freeze/winmakemakefile.py +++ b/Tools/freeze/winmakemakefile.py @@ -50,13 +50,29 @@ def makemakefile(outfp, vars, files, target): sys.stdout = save def realwork(vars, moddefns, target): - print "# Makefile for Windows (NT or 95) generated by freeze.py script" + print "# Makefile for Microsoft Visual C++ generated by freeze.py script" print print 'target = %s' % target print 'pythonhome = "%s"' % vars['prefix'] - # XXX The following line is fishy and may need manual fixing - print 'pythonlib = "%s"' % (vars['exec_prefix'] + - "/pcbuild/release/python15.lib") + print + print 'DEBUG=0 # Set to 1 to use the _d versions of Python.' + print '!IF $(DEBUG)' + print 'debug_suffix=_d' + print 'c_debug=/Zi /Od /DDEBUG /D_DEBUG' + print 'l_debug=/DEBUG' + print 'temp_dir=Build\\Debug' + print '!ELSE' + print 'debug_suffix=' + print 'c_debug=/Ox' + print 'l_debug=' + print 'temp_dir=Build\\Release' + print '!ENDIF' + print + + print '# The following line assumes you have built Python using the standard instructions' + print '# Otherwise fix the following line to point to the library.' + print 'pythonlib = "$(pythonhome)/pcbuild/python15$(debug_suffix).lib"' + print # We only ever write one "entry point" symbol - either # "main" or "WinMain". Therefore, there is no need to @@ -69,21 +85,27 @@ def realwork(vars, moddefns, target): target_link_flags = "-dll" target_ext = ".dll" - print "cdl = /MD" # XXX - Should this come from vars? User may have specific requirements... + + print "# As the target uses Python15.dll, we must use this compiler option!" + print "cdl = /MD" print - print "all: $(target)%s" % (target_ext) + print "all: $(target)$(debug_suffix)%s" % (target_ext) + print + + print '$(temp_dir):' + print ' if not exist $(temp_dir)\. mkdir $(temp_dir)' print objects = [] - libs = ["shell32.lib", "comdlg32.lib", "wsock32.lib", "user32.lib"] + libs = ["shell32.lib", "comdlg32.lib", "wsock32.lib", "user32.lib", "oleaut32.lib"] for moddefn in moddefns: print "# Module", moddefn.name for file in moddefn.sourceFiles: base = os.path.basename(file) base, ext = os.path.splitext(base) objects.append(base + ".obj") - print '%s.obj: "%s"' % (base, file) - print "\t@$(CC) -c -nologo $(cdl) /D BUILD_FREEZE", + print '$(temp_dir)\%s.obj: "%s"' % (base, file) + print "\t@$(CC) -c -nologo /Fo$* $(cdl) $(c_debug) /D BUILD_FREEZE", print "-I$(pythonhome)/Include -I$(pythonhome)/PC \\" print "\t\t$(cflags) $(cdebug) $(cinclude) \\" extra = moddefn.GetCompilerOptions() @@ -102,20 +124,20 @@ def realwork(vars, moddefns, target): print ; print print "OBJS=", - for obj in objects: print '"%s"' % (obj), + for obj in objects: print '"$(temp_dir)\%s"' % (obj), print ; print print "LIBS=", for lib in libs: print '"%s"' % (lib), print ; print - print "$(target)%s: $(OBJS)" % (target_ext) - print "\tlink -out:$(target)%s %s" % (target_ext, target_link_flags), + print "$(target)$(debug_suffix)%s: $(temp_dir) $(OBJS)" % (target_ext) + print "\tlink -out:$(target)$(debug_suffix)%s %s" % (target_ext, target_link_flags), print "\t$(OBJS) \\" print "\t$(LIBS) \\" print "\t$(ADDN_LINK_FILES) \\" - print "\t\t$(pythonlib) $(lcustom)\\" - print "\t\t$(resources)" + print "\t$(pythonlib) $(lcustom) $(l_debug)\\" + print "\t$(resources)" print print "clean:" print "\t-rm -f *.obj"