Demo/scripts/freeze.py

   1 #! /usr/local/bin/python
   2
   3 # Given a Python script, create a binary that runs the script.
   4 # The binary is 100% independent of Python libraries and binaries.
   5 # It will not contain any Python source code -- only "compiled" Python
   6 # (as initialized static variables containing marshalled code objects).
   7 # It even does the right thing for dynamically loaded modules!
   8 # The module search path of the binary is set to the current directory.
   9 #
  10 # Some problems remain:
  11 # - It's highly non-portable, since it knows about paths and libraries
  12 #   (there's a customization section though, and it knows how to
  13 #   distinguish an SGI from a Sun SPARC system -- adding knowledge
  14 #   about more systems is left as an exercise for the reader).
  15 # - You need to have the Python source tree lying around as well as
  16 #   the "libpython.a" used to generate the Python binary.
  17 # - For scripts that use many modules it generates absurdly large
  18 #   files (frozen.c and config.o as well as the final binary),
  19 #   and is consequently rather slow.
  20 #
  21 # Caveats:
  22 # - The search for modules sometimes finds modules that are never
  23 #   actually imported since the code importing them is never executed.
  24 # - If an imported module isn't found, you get a warning but the
  25 #   process of freezing continues.  The binary will fail if it
  26 #   actually tries to import one of these modules.
  27 # - This often happens with the module 'mac', which module 'os' tries
  28 #   to import (to determine whether it is running on a Macintosh).
  29 #   You can ignore the warning about this.
  30 # - If the program dynamically reads or generates Python code and
  31 #   executes it, this code may reference built-in or library modules
  32 #   that aren't present in the frozen binary, and this will fail.
  33 # - Your program may be using external data files, e.g. compiled
  34 #   forms definitions (*.fd).  These aren't incorporated.  Since
  35 #   sys.path in the resulting binary only contains '.', if your
  36 #   program searches its data files along sys.path (as the 'flp'
  37 #   modules does to find its forms definitions), you may need to
  38 #   change the program to extend the search path or instruct its users
  39 #   to set the environment variable PYTHONPATH to point to your data
  40 #   files.
  41 #
  42 # Usage hints:
  43 # - If you have a bunch of scripts that you want to freeze, instead
  44 #   of freezing each of them separately, you might consider writing
  45 #   a tiny main script that looks at sys.argv[0] and then imports
  46 #   the corresponding module.  You can then make links to the
  47 #   frozen binary named after the various scripts you support.
  48 #   Pass the additional scripts as arguments after the main script.
  49 #   A minimal script to do this is the following.
  50 #       import sys, posixpath
  51 #       exec('import ' + posixpath.basename(sys.argv[0]) + '\n')
  52
  53
  54 import os
  55 import sys
  56 import regex
  57 import getopt
  58 import regsub
  59 import string
  60 import marshal
  61
  62 # Function to join two pathnames with a slash in between
  63 j = os.path.join
  64
  65 ##################################
  66 # START OF CONFIGURATION SECTION #
  67 ##################################
  68
  69 # Attempt to guess machine architecture
  70 if os.path.exists('/usr/lib/libgl_s'): ARCH = 'sgi'
  71 elif os.path.exists('/etc/issue'): ARCH = 'sequent'
  72 else: ARCH = 'sun4'
  73
  74 # Site parametrizations (change to match your site)
  75 CC = 'cc'                               # C compiler
  76 TOP = '/ufs/guido/src'                  # Parent of all source trees
  77 PYTHON = j(TOP, 'python')               # Top of the Python source tree
  78 SRC = j(PYTHON, 'src')                  # Python source directory
  79 BLD = j(PYTHON, 'build.' + ARCH)        # Python build directory
  80 #BLD = SRC                              # Use this if you build in SRC
  81
  82 LIBINST = '/ufs/guido/src/python/irix4/tmp/lib/python/lib' # installed libraries
  83 INCLINST = '/ufs/guido/src/python/irix4/tmp/include/Py' # installed include files
  84
  85 # Other packages (change to match your site)
  86 DL = j(TOP, 'dl')                       # Top of the dl source tree
  87 DL_DLD = j(TOP, 'dl-dld')               # The dl-dld source directory
  88 DLD = j(TOP, 'dld-3.2.3')               # The dld source directory
  89 FORMS = j(TOP, 'forms')                 # Top of the FORMS source tree
  90 STDWIN = j(TOP, 'stdwin')               # Top of the STDWIN source tree
  91 READLINE = j(TOP, 'readline.' + ARCH)   # Top of the GNU Readline source tree
  92 SUN_X11 = '/usr/local/X11R5/lib/libX11.a'
  93
  94 # File names (usually no need to change)
  95 LIBP = [                                # Main Python libraries
  96         j(LIBINST, 'libPython.a'),
  97         j(LIBINST, 'libParser.a'),
  98         j(LIBINST, 'libObjects.a'),
  99         j(LIBINST, 'libModules.a')
 100        ]
 101 CONFIG_IN = j(LIBINST, 'config.c.in')   # Configuration source file
 102 FMAIN = j(LIBINST, 'frozenmain.c')      # Special main source file
 103
 104 # Libraries needed when linking.  First tuple item is built-in module
 105 # for which it is needed (or '*' for always), rest are ld arguments.
 106 # There is a separate list per architecture.
 107 libdeps_sgi = [ \
 108           ('stdwin',    j(STDWIN, 'Build/' + ARCH + '/x11/lib/lib.a')), \
 109           ('fl',        j(FORMS, 'FORMS/libforms.a'), '-lfm_s'), \
 110           ('*',         j(READLINE, 'libreadline.a'), '-ltermcap'), \
 111           ('al',        '-laudio'), \
 112           ('sv',        '-lsvideo', '-lXext'), \
 113           ('cd',        '-lcdaudio', '-lds'), \
 114           ('cl',        '-lcl'), \
 115           ('imgfile',   '-limage', '-lgutil', '-lm'), \
 116           ('mpz',       '/ufs/guido/src/gmp/libgmp.a'), \
 117           ('*',         '-lsun'), \
 118           ('*',         j(DL, 'libdl.a'), '-lmld'), \
 119           ('*',         '-lmpc'), \
 120           ('fm',        '-lfm_s'), \
 121           ('gl',        '-lgl_s', '-lX11_s'), \
 122           ('stdwin',    '-lX11_s'), \
 123           ('*',         '-lm'), \
 124           ('*',         '-lc_s'), \
 125           ]
 126 libdeps_sun4 = [ \
 127           ('*',         '-Bstatic'), \
 128           ('stdwin',    j(STDWIN, 'Build/' + ARCH + '/x11/lib/lib.a')), \
 129           ('*',         j(READLINE, 'libreadline.a')), \
 130           ('*',         '-lm'), \
 131           ('*',         j(DL_DLD,'libdl.a'), j(DLD,'libdld.a')), \
 132           ('*',         SUN_X11), \
 133           ('*',         '-ltermcap'), \
 134           ('*',         '-lc'), \
 135           ]
 136 libdeps_sequent = [ \
 137           ('*',         j(LIBINST, 'libreadline.a'), '-ltermcap'), \
 138           ('*',         '-lsocket'), \
 139           ('*',         '-linet'), \
 140           ('*',         '-lnsl'), \
 141           ('*',         '-lm'), \
 142           ('*',         '-lc'), \
 143           ]
 144 libdeps = eval('libdeps_' + ARCH)
 145
 146 ################################
 147 # END OF CONFIGURATION SECTION #
 148 ################################
 149
 150 # Exception used when scanfile fails
 151 NoSuchFile = 'NoSuchFile'
 152
 153 # Global options
 154 quiet = 0                               # -q
 155 verbose = 0                             # -v
 156 noexec = 0                              # -n
 157 nowrite = 0                             # -N
 158 ofile = 'a.out'                         # -o file
 159
 160 # Main program -- argument parsing etc.
 161 def main():
 162         global quiet, verbose, noexec, nowrite, ofile
 163         try:
 164                 opts, args = getopt.getopt(sys.argv[1:], 'nNo:qv')
 165         except getopt.error, msg:
 166                 usage(str(msg))
 167                 sys.exit(2)
 168         for o, a in opts:
 169                 if o == '-n': noexec = 1
 170                 if o == '-N': nowrite = 1
 171                 if o == '-o': ofile = a
 172                 if o == '-q': verbose = 0; quiet = 1
 173                 if o == '-v': verbose = verbose + 1; quiet = 0
 174         if len(args) < 1:
 175                 usage('please pass at least one file argument')
 176                 sys.exit(2)
 177         process(args[0], args[1:])
 178
 179 # Print usage message to stderr
 180 def usage(*msgs):
 181         sys.stdout = sys.stderr
 182         for msg in msgs: print msg
 183         print 'Usage: freeze [options] scriptfile [modulefile ...]'
 184         print '-n      : generate the files but don\'t compile and link'
 185         print '-N      : don\'t write frozen.c (do compile unless -n given)'
 186         print '-o file : binary output file (default a.out)'
 187         print '-q      : quiet (no messages at all except errors)'
 188         print '-v      : verbose (lots of extra messages)'
 189
 190 # Process the script file
 191 def process(filename, addmodules):
 192         global noexec
 193         #
 194         if not quiet: print 'Computing needed modules ...'
 195         todo = {}
 196         todo['__main__'] = filename
 197         for name in addmodules:
 198                 mod = os.path.basename(name)
 199                 if mod[-3:] == '.py': mod = mod[:-3]
 200                 todo[mod] = name
 201         try:
 202                 dict = closure(todo)
 203         except NoSuchFile, filename:
 204                 sys.stderr.write('Can\'t open file %s\n' % filename)
 205                 sys.exit(1)
 206         #
 207         mods = dict.keys()
 208         mods.sort()
 209         #
 210         if verbose:
 211                 print '%-15s %s' % ('Module', 'Filename')
 212                 for mod in mods:
 213                         print '%-15s %s' % (`mod`, dict[mod])
 214         #
 215         if not quiet: print 'Looking for dynamically linked modules ...'
 216         dlmodules = []
 217         objs = []
 218         libs = []
 219         for mod in mods:
 220                 if dict[mod][-2:] == '.o':
 221                         if verbose: print 'Found', mod, dict[mod]
 222                         dlmodules.append(mod)
 223                         objs.append(dict[mod])
 224                         libsname = dict[mod][:-2] + '.libs'
 225                         try:
 226                                 f = open(libsname, 'r')
 227                         except IOError:
 228                                 f = None
 229                         if f:
 230                                 libtext = f.read()
 231                                 f.close()
 232                                 for lib in string.split(libtext):
 233                                         if lib in libs: libs.remove(lib)
 234                                         libs.append(lib)
 235         #
 236         if not nowrite:
 237                 if not quiet: print 'Writing frozen.c ...'
 238                 writefrozen('frozen.c', dict)
 239         else:
 240                 if not quiet: print 'NOT writing frozen.c ...'
 241         #
 242 ##      if not dlmodules:
 243         if 0:
 244                 config = CONFIG
 245                 if not quiet: print 'Using existing', config, '...'
 246         else:
 247                 config = 'tmpconfig.c'
 248                 if nowrite:
 249                         if not quiet: print 'NOT writing config.c ...'
 250                 else:
 251                         if not quiet:
 252                                 print 'Writing config.c with dl modules ...'
 253                         f = open(CONFIG_IN, 'r')
 254                         g = open(config, 'w')
 255                         m1 = regex.compile('-- ADDMODULE MARKER 1 --')
 256                         m2 = regex.compile('-- ADDMODULE MARKER 2 --')
 257                         builtinmodules = []
 258                         stdmodules = ('sys', '__main__', '__builtin__',
 259                                       'marshal')
 260                         todomodules = builtinmodules + dlmodules
 261                         for mod in dict.keys():
 262                                 if dict[mod] == '<builtin>' and \
 263                                           mod not in stdmodules:
 264                                         builtinmodules.append(mod)
 265                         while 1:
 266                                 line = f.readline()
 267                                 if not line: break
 268                                 g.write(line)
 269                                 if m1.search(line) >= 0:
 270                                         if verbose: print 'Marker 1 ...'
 271                                         for mod in todomodules:
 272                                                 g.write('extern void init' + \
 273                                                   mod + '();\n')
 274                                 if m2.search(line) >= 0:
 275                                         if verbose: print 'Marker 2 ...'
 276                                         for mod in todomodules:
 277                                                 g.write('{"' + mod + \
 278                                                   '", init' + mod + '},\n')
 279                         g.close()
 280         #
 281         if not quiet:
 282                 if noexec: print 'Generating compilation commands ...'
 283                 else: print 'Starting compilation ...'
 284         defs = ['-DNO_MAIN', '-DUSE_FROZEN', '-DPYTHONPATH=\'"."\'']
 285         #
 286         incs = ['-I.', '-I' + INCLINST]
 287         if dict.has_key('stdwin'):
 288                 incs.append('-I' + j(STDWIN, 'H'))
 289         #
 290         srcs = [config, FMAIN]
 291         #
 292         if type(LIBP) == type(''):
 293                 libs.append(LIBP)
 294         else:
 295                 for lib in LIBP:
 296                         libs.append(lib)
 297         for item in libdeps:
 298                 m = item[0]
 299                 if m == '*' or dict.has_key(m):
 300                         for l in item[1:]:
 301                                 if l in libs: libs.remove(l)
 302                                 libs.append(l)
 303         #
 304         sts = 0
 305         #
 306         cmd = CC + ' -c'
 307         cmd = cmd + ' ' + string.join(defs)
 308         cmd = cmd + ' ' + string.join(incs)
 309         cmd = cmd + ' ' + string.join(srcs)
 310         print cmd
 311         #
 312         if not noexec:
 313                 sts = os.system(cmd)
 314                 if sts:
 315                         print 'Exit status', sts, '-- turning on -n'
 316                         noexec = 1
 317         #
 318         for s in srcs:
 319                 s = os.path.basename(s)
 320                 if s[-2:] == '.c': s = s[:-2]
 321                 o = s + '.o'
 322                 objs.insert(0, o)
 323         #
 324         cmd = CC
 325         cmd = cmd + ' ' + string.join(objs)
 326         cmd = cmd + ' ' + string.join(libs)
 327         cmd = cmd + ' -o ' + ofile
 328         print cmd
 329         #
 330         if not noexec:
 331                 sts = os.system(cmd)
 332                 if sts:
 333                         print 'Exit status', sts
 334                 else:
 335                         print 'Done.'
 336         #
 337         if not quiet and not noexec and sts == 0:
 338                 print 'Note: consider this:'; print '\tstrip', ofile
 339         #
 340         sys.exit(sts)
 341
 342
 343 # Generate code for a given module
 344 def makecode(filename):
 345         if filename[-2:] == '.o':
 346                 return None
 347         try:
 348                 f = open(filename, 'r')
 349         except IOError:
 350                 return None
 351         if verbose: print 'Making code from', filename, '...'
 352         text = f.read()
 353         code = compile(text, filename, 'exec')
 354         f.close()
 355         return marshal.dumps(code)
 356
 357
 358 # Write the C source file containing the frozen Python code
 359 def writefrozen(filename, dict):
 360         f = open(filename, 'w')
 361         codelist = []
 362         for mod in dict.keys():
 363                 codestring = makecode(dict[mod])
 364                 if codestring is not None:
 365                         codelist.append((mod, codestring))
 366         write = sys.stdout.write
 367         save_stdout = sys.stdout
 368         try:
 369                 sys.stdout = f
 370                 for mod, codestring in codelist:
 371                         if verbose:
 372                                 write('Writing initializer for %s\n'%mod)
 373                         print 'static unsigned char M_' + mod + '[' + \
 374                                   str(len(codestring)) + '+1] = {'
 375                         for i in range(0, len(codestring), 16):
 376                                 for c in codestring[i:i+16]:
 377                                         print str(ord(c)) + ',',
 378                                 print
 379                         print '};'
 380                 print 'struct frozen {'
 381                 print '  char *name;'
 382                 print '  unsigned char *code;'
 383                 print '  int size;'
 384                 print '} frozen_modules[] = {'
 385                 for mod, codestring in codelist:
 386                         print '  {"' + mod + '",',
 387                         print 'M_' + mod + ',',
 388                         print str(len(codestring)) + '},'
 389                 print '  {0, 0, 0} /* sentinel */'
 390                 print '};'
 391         finally:
 392                 sys.stdout = save_stdout
 393         f.close()
 394
 395
 396 # Determine the names and filenames of the modules imported by the
 397 # script, recursively.  This is done by scanning for lines containing
 398 # import statements.  (The scanning has only superficial knowledge of
 399 # Python syntax and no knowledge of semantics, so in theory the result
 400 # may be incorrect -- however this is quite unlikely if you don't
 401 # intentionally obscure your Python code.)
 402
 403 # Compute the closure of scanfile() -- special first file because of script
 404 def closure(todo):
 405         done = {}
 406         while todo:
 407                 newtodo = {}
 408                 for modname in todo.keys():
 409                         if not done.has_key(modname):
 410                                 filename = todo[modname]
 411                                 if filename is None:
 412                                         filename = findmodule(modname)
 413                                 done[modname] = filename
 414                                 if filename in ('<builtin>', '<unknown>'):
 415                                         continue
 416                                 modules = scanfile(filename)
 417                                 for m in modules:
 418                                         if not done.has_key(m):
 419                                                 newtodo[m] = None
 420                 todo = newtodo
 421         return done
 422
 423 # Scan a file looking for import statements
 424 importstr = '\(^\|:\)[ \t]*import[ \t]+\([a-zA-Z0-9_, \t]+\)'
 425 fromstr   = '\(^\|:\)[ \t]*from[ \t]+\([a-zA-Z0-9_]+\)[ \t]+import[ \t]+'
 426 isimport = regex.compile(importstr)
 427 isfrom = regex.compile(fromstr)
 428 def scanfile(filename):
 429         allmodules = {}
 430         try:
 431                 f = open(filename, 'r')
 432         except IOError, msg:
 433                 raise NoSuchFile, filename
 434         while 1:
 435                 line = f.readline()
 436                 if not line: break # EOF
 437                 while line[-2:] == '\\\n': # Continuation line
 438                         line = line[:-2] + ' '
 439                         line = line + f.readline()
 440                 if isimport.search(line) >= 0:
 441                         rawmodules = isimport.group(2)
 442                         modules = string.splitfields(rawmodules, ',')
 443                         for i in range(len(modules)):
 444                                 modules[i] = string.strip(modules[i])
 445                 elif isfrom.search(line) >= 0:
 446                         modules = [isfrom.group(2)]
 447                 else:
 448                         continue
 449                 for mod in modules:
 450                         allmodules[mod] = None
 451         f.close()
 452         return allmodules.keys()
 453
 454 # Find the file containing a module, given its name; None if not found
 455 builtins = sys.builtin_module_names + ['sys']
 456 def findmodule(modname):
 457         if modname in builtins: return '<builtin>'
 458         for dirname in sys.path:
 459                 dlfullname = os.path.join(dirname, modname + 'module.o')
 460                 try:
 461                         f = open(dlfullname, 'r')
 462                 except IOError:
 463                         f = None
 464                 if f:
 465                         f.close()
 466                         return dlfullname
 467                 fullname = os.path.join(dirname, modname + '.py')
 468                 try:
 469                         f = open(fullname, 'r')
 470                 except IOError:
 471                         continue
 472                 f.close()
 473                 return fullname
 474         if not quiet:
 475                 sys.stderr.write('Warning: module %s not found\n' % modname)
 476         return '<unknown>'
 477
 478
 479 # Call the main program
 480 main()