funguloids/mpak.py

   1 #!/usr/bin/env python
   2 """
   3         MPAK package handling utility
   4         Version 1.4 (Python-implementation)
   5         Copyright (c) 2008, Mika Halttunen. <http://www.mhgames.co.nr>
   6
   7         This command line tool allows creation and extraction of MPAK (.mpk) packages used
   8         in several of my games. MPAK is a simple WAD-like file format of mine, that allows storing
   9         the game data in one single .mpk file. I originally had a very crude command line program
  10         bit like this one (written in C++), and decided to write this Python-implementation as
  11         an Python-programming excercise. So, it's my first Python program. :)
  12
  13         Version history:
  14         v1.4: The first Python version
  15         v1.0 -- v1.31: The original C++ implementation
  16 """
  17 import getopt, sys
  18 import os
  19 import traceback
  20 import struct
  21 import binascii
  22 import fnmatch
  23 import shutil
  24 from ctypes import c_uint32
  25
  26 def usage():
  27         """
  28         Prints the program usage.
  29         """
  30         print "MPAK package handling utility"
  31         print "Version 1.4 (Python-implementation)"
  32         print "Copyright (c) 2008, Mika Halttunen."
  33         print ""
  34         print "Usage:", sys.argv[0],"[switch]","-f pakfile.mpk","[file1]","[file2]", "[...]", "[fileN]"
  35         print "where [switch] is one of the following:"
  36         print "  -f, --file=FILE   Use package FILE"
  37         print "  -c, --create      Create a new package with files 'file1' to 'fileN'"
  38         print "  -l, --list        List the files from given package"
  39         print "  -e, --extract     Extract all files (by default) from given package. If you"
  40         print "                    want to only extract some specific files, you can name"
  41         print "                    them individually, and/or use wildcards (i.e. *.png)."
  42         print "                    You can supply path where to extract with -p."
  43         print "  -p, --path=PATH   Extract to PATH (created if doesn't exist)"
  44         print "  -h, --help        Print this usage text"
  45         print ""
  46
  47
  48 def errorMsg(msg):
  49         """
  50         Prints an error message and exits.
  51         """
  52         try:
  53                 pos = traceback.extract_stack(limit=2)
  54                 if pos:
  55                         print "ERROR: In %s:%s, line %d:" % (pos[0][0], pos[0][2], pos[0][1])
  56                 else: print "ERROR:"
  57                 print "\t",msg
  58         except:
  59                 if __debug__:
  60                         traceback.print_exc()
  61                 pass
  62         sys.exit(2)
  63
  64
  65 def separator():
  66         """
  67         Prints the separator line.
  68         """
  69         print "-"*75
  70
  71
  72 def computeCRC(file, offset):
  73         """
  74         Computes the CRC32 for the file, starting at given offset.
  75         """
  76         f = open(file, "rb")
  77         f.seek(offset)
  78         crc = 0
  79
  80         # Compute a running CRC32 for the file in 16kb chunks
  81         while True:
  82                 buffer = f.read(16384)
  83                 if buffer == "": break          # End of file
  84
  85                 crc = binascii.crc32(buffer, crc)
  86
  87         f.close()
  88         return crc
  89
  90
  91 def createPackage(pakFile, files):
  92         """
  93         Creates a new MPAK package.
  94
  95         This copies the given files into the new package file, writes the file table
  96         and closes the package. MPAK doesn't support adding new files to an existing
  97         package.
  98         """
  99         print "Creating '%s'.." % pakFile
 100         if len(files) < 1: errorMsg("No input files specified!")
 101         separator()
 102
 103         # Open the package file for writing
 104         out = open(pakFile, "wb")
 105
 106         # Write the header and reserve 4+4 bytes for CRC32 and file table offset
 107         out.write("MPK1")
 108         out.write("."*8)
 109
 110         # Write each file
 111         package = { "fileNames": [], "fileOffsets": [] }
 112         count = 0
 113         for file in files:
 114                 # Get the basename
 115                 filename = os.path.basename(file)
 116                 print " <",filename,"...",
 117                 package["fileNames"].append(filename)
 118
 119                 # Get the file size in bytes
 120                 stats = os.stat(file)
 121
 122                 # Store the current offset
 123                 package["fileOffsets"].append(out.tell())
 124
 125                 # Open the file and copy its contents
 126                 f = open(file, "rb")
 127                 shutil.copyfileobj(f, out, 16384)
 128                 f.close()
 129
 130                 print "OK. (%.1f KB)" % (stats.st_size / 1024.0)
 131                 count = count + 1
 132
 133         separator()
 134
 135         # Grab the file table offset and write the table
 136         ftOffset = out.tell()
 137
 138         # Write the number of files
 139         out.write(struct.pack("<L", count))
 140
 141         # Write the file information
 142         for i in range(count):
 143                 # File name length
 144                 length = len(package["fileNames"][i]) + 1
 145                 out.write(struct.pack("B", length))
 146                 # File name, plus one zero for the C++ implementation
 147                 out.write(package["fileNames"][i])
 148                 out.write(struct.pack("B", 0))
 149                 # File offset
 150                 out.write(struct.pack("<L", package["fileOffsets"][i]))
 151
 152         # Update the header to have the correct file table offset
 153         out.seek(8)
 154         out.write(struct.pack("<L", ftOffset))
 155
 156         # Compute the CRC32 and write it to the header
 157         out.flush()
 158         crc32 = c_uint32(0)
 159         crc32.value = computeCRC(pakFile, 8)
 160         out.seek(4)
 161         out.write(struct.pack("<L", crc32.value))
 162
 163         print "Added %d files to %s" % (count, pakFile)
 164         print "Package '%s' created successfully. CRC32 checksum is %s." % (pakFile, hex(crc32.value))
 165         out.close()
 166
 167
 168 def readPackage(pakFile):
 169         """
 170         Opens the given MPAK package, reads its information and stores it to a
 171         package dictionary. Returns the dictionary.
 172         """
 173         packageInfo = { "filename": pakFile }
 174
 175         f = open(pakFile, 'rb')
 176         if f.read(4) != "MPK1": errorMsg("Unsupported file format!")
 177
 178         # Read the CRC32 checksum and the file table header offset
 179         buffer = f.read(8)
 180         crc32, headerOffset = struct.unpack("<LL", buffer)
 181         crc32 = int(crc32)
 182         packageInfo["crc"] = crc32
 183
 184         # Check that the CRC32 matches
 185         checksum = c_uint32(0)
 186         checksum.value = computeCRC(pakFile, 8)
 187         if checksum.value != crc32:
 188                 f.close()
 189                 errorMsg("Checksum doesn't match; perhaps a corrupted package?")
 190
 191         # Seek to the file table, and read the number of files
 192         f.seek(headerOffset)
 193         numFiles = struct.unpack("<L", f.read(4))[0]
 194         packageInfo["numFiles"] = numFiles
 195
 196         # Read the file information
 197         fileNames = []
 198         fileOffsets = []
 199         for i in range(numFiles):
 200                 namelen = struct.unpack("B", f.read(1))[0]
 201                 file = f.read(namelen)
 202                 offset = struct.unpack("<L", f.read(4))[0]
 203                 fileNames.append(file[:-1])             # Remove the trailing null character
 204                 fileOffsets.append(offset)
 205
 206         # Compute the file sizes from the offsets
 207         fileSizes = []
 208         for i in range(numFiles-1):
 209                 fileSizes.append(fileOffsets[i+1] - fileOffsets[i])
 210         fileSizes.append(headerOffset - fileOffsets[numFiles-1])
 211
 212         # Store the information
 213         packageInfo["fileNames"] = fileNames
 214         packageInfo["fileOffsets"] = fileOffsets
 215         packageInfo["fileSizes"] = fileSizes
 216         f.close()
 217         return packageInfo
 218
 219
 220 def listPackage(pakFile):
 221         """
 222         Lists the contents of a MPAK package.
 223         """
 224         print "Listing '%s'.." % pakFile
 225         package = readPackage(pakFile)
 226
 227         # Print the listing
 228         numFiles = package["numFiles"]
 229         print "'%s' (CRC32: %s) contains %d files:" % (pakFile, hex(package["crc"]), numFiles)
 230         print ""
 231         print " NUM : FILE                           : SIZE(KB)   : OFFSET"
 232         separator()
 233         for i in range(numFiles):
 234                 print " %3d : %30s : %-10.1f : (at %s)" % (i+1, package["fileNames"][i], package["fileSizes"][i] / 1024.0, hex(package["fileOffsets"][i]))
 235
 236         separator()
 237         print " NUM : FILE                           : SIZE(KB)   : OFFSET"
 238
 239
 240 def extractPackage(pakFile, path, filters):
 241         """
 242         Extracts files from a package to given path.
 243
 244         By default extracts all the files. Can be given list of wildcards (i.e. *.png) to
 245         extract only the files that match given wildcards. Wildcards can also be file names
 246         from the package.
 247
 248         The given path is created if it doesn't exist.
 249         If the path is just a single directory name, it's assumed to exist in the current
 250         working directory.
 251         """
 252         print "Extracting files from '%s' to %s.." % (pakFile, path)
 253         package = readPackage(pakFile)
 254
 255         # Try to create the path if it doesn't exist
 256         path = os.path.abspath(path)
 257         if not os.path.exists(path):
 258                 print "Path",path,"doesn't exist, creating it.."
 259                 try:
 260                         os.makedirs(path)
 261                 except:
 262                         errorMsg("Unable to create directory " + path + "!");
 263
 264         separator()
 265
 266         # Open the file, and extract all the individual files from it
 267         count = 0
 268         f = open(pakFile, "rb")
 269         for i in range(package["numFiles"]):
 270                 # Test if the file name matches the given wildcard
 271                 if len(filters) > 0:
 272                         for filter in filters:
 273                                 if fnmatch.fnmatch(package["fileNames"][i], filter):
 274                                         break
 275                         else: continue
 276
 277                 print " >", package["fileNames"][i],"...",
 278                 # Seek to the correct offset
 279                 f.seek(package["fileOffsets"][i])
 280
 281                 # Open a new file for writing, and write the file out in 16kb chunks
 282                 out = open(os.path.join(path, package["fileNames"][i]), "wb")
 283                 bytesWritten = 0
 284                 bytesTotal = package["fileSizes"][i];
 285                 while True:
 286                         # We have to watch not to write too much
 287                         bytesLeft = bytesTotal - bytesWritten
 288                         if bytesLeft > 16384: bytesLeft = 16384
 289
 290                         buffer = f.read(bytesLeft)
 291                         out.write(buffer)
 292                         bytesWritten = bytesWritten + bytesLeft
 293
 294                         if bytesWritten == bytesTotal:
 295                                 break
 296
 297                 out.close()
 298                 print "OK."
 299                 count = count + 1
 300
 301         f.close()
 302         separator()
 303         print "%d (of %d) files extracted to %s." % (count, package["numFiles"], path)
 304
 305
 306 def main():
 307         """
 308         Main method.
 309         """
 310         try:
 311                 # Get the optiosn
 312                 opts, args = getopt.getopt(sys.argv[1:], "f:clep:h", ["file=", "create", "list", "extract", "path=", "help"])
 313         except getopt.GetoptError, err:
 314                 # Print the program usage and exit
 315                 print "ERROR:", str(err)
 316                 usage()
 317                 sys.exit(2)
 318
 319         extractPath = os.getcwd()
 320         pakFile = None
 321         action = None
 322
 323         # Handle the options
 324         for o, a in opts:
 325                 if o in ("-f", "--file"):
 326                         pakFile = a                                     # Grab the pakfile
 327                 elif o in ("-c", "--create"):
 328                         action = "create"
 329                 elif o in ("-l", "--list"):
 330                         action = "list"
 331                 elif o in ("-e", "--extract"):
 332                         action = "extract"
 333                 elif o in ("-p", "--path"):
 334                         extractPath = a                         # Grab the path
 335                 elif o in ("-h", "--help"):
 336                         usage()
 337                         sys.exit()
 338                 else:
 339                         assert False, "Unhandled option"
 340
 341         # Check that we got a pakfile
 342         if pakFile == None:
 343                 usage()
 344                 sys.exit(2)
 345
 346         if action == "create": createPackage(pakFile, args)
 347         elif action == "list": listPackage(pakFile)
 348         elif action == "extract": extractPackage(pakFile, extractPath, args)
 349         else: usage()
 350         sys.exit()
 351
 352 if __name__ == "__main__":
 353         main()
 354