tools/update-packaging/make_incremental_updates.py

   1 import os
   2 import shutil
   3 import sha
   4 import sha
   5 from os.path import join, getsize
   6 from stat import *
   7 import re
   8 import sys
   9 import getopt
  10 import time
  11 import datetime
  12 import bz2
  13 import string
  14 import tempfile
  15
  16 class PatchInfo:
  17     """ Represents the meta-data associated with a patch
  18         work_dir = working dir where files are stored for this patch
  19         archive_files = list of files to include in this patch
  20         manifest = set of patch instructions
  21         file_exclusion_list =
  22         files to exclude from this patch. names without slashes will be
  23                               excluded anywhere in the directory hiearchy.   names with slashes
  24                               will only be excluded at that exact path
  25         """
  26     def __init__(self, work_dir, file_exclusion_list, path_exclusion_list):
  27         self.work_dir=work_dir
  28         self.archive_files=[]
  29         self.manifest=[]
  30         self.file_exclusion_list=file_exclusion_list
  31         self.path_exclusion_list=path_exclusion_list
  32
  33     def append_add_instruction(self, filename):
  34         """ Appends an add instruction for this patch.
  35             if the filename starts with extensions/ adds an add-if instruction
  36             to test the existence of the subdirectory.  This was ported from
  37             mozilla/tools/update-packaging/common.sh/make_add_instruction
  38         """
  39         if filename.startswith("extensions/"):
  40             testdir = "extensions/"+filename.split("/")[1]  # Dir immediately following extensions is used for the test
  41             self.manifest.append('add-if "'+testdir+'" "'+filename+'"')
  42         else:
  43             self.manifest.append('add "'+filename+'"')
  44
  45     def append_patch_instruction(self, filename, patchname):
  46         """ Appends an patch instruction for this patch.
  47
  48             filename = file to patch
  49             patchname = patchfile to apply to file
  50
  51             if the filename starts with extensions/ adds a patch-if instruction
  52             to test the existence of the subdirectory.
  53             if the filename starts with searchplugins/ add a add-if instruction for the filename
  54             This was ported from
  55             mozilla/tools/update-packaging/common.sh/make_patch_instruction
  56         """
  57         if filename.startswith("extensions/"):
  58             testdir = "extensions/"+filename.split("/")[1]
  59             self.manifest.append('patch-if "'+testdir+'" "'+patchname+'" "'+filename+'"')
  60         elif filename.startswith("searchplugins/"):
  61             self.manifest.append('patch-if "'+filename+'" "'+patchname+'" "'+filename+'"')
  62         else:
  63             self.manifest.append('patch "'+patchname+'" "'+filename+'"')
  64
  65     def append_remove_instruction(self, filename):
  66         """ Appends an remove instruction for this patch.
  67             This was ported from
  68             mozilla/tools/update-packaging/common.sh/make_remove_instruction
  69         """
  70         self.manifest.append('remove "'+filename+'"')
  71
  72     def create_manifest_file(self):
  73         """ Createst the manifest file into to the root of the work_dir """
  74         manifest_file_path = os.path.join(self.work_dir,"update.manifest")
  75         manifest_file = open(manifest_file_path, "w")
  76         manifest_file.writelines(string.join(self.manifest, '\n'))
  77         manifest_file.writelines("\n")
  78         manifest_file.close()
  79
  80         bzip_file(manifest_file_path)
  81         self.archive_files.append('"update.manifest"')
  82
  83
  84     def build_marfile_entry_hash(self, root_path):
  85          """ Iterates through the root_path, creating a MarFileEntry for each file
  86              in that path.  Excluseds any filenames in the file_exclusion_list
  87          """
  88          mar_entry_hash = {}
  89          filename_set = set()
  90          for root, dirs, files in os.walk(root_path):
  91              for name in files:
  92                  # filename is relative path from root directory
  93                  partial_path = root[len(root_path)+1:]
  94                  if name not in self.file_exclusion_list:
  95                      filename = os.path.join(partial_path, name)
  96                      if "/"+filename not in self.path_exclusion_list:
  97                          mar_entry_hash[filename]=MarFileEntry(root_path, filename)
  98                          filename_set.add(filename)
  99          return mar_entry_hash, filename_set
 100
 101
 102 class MarFileEntry:
 103     """Represents a file inside a Mozilla Archive Format (MAR)
 104         abs_path = abspath to the the file
 105         name =  relative path within the mar.  e.g.
 106           foo.mar/dir/bar.txt extracted into /tmp/foo:
 107             abs_path=/tmp/foo/dir/bar.txt
 108             name = dir/bar.txt
 109     """
 110     def __init__(self, root, name):
 111         """root = path the the top of the mar
 112            name = relative path within the mar"""
 113         self.name=name
 114         self.abs_path=os.path.join(root,name)
 115         self.sha_cache=None
 116
 117     def __str__(self):
 118         return 'Name: %s FullPath: %s' %(self.name,self.abs_path)
 119
 120     def calc_file_sha_digest(self, filename):
 121         """ Returns sha digest of given filename"""
 122         file_content = open(filename, 'r').read()
 123         return sha.new(file_content).digest()
 124
 125     def sha(self):
 126         """ Returns sha digest of file repreesnted by this _marfile_entry\x10"""
 127         if not self.sha_cache:
 128             self.sha_cache=self.calc_file_sha_digest(self.abs_path)
 129         return self.sha_cache
 130
 131 def exec_shell_cmd(cmd):
 132     """Execs shell cmd and raises an exception if the cmd fails"""
 133     if (os.system(cmd)):
 134         raise Exception, "cmd failed "+cmd
 135
 136
 137 def copy_file(src_file_abs_path, dst_file_abs_path):
 138     """ Copies src to dst creating any parent dirs required in dst first """
 139     dst_file_dir=os.path.dirname(dst_file_abs_path)
 140     if not os.path.exists(dst_file_dir):
 141          os.makedirs(dst_file_dir)
 142     # Copy the file over
 143     shutil.copy2(src_file_abs_path, dst_file_abs_path)
 144
 145 def bzip_file(filename):
 146     """ Bzip's the file in place.  The original file is replaced with a bzip'd version of itself
 147         assumes the path is absolute"""
 148     exec_shell_cmd('bzip2 -z9 "' + filename+'"')
 149     os.rename(filename+".bz2",filename)
 150
 151 def bunzip_file(filename):
 152     """ Bzip's the file in palce.   The original file is replaced with a bunzip'd version of itself.
 153         doesn't matter if the filename ends in .bz2 or not"""
 154     if not filename.endswith(".bz2"):
 155         os.rename(filename, filename+".bz2")
 156         filename=filename+".bz2"
 157     exec_shell_cmd('bzip2 -d "' + filename+'"')
 158
 159
 160 def extract_mar(filename, work_dir):
 161     """ Extracts the marfile intot he work_dir
 162         assumes work_dir already exists otherwise will throw osError"""
 163     print "Extracting "+filename+" to "+work_dir
 164     saved_path = os.getcwd()
 165     try:
 166         os.chdir(work_dir)
 167         exec_shell_cmd("mar -x "+filename)
 168     finally:
 169         os.chdir(saved_path)
 170
 171 def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info):
 172     """ Creates the partial patch file and manifest entry for the pair of files passed in
 173     """
 174     if not (from_marfile_entry.sha(),to_marfile_entry.sha()) in shas:
 175         print "diffing: " + from_marfile_entry.name
 176
 177         #bunzip to/from
 178         bunzip_file(from_marfile_entry.abs_path)
 179         bunzip_file(to_marfile_entry.abs_path)
 180
 181         # The patch file will be created in the working directory with the
 182         # name of the file in the mar + .patch
 183         patch_file_abs_path = os.path.join(patch_info.work_dir,from_marfile_entry.name+".patch")
 184         patch_file_dir=os.path.dirname(patch_file_abs_path)
 185         if not os.path.exists(patch_file_dir):
 186             os.makedirs(patch_file_dir)
 187
 188         # Create bzip'd patch file
 189         exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
 190         bzip_file(patch_file_abs_path)
 191
 192         # Create bzip's full file
 193         full_file_abs_path =  os.path.join(patch_info.work_dir, to_marfile_entry.name)
 194         shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
 195         bzip_file(full_file_abs_path)
 196
 197         ## TOODO NEED TO ADD HANDLING FOR FORCED UPDATES
 198         if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
 199             # Patch is smaller than file.  Remove the file and add patch to manifest
 200             os.remove(full_file_abs_path)
 201             file_in_manifest_name = from_marfile_entry.name+".patch"
 202             file_in_manifest_abspath = patch_file_abs_path
 203             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 204         else:
 205             # File is smaller than patch.  Remove the patch and add file to manifest
 206             os.remove(patch_file_abs_path)
 207             file_in_manifest_name = from_marfile_entry.name
 208             file_in_manifest_abspath = full_file_abs_path
 209             patch_info.append_add_instruction(file_in_manifest_name)
 210
 211         shas[from_marfile_entry.sha(),to_marfile_entry.sha()] = (file_in_manifest_name,file_in_manifest_abspath)
 212         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
 213     else:
 214         print "skipping diff: " + from_marfile_entry.name
 215         filename, src_file_abs_path = shas[from_marfile_entry.sha(),to_marfile_entry.sha()]
 216         # We've already calculated the patch for this pair of files.
 217         if (filename.endswith(".patch")):
 218             # Patch was smaller than file - add patch instruction to manifest
 219             file_in_manifest_name = to_marfile_entry.name+'.patch';
 220             patch_info.append_patch_instruction(to_marfile_entry.name, file_in_manifest_name)
 221         else:
 222             # File was smaller than file - add file to manifest
 223             file_in_manifest_name = to_marfile_entry.name
 224             patch_info.append_add_instruction(file_in_manifest_name)
 225         # Copy the pre-calculated file into our new patch work aread
 226         copy_file(src_file_abs_path, os.path.join(patch_info.work_dir, file_in_manifest_name))
 227         patch_info.archive_files.append('"'+file_in_manifest_name+'"')
 228
 229 def create_add_patch_for_file(to_marfile_entry, patch_info):
 230     """  Copy the file to the working dir, add the add instruction, and add it to the list of archive files """
 231     print "Adding New File " + to_marfile_entry.name
 232     copy_file(to_marfile_entry.abs_path, os.path.join(patch_info.work_dir, to_marfile_entry.name))
 233     patch_info.append_add_instruction(to_marfile_entry.name)
 234     patch_info.archive_files.append('"'+to_marfile_entry.name+'"')
 235
 236 def process_explicit_remove_files(dir_path, patch_info):
 237     """ Looks for a 'removed-files' file in the dir_path.  If the removed-files does not exist
 238     this will throw.  If found adds the removed-files
 239     found in that file to the patch_info"""
 240
 241     # Windows and linux have this file at the root of the dir
 242     list_file_path = os.path.join(dir_path, "removed-files")
 243     prefix=""
 244     if not os.path.exists(list_file_path):
 245         # Mac has is in Contents/MacOS/
 246         prefix= "Contents/MacOS"
 247         list_file_path = os.path.join(dir_path, prefix+"/removed-files")
 248
 249     if (os.path.exists(list_file_path)):
 250         list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
 251
 252         for line in list_file:
 253             line = line.strip()
 254             # Exclude any blank lines or any lines ending with a slash, which indicate
 255             # directories.  The updater doesn't know how to remove entire directories.
 256             if line and not line.endswith("/"):
 257                 patch_info.append_remove_instruction(os.path.join(prefix,line))
 258
 259 def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch_info):
 260     """ Builds a partial patch by comparing the files in from_dir_path to thoes of to_dir_path"""
 261     # Cannocolize the paths for safey
 262     from_dir_path = os.path.abspath(from_dir_path)
 263     to_dir_path = os.path.abspath(to_dir_path)
 264     # First create a hashtable of the from  and to directories
 265     from_dir_hash,from_dir_set = patch_info.build_marfile_entry_hash(from_dir_path)
 266     to_dir_hash,to_dir_set = patch_info.build_marfile_entry_hash(to_dir_path)
 267
 268     # Files which exist in both sets need to be patched
 269     patch_filenames = list(from_dir_set.intersection(to_dir_set))
 270     patch_filenames.sort()
 271     for filename in patch_filenames:
 272         from_marfile_entry = from_dir_hash[filename]
 273         to_marfile_entry = to_dir_hash[filename]
 274         if from_marfile_entry.sha() != to_marfile_entry.sha():
 275             # Not the same - calculate a patch
 276             create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, patch_info)
 277
 278     # files in from_dir not in to_dir need to be removed
 279     remove_filenames = list(from_dir_set - to_dir_set)
 280     remove_filenames.sort()
 281     for filename in remove_filenames:
 282         patch_info.append_remove_instruction(from_dir_hash[filename].name)
 283
 284     # files in to_dir not in from_dir need to added
 285     add_filenames = list(to_dir_set - from_dir_set)
 286     add_filenames.sort()
 287     for filename in add_filenames:
 288         create_add_patch_for_file(to_dir_hash[filename], patch_info)
 289
 290     process_explicit_remove_files(to_dir_path, patch_info)
 291
 292     # Construct Manifest file
 293     patch_info.create_manifest_file()
 294
 295     # And construct the mar
 296     mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
 297     exec_shell_cmd(mar_cmd)
 298
 299     # Copy mar to final destination
 300     patch_file_dir = os.path.split(patch_filename)[0]
 301     if not os.path.exists(patch_file_dir):
 302         os.makedirs(patch_file_dir)
 303     shutil.copy2(os.path.join(patch_info.work_dir,"output.mar"), patch_filename)
 304     return patch_filename
 305
 306 def usage():
 307     print "-h for help"
 308     print "-f for patchlist_file"
 309
 310 def get_buildid(work_dir, platform):
 311     """ extracts buildid from MAR
 312         TODO: this should handle 1.8 branch too
 313     """
 314     if platform == 'mac':
 315       ini = '%s/Contents/MacOS/application.ini' % work_dir
 316     else:
 317       ini = '%s/application.ini' % work_dir
 318     if not os.path.exists(ini):
 319         print 'WARNING: application.ini not found, cannot find build ID'
 320         return ''
 321     file = bz2.BZ2File(ini)
 322     for line in file:
 323       if line.find('BuildID') == 0:
 324         return line.strip().split('=')[1]
 325     print 'WARNING: cannot find build ID in application.ini'
 326     return ''
 327
 328 def decode_filename(filepath):
 329     """ Breaks filename/dir structure into component parts based on regex
 330         for example: firefox-3.0b3pre.en-US.linux-i686.complete.mar
 331         Or linux-i686/en-US/firefox-3.0b3.complete.mar
 332         Returns dict with keys product, version, locale, platform, type
 333     """
 334     try:
 335       m = re.search(
 336         '(?P<product>\w+)(-)(?P<version>\w+\.\w+)(\.)(?P<locale>.+?)(\.)(?P<platform>.+?)(\.)(?P<type>\w+)(.mar)',
 337       os.path.basename(filepath))
 338       return m.groupdict()
 339     except Exception, exc:
 340       try:
 341         m = re.search(
 342           '(?P<platform>.+?)\/(?P<locale>.+?)\/(?P<product>\w+)-(?P<version>\w+\.\w+)\.(?P<type>\w+).mar',
 343         filepath)
 344         return m.groupdict()
 345       except:
 346         raise Exception("could not parse filepath %s: %s" % (filepath, exc))
 347
 348 def create_partial_patches(patches):
 349     """ Given the patches generates a set of partial patches"""
 350     shas = {}
 351
 352     work_dir_root = None
 353     metadata = []
 354     try:
 355         work_dir_root = tempfile.mkdtemp('-fastmode', 'tmp', os.getcwd())
 356         print "Building patches using work dir: %s" % (work_dir_root)
 357
 358         # Iterate through every patch set in the patch file
 359         patch_num = 1
 360         for patch in patches:
 361             startTime = time.time()
 362
 363             from_filename,to_filename,patch_filename,forced_updates = patch.split(",")
 364             from_filename,to_filename,patch_filename = os.path.abspath(from_filename),os.path.abspath(to_filename),os.path.abspath(patch_filename)
 365
 366             # Each patch iteration uses its own work dir
 367             work_dir = os.path.join(work_dir_root,str(patch_num))
 368             os.mkdir(work_dir)
 369
 370             # Extract from mar into from dir
 371             work_dir_from =  os.path.join(work_dir,"from");
 372             os.mkdir(work_dir_from)
 373             extract_mar(from_filename,work_dir_from)
 374             from_decoded = decode_filename(from_filename)
 375             from_buildid = get_buildid(work_dir_from, from_decoded['platform'])
 376             from_shasum = sha.sha(open(from_filename).read()).hexdigest()
 377             from_size = str(os.path.getsize(to_filename))
 378
 379             # Extract to mar into to dir
 380             work_dir_to =  os.path.join(work_dir,"to")
 381             os.mkdir(work_dir_to)
 382             extract_mar(to_filename, work_dir_to)
 383             to_decoded = decode_filename(from_filename)
 384             to_buildid = get_buildid(work_dir_to, to_decoded['platform'])
 385             to_shasum = sha.sha(open(to_filename).read()).hexdigest()
 386             to_size = str(os.path.getsize(to_filename))
 387
 388             mar_extract_time = time.time()
 389
 390             partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['channel-prefs.js','update.manifest','removed-files'],['/readme.txt']))
 391             partial_buildid = to_buildid
 392             partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
 393             partial_size = str(os.path.getsize(partial_filename))
 394
 395             metadata.append({
 396              'to_filename': os.path.basename(to_filename),
 397              'from_filename': os.path.basename(from_filename),
 398              'partial_filename': os.path.basename(partial_filename),
 399              'to_buildid':to_buildid,
 400              'from_buildid':from_buildid,
 401              'to_sha1sum':to_shasum,
 402              'from_sha1sum':from_shasum,
 403              'partial_sha1sum':partial_shasum,
 404              'to_size':to_size,
 405              'from_size':from_size,
 406              'partial_size':partial_size,
 407              'to_version':to_decoded['version'],
 408              'from_version':from_decoded['version'],
 409              'locale':from_decoded['locale'],
 410              'platform':from_decoded['platform'],
 411             })
 412             print "done with patch %s/%s time (%.2fs/%.2fs/%.2fs) (mar/patch/total)" % (str(patch_num),str(len(patches)),mar_extract_time-startTime,time.time()-mar_extract_time,time.time()-startTime)
 413             patch_num += 1
 414         return metadata
 415     finally:
 416         # If we fail or get a ctrl-c during run be sure to clean up temp dir
 417         if (work_dir_root and os.path.exists(work_dir_root)):
 418             shutil.rmtree(work_dir_root)
 419
 420 def main(argv):
 421     patchlist_file = None
 422     try:
 423          opts, args = getopt.getopt(argv, "hf:", ["help", "patchlist_file="])
 424          for opt, arg in opts:
 425             if opt in ("-h", "--help"):
 426                 usage()
 427                 sys.exit()
 428             elif opt in ("-f", "--patchlist_file"):
 429                 patchlist_file = arg
 430     except getopt.GetoptError:
 431           usage()
 432           sys.exit(2)
 433
 434     if not patchlist_file:
 435         usage()
 436         sys.exit(2)
 437
 438     patches = []
 439     f = open(patchlist_file, 'r')
 440     for line in f.readlines():
 441         patches.append(line)
 442     f.close()
 443     create_partial_patches(patches)
 444
 445 if __name__ == "__main__":
 446     main(sys.argv[1:])
 447