ACE/bin/make_release.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # @file make_release.py
   5 # @author William R. Otte <wotte@dre.vanderbilt.edu>
   6 #
   7 # Packaging script for ACE/TAO
   8 #
   9 # Requires Python 3.4+
  10
  11 from time import strftime
  12 import re
  13 import subprocess
  14 import shlex
  15 import multiprocessing
  16 import sys
  17 import enum
  18 import os
  19 from os.path import join
  20 import shutil
  21
  22 ##################################################
  23 #### Global variables
  24 ##################################################
  25 """ Options from the command line """
  26 opts=None
  27
  28 """ Absolute path from the git workspace to be used for the
  29 release"""
  30 doc_root=None
  31
  32 """ A dict containing version information used for the release.
  33 This dict contains entries of the form
  34 COMPONENT_version
  35 COMPONENT_micro
  36 COMPONENT_minor
  37 COMPONENT_major
  38 COMPONENT_code """
  39 comp_versions = {}
  40 old_comp_versions = {}
  41
  42 release_date = strftime (# ie: Mon Jan 23 00:35:37 CST 2006
  43                               "%a %b %d %H:%M:%S %Z %Y")
  44 cpu_count = multiprocessing.cpu_count()
  45
  46 # Packaging configuration
  47
  48 """ This is a regex that detects files that SHOULD NOT have line endings
  49 converted to CRLF when being put into a ZIP file """
  50 bin_regex = re.compile ("\.(mak|mdp|ide|exe|ico|gz|zip|xls|sxd|gif|vcp|vcproj|vcw|sln|dfm|jpg|png|vsd|bz2|pdf|ppt|graffle|pptx|odt|sh)$")
  51 version_restr = r'(\d+)(?:\.(\d+)(?:\.(\d+))?)?'
  52 version_re = re.compile(version_restr)
  53
  54 ##################################################
  55 #### Utility Methods
  56 ##################################################
  57
  58 class ReleaseType(enum.Enum):
  59     major = enum.auto()
  60     minor = enum.auto()
  61     micro = enum.auto()
  62
  63
  64 def parse_args ():
  65     from argparse import ArgumentParser
  66     parser = ArgumentParser ()
  67
  68     mutex_args = parser.add_mutually_exclusive_group(required=True)
  69     for rt in ReleaseType:
  70         mutex_args.add_argument ('--' + rt.name,
  71             dest="release_type", default=None, action="store_const", const=rt,
  72             help="Create a " + rt.name + " release.")
  73     mutex_args.add_argument ("--kit",
  74         dest="action", default=None, action="store_const", const="kit",
  75         help="Create kits.")
  76
  77     parser.add_argument ("--tag", action="store_true",
  78         help="Update tags and branches of the repositories", default=False)
  79     parser.add_argument ("--update", action="store_true",
  80         help="Update the version numbers", default=False)
  81     parser.add_argument ("--push", action="store_true",
  82         help="Push all changes to remote", default=False)
  83
  84     parser.add_argument ("--dest", dest="package_dir",
  85         help="Specify destination for the created packages.", default=None)
  86
  87     parser.add_argument ("--root", dest="repo_root",
  88         help="Specify an alternate repository root",
  89         default="https://github.com/DOCGroup/ACE_TAO.git")
  90     parser.add_argument ("--ace-tao-branch",
  91         help="ACE/TAO branch to update", default="master")
  92
  93     parser.add_argument ("--mpc_root",
  94         help="Specify an alternate MPC repository root",
  95         default="https://github.com/DOCGroup/MPC.git")
  96     parser.add_argument ("--mpc-branch",
  97         help="MPC branch to update", default="master")
  98
  99     parser.add_argument ("-n", dest="take_action", action="store_false",
 100         help="Take no action", default=True)
 101     parser.add_argument ("--verbose", action="store_true",
 102         help="Print out actions as they are being performed",
 103         default=False)
 104
 105     options = parser.parse_args ()
 106
 107     if options.tag:
 108         if not options.update:
 109             print ("Warning: You are tagging a release, but not requesting a version increment")
 110
 111         if not options.push:
 112             print ("Warning: You are tagging a release, but not requesting a push to remote")
 113
 114     return options
 115
 116
 117 def ex (command, allow_fail=False):
 118     if vprint ("Executing", command, take_action=True):
 119         return
 120
 121     status = os.system(command)
 122     if status != 0:
 123         print (("ERROR" if allow_fail else "WARNING") +
 124           ": Nonzero return value from " + command, file=sys.stderr)
 125         if not allow_fail:
 126             raise Exception
 127
 128
 129 def vprint (*args, take_action=False, **kwargs):
 130     """Prints the supplied message if verbose is enabled or this is a dry-run
 131     print statement. Return a bool of the latter case, so the caller can
 132     act differently.
 133     """
 134
 135     take_action = take_action and not opts.take_action
 136
 137     if opts.verbose or take_action:
 138         print (*args, **kwargs)
 139
 140     return take_action
 141
 142
 143 def get_tag (verdict, component):
 144     return "ACE+TAO-%d_%d_%d" % (
 145         verdict[component + '_major'], verdict[component + '_minor'], verdict[component + '_micro'])
 146
 147
 148 def get_path (*args):
 149     if not args:
 150         args = ('',)
 151     return join (doc_root, 'ACE_TAO', *args)
 152
 153 ##################################################
 154 #### Tagging methods
 155 ##################################################
 156 def commit (files):
 157     """ Commits the supplied list of files to the repository. """
 158
 159     version = get_tag(comp_versions, 'ACE')
 160     root_path = get_path()
 161     files = [i[len(root_path):] if i.startswith(root_path) else i for i in files]
 162
 163     print ("Committing the following files for " + version + ':', " ".join (files))
 164
 165     if opts.take_action:
 166         for file in files:
 167             print ("Adding file " + file + " to commit")
 168             ex ("cd $DOC_ROOT/ACE_TAO && git add " + file)
 169
 170         ex ("cd $DOC_ROOT/ACE_TAO && git commit -m\"" + version + "\"")
 171
 172 def check_workspace ():
 173     """ Checks that the DOC and MPC repositories are up to date.  """
 174
 175     try:
 176         ex ("cd $DOC_ROOT/ACE_TAO && git pull -p")
 177         print ("Successfully updated ACE/TAO working copy")
 178     except:
 179         print ("Unable to update ACE/TAO workspace at " + doc_root)
 180         raise
 181
 182     try:
 183         ex ("cd $DOC_ROOT/MPC && git pull -p")
 184         print ("Successfully updated MPC working copy to revision ")
 185     except:
 186         print ("Unable to update the MPC workspace at " + doc_root + "/ACE/MPC")
 187         raise
 188
 189     vprint ("Repos root URL = " + opts.repo_root + "\n")
 190     vprint ("Repos MPC root URL = " + opts.mpc_root + "\n")
 191
 192 def update_version_files (component):
 193     """ Updates the version files for a given component.  This includes
 194     Version.h, the PRF, and the VERSION.txt file."""
 195
 196     vprint ("Updating version files for " + component)
 197
 198     retval = []
 199
 200     ## Update component/VERSION.txt
 201     path = get_path(component, "VERSION.txt")
 202     with open (path, "r+") as version_file:
 203         new_version = re.sub (component + " version .*",
 204                               "%s version %s, released %s" % (component,
 205                                                               comp_versions[component + "_version"],
 206                                                               release_date),
 207                               version_file.read ())
 208         if opts.take_action:
 209             version_file.seek (0)
 210             version_file.truncate (0)
 211             version_file.write (new_version)
 212         else:
 213             print ("New version file for " + component)
 214             print (new_version)
 215
 216         vprint ("Updating Version.h for " + component)
 217
 218     retval.append(path)
 219
 220     ## Update COMPONENT/component/Version.h
 221     comp_l = len(component + "_")
 222     parts = {k[comp_l:]:v for (k, v) in comp_versions.items() if k.startswith(component)}
 223     parts["comp"]  = component
 224     version_header = """
 225 // -*- C++ -*-
 226 // This is file was automatically generated by $ACE_ROOT/bin/make_release.py
 227
 228 #define {comp}_MAJOR_VERSION {major}
 229 #define {comp}_MINOR_VERSION {minor}
 230 #define {comp}_MICRO_VERSION {micro}
 231 #define {comp}_VERSION \"{version}\"
 232 #define {comp}_VERSION_CODE 0x{code:x}
 233 #define {comp}_MAKE_VERSION_CODE(a,b,c) (((a) << 16) + ((b) << 8) + (c))
 234 """.format(**parts)
 235
 236     path = get_path(component, component.lower (), "Version.h")
 237     if opts.take_action:
 238         with open (path, 'w+') as version_h:
 239             version_h.write (version_header)
 240     else:
 241         print ("New Version.h for " + component)
 242         print (version_header)
 243
 244     retval.append(path)
 245
 246     # Update component/PROBLEM-REPORT-FORM
 247     vprint ("Updating PRF for " + component)
 248
 249     version_line_re = re.compile (r"^\s*(\w+) +VERSION ?:")
 250     path = get_path(component, "PROBLEM-REPORT-FORM")
 251
 252     with open (path, 'r+') as prf:
 253         new_prf = ""
 254         for line in prf.readlines ():
 255             match = version_line_re.search (line)
 256             if match is not None:
 257                 vprint ("Found PRF Version for " + match.group (1))
 258                 new_version = comp_versions[match.group(1) + "_version"]
 259                 line = version_re.sub (new_version, line)
 260
 261             new_prf += line
 262
 263         if opts.take_action:
 264             prf.seek (0)
 265             prf.truncate (0)
 266             prf.writelines (new_prf)
 267         else:
 268             print ("New PRF for " + component)
 269             print ("".join (new_prf))
 270
 271     retval.append(path)
 272
 273     return retval
 274
 275
 276 def update_spec_file ():
 277     path = get_path('ACE', "rpmbuild", "ace-tao.spec")
 278     with open (path, 'r+') as spec_file:
 279         new_spec = ""
 280         for line in spec_file.readlines ():
 281             if line.find ("define ACEVER ") != -1:
 282                 line = "%define ACEVER  " + comp_versions["ACE_version"] + "\n"
 283             if line.find ("define TAOVER ") != -1:
 284                 line = "%define TAOVER  " + comp_versions["TAO_version"] + "\n"
 285             if line.find ("define is_major_ver") != -1:
 286                 line = "%define is_major_ver {}\n".format(
 287                     int(opts.release_type != ReleaseType.micro))
 288
 289             new_spec += line
 290
 291         if opts.take_action:
 292             spec_file.seek (0)
 293             spec_file.truncate (0)
 294             spec_file.writelines (new_spec)
 295         else:
 296             print ("New spec file:")
 297             print ("".join (new_spec))
 298
 299     return [path]
 300
 301 def update_debianbuild ():
 302     """ Updates ACE_ROOT/debian directory.
 303     - renames all files with version numbers in name; if file contains
 304       lintian overrides, update version numbers inside file
 305     - updates version numbers inside file debian/control
 306     Currently ONLY ACE is handled here """
 307
 308     prev_ace_ver = None
 309
 310     path = get_path('ACE', 'debian', 'control')
 311
 312     mask = re.compile ("(libace|libACE|libkokyu|libKokyu|libnetsvcs)([^\s,:]*-)(\d+\.\d+\.\d+)([^\s,:]*)")
 313
 314     def update_ver (match):
 315         return match.group (1) + match.group (2) + comp_versions["ACE_version"] + match.group (4)
 316
 317     # update debian/control
 318     with open (path, 'r+') as control_file:
 319         new_ctrl = ""
 320         for line in control_file.readlines ():
 321             if re.search ("^(Package|Depends|Suggests):", line) is not None:
 322                 line = mask.sub (update_ver, line)
 323             elif re.search ('^Replaces:', line) is not None:
 324                 line = line.replace (old_comp_versions["ACE_version"], comp_versions["ACE_version"])
 325
 326             new_ctrl += line
 327
 328         if opts.take_action:
 329             control_file.seek (0)
 330             control_file.truncate (0)
 331             control_file.writelines (new_ctrl)
 332         else:
 333             print ("New control file:")
 334             print ("".join (new_ctrl))
 335
 336     return [path]
 337
 338 def get_and_update_versions ():
 339     """ Gets current version information for each component,
 340     updates the version files, creates changelog entries,
 341     and commit the changes into the repository."""
 342
 343     try:
 344         get_comp_versions ("ACE")
 345         get_comp_versions ("TAO")
 346
 347         if opts.update:
 348             files = []
 349             files += update_version_files ("ACE")
 350             files += update_version_files ("TAO")
 351             files += create_changelog ("ACE")
 352             files += create_changelog ("TAO")
 353             files += update_spec_file ()
 354             files += update_debianbuild ()
 355
 356             commit (files)
 357
 358     except:
 359         print ("Fatal error in get_and_update_versions.")
 360         raise
 361
 362 def create_changelog (component):
 363     """ Creates a changelog entry for the supplied component that includes
 364     the version number being released"""
 365     vprint ("Creating ChangeLog entry for " + component)
 366
 367     old_tag = get_tag (old_comp_versions, 'ACE')
 368
 369     # Generate changelogs per component
 370     path = get_path(component, "ChangeLogs", component + "-" + comp_versions[component + "_version_"])
 371     ex ("cd $DOC_ROOT/ACE_TAO && git log " + old_tag + "..HEAD " + component + " > " + path)
 372
 373     return [path]
 374
 375 def get_comp_versions (component):
 376     """ Extracts the current version number from the VERSION.txt
 377     file and increments it appropriately for the release type
 378     requested."""
 379     vprint ("Detecting current version for " + component)
 380
 381     regex = re.compile (r"version " + version_restr)
 382     major = component + "_major"
 383     minor = component + "_minor"
 384     micro = component + "_micro"
 385
 386
 387     version = (None, None, None)
 388     with open (doc_root + "/ACE_TAO/" + component + "/VERSION.txt") as version_file:
 389         for line in version_file:
 390             match = regex.search (line)
 391             if match is not None:
 392                 version = match.groups(default=0)
 393
 394                 vprint ("Detected version %s.%s.%s" % version)
 395
 396                 comp_versions[major] = int (version[0])
 397                 comp_versions[minor] = int (version[1])
 398                 comp_versions[micro] = int (version[2])
 399
 400                 break
 401
 402             print ("FATAL ERROR: Unable to locate current version for " + component)
 403             raise Exception
 404
 405     # Also store the current release (old from now)
 406     old_comp_versions[major] = comp_versions[major]
 407     old_comp_versions[minor] = comp_versions[minor]
 408     old_comp_versions[micro] = comp_versions[micro]
 409
 410     if opts.update:
 411         if opts.release_type == ReleaseType.major:
 412             comp_versions[major] += 1
 413             comp_versions[minor] = 0
 414             comp_versions[micro] = 0
 415         elif opts.release_type == ReleaseType.minor:
 416             comp_versions[minor] += 1
 417             comp_versions[micro] = 0
 418         elif opts.release_type == ReleaseType.micro:
 419             comp_versions[micro] += 1
 420
 421     def make_version (versions, joiner):
 422         return joiner.join ([
 423           str (versions[component + '_' + x]) for x in ReleaseType.__members__.keys ()
 424         ])
 425
 426     comp_versions [component + "_version"] = make_version (comp_versions, '.')
 427     comp_versions [component + "_version_"] = make_version (comp_versions, '_')
 428
 429     comp_versions [component + "_code"] = \
 430         (comp_versions[major] << 16) + \
 431         (comp_versions[minor] << 8) + \
 432         comp_versions[micro]
 433
 434     old_comp_versions [component + "_version"] = make_version (old_comp_versions, '.')
 435     old_comp_versions [component + "_version_"] = make_version (old_comp_versions, '_')
 436
 437     if opts.update:
 438       vprint ("Updating from version %s to version %s" %
 439                   (old_comp_versions [component + "_version"], comp_versions [component + "_version"]))
 440     else:
 441       vprint ("Found version %s" %
 442                   (comp_versions [component + "_version"]))
 443
 444     # else:
 445     #     comp_versions [component + "_version"] = \
 446     #                   str (comp_versions[major])  + '.' + \
 447     #                   str (comp_versions[minor])
 448
 449
 450 def update_latest_branch (product, which, main_branch):
 451     """Update one of the Latest_ACE7TAO3_* branches to point to the new release.
 452     """
 453
 454     name = "Latest_ACE7TAO3_" + which
 455
 456     vprint ('Fast-forwarding', name, 'to', main_branch)
 457     ex ("cd $DOC_ROOT/" + product + " && git fetch . " + main_branch +  ":" + name)
 458
 459
 460 def push_latest_branch (product, which, main_branch):
 461     """Update one of the remote Latest_ACE7TAO3_* branches to point to the new release.
 462     """
 463
 464     name = "Latest_ACE7TAO3_" + which
 465
 466     if opts.push:
 467         vprint ("Pushing branch", name)
 468         ex ("cd $DOC_ROOT/" + product + " && git push origin refs/heads/" + name,
 469             allow_fail=True)
 470
 471
 472 def latest_branch_helper (fn, release_type):
 473     release_types = tuple(ReleaseType.__members__.values())
 474     do = release_types[release_types.index(release_type):]
 475     if ReleaseType.micro in do:
 476         fn ("ACE_TAO", "Micro", opts.ace_tao_branch)
 477         fn ("MPC", "Micro", opts.mpc_branch)
 478     if ReleaseType.minor in do:
 479         fn ("ACE_TAO", "Minor", opts.ace_tao_branch)
 480         fn ("MPC", "Minor", opts.mpc_branch)
 481     if ReleaseType.major in do:
 482         fn ("ACE_TAO", "Major", opts.ace_tao_branch)
 483         fn ("MPC", "Major", opts.mpc_branch)
 484
 485
 486 def tag ():
 487     """Add the release tag and fast-forward the release branches on DOC and MPC
 488     repositories.
 489     """
 490
 491     tagname = get_tag(comp_versions, 'ACE')
 492
 493     if opts.tag:
 494         if opts.take_action:
 495             vprint ("Placing tag %s on ACE_TAO" % (tagname))
 496             ex ("cd $DOC_ROOT/ACE_TAO && git tag -a " + tagname + " -m\"" + tagname + "\"")
 497
 498             vprint ("Placing tag %s on MPC" % (tagname))
 499             ex ("cd $DOC_ROOT/MPC && git tag -a " + tagname + " -m\"" + tagname + "\"")
 500
 501             # Update release branches
 502             latest_branch_helper (update_latest_branch, opts.release_type)
 503         else:
 504             vprint ("Placing tag %s on ACE_TAO" % (tagname))
 505             vprint ("Placing tag %s on MPC" % (tagname))
 506             print ("Creating tags:\n")
 507             print ("Placing tag " + tagname + "\n")
 508
 509
 510 def push ():
 511     """Push the release tag and the fast-forwarded release branches on DOC and
 512     MPC repositories.
 513     """
 514
 515     tagname = get_tag (comp_versions, 'ACE')
 516
 517     if opts.push:
 518         if opts.take_action:
 519             vprint ("Pushing ACE_TAO", opts.ace_tao_branch, "to origin")
 520             ex ("cd $DOC_ROOT/ACE_TAO && git push origin " + opts.ace_tao_branch)
 521
 522             vprint ("Pushing tag %s on ACE_TAO" % (tagname))
 523             ex ("cd $DOC_ROOT/ACE_TAO && git push origin tag " + tagname)
 524
 525             vprint ("Pushing tag %s on MPC" % (tagname))
 526             ex ("cd $DOC_ROOT/MPC && git push origin tag " + tagname)
 527
 528             # Push release branches
 529             latest_branch_helper (push_latest_branch, opts.release_type)
 530         else:
 531             vprint ("Pushing tag %s on ACE_TAO" % (tagname))
 532             vprint ("Pushing tag %s on MPC" % (tagname))
 533             print ("Pushing tags:\n")
 534             print ("Pushing tag " + tagname + "\n")
 535
 536
 537 ##################################################
 538 #### Packaging methods
 539 ##################################################
 540 def export_wc (stage_dir):
 541
 542     tag = get_tag (comp_versions, 'ACE')
 543
 544     # Clone the ACE repository with the needed tag
 545     print ("Retrieving ACE with tag " + tag)
 546     ex ("git clone --depth 1 --branch " + tag + " " + opts.repo_root + " " + stage_dir + "/ACE_TAO")
 547
 548     # Clone the MPC repository with the needed tag
 549     print ("Retrieving MPC with tag " + tag)
 550     ex ("git clone --depth 1 --branch " + tag + " " + opts.mpc_root + " " + stage_dir + "/MPC")
 551
 552     # Setting up stage_dir
 553     print ("Moving ACE")
 554     ex ("mv " + stage_dir + "/ACE_TAO/ACE " + stage_dir + "/ACE_wrappers")
 555     print ("Moving TAO")
 556     ex ("mv " + stage_dir + "/ACE_TAO/TAO " + stage_dir + "/ACE_wrappers/TAO")
 557     print ("Moving MPC")
 558     ex ("mv " + stage_dir + "/MPC " + stage_dir + "/ACE_wrappers/MPC")
 559
 560 def update_packages (text_files_list, bin_files_list, stage_dir, package_dir):
 561     stream_encoding = 'utf-8'
 562     list_to_bytes = lambda l: ('\n'.join (l)).encode (stream_encoding)
 563     text_files = list_to_bytes (text_files_list)
 564     bin_files = list_to_bytes (bin_files_list)
 565
 566     print ("Updating packages....")
 567     os.chdir (stage_dir)
 568
 569     # -g appends, -q for quiet operation
 570     zip_base_args = " -gqu "
 571     # -l causes line ending conversion for windows
 572     zip_text_args = " -l "
 573     zip_file = stage_dir + "/zip-archive.zip"
 574
 575     # -r appends, -f specifies file.
 576     tar_args = "-uf "
 577     tar_file = stage_dir + "/tar-archive.tar"
 578
 579     # Zip binary files
 580     print ("\tAdding binary files to zip....")
 581     p = subprocess.Popen (
 582         shlex.split ("xargs zip " + zip_base_args + zip_file),
 583         stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
 584     instream, outstream = (p.stdin, p.stdout)
 585
 586     instream.write (bin_files)
 587
 588     instream.close ()
 589     outstream.close ()
 590
 591     # Need to wait for zip process spawned by popen2 to complete
 592     # before proceeding.
 593     os.wait ()
 594
 595     print ("\tAdding text files to zip.....")
 596     p = subprocess.Popen (
 597         shlex.split ("xargs zip " + zip_base_args + zip_text_args + zip_file),
 598         stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
 599     instream, outstream = (p.stdin, p.stdout)
 600
 601     instream.write (text_files)
 602
 603     instream.close ()
 604     outstream.close ()
 605
 606     # Need to wait for zip process spawned by popen2 to complete
 607     # before proceeding.
 608     os.wait ()
 609
 610     # Tar files
 611     print ("\tAdding to tar file....")
 612     if not os.path.exists (tar_file):
 613         open(tar_file, 'w').close ()
 614
 615     p = subprocess.Popen (
 616         shlex.split ("xargs tar " + tar_args + tar_file),
 617         stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
 618     instream, outstream = (p.stdin, p.stdout)
 619
 620     instream.write (b' ' + bin_files + b' ' + text_files)
 621
 622     instream.close ()
 623
 624     print (outstream.read ().decode(stream_encoding))
 625     outstream.close ()
 626
 627     os.wait ()
 628
 629 def move_packages (name, stage_dir, package_dir):
 630     """ Copies the temporary files from the stage_dir to the package_dir.
 631         Renames them to name.tar and name.zip, respectively, and compresses
 632         the tarfile with gzip and bzip2. """
 633
 634     print ("Storing packages for ", name)
 635
 636     # Take care of the zip file
 637     print ("\tZip file...")
 638     target_file = join (package_dir, name + ".zip")
 639     shutil.copy (join (stage_dir, "zip-archive.zip"), target_file)
 640     ex ("md5sum " + target_file + " > " + target_file + ".md5")
 641
 642
 643     tar_file = join (stage_dir, "tar-archive.tar")
 644     target_file = join (package_dir, name + ".tar")
 645
 646     # bzip
 647     print ("\tBzip2 file.....")
 648     shutil.copy (tar_file, target_file)
 649     ex ("bzip2 " + target_file)
 650     ex ("md5sum " + target_file + ".bz2 > " + target_file + ".bz2.md5")
 651
 652     print ("\tgzip file.....")
 653     shutil.copy (tar_file, target_file)
 654     ex ("gzip " + target_file)
 655     ex ("md5sum " + target_file + ".gz > " + target_file + ".gz.md5")
 656
 657 def create_file_lists (base_dir, prefix, exclude):
 658     """ Creates two lists of files:  files that need CR->CRLF
 659     conversions (useful for zip files) and those that don't,
 660     excluding files/directories found in exclude. """
 661
 662     text_files = list ()
 663     bin_files = list ()
 664
 665     for root, dirs, files in os.walk (base_dir, topdown=True):
 666 #        print "root", root
 667
 668         relroot = root.replace (base_dir, "")
 669
 670 #        print "relroot", relroot
 671
 672         if relroot and relroot[0] == '/':
 673             relroot = relroot [1:]
 674
 675         excluded = False
 676         for item in exclude:
 677             dir_item = item + '/'
 678             if relroot.startswith (dir_item) or relroot.startswith (item):
 679 #                print "excluding", relroot
 680                 excluded = True
 681 #            else:
 682 #                print relroot, "does not start with", dir_item, "or", item
 683
 684         if excluded:
 685             continue
 686
 687         # Remove dirs that are listed in our exclude pattern
 688         for item in dirs:
 689 #            print "item", item
 690             # Remove our excludes
 691             if (item) in exclude:
 692 #                print "Removing " + item + " from consideration...."
 693                 dirs.remove (item)
 694
 695         # Remove files that are listed in our exclude pattern
 696         for item in files:
 697             fullitem = os.path.join (relroot, item)
 698             if fullitem in exclude or item in exclude:
 699 #                print "Removing " + fullitem + " from consideration...."
 700                 continue
 701             else:
 702                 if bin_regex.search (fullitem) is not None:
 703                     bin_files.append ('"' + os.path.join (prefix, fullitem) + '"')
 704                 else:
 705                     text_files.append ('"' + os.path.join (prefix, fullitem) + '"')
 706
 707     return (text_files, bin_files)
 708
 709 def write_file_lists (comp, text, bin):
 710     with open (comp + ".files", 'w') as outfile:
 711         outfile.write ("\n".join (text))
 712         outfile.write (".............\nbin files\n.............\n")
 713         outfile.write ("\n".join (bin))
 714
 715 def package (stage_dir, package_dir, decorator):
 716     """ Packages ACE, ACE+TAO releases of current
 717         staged tree, with decorator appended to the name of the archive. """
 718
 719     os.chdir (stage_dir)
 720
 721     # Erase our old temp files
 722     try:
 723 #        print "removing files", join (stage_dir, "zip-archive.zip"), join (stage_dir, "tar-archive.tar")
 724         os.remove (join (stage_dir, "zip-archive.zip"))
 725         os.remove (join (stage_dir, "tar-archive.tar"))
 726     except:
 727         print ("error removing files", join (stage_dir, "zip-archive.zip"), join (stage_dir, "tar-archive.tar"))
 728         pass # swallow any errors
 729
 730     text_files, bin_files = create_file_lists (join (stage_dir, "ACE_wrappers"),
 731                                                "ACE_wrappers", ["TAO", ".gitignore", ".git"])
 732
 733 #    write_file_lists ("fACE" + decorator, text_files, bin_files)
 734     update_packages (text_files, bin_files, stage_dir, package_dir)
 735
 736     move_packages ("ACE" + decorator, stage_dir, package_dir)
 737
 738     # for TAO:
 739     text_files, bin_files = create_file_lists (join (stage_dir, "ACE_wrappers/TAO"),
 740                                                      "ACE_wrappers/TAO", [".gitignore", ".git"])
 741
 742 #    write_file_lists ("fTAO" + decorator, text_files, bin_files)
 743     update_packages (text_files, bin_files, stage_dir, package_dir)
 744
 745     move_packages ("ACE+TAO" + decorator, stage_dir, package_dir)
 746
 747 def generate_workspaces (stage_dir):
 748     """ Generates workspaces in the given stage_dir """
 749     print ("Generating workspaces...")
 750
 751     # Make sure we are in the right directory...
 752     os.chdir (os.path.join (stage_dir, "ACE_wrappers"))
 753
 754     # Set up our environment
 755     os.putenv ("ACE_ROOT", os.path.join (stage_dir, "ACE_wrappers"))
 756     os.putenv ("MPC_ROOT", os.path.join (stage_dir, "ACE_wrappers", "MPC"))
 757     os.putenv ("TAO_ROOT", os.path.join (stage_dir, "ACE_wrappers", "TAO"))
 758     os.putenv ("DDS_ROOT", "")
 759
 760     # Create option strings
 761     mpc_command = os.path.join (stage_dir, "ACE_wrappers", "bin", "mwc.pl")
 762     exclude_option = ' -exclude TAO/TAO_*.mwc '
 763     workers_option = ' -workers ' + str(cpu_count)
 764     mpc_option = ' -recurse -hierarchy -relative ACE_ROOT=' + stage_dir + '/ACE_wrappers '
 765     mpc_option += ' -relative TAO_ROOT=' + stage_dir + '/ACE_wrappers/TAO '
 766     msvc_exclude_option = ' '
 767     vs2017_option = ' -name_modifier *_vs2017 '
 768     vs2019_option = ' -name_modifier *_vs2019 '
 769
 770     redirect_option = str ()
 771     if not opts.verbose:
 772         redirect_option = " >> ../mpc.log 2>&1"
 773
 774     print ("\tGenerating GNUmakefiles....")
 775     ex (mpc_command + " -type gnuace " + \
 776         exclude_option + workers_option + mpc_option + redirect_option)
 777
 778     print ("\tGenerating VS2017 solutions...")
 779     ex (mpc_command + " -type vs2017 "  + \
 780         msvc_exclude_option + mpc_option + workers_option + vs2017_option + redirect_option)
 781
 782     print ("\tGenerating VS2019 solutions...")
 783     ex (mpc_command + " -type vs2019 " + \
 784         msvc_exclude_option + mpc_option + workers_option + vs2019_option + redirect_option)
 785
 786     print ("\tCorrecting permissions for all generated files...")
 787     regex = [
 788         '*.vc[p,w]',
 789         '*.bmak',
 790         '*.vcproj',
 791         '*.sln',
 792         '*.vcxproj',
 793         '*.filters',
 794         'GNUmake*',
 795     ]
 796     ex ("find ./ " + ' -or '.join(["-name '%s'" % (i,) for i in regex]) + " | xargs chmod 0644")
 797
 798 def create_kit ():
 799     """ Creates kits """
 800
 801     # Get version numbers for this working copy, note this will
 802     # not update the numbers.
 803     print ("Getting current version information....")
 804
 805     get_comp_versions ("ACE")
 806     get_comp_versions ("TAO")
 807
 808     print ("Creating working directories....")
 809     stage_dir, package_dir = make_working_directories ()
 810
 811     print ("Exporting working copy...")
 812     export_wc (stage_dir)
 813
 814     ### make source only packages
 815     package (stage_dir, package_dir, "-src")
 816
 817     generate_workspaces (stage_dir)
 818
 819     ### create standard packages.
 820     package (stage_dir, package_dir, "")
 821
 822 def make_working_directories ():
 823     """ Creates directories that we will be working in.
 824     In particular, we will have DOC_ROOT/stage-PID and
 825     DOC_ROOT/packages-PID """
 826
 827     stage_dir = join (doc_root, "stage-" + str (os.getpid ()))
 828     package_dir = join (doc_root, "package-" + str (os.getpid ()))
 829
 830     os.mkdir (stage_dir)
 831     os.mkdir (package_dir)
 832
 833     return (stage_dir, package_dir)
 834
 835 def main ():
 836
 837     if opts.action == "kit":
 838         print ("Creating a kit.")
 839         input ("Press enter to continue")
 840
 841         create_kit ()
 842
 843     else:
 844         print ("Making a " + opts.release_type.name + " release.")
 845         input ("Press enter to continue")
 846
 847         get_and_update_versions ()
 848         tag ()
 849         push ()
 850
 851 if __name__ == "__main__":
 852     opts = parse_args ()
 853
 854     doc_root = os.getenv ("DOC_ROOT")
 855     if doc_root is None:
 856         sys.exit ("ERROR: Environment DOC_ROOT must be defined.")
 857
 858     main ()