release.py

   1 # Copyright (C) 2007, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os, sys, subprocess, shutil, tarfile, tempfile
   5 from zeroinstall import SafeException
   6 from zeroinstall.injector import reader, model
   7 from logging import info
   8
   9 import support
  10 from scm import GIT
  11
  12 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
  13
  14 release_status_file = 'release-status'
  15
  16 valid_phases = ['commit-release', 'generate-archive']
  17
  18 TMP_BRANCH_NAME = '0release-tmp'
  19
  20 def run_unit_tests(impl):
  21         self_test = impl.metadata.get('self-test', None)
  22         if self_test is None:
  23                 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
  24                 return
  25         self_test = os.path.join(impl.id, self_test)
  26         print "Running self-test:", self_test
  27         exitstatus = subprocess.call([self_test], cwd = os.path.dirname(self_test))
  28         if exitstatus:
  29                 raise SafeException("Self-test failed with exit status %d" % exitstatus)
  30
  31 class Status(object):
  32         __slots__ = ['old_snapshot_version', 'release_version', 'head_before_release', 'new_snapshot_version', 'head_at_release', 'created_archive', 'tagged']
  33         def __init__(self):
  34                 for name in self.__slots__:
  35                         setattr(self, name, None)
  36
  37                 if os.path.isfile(release_status_file):
  38                         for line in file(release_status_file):
  39                                 assert line.endswith('\n')
  40                                 line = line[:-1]
  41                                 name, value = line.split('=')
  42                                 setattr(self, name, value)
  43                                 info("Loaded status %s=%s", name, value)
  44
  45         def save(self):
  46                 tmp_name = release_status_file + '.new'
  47                 tmp = file(tmp_name, 'w')
  48                 try:
  49                         lines = ["%s=%s\n" % (name, getattr(self, name)) for name in self.__slots__ if getattr(self, name)]
  50                         tmp.write(''.join(lines))
  51                         tmp.close()
  52                         os.rename(tmp_name, release_status_file)
  53                         info("Wrote status to %s", release_status_file)
  54                 except:
  55                         os.unlink(tmp_name)
  56                         raise
  57
  58 def do_release(local_iface, options):
  59         status = Status()
  60         local_impl = support.get_singleton_impl(local_iface)
  61
  62         local_impl_dir = local_impl.id
  63         assert local_impl_dir.startswith('/')
  64         local_impl_dir = os.path.realpath(local_impl_dir)
  65         assert os.path.isdir(local_impl_dir)
  66         assert local_iface.uri.startswith(local_impl_dir + '/')
  67         local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
  68         assert not local_iface_rel_path.startswith('/')
  69         assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
  70
  71         phase_actions = {}
  72         for phase in valid_phases:
  73                 phase_actions[phase] = []       # List of <release:action> elements
  74
  75         release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
  76         if len(release_management) == 1:
  77                 info("Found <release:management> element.")
  78                 release_management = release_management[0]
  79                 for x in release_management.childNodes:
  80                         if x.uri == XMLNS_RELEASE and x.name == 'action':
  81                                 phase = x.getAttribute('phase')
  82                                 if phase not in valid_phases:
  83                                         raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
  84                                 phase_actions[phase].append(x.content)
  85                         else:
  86                                 warn("Unknown <release:management> element: %s", x)
  87         elif len(release_management) > 1:
  88                 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
  89         else:
  90                 info("No <release:management> element found in local feed.")
  91
  92         scm = GIT(local_iface, options)
  93
  94         def run_hooks(phase, cwd, env):
  95                 info("Running hooks for phase '%s'" % phase)
  96                 full_env = os.environ.copy()
  97                 full_env.update(env)
  98                 for x in phase_actions[phase]:
  99                         print "[%s]: %s" % (phase, x)
 100                         support.check_call(x, shell = True, cwd = cwd, env = full_env)
 101
 102         def set_to_release():
 103                 print "Snapshot version is " + local_impl.get_version()
 104                 suggested = support.suggest_release_version(local_impl.get_version())
 105                 release_version = raw_input("Version number for new release [%s]: " % suggested)
 106                 if not release_version:
 107                         release_version = suggested
 108
 109                 scm.ensure_no_tag(release_version)
 110
 111                 status.head_before_release = scm.get_head_revision()
 112                 status.save()
 113
 114                 working_copy = local_impl.id
 115                 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
 116
 117                 print "Releasing version", release_version
 118                 support.publish(local_iface.uri, set_released = 'today', set_version = release_version)
 119
 120                 status.old_snapshot_version = local_impl.get_version()
 121                 status.release_version = release_version
 122                 status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
 123                 status.save()
 124
 125         def set_to_snapshot(snapshot_version):
 126                 assert snapshot_version.endswith('-post')
 127                 support.publish(local_iface.uri, set_released = '', set_version = snapshot_version)
 128                 scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
 129                 status.new_snapshot_version = scm.get_head_revision()
 130                 status.save()
 131
 132         def ensure_ready_to_release():
 133                 scm.ensure_committed()
 134                 scm.ensure_versioned(local_iface_rel_path)
 135                 info("No uncommitted changes. Good.")
 136                 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
 137                 #run_unit_tests(local_impl)
 138
 139         def create_feed(local_iface_stream, archive_file, archive_name):
 140                 tmp = tempfile.NamedTemporaryFile(prefix = '0release-')
 141                 shutil.copyfileobj(local_iface_stream, tmp)
 142                 tmp.flush()
 143
 144                 support.publish(tmp.name,
 145                         archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
 146                         archive_file = archive_file,
 147                         archive_extract = archive_name)
 148                 return tmp
 149
 150         def unpack_tarball(archive_file, archive_name):
 151                 tar = tarfile.open(archive_file, 'r:bz2')
 152                 members = [m for m in tar.getmembers() if m.name != 'pax_global_header']
 153                 tar.extractall('.', members = members)
 154
 155         def get_previous_release(this_version):
 156                 """Return the highest numbered verison in the master feed before this_version.
 157                 @return: version, or None if there wasn't one"""
 158                 parsed_release_version = model.parse_version(this_version)
 159
 160                 if os.path.exists(options.master_feed_file):
 161                         master = model.Interface(os.path.realpath(options.master_feed_file))
 162                         reader.update(master, master.uri, local = True)
 163                         versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version]
 164                         if versions:
 165                                 return model.format_version(max(versions))
 166                 return None
 167
 168         def export_changelog(previous_release = None):
 169                 changelog = file('changelog-%s' % status.release_version, 'w')
 170                 try:
 171                         try:
 172                                 scm.export_changelog(previous_release, status.head_before_release, changelog)
 173                         except SafeException, ex:
 174                                 print "WARNING: Failed to generate changelog: " + str(ex)
 175                         else:
 176                                 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
 177                 finally:
 178                         changelog.close()
 179
 180         def fail_candidate(archive_file):
 181                 support.backup_if_exists(archive_file)
 182                 scm.delete_branch(TMP_BRANCH_NAME)
 183                 os.unlink(release_status_file)
 184                 print "Restored to state before starting release. Make your fixes and try again..."
 185
 186         def accept_and_publish(archive_file, archive_name, local_iface_rel_path):
 187                 assert options.master_feed_file
 188
 189                 master = model.Interface(os.path.realpath(options.master_feed_file))
 190                 reader.update(master, master.uri, local = True)
 191                 existing_releases = [impl for impl in master.implementations.values() if impl.get_version() == status.release_version]
 192                 if len(existing_releases):
 193                         raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options.master_feed_file, status.release_version))
 194
 195                 if status.tagged:
 196                         print "Already tagged and added to master feed."
 197                 else:
 198                         scm.ensure_committed()
 199                         head = scm.get_head_revision()
 200                         if head != status.head_before_release:
 201                                 raise SafeException("Changes committed since we started!\n" +
 202                                                     "HEAD was " + status.head_before_release + "\n"
 203                                                     "HEAD now " + head)
 204
 205                         tar = tarfile.open(archive_file, 'r:bz2')
 206                         stream = tar.extractfile(tar.getmember(archive_name + '/' + local_iface_rel_path))
 207                         remote_dl_iface = create_feed(stream, archive_file, archive_name)
 208                         stream.close()
 209
 210                         support.publish(options.master_feed_file, local = remote_dl_iface.name, xmlsign = True, key = options.key)
 211                         remote_dl_iface.close()
 212
 213                         scm.tag(status.release_version, status.head_at_release)
 214                         scm.reset_hard(TMP_BRANCH_NAME)
 215                         scm.delete_branch(TMP_BRANCH_NAME)
 216
 217                         status.tagged = 'true'
 218                         status.save()
 219
 220                 # Copy files...
 221                 print "Upload %s as %s" % (archive_file, options.archive_dir_public_url + '/' + os.path.basename(archive_file))
 222                 cmd = options.archive_upload_command.strip()
 223                 if cmd:
 224                         support.show_and_run(cmd, [archive_file])
 225                 else:
 226                         print "NOTE: No upload command set => you'll have to upload it yourself!"
 227
 228                 assert len(local_iface.feed_for) == 1
 229                 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
 230                 feed_files = [options.master_feed_file]
 231                 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
 232                 cmd = options.master_feed_upload_command.strip()
 233                 if cmd:
 234                         support.show_and_run(cmd, feed_files)
 235                 else:
 236                         print "NOTE: No feed upload command set => you'll have to upload them yourself!"
 237
 238                 print "Push changes to public SCM repository..."
 239                 public_repos = options.public_scm_repository
 240                 if public_repos:
 241                         scm.push_head_and_release(status.release_version)
 242                 else:
 243                         print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
 244
 245                 os.unlink(release_status_file)
 246
 247         if status.head_before_release:
 248                 head = scm.get_head_revision()
 249                 if status.release_version:
 250                         print "RESUMING release of %s %s" % (local_iface.get_name(), status.release_version)
 251                 elif head == status.head_before_release:
 252                         print "Restarting release of %s (HEAD revision has not changed)" % local_iface.get_name()
 253                 else:
 254                         raise SafeException("Something went wrong with the last run:\n" +
 255                                             "HEAD revision for last run was " + status.head_before_release + "\n" +
 256                                             "HEAD revision now is " + head + "\n" +
 257                                             "You should revert your working copy to the previous head and try again.\n" +
 258                                             "If you're sure you want to release from the current head, delete '" + release_status_file + "'")
 259         else:
 260                 print "Releasing", local_iface.get_name()
 261
 262         ensure_ready_to_release()
 263
 264         if status.release_version:
 265                 need_set_snapshot = False
 266                 if status.new_snapshot_version:
 267                         head = scm.get_head_revision()
 268                         if head != status.head_before_release:
 269                                 raise SafeException("There are more commits since we started!\n"
 270                                                     "HEAD was " + status.head_before_release + "\n"
 271                                                     "HEAD now " + head + "\n"
 272                                                     "To include them, delete '" + release_status_file + "' and try again.\n"
 273                                                     "To leave them out, put them on a new branch and reset HEAD to the release version.")
 274                 else:
 275                         raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
 276                                             "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, release_status_file))
 277         else:
 278                 set_to_release()
 279                 assert status.release_version
 280                 need_set_snapshot = True
 281
 282         archive_name = support.make_archive_name(local_iface.get_name(), status.release_version)
 283         archive_file = archive_name + '.tar.bz2'
 284
 285         if status.created_archive and os.path.isfile(archive_file):
 286                 print "Archive already created"
 287         else:
 288                 support.backup_if_exists(archive_file)
 289                 scm.export(archive_name, archive_file, status.head_at_release)
 290
 291                 if phase_actions['generate-archive']:
 292                         try:
 293                                 unpack_tarball(archive_file, archive_name)
 294                                 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
 295                                 info("Regenerating archive (may have been modified by generate-archive hooks...")
 296                                 support.check_call(['tar', 'cjf', archive_file, archive_name])
 297                         except SafeException:
 298                                 fail_candidate(archive_file)
 299                                 raise
 300
 301                 status.created_archive = 'true'
 302                 status.save()
 303
 304         if need_set_snapshot:
 305                 set_to_snapshot(status.release_version + '-post')
 306                 # Revert back to the original revision, so that any fixes the user makes
 307                 # will get applied before the tag
 308                 scm.reset_hard(scm.get_current_branch())
 309
 310         #backup_if_exists(archive_name)
 311         unpack_tarball(archive_file, archive_name)
 312         if local_impl.main:
 313                 main = os.path.join(archive_name, local_impl.main)
 314                 if not os.path.exists(main):
 315                         raise SafeException("Main executable '%s' not found after unpacking archive!" % main)
 316
 317         extracted_iface_path = os.path.abspath(os.path.join(archive_name, local_iface_rel_path))
 318         assert os.path.isfile(extracted_iface_path), "Local feed not in archive! Is it under version control?"
 319         extracted_iface = model.Interface(extracted_iface_path)
 320         reader.update(extracted_iface, extracted_iface_path, local = True)
 321         extracted_impl = support.get_singleton_impl(extracted_iface)
 322
 323         try:
 324                 run_unit_tests(extracted_impl)
 325         except SafeException:
 326                 print "(leaving extracted directory for examination)"
 327                 fail_candidate(archive_file)
 328                 raise
 329         # Unpack it again in case the unit-tests changed anything
 330         shutil.rmtree(archive_name)
 331         unpack_tarball(archive_file, archive_name)
 332
 333         previous_release = get_previous_release(status.release_version)
 334         export_changelog()
 335
 336         print "\nCandidate release archive:", archive_file
 337         print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
 338
 339         print "\nPlease check candidate and select an action:"
 340         print "P) Publish candidate (accept)"
 341         print "F) Fail candidate (untag)"
 342         if previous_release:
 343                 print "D) Diff against release archive for %s" % previous_release
 344                 maybe_diff = ['Diff']
 345         else:
 346                 maybe_diff = []
 347         print "(you can also hit CTRL-C and resume this script when done)"
 348
 349         while True:
 350                 choice = support.get_choice(['Publish', 'Fail'] + maybe_diff)
 351                 if choice == 'Diff':
 352                         previous_archive_name = support.make_archive_name(local_iface.get_name(), previous_release)
 353                         previous_archive_file = previous_archive_name + '.tar.bz2'
 354                         if os.path.isfile(previous_archive_file):
 355                                 unpack_tarball(previous_archive_file, previous_archive_name)
 356                                 try:
 357                                         support.show_diff(previous_archive_name, archive_name)
 358                                 finally:
 359                                         shutil.rmtree(previous_archive_name)
 360                         else:
 361                                 # TODO: download it?
 362                                 print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
 363                 else:
 364                         break
 365
 366         info("Deleting extracted archive %s", archive_name)
 367         shutil.rmtree(archive_name)
 368
 369         if choice == 'Publish':
 370                 accept_and_publish(archive_file, archive_name, local_iface_rel_path)
 371         else:
 372                 assert choice == 'Fail'
 373                 fail_candidate(archive_file)