release.py

   1 # Copyright (C) 2007, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os, sys, subprocess, shutil, tarfile, tempfile
   5 from zeroinstall import SafeException
   6 from zeroinstall.injector import reader, model
   7 from logging import info
   8 from scm import GIT
   9
  10 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
  11
  12 release_status_file = 'release-status'
  13
  14 valid_phases = ['commit-release', 'generate-archive']
  15
  16 def run_unit_tests(impl):
  17         self_test = impl.metadata.get('self-test', None)
  18         if self_test is None:
  19                 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
  20                 return
  21         self_test = os.path.join(impl.id, self_test)
  22         print "Running self-test:", self_test
  23         exitstatus = subprocess.call([self_test], cwd = os.path.dirname(self_test))
  24         if exitstatus:
  25                 raise SafeException("Self-test failed with exit status %d" % exitstatus)
  26
  27 def show_and_run(cmd, args):
  28         print "Executing: %s %s" % (cmd, ' '.join("[%s]" % x for x in args))
  29         subprocess.check_call(['sh', '-c', cmd, '-'] + args)
  30
  31 def suggest_release_version(snapshot_version):
  32         """Given a snapshot version, suggest a suitable release version.
  33         >>> suggest_release_version('1.0-pre')
  34         '1.0'
  35         >>> suggest_release_version('0.9-post')
  36         '0.10'
  37         >>> suggest_release_version('3')
  38         Traceback (most recent call last):
  39                 ...
  40         SafeException: Version '3' is not a snapshot version (should end in -pre or -post)
  41         """
  42         version = model.parse_version(snapshot_version)
  43         mod = version[-1]
  44         if mod == 0:
  45                 raise SafeException("Version '%s' is not a snapshot version (should end in -pre or -post)" % snapshot_version)
  46         if mod > 0:
  47                 # -post, so increment the number
  48                 version[-2][-1] += 1
  49         version[-1] = 0 # Remove the modifier
  50         return model.format_version(version)
  51
  52 def publish(iface, **kwargs):
  53         args = [os.environ['0PUBLISH']]
  54         for k in kwargs:
  55                 value = kwargs[k]
  56                 if value is True:
  57                         args += ['--' + k.replace('_', '-')]
  58                 elif value is not None:
  59                         args += ['--' + k.replace('_', '-'), value]
  60         args.append(iface)
  61         info("Executing %s", args)
  62         subprocess.check_call(args)
  63
  64 def get_singleton_impl(iface):
  65         impls = iface.implementations
  66         if len(impls) != 1:
  67                 raise SafeException("Local feed '%s' contains %d versions! I need exactly one!" % (iface.uri, len(impls)))
  68         return impls.values()[0]
  69
  70 def backup_if_exists(name):
  71         if not os.path.exists(name):
  72                 return
  73         backup = name + '~'
  74         if os.path.exists(backup):
  75                 print "(deleting old backup %s)" % backup
  76                 if os.path.isdir(backup):
  77                         shutil.rmtree(backup)
  78                 else:
  79                         os.unlink(backup)
  80         os.rename(name, backup)
  81         print "(renamed old %s as %s; will delete on next run)" % (name, backup)
  82
  83 class Status(object):
  84         __slots__ = ['old_snapshot_version', 'release_version', 'head_before_release', 'new_snapshot_version', 'head_at_release', 'created_archive', 'tagged']
  85         def __init__(self):
  86                 for name in self.__slots__:
  87                         setattr(self, name, None)
  88
  89                 if os.path.isfile(release_status_file):
  90                         for line in file(release_status_file):
  91                                 assert line.endswith('\n')
  92                                 line = line[:-1]
  93                                 name, value = line.split('=')
  94                                 setattr(self, name, value)
  95                                 info("Loaded status %s=%s", name, value)
  96
  97         def save(self):
  98                 tmp_name = release_status_file + '.new'
  99                 tmp = file(tmp_name, 'w')
 100                 try:
 101                         lines = ["%s=%s\n" % (name, getattr(self, name)) for name in self.__slots__ if getattr(self, name)]
 102                         tmp.write(''.join(lines))
 103                         tmp.close()
 104                         os.rename(tmp_name, release_status_file)
 105                         info("Wrote status to %s", release_status_file)
 106                 except:
 107                         os.unlink(tmp_name)
 108                         raise
 109
 110 def get_choice(*options):
 111         while True:
 112                 choice = raw_input('/'.join(options) + ': ').lower()
 113                 if not choice: continue
 114                 for o in options:
 115                         if o.lower().startswith(choice):
 116                                 return o
 117
 118 def do_release(local_iface, options):
 119         status = Status()
 120         local_impl = get_singleton_impl(local_iface)
 121
 122         local_impl_dir = local_impl.id
 123         assert local_impl_dir.startswith('/')
 124         local_impl_dir = os.path.realpath(local_impl_dir)
 125         assert os.path.isdir(local_impl_dir)
 126         assert local_iface.uri.startswith(local_impl_dir + '/')
 127         local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
 128         assert not local_iface_rel_path.startswith('/')
 129         assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
 130
 131         phase_actions = {}
 132         for phase in valid_phases:
 133                 phase_actions[phase] = []       # List of <release:action> elements
 134
 135         release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
 136         if len(release_management) == 1:
 137                 info("Found <release:management> element.")
 138                 release_management = release_management[0]
 139                 for x in release_management.childNodes:
 140                         if x.uri == XMLNS_RELEASE and x.name == 'action':
 141                                 phase = x.getAttribute('phase')
 142                                 if phase not in valid_phases:
 143                                         raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
 144                                 phase_actions[phase].append(x.content)
 145                         else:
 146                                 warn("Unknown <release:management> element: %s", x)
 147         elif len(release_management) > 1:
 148                 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
 149         else:
 150                 info("No <release:management> element found in local feed.")
 151
 152         scm = GIT(local_iface, options)
 153
 154         def run_hooks(phase, cwd, env):
 155                 info("Running hooks for phase '%s'" % phase)
 156                 full_env = os.environ.copy()
 157                 full_env.update(env)
 158                 for x in phase_actions[phase]:
 159                         print "[%s]: %s" % (phase, x)
 160                         subprocess.check_call(x, shell = True, cwd = cwd, env = full_env)
 161
 162         def set_to_release():
 163                 print "Snapshot version is " + local_impl.get_version()
 164                 suggested = suggest_release_version(local_impl.get_version())
 165                 release_version = raw_input("Version number for new release [%s]: " % suggested)
 166                 if not release_version:
 167                         release_version = suggested
 168
 169                 scm.ensure_no_tag(release_version)
 170
 171                 status.head_before_release = scm.get_head_revision()
 172                 status.save()
 173
 174                 working_copy = local_impl.id
 175                 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
 176
 177                 print "Releasing version", release_version
 178                 publish(local_iface.uri, set_released = 'today', set_version = release_version)
 179
 180                 status.old_snapshot_version = local_impl.get_version()
 181                 status.release_version = release_version
 182                 scm.commit('Release %s' % release_version)
 183                 status.head_at_release = scm.get_head_revision()
 184                 status.save()
 185
 186                 return release_version
 187
 188         def set_to_snapshot(snapshot_version):
 189                 assert snapshot_version.endswith('-post')
 190                 publish(local_iface.uri, set_released = '', set_version = snapshot_version)
 191                 scm.commit('Start development series %s' % snapshot_version)
 192                 status.new_snapshot_version = scm.get_head_revision()
 193                 status.save()
 194
 195         def ensure_ready_to_release():
 196                 scm.ensure_committed()
 197                 info("No uncommitted changes. Good.")
 198                 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
 199                 #run_unit_tests(local_impl)
 200
 201         def create_feed(local_iface_stream, archive_file, archive_name, version):
 202                 tmp = tempfile.NamedTemporaryFile(prefix = '0release-')
 203                 shutil.copyfileobj(local_iface_stream, tmp)
 204                 tmp.flush()
 205
 206                 publish(tmp.name,
 207                         archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
 208                         archive_file = archive_file,
 209                         archive_extract = archive_name)
 210                 return tmp
 211
 212         def unpack_tarball(archive_file, archive_name):
 213                 tar = tarfile.open(archive_file, 'r:bz2')
 214                 members = [m for m in tar.getmembers() if m.name != 'pax_global_header']
 215                 tar.extractall('.', members = members)
 216
 217         def export_changelog():
 218                 parsed_release_version = model.parse_version(status.release_version)
 219
 220                 previous_release = None
 221                 if os.path.exists(options.master_feed_file):
 222                         master = model.Interface(os.path.realpath(options.master_feed_file))
 223                         reader.update(master, master.uri, local = True)
 224                         versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version]
 225                         if versions:
 226                                 previous_release = model.format_version(max(versions))
 227
 228                 changelog = file('changelog-%s' % status.release_version, 'w')
 229                 try:
 230                         try:
 231                                 scm.export_changelog(previous_release, status.head_before_release, changelog)
 232                         except SafeException, ex:
 233                                 print "WARNING: Failed to generate changelog: " + str(ex)
 234                         else:
 235                                 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
 236                 finally:
 237                         changelog.close()
 238
 239         def fail_candidate(archive_file):
 240                 backup_if_exists(archive_file)
 241                 head = scm.get_head_revision()
 242                 if head != status.new_snapshot_version:
 243                         raise SafeException("There have been commits since starting the release! Please rebase them onto %s" % status.head_before_release)
 244                 # Check no uncommitted changes
 245                 scm.ensure_committed()
 246                 scm.reset_hard(status.head_before_release)
 247                 os.unlink(release_status_file)
 248                 print "Restored to state before starting release. Make your fixes and try again..."
 249
 250         def accept_and_publish(archive_file, archive_name, local_iface_rel_path):
 251                 assert options.master_feed_file
 252
 253                 if status.tagged:
 254                         print "Already tagged and added to master feed."
 255                 else:
 256                         tar = tarfile.open(archive_file, 'r:bz2')
 257                         stream = tar.extractfile(tar.getmember(archive_name + '/' + local_iface_rel_path))
 258                         remote_dl_iface = create_feed(stream, archive_file, archive_name, version)
 259                         stream.close()
 260
 261                         publish(options.master_feed_file, local = remote_dl_iface.name, xmlsign = True, key = options.key)
 262                         remote_dl_iface.close()
 263
 264                         scm.tag(status.release_version, status.head_at_release)
 265
 266                         status.tagged = 'true'
 267                         status.save()
 268
 269                 # Copy files...
 270                 print "Upload %s as %s" % (archive_file, options.archive_dir_public_url + '/' + os.path.basename(archive_file))
 271                 cmd = options.archive_upload_command.strip()
 272                 if cmd:
 273                         show_and_run(cmd, [archive_file])
 274                 else:
 275                         print "NOTE: No upload command set => you'll have to upload it yourself!"
 276
 277                 assert len(local_iface.feed_for) == 1
 278                 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
 279                 feed_files = [options.master_feed_file]
 280                 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
 281                 cmd = options.master_feed_upload_command.strip()
 282                 if cmd:
 283                         show_and_run(cmd, feed_files)
 284                 else:
 285                         print "NOTE: No feed upload command set => you'll have to upload them yourself!"
 286
 287                 print "Push changes to public SCM repository..."
 288                 public_repos = options.public_scm_repository
 289                 if public_repos:
 290                         scm.push_head_and_release(status.release_version)
 291                 else:
 292                         print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
 293
 294                 os.unlink(release_status_file)
 295
 296         if status.head_before_release:
 297                 head = scm.get_head_revision()
 298                 if status.release_version:
 299                         print "RESUMING release of version %s" % status.release_version
 300                 elif head == status.head_before_release:
 301                         print "Restarting release (HEAD revision has not changed)"
 302                 else:
 303                         raise SafeException("Something went wrong with the last run:\n" +
 304                                             "HEAD revision for last run was " + status.head_before_release + "\n" +
 305                                             "HEAD revision now is " + head + "\n" +
 306                                             "You should revert your working copy to the previous head and try again.\n" +
 307                                             "If you're sure you want to release from the current head, delete '" + release_status_file + "'")
 308
 309         print "Releasing", local_iface.get_name()
 310
 311         ensure_ready_to_release()
 312
 313         if status.release_version:
 314                 version = status.release_version
 315                 need_set_snapshot = False
 316                 if status.new_snapshot_version:
 317                         head = scm.get_head_revision()
 318                         if head != status.new_snapshot_version:
 319                                 print "WARNING: there are more commits since we tagged; they will not be included in the release!"
 320                 else:
 321                         raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
 322                                             "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, release_status_file))
 323         else:
 324                 version = set_to_release()
 325                 need_set_snapshot = True
 326
 327         archive_name = local_iface.get_name().lower().replace(' ', '-') + '-' + version
 328         archive_file = archive_name + '.tar.bz2'
 329
 330         if status.created_archive and os.path.isfile(archive_file):
 331                 print "Archive already created"
 332         else:
 333                 backup_if_exists(archive_file)
 334                 scm.export(archive_name, archive_file)
 335
 336                 if phase_actions['generate-archive']:
 337                         try:
 338                                 unpack_tarball(archive_file, archive_name)
 339                                 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
 340                                 info("Regenerating archive (may have been modified by generate-archive hooks...")
 341                                 subprocess.check_call(['tar', 'cjf', archive_file, archive_name])
 342                         except SafeException:
 343                                 fail_candidate(archive_file)
 344                                 raise
 345
 346                 status.created_archive = 'true'
 347                 status.save()
 348
 349         if need_set_snapshot:
 350                 set_to_snapshot(version + '-post')
 351
 352         #backup_if_exists(archive_name)
 353         unpack_tarball(archive_file, archive_name)
 354         if local_impl.main:
 355                 main = os.path.join(archive_name, local_impl.main)
 356                 if not os.path.exists(main):
 357                         raise SafeException("Main executable '%s' not found after unpacking archive!" % main)
 358
 359         extracted_iface_path = os.path.abspath(os.path.join(archive_name, local_iface_rel_path))
 360         extracted_iface = model.Interface(extracted_iface_path)
 361         reader.update(extracted_iface, extracted_iface_path, local = True)
 362         extracted_impl = get_singleton_impl(extracted_iface)
 363
 364         try:
 365                 run_unit_tests(extracted_impl)
 366         except SafeException:
 367                 print "(leaving extracted directory for examination)"
 368                 fail_candidate(archive_file)
 369                 raise
 370         # Unpack it again in case the unit-tests changed anything
 371         shutil.rmtree(archive_name)
 372         unpack_tarball(archive_file, archive_name)
 373
 374         export_changelog()
 375
 376         print "\nCandidate release archive:", archive_file
 377         print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
 378
 379         print "\nPlease check candidate and select an action:"
 380         print "P) Publish candidate (accept)"
 381         print "F) Fail candidate (untag)"
 382         print "(you can also hit CTRL-C and resume this script when done)"
 383         choice = get_choice('Publish', 'Fail')
 384
 385         info("Deleting extracted archive %s", archive_name)
 386         shutil.rmtree(archive_name)
 387
 388         if choice == 'Publish':
 389                 accept_and_publish(archive_file, archive_name, local_iface_rel_path)
 390         else:
 391                 assert choice == 'Fail'
 392                 fail_candidate(archive_file)
 393
 394
 395 if __name__ == "__main__":
 396     import doctest
 397     doctest.testmod()