stgit/lib/stackupgrade.py

   1 import json
   2 import os
   3 import shutil
   4
   5 from stgit.config import config
   6 from stgit.exception import StackException
   7 from stgit.lib.git.objects import BlobData, CommitData, TreeData
   8 from stgit.out import out
   9 from stgit.run import RunException
  10
  11 # The current StGit metadata format version.
  12 FORMAT_VERSION = 5
  13
  14
  15 def _format_version_key(branch):
  16     return 'branch.%s.stgit.stackformatversion' % branch
  17
  18
  19 def _read_strings(filename):
  20     """Reads the lines from a file"""
  21     with open(filename, encoding='utf-8') as f:
  22         return [line.strip() for line in f.readlines()]
  23
  24
  25 def _read_string(filename):
  26     """Reads the first line from a file"""
  27     with open(filename, encoding='utf-8') as f:
  28         return f.readline().strip()
  29
  30
  31 def _try_rm(f):
  32     if os.path.exists(f):
  33         os.remove(f)
  34
  35
  36 def update_to_current_format_version(repository, branch):
  37     """Update a potentially older StGit directory structure to the latest version.
  38
  39     Note: This function should depend as little as possible on external functions that
  40     may change during a format version bump, since it must remain able to process older
  41     formats.
  42
  43     """
  44
  45     patches_dir = os.path.join(repository.directory, 'patches')
  46     branch_dir = os.path.join(patches_dir, branch)
  47     old_format_key = _format_version_key(branch)
  48     older_format_key = 'branch.%s.stgitformatversion' % branch
  49
  50     def get_meta_file_version():
  51         """Get format version from the ``meta`` file in the stack log branch."""
  52         new_version = get_stack_json_file_version()
  53         if new_version is not None:
  54             return new_version
  55
  56         old_version = get_old_meta_file_version()
  57         if old_version is not None:
  58             return old_version
  59
  60     def get_old_meta_file_version():
  61         """Get format version from the ``meta`` file in the stack log branch."""
  62         stack_ref = 'refs/heads/%s.stgit:meta' % branch
  63         try:
  64             lines = (
  65                 repository.run(['git', 'show', stack_ref])
  66                 .discard_stderr()
  67                 .output_lines()
  68             )
  69         except RunException:
  70             return None
  71
  72         for line in lines:
  73             if line.startswith('Version: '):
  74                 return int(line.split('Version: ', 1)[1])
  75         else:
  76             return None
  77
  78     def get_stack_json_file_version():
  79         stack_ref = 'refs/stacks/%s:stack.json' % branch
  80         try:
  81             data = (
  82                 repository.run(['git', 'show', stack_ref])
  83                 .decoding('utf-8')
  84                 .discard_stderr()
  85                 .raw_output()
  86             )
  87         except RunException:
  88             return None
  89
  90         try:
  91             stack_json = json.loads(data)
  92         except json.JSONDecodeError:
  93             return None
  94         else:
  95             return stack_json.get('version')
  96
  97     def get_format_version():
  98         """Return the integer format version number.
  99
 100         :returns: the format version number or None if the branch does not have any
 101                   StGit metadata at all, of any version
 102
 103         """
 104         mfv = get_meta_file_version()
 105         if mfv is not None and mfv >= 4:
 106             # Modern-era format version found in branch meta blob.
 107             return mfv
 108
 109         # Older format versions were stored in the Git config.
 110         fv = config.get(old_format_key)
 111         ofv = config.get(older_format_key)
 112         if fv:
 113             # Great, there's an explicitly recorded format version
 114             # number, which means that the branch is initialized and
 115             # of that exact version.
 116             return int(fv)
 117         elif ofv:
 118             # Old name for the version info: upgrade it.
 119             config.set(old_format_key, ofv)
 120             config.unset(older_format_key)
 121             return int(ofv)
 122         elif os.path.isdir(os.path.join(branch_dir, 'patches')):
 123             # There's a .git/patches/<branch>/patches dirctory, which
 124             # means this is an initialized version 1 branch.
 125             return 1
 126         elif os.path.isdir(branch_dir):
 127             # There's a .git/patches/<branch> directory, which means
 128             # this is an initialized version 0 branch.
 129             return 0
 130         else:
 131             # The branch doesn't seem to be initialized at all.
 132             return None
 133
 134     def set_format_version_in_config(v):
 135         out.info('Upgraded branch %s to format version %d' % (branch, v))
 136         config.set(old_format_key, '%d' % v)
 137
 138     def rm_ref(ref):
 139         if repository.refs.exists(ref):
 140             repository.refs.delete(ref)
 141
 142     # Update 0 -> 1.
 143     if get_format_version() == 0:
 144         os.makedirs(os.path.join(branch_dir, 'trash'), exist_ok=True)
 145         patch_dir = os.path.join(branch_dir, 'patches')
 146         os.makedirs(patch_dir, exist_ok=True)
 147         refs_base = 'refs/patches/%s' % branch
 148         with open(os.path.join(branch_dir, 'unapplied')) as f:
 149             patches = f.readlines()
 150         with open(os.path.join(branch_dir, 'applied')) as f:
 151             patches.extend(f.readlines())
 152         for patch in patches:
 153             patch = patch.strip()
 154             os.rename(os.path.join(branch_dir, patch), os.path.join(patch_dir, patch))
 155             topfield = os.path.join(patch_dir, patch, 'top')
 156             if os.path.isfile(topfield):
 157                 top = _read_string(topfield)
 158             else:
 159                 top = None
 160             if top:
 161                 repository.refs.set(
 162                     refs_base + '/' + patch,
 163                     repository.get_commit(top),
 164                     'StGit upgrade',
 165                 )
 166         set_format_version_in_config(1)
 167
 168     # Update 1 -> 2.
 169     if get_format_version() == 1:
 170         desc_file = os.path.join(branch_dir, 'description')
 171         if os.path.isfile(desc_file):
 172             desc = _read_string(desc_file)
 173             if desc:
 174                 config.set('branch.%s.description' % branch, desc)
 175             _try_rm(desc_file)
 176         _try_rm(os.path.join(branch_dir, 'current'))
 177         rm_ref('refs/bases/%s' % branch)
 178         set_format_version_in_config(2)
 179
 180     # Update 2 -> 3
 181     if get_format_version() == 2:
 182         protect_file = os.path.join(branch_dir, 'protected')
 183         if os.path.isfile(protect_file):
 184             config.set('branch.%s.stgit.protect' % branch, 'true')
 185             os.remove(protect_file)
 186         set_format_version_in_config(3)
 187
 188     # compatibility with the new infrastructure. The changes here do not
 189     # affect the compatibility with the old infrastructure (format version 2)
 190     if get_format_version() == 3:
 191         os.makedirs(branch_dir, exist_ok=True)
 192         hidden_file = os.path.join(branch_dir, 'hidden')
 193         if not os.path.isfile(hidden_file):
 194             open(hidden_file, 'w+', encoding='utf-8').close()
 195
 196         applied_file = os.path.join(branch_dir, 'applied')
 197         unapplied_file = os.path.join(branch_dir, 'unapplied')
 198
 199         applied = _read_strings(applied_file)
 200         unapplied = _read_strings(unapplied_file)
 201         hidden = _read_strings(hidden_file)
 202
 203         state_ref = 'refs/heads/%s.stgit' % branch
 204
 205         head = repository.refs.get('refs/heads/%s' % branch)
 206         parents = [head]
 207         meta_lines = [
 208             'Version: 4',
 209             'Previous: None',
 210             'Head: %s' % head.sha1,
 211         ]
 212
 213         patches_tree = {}
 214
 215         for patch_list, title in [
 216             (applied, 'Applied'),
 217             (unapplied, 'Unapplied'),
 218             (hidden, 'Hidden'),
 219         ]:
 220             meta_lines.append('%s:' % title)
 221             for i, pn in enumerate(patch_list):
 222                 patch_ref = 'refs/patches/%s/%s' % (branch, pn)
 223                 commit = repository.refs.get(patch_ref)
 224                 meta_lines.append('  %s: %s' % (pn, commit.sha1))
 225                 if title != 'Applied' or i == len(patch_list) - 1:
 226                     if commit not in parents:
 227                         parents.append(commit)
 228                 cd = commit.data
 229                 patch_meta = '\n'.join(
 230                     [
 231                         'Bottom: %s' % cd.parent.data.tree.sha1,
 232                         'Top:    %s' % cd.tree.sha1,
 233                         'Author: %s' % cd.author.name_email,
 234                         'Date:   %s' % cd.author.date,
 235                         '',
 236                         cd.message_str,
 237                     ]
 238                 ).encode('utf-8')
 239                 patches_tree[pn] = repository.commit(BlobData(patch_meta))
 240         meta_lines.append('')
 241
 242         meta = '\n'.join(meta_lines).encode('utf-8')
 243         tree = repository.commit(
 244             TreeData(
 245                 {
 246                     'meta': repository.commit(BlobData(meta)),
 247                     'patches': repository.commit(TreeData(patches_tree)),
 248                 }
 249             )
 250         )
 251         state_commit = repository.commit(
 252             CommitData(
 253                 tree=tree,
 254                 message='stack upgrade to version 4',
 255                 parents=parents,
 256             )
 257         )
 258         repository.refs.set(state_ref, state_commit, 'stack upgrade to v4')
 259
 260         for patch_list in [applied, unapplied, hidden]:
 261             for pn in patch_list:
 262                 patch_log_ref = 'refs/patches/%s/%s.log' % (branch, pn)
 263                 if repository.refs.exists(patch_log_ref):
 264                     repository.refs.delete(patch_log_ref)
 265
 266         config.unset(old_format_key)
 267
 268         shutil.rmtree(branch_dir)
 269         try:
 270             # .git/patches will be removed after the last stack is converted
 271             os.rmdir(patches_dir)
 272         except OSError:
 273             pass
 274         out.info('Upgraded branch %s to format version %d' % (branch, 4))
 275
 276     # Metadata moves from refs/heads/<branch>.stgit to refs/stacks/<branch>.
 277     # Also, metadata file format is JSON instead of custom format.
 278     if get_format_version() == 4:
 279         old_state_ref = 'refs/heads/%s.stgit' % branch
 280         old_state = repository.refs.get(old_state_ref)
 281         old_meta = old_state.data.tree.data['meta'][1].data.bytes
 282         lines = old_meta.decode('utf-8').splitlines()
 283         if not lines[0].startswith('Version: 4'):
 284             raise StackException('Malformed metadata (expected version 4)')
 285
 286         parsed = {}
 287         key = None
 288         for line in lines:
 289             if line.startswith(' '):
 290                 assert key is not None
 291                 parsed[key].append(line.strip())
 292             else:
 293                 key, val = [x.strip() for x in line.split(':', 1)]
 294                 if val:
 295                     parsed[key] = val
 296                 else:
 297                     parsed[key] = []
 298
 299         head = repository.refs.get('refs/heads/%s' % branch)
 300
 301         new_meta = dict(
 302             version=5,
 303             prev=parsed['Previous'],
 304             head=head.sha1,
 305             applied=[],
 306             unapplied=[],
 307             hidden=[],
 308             patches=dict(),
 309         )
 310
 311         if parsed['Head'] != new_meta['head']:
 312             raise StackException('Unexpected head mismatch')
 313
 314         for patch_list_name in ['Applied', 'Unapplied', 'Hidden']:
 315             for entry in parsed[patch_list_name]:
 316                 pn, sha1 = [x.strip() for x in entry.split(':')]
 317                 new_patch_list_name = patch_list_name.lower()
 318                 new_meta[new_patch_list_name].append(pn)
 319                 new_meta['patches'][pn] = dict(oid=sha1)
 320
 321         meta_bytes = json.dumps(new_meta, indent=2).encode('utf-8')
 322
 323         tree = repository.commit(
 324             TreeData(
 325                 {
 326                     'stack.json': repository.commit(BlobData(meta_bytes)),
 327                     'patches': old_state.data.tree.data['patches'],
 328                 }
 329             )
 330         )
 331
 332         repository.refs.set(
 333             'refs/stacks/%s' % branch,
 334             repository.commit(
 335                 CommitData(
 336                     tree=tree,
 337                     message='stack upgrade to version 5',
 338                     parents=[head],
 339                 )
 340             ),
 341             'stack upgrade to v5',
 342         )
 343
 344         repository.refs.delete(old_state_ref)
 345         out.info('Upgraded branch %s to format version %d' % (branch, 5))
 346
 347     # Make sure we're at the latest version.
 348     fv = get_format_version()
 349     if fv not in [None, FORMAT_VERSION]:
 350         raise StackException(
 351             'Branch %s is at format version %d, expected %d'
 352             % (branch, fv, FORMAT_VERSION)
 353         )
 354     return fv is not None  # true if branch is initialized