maintainers/scripts/update.py

   1 from __future__ import annotations
   2 from typing import Dict, Generator, List, Optional, Tuple
   3 import argparse
   4 import asyncio
   5 import contextlib
   6 import json
   7 import os
   8 import re
   9 import subprocess
  10 import sys
  11 import tempfile
  12
  13 class CalledProcessError(Exception):
  14     process: asyncio.subprocess.Process
  15     stderr: Optional[bytes]
  16
  17 class UpdateFailedException(Exception):
  18     pass
  19
  20 def eprint(*args, **kwargs):
  21     print(*args, file=sys.stderr, **kwargs)
  22
  23 async def check_subprocess_output(*args, **kwargs):
  24     """
  25     Emulate check and capture_output arguments of subprocess.run function.
  26     """
  27     process = await asyncio.create_subprocess_exec(*args, **kwargs)
  28     # We need to use communicate() instead of wait(), as the OS pipe buffers
  29     # can fill up and cause a deadlock.
  30     stdout, stderr = await process.communicate()
  31
  32     if process.returncode != 0:
  33         error = CalledProcessError()
  34         error.process = process
  35         error.stderr = stderr
  36
  37         raise error
  38
  39     return stdout
  40
  41 async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_dir: Optional[Tuple[str, str]], package: Dict, keep_going: bool):
  42     worktree: Optional[str] = None
  43
  44     update_script_command = package['updateScript']
  45
  46     if temp_dir is not None:
  47         worktree, _branch = temp_dir
  48
  49         # Ensure the worktree is clean before update.
  50         await check_subprocess_output('git', 'reset', '--hard', '--quiet', 'HEAD', cwd=worktree)
  51
  52         # Update scripts can use $(dirname $0) to get their location but we want to run
  53         # their clones in the git worktree, not in the main nixpkgs repo.
  54         update_script_command = map(lambda arg: re.sub(r'^{0}'.format(re.escape(nixpkgs_root)), worktree, arg), update_script_command)
  55
  56     eprint(f" - {package['name']}: UPDATING ...")
  57
  58     try:
  59         update_info = await check_subprocess_output(
  60             'env',
  61             f"UPDATE_NIX_NAME={package['name']}",
  62             f"UPDATE_NIX_PNAME={package['pname']}",
  63             f"UPDATE_NIX_OLD_VERSION={package['oldVersion']}",
  64             f"UPDATE_NIX_ATTR_PATH={package['attrPath']}",
  65             *update_script_command,
  66             stdout=asyncio.subprocess.PIPE,
  67             stderr=asyncio.subprocess.PIPE,
  68             cwd=worktree,
  69         )
  70         await merge_changes(merge_lock, package, update_info, temp_dir)
  71     except KeyboardInterrupt as e:
  72         eprint('Cancelling…')
  73         raise asyncio.exceptions.CancelledError()
  74     except CalledProcessError as e:
  75         eprint(f" - {package['name']}: ERROR")
  76         eprint()
  77         eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
  78         eprint()
  79         eprint(e.stderr.decode('utf-8'))
  80         with open(f"{package['pname']}.log", 'wb') as logfile:
  81             logfile.write(e.stderr)
  82         eprint()
  83         eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
  84
  85         if not keep_going:
  86             raise UpdateFailedException(f"The update script for {package['name']} failed with exit code {e.process.returncode}")
  87
  88 @contextlib.contextmanager
  89 def make_worktree() -> Generator[Tuple[str, str], None, None]:
  90     with tempfile.TemporaryDirectory() as wt:
  91         branch_name = f'update-{os.path.basename(wt)}'
  92         target_directory = f'{wt}/nixpkgs'
  93
  94         subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory])
  95         try:
  96             yield (target_directory, branch_name)
  97         finally:
  98             subprocess.run(['git', 'worktree', 'remove', '--force', target_directory])
  99             subprocess.run(['git', 'branch', '-D', branch_name])
 100
 101 async def commit_changes(name: str, merge_lock: asyncio.Lock, worktree: str, branch: str, changes: List[Dict]) -> None:
 102     for change in changes:
 103         # Git can only handle a single index operation at a time
 104         async with merge_lock:
 105             await check_subprocess_output('git', 'add', *change['files'], cwd=worktree)
 106             commit_message = '{attrPath}: {oldVersion} -> {newVersion}'.format(**change)
 107             if 'commitMessage' in change:
 108                 commit_message = change['commitMessage']
 109             elif 'commitBody' in change:
 110                 commit_message = commit_message + '\n\n' + change['commitBody']
 111             await check_subprocess_output('git', 'commit', '--quiet', '-m', commit_message, cwd=worktree)
 112             await check_subprocess_output('git', 'cherry-pick', branch)
 113
 114 async def check_changes(package: Dict, worktree: str, update_info: str):
 115     if 'commit' in package['supportedFeatures']:
 116         changes = json.loads(update_info)
 117     else:
 118         changes = [{}]
 119
 120     # Try to fill in missing attributes when there is just a single change.
 121     if len(changes) == 1:
 122         # Dynamic data from updater take precedence over static data from passthru.updateScript.
 123         if 'attrPath' not in changes[0]:
 124             # update.nix is always passing attrPath
 125             changes[0]['attrPath'] = package['attrPath']
 126
 127         if 'oldVersion' not in changes[0]:
 128             # update.nix is always passing oldVersion
 129             changes[0]['oldVersion'] = package['oldVersion']
 130
 131         if 'newVersion' not in changes[0]:
 132             attr_path = changes[0]['attrPath']
 133             obtain_new_version_output = await check_subprocess_output('nix-instantiate', '--expr', f'with import ./. {{}}; lib.getVersion {attr_path}', '--eval', '--strict', '--json', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
 134             changes[0]['newVersion'] = json.loads(obtain_new_version_output.decode('utf-8'))
 135
 136         if 'files' not in changes[0]:
 137             changed_files_output = await check_subprocess_output('git', 'diff', '--name-only', 'HEAD', stdout=asyncio.subprocess.PIPE, cwd=worktree)
 138             changed_files = changed_files_output.splitlines()
 139             changes[0]['files'] = changed_files
 140
 141             if len(changed_files) == 0:
 142                 return []
 143
 144     return changes
 145
 146 async def merge_changes(merge_lock: asyncio.Lock, package: Dict, update_info: str, temp_dir: Optional[Tuple[str, str]]) -> None:
 147     if temp_dir is not None:
 148         worktree, branch = temp_dir
 149         changes = await check_changes(package, worktree, update_info)
 150
 151         if len(changes) > 0:
 152             await commit_changes(package['name'], merge_lock, worktree, branch, changes)
 153         else:
 154             eprint(f" - {package['name']}: DONE, no changes.")
 155     else:
 156         eprint(f" - {package['name']}: DONE.")
 157
 158 async def updater(nixpkgs_root: str, temp_dir: Optional[Tuple[str, str]], merge_lock: asyncio.Lock, packages_to_update: asyncio.Queue[Optional[Dict]], keep_going: bool, commit: bool):
 159     while True:
 160         package = await packages_to_update.get()
 161         if package is None:
 162             # A sentinel received, we are done.
 163             return
 164
 165         if not ('commit' in package['supportedFeatures'] or 'attrPath' in package):
 166             temp_dir = None
 167
 168         await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
 169
 170 async def start_updates(max_workers: int, keep_going: bool, commit: bool, packages: List[Dict]):
 171     merge_lock = asyncio.Lock()
 172     packages_to_update: asyncio.Queue[Optional[Dict]] = asyncio.Queue()
 173
 174     with contextlib.ExitStack() as stack:
 175         temp_dirs: List[Optional[Tuple[str, str]]] = []
 176
 177         # Do not create more workers than there are packages.
 178         num_workers = min(max_workers, len(packages))
 179
 180         nixpkgs_root_output = await check_subprocess_output('git', 'rev-parse', '--show-toplevel', stdout=asyncio.subprocess.PIPE)
 181         nixpkgs_root = nixpkgs_root_output.decode('utf-8').strip()
 182
 183         # Set up temporary directories when using auto-commit.
 184         for i in range(num_workers):
 185             temp_dir = stack.enter_context(make_worktree()) if commit else None
 186             temp_dirs.append(temp_dir)
 187
 188         # Fill up an update queue,
 189         for package in packages:
 190             await packages_to_update.put(package)
 191
 192         # Add sentinels, one for each worker.
 193         # A workers will terminate when it gets sentinel from the queue.
 194         for i in range(num_workers):
 195             await packages_to_update.put(None)
 196
 197         # Prepare updater workers for each temp_dir directory.
 198         # At most `num_workers` instances of `run_update_script` will be running at one time.
 199         updaters = asyncio.gather(*[updater(nixpkgs_root, temp_dir, merge_lock, packages_to_update, keep_going, commit) for temp_dir in temp_dirs])
 200
 201         try:
 202             # Start updater workers.
 203             await updaters
 204         except asyncio.exceptions.CancelledError:
 205             # When one worker is cancelled, cancel the others too.
 206             updaters.cancel()
 207         except UpdateFailedException as e:
 208             # When one worker fails, cancel the others, as this exception is only thrown when keep_going is false.
 209             updaters.cancel()
 210             eprint(e)
 211             sys.exit(1)
 212
 213 def main(max_workers: int, keep_going: bool, commit: bool, packages_path: str, skip_prompt: bool) -> None:
 214     with open(packages_path) as f:
 215         packages = json.load(f)
 216
 217     eprint()
 218     eprint('Going to be running update for following packages:')
 219     for package in packages:
 220         eprint(f" - {package['name']}")
 221     eprint()
 222
 223     confirm = '' if skip_prompt else input('Press Enter key to continue...')
 224
 225     if confirm == '':
 226         eprint()
 227         eprint('Running update for:')
 228
 229         asyncio.run(start_updates(max_workers, keep_going, commit, packages))
 230
 231         eprint()
 232         eprint('Packages updated!')
 233         sys.exit()
 234     else:
 235         eprint('Aborting!')
 236         sys.exit(130)
 237
 238 parser = argparse.ArgumentParser(description='Update packages')
 239 parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
 240 parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
 241 parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
 242 parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
 243 parser.add_argument('--skip-prompt', '-s', dest='skip_prompt', action='store_true', help='Do not stop for prompts')
 244
 245 if __name__ == '__main__':
 246     args = parser.parse_args()
 247
 248     try:
 249         main(args.max_workers, args.keep_going, args.commit, args.packages, args.skip_prompt)
 250     except KeyboardInterrupt as e:
 251         # Let’s cancel outside of the main loop too.
 252         sys.exit(130)