packaging/pkglib.py

   1 import os, sys, re, subprocess, argparse
   2
   3 # This python3 library provides a few helpful routines that are
   4 # used by the latest packaging scripts.
   5
   6 default_encoding = 'utf-8'
   7
   8 # Output the msg args to stderr.  Accepts all the args that print() accepts.
   9 def warn(*msg):
  10     print(*msg, file=sys.stderr)
  11
  12
  13 # Output the msg args to stderr and die with a non-zero return-code.
  14 # Accepts all the args that print() accepts.
  15 def die(*msg):
  16     warn(*msg)
  17     sys.exit(1)
  18
  19
  20 # Set this to an encoding name or set it to None to avoid the default encoding idiom.
  21 def set_default_encoding(enc):
  22     default_encoding = enc
  23
  24
  25 # Set shell=True if the cmd is a string; sets a default encoding unless raw=True was specified.
  26 def _tweak_opts(cmd, opts, **maybe_set_args):
  27     def _maybe_set(o, **msa): # Only set a value if the user didn't already set it.
  28         for var, val in msa.items():
  29             if var not in o:
  30                 o[var] = val
  31
  32     opts = opts.copy()
  33     _maybe_set(opts, **maybe_set_args)
  34
  35     if isinstance(cmd, str):
  36         _maybe_set(opts, shell=True)
  37
  38     want_raw = opts.pop('raw', False)
  39     if default_encoding and not want_raw:
  40         _maybe_set(opts, encoding=default_encoding)
  41
  42     capture = opts.pop('capture', None)
  43     if capture:
  44         if capture == 'stdout':
  45             _maybe_set(opts, stdout=subprocess.PIPE)
  46         elif capture == 'stderr':
  47             _maybe_set(opts, stderr=subprocess.PIPE)
  48         elif capture == 'output':
  49             _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  50         elif capture == 'combined':
  51             _maybe_set(opts, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
  52
  53     discard = opts.pop('discard', None)
  54     if discard:
  55         # We DO want to override any already set stdout|stderr values (unlike above).
  56         if discard == 'stdout' or discard == 'output':
  57             opts['stdout'] = subprocess.DEVNULL
  58         if discard == 'stderr' or discard == 'output':
  59             opts['stderr'] = subprocess.DEVNULL
  60
  61     return opts
  62
  63
  64 # This does a normal subprocess.run() with some auto-args added to make life easier.
  65 def cmd_run(cmd, **opts):
  66     return subprocess.run(cmd, **_tweak_opts(cmd, opts))
  67
  68
  69 # Like cmd_run() with a default check=True specified.
  70 def cmd_chk(cmd, **opts):
  71     return subprocess.run(cmd, **_tweak_opts(cmd, opts, check=True))
  72
  73
  74 # Capture stdout in a string and return an object with out, err, and rc (return code).
  75 # It defaults to capture='stdout' (so err is empty) but can be overridden using
  76 # capture='combined' or capture='output' (the latter populates the err value).
  77 def cmd_txt(cmd, **opts):
  78     input = opts.pop('input', None)
  79     if input is not None:
  80         opts['stdin'] = subprocess.PIPE
  81     proc = subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))
  82     out, err = proc.communicate(input=input)
  83     return argparse.Namespace(out=out, err=err, rc=proc.returncode)
  84
  85
  86 # Just like calling cmd_txt() except that it raises an error if the command has a non-0 return code.
  87 # The raised error includes the cmd, the return code, and the captured output.
  88 def cmd_txt_chk(cmd, **opts):
  89     ct = cmd_txt(cmd, **opts)
  90     if ct.rc != 0:
  91         cmd_err = f'Command "{cmd}" returned non-0 exit status "{ct.rc}" and output:\n{ct.out}{ct.err}'
  92         raise Exception(cmd_err)
  93     return ct
  94
  95
  96 # Starts a piped-output command of stdout (by default) and leaves it up to you to read
  97 # the output and call communicate() on the returned object.
  98 def cmd_pipe(cmd, **opts):
  99     return subprocess.Popen(cmd, **_tweak_opts(cmd, opts, capture='stdout'))
 100
 101
 102 # Runs a "git status" command and dies if the checkout is not clean (the
 103 # arg fatal_unless_clean can be used to make that non-fatal.  Returns a
 104 # tuple of the current branch, the is_clean flag, and the status text.
 105 def check_git_status(fatal_unless_clean=True, subdir='.'):
 106     status_txt = cmd_txt_chk(f"cd '{subdir}' && git status").out
 107     is_clean = re.search(r'\nnothing to commit.+working (directory|tree) clean', status_txt) != None
 108
 109     if not is_clean and fatal_unless_clean:
 110         if subdir == '.':
 111             subdir = ''
 112         else:
 113             subdir = f" *{subdir}*"
 114         die(f"The{subdir} checkout is not clean:\n" + status_txt)
 115
 116     m = re.match(r'^(?:# )?On branch (.+)\n', status_txt)
 117     cur_branch = m[1] if m else None
 118
 119     return (cur_branch, is_clean, status_txt)
 120
 121
 122 # Calls check_git_status() on the current git checkout and (optionally) a subdir path's
 123 # checkout. Use fatal_unless_clean to indicate if an unclean checkout is fatal or not.
 124 # The master_branch arg indicates what branch we want both checkouts to be using, and
 125 # if the branch is wrong the user is given the option of either switching to the right
 126 # branch or aborting.
 127 def check_git_state(master_branch, fatal_unless_clean=True, check_extra_dir=None):
 128     cur_branch = check_git_status(fatal_unless_clean)[0]
 129     branch = re.sub(r'^patch/([^/]+)/[^/]+$', r'\1', cur_branch) # change patch/BRANCH/PATCH_NAME into BRANCH
 130     if branch != master_branch:
 131         print(f"The checkout is not on the {master_branch} branch.")
 132         if master_branch != 'master':
 133             sys.exit(1)
 134         ans = input(f"Do you want me to continue with --branch={branch}? [n] ")
 135         if not ans or not re.match(r'^y', ans, flags=re.I):
 136             sys.exit(1)
 137         master_branch = branch
 138
 139     if check_extra_dir and os.path.isdir(os.path.join(check_extra_dir, '.git')):
 140         branch = check_git_status(fatal_unless_clean, check_extra_dir)[0]
 141         if branch != master_branch:
 142             print(f"The *{check_extra_dir}* checkout is on branch {branch}, not branch {master_branch}.")
 143             ans = input(f"Do you want to change it to branch {master_branch}? [n] ")
 144             if not ans or not re.match(r'^y', ans, flags=re.I):
 145                 sys.exit(1)
 146             subdir.check_call(f"cd {check_extra_dir} && git checkout '{master_branch}'", shell=True)
 147
 148     return (cur_branch, master_branch)
 149
 150
 151 # Return the git hash of the most recent commit.
 152 def latest_git_hash(branch):
 153     out = cmd_txt_chk(['git', 'log', '-1', '--no-color', branch]).out
 154     m = re.search(r'^commit (\S+)', out, flags=re.M)
 155     if not m:
 156         die(f"Unable to determine commit hash for master branch: {branch}")
 157     return m[1]
 158
 159
 160 # Return a set of all branch names that have the format "patch/BASE_BRANCH/NAME"
 161 # for the given base_branch string.  Just the NAME portion is put into the set.
 162 def get_patch_branches(base_branch):
 163     branches = set()
 164     proc = cmd_pipe('git branch -l'.split())
 165     for line in proc.stdout:
 166         m = re.search(r' patch/([^/]+)/(.+)', line)
 167         if m and m[1] == base_branch:
 168             branches.add(m[2])
 169     proc.communicate()
 170     return branches
 171
 172
 173 def mandate_gensend_hook():
 174     hook = '.git/hooks/pre-push'
 175     if not os.path.exists(hook):
 176         print('Creating hook file:', hook)
 177         cmd_chk(['./rsync', '-a', 'packaging/pre-push', hook])
 178     else:
 179         ct = cmd_txt(['fgrep', 'make gensend', hook], discard='output')
 180         if ct.rc:
 181             die('Please add a "make gensend" into your', hook, 'script.')
 182
 183
 184 # Snag the GENFILES values out of the Makefile file and return them as a list.
 185 def get_gen_files(want_dir_plus_list=False):
 186     cont_re = re.compile(r'\\\n')
 187
 188     gen_files = [ ]
 189
 190     auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').out.strip().replace('/', '%'))
 191
 192     with open(auto_dir + '/Makefile', 'r', encoding='utf-8') as fh:
 193         for line in fh:
 194             if not gen_files:
 195                 chk = re.sub(r'^GENFILES=', '', line)
 196                 if line == chk:
 197                     continue
 198                 line = chk
 199             m = re.search(r'\\$', line)
 200             line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line)
 201             gen_files += line.split()
 202             if not m:
 203                 break
 204
 205     if want_dir_plus_list:
 206         return (auto_dir, gen_files)
 207
 208     return [ os.path.join(auto_dir, fn) for fn in gen_files ]
 209
 210
 211 def get_rsync_version():
 212     with open('version.h', 'r', encoding='utf-8') as fh:
 213         txt = fh.read()
 214     m = re.match(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', txt)
 215     if m:
 216         return m[1]
 217     die("Unable to find RSYNC_VERSION define in version.h")
 218
 219
 220 def get_NEWS_version_info():
 221     rel_re = re.compile(r'^\| \S{2} \w{3} \d{4}\s+\|\s+(?P<ver>\d+\.\d+\.\d+)\s+\|\s+(?P<pdate>\d{2} \w{3} \d{4})?\s+\|\s+(?P<pver>\d+)\s+\|')
 222     last_version = last_protocol_version = None
 223     pdate = { }
 224
 225     with open('NEWS.md', 'r', encoding='utf-8') as fh:
 226         for line in fh:
 227             if not last_version: # Find the first non-dev|pre version with a release date.
 228                 m = re.search(r'rsync (\d+\.\d+\.\d+) .*\d\d\d\d', line)
 229                 if m:
 230                     last_version = m[1]
 231             m = rel_re.match(line)
 232             if m:
 233                 if m['pdate']:
 234                     pdate[m['ver']] = m['pdate']
 235                 if m['ver'] == last_version:
 236                     last_protocol_version = m['pver']
 237
 238     if not last_protocol_version:
 239         die(f"Unable to determine protocol_version for {last_version}.")
 240
 241     return last_version, last_protocol_version, pdate
 242
 243
 244 def get_protocol_versions():
 245     protocol_version = subprotocol_version = None
 246
 247     with open('rsync.h', 'r', encoding='utf-8') as fh:
 248         for line in fh:
 249             m = re.match(r'^#define\s+PROTOCOL_VERSION\s+(\d+)', line)
 250             if m:
 251                 protocol_version = m[1]
 252                 continue
 253             m = re.match(r'^#define\s+SUBPROTOCOL_VERSION\s+(\d+)', line)
 254             if m:
 255                 subprotocol_version = m[1]
 256                 break
 257
 258     if not protocol_version:
 259         die("Unable to determine the current PROTOCOL_VERSION.")
 260
 261     if not subprotocol_version:
 262         die("Unable to determine the current SUBPROTOCOL_VERSION.")
 263
 264     return protocol_version, subprotocol_version
 265
 266 # vim: sw=4 et