tools/github_release.py

   1 #!/usr/bin/env python3
   2 """Prepare release and upload Windows and macOS artifacts."""
   3 import argparse
   4 import hashlib
   5 import os
   6 import re
   7 import sys
   8 import zipfile
   9
  10 import magic  # use python-magic (not compatible with filemagic)
  11 import requests
  12 from github3 import login
  13 from jinja2 import Template
  14
  15
  16 def debug_requests():
  17     """Turn requests debug on."""
  18     # These two lines enable debugging at httplib level (requests->urllib3->http.client)
  19     # You will see the REQUEST, including HEADERS and DATA, and RESPONSE with HEADERS but without DATA.
  20     # The only thing missing will be the response.body which is not logged.
  21     import http.client as http_client
  22     import logging
  23     http_client.HTTPConnection.debuglevel = 1
  24
  25     # You must initialize logging, otherwise you'll not see debug output.
  26     logging.basicConfig()
  27     logging.getLogger().setLevel(logging.DEBUG)
  28     requests_log = logging.getLogger("requests.packages.urllib3")
  29     requests_log.setLevel(logging.DEBUG)
  30     requests_log.propagate = True
  31
  32
  33 def error_exit(msg, code=1):
  34     """Print msg and exit with code."""
  35     print(msg, file=sys.stderr)
  36     sys.exit(code)
  37
  38
  39 def download_items(urls, prefix):
  40     print("D: downloading %s" % urls)
  41     for url in urls:
  42         print("I: downloading %s" % url)
  43         filename = url.split('/')[-1]
  44         output = os.path.join("_build", "{}-{}".format(prefix, filename))
  45         with requests.get(url, stream=True) as r:
  46             with open(output, "wb") as f:
  47                 for chunk in r.iter_content(chunk_size=1000000):
  48                     f.write(chunk)
  49
  50
  51 def download_mac_github(github_workflow, prefix, version):
  52     """Download mac workflow artifacts from github and exit."""
  53     headers = {'Accept': 'application/vnd.github+json', 'Authorization': 'token %s' % github_token}
  54
  55     print("I: downloading release artifacts for workflow %d" % github_workflow)
  56     r = requests.get("https://api.github.com/repos/gpodder/gpodder/actions/artifacts", headers=headers)
  57     if not r.ok:
  58         error_exit('ERROR: API fetch failed %d %s' % (r.status_code, r.reason))
  59     artifacts = r.json()
  60     artifact = [(a['id'], a['archive_download_url']) for a in artifacts['artifacts'] if a['workflow_run']['id'] == github_workflow]
  61     if len(artifact) != 1:
  62         error_exit("Nothing found to download")
  63     id, url = artifact[0]
  64     print("I: found artifact %d" % id)
  65
  66     print("I: downloading %s" % url)
  67     output = os.path.join("_build", "{}-artifact.zip".format(prefix))
  68     with requests.get(url, stream=True, headers=headers) as r:
  69         if not r.ok:
  70             error_exit('ERROR: artifact fetch failed %d %s' % (r.status_code, r.reason))
  71         with open(output, "wb") as f:
  72             for chunk in r.iter_content(chunk_size=1000000):
  73                 f.write(chunk)
  74     print("I: unzipping %s" % output)
  75     with zipfile.ZipFile(output, 'r') as z:
  76         z.extractall('_build')
  77     os.remove(output)
  78     os.remove(os.path.join("_build", "{}-gPodder-{}.zip.md5".format(prefix, version)))
  79     os.remove(os.path.join("_build", "{}-gPodder-{}.zip.sha256".format(prefix, version)))
  80     checksums()
  81
  82
  83 def download_appveyor(appveyor_build, prefix):
  84     """Download build artifacts from appveyor and exit."""
  85     print("I: downloading release artifacts from appveyor")
  86     build = requests.get("https://ci.appveyor.com/api/projects/elelay/gpodder/build/%s" % appveyor_build).json()
  87     job_id = build.get("build", {}).get("jobs", [{}])[0].get("jobId")
  88     if job_id:
  89         job_url = "https://ci.appveyor.com/api/buildjobs/{}/artifacts".format(job_id)
  90         artifacts = requests.get(job_url).json()
  91         items = ["{}/{}".format(job_url, f["fileName"]) for f in artifacts if f["type"] == "File"]
  92         if len(items) == 0:
  93             error_exit("Nothing found to download")
  94         download_items(items, prefix)
  95     else:
  96         error_exit("no jobId in {}".format(build))
  97
  98
  99 def checksums():
 100     """Compute artifact checksums."""
 101     ret = {}
 102     for f in os.listdir("_build"):
 103         archive = os.path.join("_build", f)
 104         m = hashlib.md5()
 105         s = hashlib.sha256()
 106         with open(archive, "rb") as f:
 107             block = f.read(4096)
 108             while block:
 109                 m.update(block)
 110                 s.update(block)
 111                 block = f.read(4096)
 112         ret[os.path.basename(archive)] = {'md5': m.hexdigest(), 'sha256': s.hexdigest()}
 113     return ret
 114
 115
 116 def get_contributors(tag, previous_tag):
 117     """List contributor logins '@...' for every commit in range."""
 118     cmp = repo.compare_commits(previous_tag, tag)
 119     logins = [c.author.login for c in cmp.commits() if c.author] + [c.committer.login for c in cmp.commits()]
 120     return sorted({"@{}".format(n) for n in logins})
 121
 122
 123 def get_previous_tag():
 124     latest_release = repo.latest_release()
 125     return latest_release.tag_name
 126
 127
 128 def release_text(tag, previous_tag, mac_github=None, appveyor=None):
 129     t = Template("""
 130 Linux, macOS and Windows are supported.
 131
 132 Thanks to {{contributors[0]}}{% for c in contributors[1:-1] %}, {{c}}{% endfor %} and {{contributors[-1]}} for contributing to this release!
 133
 134 [Changes](https://github.com/gpodder/gpodder/compare/{{previous_tag}}...{{tag}}) since **{{previous_tag}}**:
 135
 136
 137 ## New features
 138  - ...
 139
 140 ## Improvements
 141  - ...
 142
 143 ## Bug fixes
 144  - ...
 145
 146 ## Translations
 147  - ...
 148
 149 ## CI references
 150  - macOS GitHub build [{{mac_github}}](https://github.com/gpodder/gpodder/actions/runs/{{mac_github}})
 151  - Windows Appveyor build [{{appveyor}}](https://ci.appveyor.com/project/elelay/gpodder/build/{{appveyor}})
 152
 153 ## Checksums
 154 {% for f, c in checksums.items() %}
 155  - {{f}}
 156 md5:<i>{{c.md5}}</i>
 157 sha256:<i>{{c.sha256}}</i>
 158 {% endfor %}
 159 """)  # noqa: W291
 160     args = {
 161             'contributors': get_contributors(tag, previous_tag),
 162             'tag': tag,
 163             'previous_tag': previous_tag,
 164             'mac_github': mac_github,
 165             'appveyor': appveyor,
 166             'checksums': checksums()
 167     }
 168     return t.render(args)
 169
 170
 171 def upload(repo, tag, previous_tag, mac_github, appveyor):
 172     """Create github release (draft) and upload assets."""
 173     print("I: creating release %s" % tag)
 174     items = os.listdir('_build')
 175     if len(items) == 0:
 176         error_exit("Nothing found to upload")
 177     try:
 178         release = repo.create_release(tag, name=tag, draft=True)
 179     except Exception as e:
 180         error_exit("Error creating release '%s' (%r)" % (tag, e))
 181     print("I: updating release description from template")
 182     text = release_text(tag, previous_tag, mac_github=mac_github, appveyor=appveyor)
 183     print(text)
 184     if release.edit(body=text):
 185         print("I: updated release description")
 186     else:
 187         error_exit("E: updating release description")
 188     print("D: uploading items\n - %s" % "\n - ".join(items))
 189     m = magic.Magic(mime=True)
 190     for itm in items:
 191         filename = os.path.join("_build", itm)
 192         content_type = m.from_file(filename)
 193         print("I: uploading %s..." % itm)
 194         with open(filename, "rb") as f:
 195             try:
 196                 _ = release.upload_asset(content_type, itm, f)
 197             except Exception as e:
 198                 error_exit("Error uploading asset '%s' (%r)" % (itm, e))
 199     print("I: upload success")
 200
 201
 202 if __name__ == "__main__":
 203     parser = argparse.ArgumentParser(description='upload gpodder-osx-bundle artifacts to a github release\n'
 204         'Example usage: \n'
 205         '    GITHUB_TOKEN=xxx python github_release.py --download --mac-github-workflow 1234567890 --appveyor 1.0.104 3.10.4\n'
 206         '    GITHUB_TOKEN=xxx python github_release.py --mac-github-workflow 1234567890 --appveyor 1.0.104 3.10.4\n',
 207         formatter_class=argparse.RawTextHelpFormatter)
 208     parser.add_argument('tag', type=str, help='gPodder git tag to create a release from')
 209     parser.add_argument('--download', action='store_true', help='download artifacts')
 210     parser.add_argument('--mac-github-workflow', type=int, help='mac github workflow number')
 211     parser.add_argument('--appveyor', type=str, help='appveyor build number')
 212     parser.add_argument('--debug', '-d', action='store_true', help='debug requests')
 213
 214     args = parser.parse_args()
 215
 216     if args.debug:
 217         debug_requests()
 218
 219     github_token = os.environ.get("GITHUB_TOKEN")
 220     if not github_token:
 221         error_exit("E: set GITHUB_TOKEN environment", -1)
 222
 223     gh = login(token=github_token)
 224     repo = gh.repository('gpodder', 'gpodder')
 225
 226     if args.download:
 227         if not args.mac_github_workflow:
 228             error_exit("E: --download requires --mac-github-workflow number")
 229         elif not args.appveyor:
 230             error_exit("E: --download requires --appveyor number")
 231         if os.path.isdir("_build"):
 232             error_exit("E: _build directory exists", -1)
 233         os.mkdir("_build")
 234         download_mac_github(args.mac_github_workflow, "macOS", args.tag)
 235         download_appveyor(args.appveyor, "windows")
 236         print("I: download success.")
 237     else:
 238         if not os.path.exists("_build"):
 239             error_exit("E: _build directory doesn't exist. You need to download build artifacts (see Usage)", -1)
 240
 241     previous_tag = get_previous_tag()
 242     upload(repo, args.tag, previous_tag, args.mac_github_workflow, args.appveyor)