pkgs/tools/networking/maubot/plugins/update.py

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"
   3
   4 import git
   5 import json
   6 import os
   7 import subprocess
   8 import ruamel.yaml
   9 import sys
  10 import toml
  11 import zipfile
  12
  13 from typing import Dict, List
  14
  15 HOSTNAMES = {
  16     'git.skeg1.se': 'gitlab',
  17     'edugit.org': 'gitlab',
  18     'codeberg.org': 'gitea',
  19 }
  20 PLUGINS: Dict[str, dict] = {}
  21
  22 yaml = ruamel.yaml.YAML(typ='safe')
  23
  24 TMP = os.environ.get('TEMPDIR', '/tmp')
  25
  26 def process_repo(path: str, official: bool):
  27     global PLUGINS
  28     with open(path, 'rt') as f:
  29         data = yaml.load(f)
  30     name, repourl, license, desc = data['name'], data['repo'], data['license'], data['description']
  31     origurl = repourl
  32     if '/' in name or ' ' in name:
  33         name = os.path.split(path)[-1].removesuffix('.yaml')
  34     name = name.replace('_', '-').lower()
  35     if name in PLUGINS.keys():
  36         raise ValueError(f'Duplicate plugin {name}, refusing to continue')
  37     repodir = os.path.join(TMP, 'maubot-plugins', name)
  38     plugindir = repodir
  39     if '/tree/' in repourl:
  40         repourl, rev_path = repourl.split('/tree/')
  41         rev, subdir = rev_path.strip('/').split('/')
  42         plugindir = os.path.join(plugindir, subdir)
  43     else:
  44         rev = None
  45         subdir = None
  46
  47     if repourl.startswith('http:'):
  48         repourl = 'https' + repourl[4:]
  49     repourl = repourl.rstrip('/')
  50     if not os.path.exists(repodir):
  51         print('Fetching', name)
  52         repo = git.Repo.clone_from(repourl + '.git', repodir)
  53     else:
  54         repo = git.Repo(repodir)
  55     tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
  56     tags = list(filter(lambda x: 'rc' not in str(x), tags))
  57     if tags:
  58         repo.git.checkout(tags[-1])
  59         rev = str(tags[-1])
  60     else:
  61         rev = str(repo.commit('HEAD'))
  62     ret: dict = {'attrs':{}}
  63     if subdir:
  64         ret['attrs']['postPatch'] = f'cd {subdir}'
  65     domain, query = repourl.removeprefix('https://').split('/', 1)
  66     hash = subprocess.run([
  67         'nurl',
  68         '--hash',
  69         f'file://{repodir}',
  70         rev
  71     ], capture_output=True, check=True).stdout.decode('utf-8')
  72     ret['attrs']['meta'] = {
  73         'description': desc,
  74         'homepage': origurl,
  75     }
  76     if domain == 'github.com':
  77         owner, repo = query.split('/')
  78         ret['github'] = {
  79             'owner': owner,
  80             'repo': repo,
  81             'rev': rev,
  82             'hash': hash,
  83         }
  84         ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
  85         ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
  86         repobase = f'{repourl}/blob/{rev}'
  87     elif HOSTNAMES.get(domain, 'gitea' if 'gitea.' in domain or 'forgejo.' in domain else None) == 'gitea':
  88         owner, repo = query.split('/')
  89         ret['gitea'] = {
  90             'domain': domain,
  91             'owner': owner,
  92             'repo': repo,
  93             'rev': rev,
  94             'hash': hash,
  95         }
  96         repobase = f'{repourl}/src/commit/{rev}'
  97         ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'
  98         ret['attrs']['meta']['changelog'] = f'{repourl}/releases'
  99     elif HOSTNAMES.get(domain, 'gitlab' if 'gitlab.' in domain else None) == 'gitlab':
 100         owner, repo = query.split('/')
 101         ret['gitlab'] = {
 102             'owner': owner,
 103             'repo': repo,
 104             'rev': rev,
 105             'hash': hash,
 106         }
 107         if domain != 'gitlab.com':
 108             ret['gitlab']['domain'] = domain
 109         repobase = f'{repourl}/-/blob/{rev}'
 110     else:
 111         raise ValueError(f'Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!')
 112     if os.path.exists(os.path.join(plugindir, 'CHANGELOG.md')):
 113         ret['attrs']['meta']['changelog'] = f'{repobase}/CHANGELOG.md'
 114     if os.path.exists(os.path.join(plugindir, 'maubot.yaml')):
 115         with open(os.path.join(plugindir, 'maubot.yaml'), 'rt') as f:
 116             ret['manifest'] = yaml.load(f)
 117     elif os.path.exists(os.path.join(plugindir, 'pyproject.toml')):
 118         ret['isPoetry'] = True
 119         with open(os.path.join(plugindir, 'pyproject.toml'), 'rt') as f:
 120             data = toml.load(f)
 121         deps = []
 122         for key, val in data['tool']['poetry'].get('dependencies', {}).items():
 123             if key in ['maubot', 'mautrix', 'python']:
 124                 continue
 125             reqs = []
 126             for req in val.split(','):
 127                 reqs.extend(poetry_to_pep(req))
 128             deps.append(key + ', '.join(reqs))
 129         ret['manifest'] = data['tool']['maubot']
 130         ret['manifest']['id'] = data['tool']['poetry']['name']
 131         ret['manifest']['version'] = data['tool']['poetry']['version']
 132         ret['manifest']['license'] = data['tool']['poetry']['license']
 133         if deps:
 134             ret['manifest']['dependencies'] = deps
 135     else:
 136         raise ValueError(f'No maubot.yaml or pyproject.toml found in {repodir}')
 137     # normalize non-spdx-conformant licenses this way
 138     # (and fill out missing license info)
 139     if 'license' not in ret['manifest'] or ret['manifest']['license'] in ['GPLv3', 'AGPL 3.0']:
 140         ret['attrs']['meta']['license'] = license
 141     elif ret['manifest']['license'] != license:
 142         print(f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}")
 143     if official:
 144         ret['isOfficial'] = official
 145     PLUGINS[name] = ret
 146
 147 def next_incomp(ver_s: str) -> str:
 148     ver = ver_s.split('.')
 149     zero = False
 150     for i in range(len(ver)):
 151         try:
 152             seg = int(ver[i])
 153         except ValueError:
 154             if zero:
 155                 ver = ver[:i]
 156                 break
 157             continue
 158         if zero:
 159             ver[i] = '0'
 160         elif seg:
 161             ver[i] = str(seg + 1)
 162             zero = True
 163     return '.'.join(ver)
 164
 165 def poetry_to_pep(ver_req: str) -> List[str]:
 166     if '*' in ver_req:
 167         raise NotImplementedError('Wildcard poetry versions not implemented!')
 168     if ver_req.startswith('^'):
 169         return ['>=' + ver_req[1:], '<' + next_incomp(ver_req[1:])]
 170     if ver_req.startswith('~'):
 171         return ['~=' + ver_req[1:]]
 172     return [ver_req]
 173
 174 def main():
 175     cache_path = os.path.join(TMP, 'maubot-plugins')
 176     if not os.path.exists(cache_path):
 177         os.makedirs(cache_path)
 178         git.Repo.clone_from('https://github.com/maubot/plugins.maubot.xyz', os.path.join(cache_path, '_repo'))
 179     else:
 180         pass
 181
 182     repodir = os.path.join(cache_path, '_repo')
 183
 184     for suffix, official in (('official', True), ('thirdparty', False)):
 185         directory = os.path.join(repodir, 'data', 'plugins', suffix)
 186         for plugin_name in os.listdir(directory):
 187             process_repo(os.path.join(directory, plugin_name), official)
 188
 189     if os.path.isdir('pkgs/tools/networking/maubot/plugins'):
 190         generated = 'pkgs/tools/networking/maubot/plugins/generated.json'
 191     else:
 192         script_dir = os.path.dirname(os.path.realpath(__file__))
 193         generated = os.path.join(script_dir, 'generated.json')
 194
 195     with open(generated, 'wt') as file:
 196         json.dump(PLUGINS, file, indent='  ', separators=(',', ': '), sort_keys=True)
 197         file.write('\n')
 198
 199 if __name__ == '__main__':
 200     main()