utils/benchmark/mingw.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3
   4 import argparse
   5 import errno
   6 import logging
   7 import os
   8 import platform
   9 import re
  10 import sys
  11 import subprocess
  12 import tempfile
  13
  14 try:
  15     import winreg
  16 except ImportError:
  17     import _winreg as winreg
  18 try:
  19     import urllib.request as request
  20 except ImportError:
  21     import urllib as request
  22 try:
  23     import urllib.parse as parse
  24 except ImportError:
  25     import urlparse as parse
  26
  27 class EmptyLogger(object):
  28     '''
  29     Provides an implementation that performs no logging
  30     '''
  31     def debug(self, *k, **kw):
  32         pass
  33     def info(self, *k, **kw):
  34         pass
  35     def warn(self, *k, **kw):
  36         pass
  37     def error(self, *k, **kw):
  38         pass
  39     def critical(self, *k, **kw):
  40         pass
  41     def setLevel(self, *k, **kw):
  42         pass
  43
  44 urls = (
  45     'http://downloads.sourceforge.net/project/mingw-w64/Toolchains%20'
  46         'targetting%20Win32/Personal%20Builds/mingw-builds/installer/'
  47         'repository.txt',
  48     'http://downloads.sourceforge.net/project/mingwbuilds/host-windows/'
  49         'repository.txt'
  50 )
  51 '''
  52 A list of mingw-build repositories
  53 '''
  54
  55 def repository(urls = urls, log = EmptyLogger()):
  56     '''
  57     Downloads and parse mingw-build repository files and parses them
  58     '''
  59     log.info('getting mingw-builds repository')
  60     versions = {}
  61     re_sourceforge = re.compile(r'http://sourceforge.net/projects/([^/]+)/files')
  62     re_sub = r'http://downloads.sourceforge.net/project/\1'
  63     for url in urls:
  64         log.debug(' - requesting: %s', url)
  65         socket = request.urlopen(url)
  66         repo = socket.read()
  67         if not isinstance(repo, str):
  68             repo = repo.decode();
  69         socket.close()
  70         for entry in repo.split('\n')[:-1]:
  71             value = entry.split('|')
  72             version = tuple([int(n) for n in value[0].strip().split('.')])
  73             version = versions.setdefault(version, {})
  74             arch = value[1].strip()
  75             if arch == 'x32':
  76                 arch = 'i686'
  77             elif arch == 'x64':
  78                 arch = 'x86_64'
  79             arch = version.setdefault(arch, {})
  80             threading = arch.setdefault(value[2].strip(), {})
  81             exceptions = threading.setdefault(value[3].strip(), {})
  82             revision = exceptions.setdefault(int(value[4].strip()[3:]),
  83                 re_sourceforge.sub(re_sub, value[5].strip()))
  84     return versions
  85
  86 def find_in_path(file, path=None):
  87     '''
  88     Attempts to find an executable in the path
  89     '''
  90     if platform.system() == 'Windows':
  91         file += '.exe'
  92     if path is None:
  93         path = os.environ.get('PATH', '')
  94     if type(path) is type(''):
  95         path = path.split(os.pathsep)
  96     return list(filter(os.path.exists,
  97         map(lambda dir, file=file: os.path.join(dir, file), path)))
  98
  99 def find_7zip(log = EmptyLogger()):
 100     '''
 101     Attempts to find 7zip for unpacking the mingw-build archives
 102     '''
 103     log.info('finding 7zip')
 104     path = find_in_path('7z')
 105     if not path:
 106         key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\7-Zip')
 107         path, _ = winreg.QueryValueEx(key, 'Path')
 108         path = [os.path.join(path, '7z.exe')]
 109     log.debug('found \'%s\'', path[0])
 110     return path[0]
 111
 112 find_7zip()
 113
 114 def unpack(archive, location, log = EmptyLogger()):
 115     '''
 116     Unpacks a mingw-builds archive
 117     '''
 118     sevenzip = find_7zip(log)
 119     log.info('unpacking %s', os.path.basename(archive))
 120     cmd = [sevenzip, 'x', archive, '-o' + location, '-y']
 121     log.debug(' - %r', cmd)
 122     with open(os.devnull, 'w') as devnull:
 123         subprocess.check_call(cmd, stdout = devnull)
 124
 125 def download(url, location, log = EmptyLogger()):
 126     '''
 127     Downloads and unpacks a mingw-builds archive
 128     '''
 129     log.info('downloading MinGW')
 130     log.debug(' - url: %s', url)
 131     log.debug(' - location: %s', location)
 132
 133     re_content = re.compile(r'attachment;[ \t]*filename=(")?([^"]*)(")?[\r\n]*')
 134
 135     stream = request.urlopen(url)
 136     try:
 137         content = stream.getheader('Content-Disposition') or ''
 138     except AttributeError:
 139         content = stream.headers.getheader('Content-Disposition') or ''
 140     matches = re_content.match(content)
 141     if matches:
 142         filename = matches.group(2)
 143     else:
 144         parsed = parse.urlparse(stream.geturl())
 145         filename = os.path.basename(parsed.path)
 146
 147     try:
 148         os.makedirs(location)
 149     except OSError as e:
 150         if e.errno == errno.EEXIST and os.path.isdir(location):
 151             pass
 152         else:
 153             raise
 154
 155     archive = os.path.join(location, filename)
 156     with open(archive, 'wb') as out:
 157         while True:
 158             buf = stream.read(1024)
 159             if not buf:
 160                 break
 161             out.write(buf)
 162     unpack(archive, location, log = log)
 163     os.remove(archive)
 164
 165     possible = os.path.join(location, 'mingw64')
 166     if not os.path.exists(possible):
 167         possible = os.path.join(location, 'mingw32')
 168         if not os.path.exists(possible):
 169             raise ValueError('Failed to find unpacked MinGW: ' + possible)
 170     return possible
 171
 172 def root(location = None, arch = None, version = None, threading = None,
 173         exceptions = None, revision = None, log = EmptyLogger()):
 174     '''
 175     Returns the root folder of a specific version of the mingw-builds variant
 176     of gcc. Will download the compiler if needed
 177     '''
 178
 179     # Get the repository if we don't have all the information
 180     if not (arch and version and threading and exceptions and revision):
 181         versions = repository(log = log)
 182
 183     # Determine some defaults
 184     version = version or max(versions.keys())
 185     if not arch:
 186         arch = platform.machine().lower()
 187         if arch == 'x86':
 188             arch = 'i686'
 189         elif arch == 'amd64':
 190             arch = 'x86_64'
 191     if not threading:
 192         keys = versions[version][arch].keys()
 193         if 'posix' in keys:
 194             threading = 'posix'
 195         elif 'win32' in keys:
 196             threading = 'win32'
 197         else:
 198             threading = keys[0]
 199     if not exceptions:
 200         keys = versions[version][arch][threading].keys()
 201         if 'seh' in keys:
 202             exceptions = 'seh'
 203         elif 'sjlj' in keys:
 204             exceptions = 'sjlj'
 205         else:
 206             exceptions = keys[0]
 207     if revision == None:
 208         revision = max(versions[version][arch][threading][exceptions].keys())
 209     if not location:
 210         location = os.path.join(tempfile.gettempdir(), 'mingw-builds')
 211
 212     # Get the download url
 213     url = versions[version][arch][threading][exceptions][revision]
 214
 215     # Tell the user whatzzup
 216     log.info('finding MinGW %s', '.'.join(str(v) for v in version))
 217     log.debug(' - arch: %s', arch)
 218     log.debug(' - threading: %s', threading)
 219     log.debug(' - exceptions: %s', exceptions)
 220     log.debug(' - revision: %s', revision)
 221     log.debug(' - url: %s', url)
 222
 223     # Store each specific revision differently
 224     slug = '{version}-{arch}-{threading}-{exceptions}-rev{revision}'
 225     slug = slug.format(
 226         version = '.'.join(str(v) for v in version),
 227         arch = arch,
 228         threading = threading,
 229         exceptions = exceptions,
 230         revision = revision
 231     )
 232     if arch == 'x86_64':
 233         root_dir = os.path.join(location, slug, 'mingw64')
 234     elif arch == 'i686':
 235         root_dir = os.path.join(location, slug, 'mingw32')
 236     else:
 237         raise ValueError('Unknown MinGW arch: ' + arch)
 238
 239     # Download if needed
 240     if not os.path.exists(root_dir):
 241         downloaded = download(url, os.path.join(location, slug), log = log)
 242         if downloaded != root_dir:
 243             raise ValueError('The location of mingw did not match\n%s\n%s'
 244                 % (downloaded, root_dir))
 245
 246     return root_dir
 247
 248 def str2ver(string):
 249     '''
 250     Converts a version string into a tuple
 251     '''
 252     try:
 253         version = tuple(int(v) for v in string.split('.'))
 254         if len(version) is not 3:
 255             raise ValueError()
 256     except ValueError:
 257         raise argparse.ArgumentTypeError(
 258             'please provide a three digit version string')
 259     return version
 260
 261 def main():
 262     '''
 263     Invoked when the script is run directly by the python interpreter
 264     '''
 265     parser = argparse.ArgumentParser(
 266         description = 'Downloads a specific version of MinGW',
 267         formatter_class = argparse.ArgumentDefaultsHelpFormatter
 268     )
 269     parser.add_argument('--location',
 270         help = 'the location to download the compiler to',
 271         default = os.path.join(tempfile.gettempdir(), 'mingw-builds'))
 272     parser.add_argument('--arch', required = True, choices = ['i686', 'x86_64'],
 273         help = 'the target MinGW architecture string')
 274     parser.add_argument('--version', type = str2ver,
 275         help = 'the version of GCC to download')
 276     parser.add_argument('--threading', choices = ['posix', 'win32'],
 277         help = 'the threading type of the compiler')
 278     parser.add_argument('--exceptions', choices = ['sjlj', 'seh', 'dwarf'],
 279         help = 'the method to throw exceptions')
 280     parser.add_argument('--revision', type=int,
 281         help = 'the revision of the MinGW release')
 282     group = parser.add_mutually_exclusive_group()
 283     group.add_argument('-v', '--verbose', action='store_true',
 284         help='increase the script output verbosity')
 285     group.add_argument('-q', '--quiet', action='store_true',
 286         help='only print errors and warning')
 287     args = parser.parse_args()
 288
 289     # Create the logger
 290     logger = logging.getLogger('mingw')
 291     handler = logging.StreamHandler()
 292     formatter = logging.Formatter('%(message)s')
 293     handler.setFormatter(formatter)
 294     logger.addHandler(handler)
 295     logger.setLevel(logging.INFO)
 296     if args.quiet:
 297         logger.setLevel(logging.WARN)
 298     if args.verbose:
 299         logger.setLevel(logging.DEBUG)
 300
 301     # Get MinGW
 302     root_dir = root(location = args.location, arch = args.arch,
 303         version = args.version, threading = args.threading,
 304         exceptions = args.exceptions, revision = args.revision,
 305         log = logger)
 306
 307     sys.stdout.write('%s\n' % os.path.join(root_dir, 'bin'))
 308
 309 if __name__ == '__main__':
 310     try:
 311         main()
 312     except IOError as e:
 313         sys.stderr.write('IO error: %s\n' % e)
 314         sys.exit(1)
 315     except OSError as e:
 316         sys.stderr.write('OS error: %s\n' % e)
 317         sys.exit(1)
 318     except KeyboardInterrupt as e:
 319         sys.stderr.write('Killed\n')
 320         sys.exit(1)