Merge Chromium + Blink git repositories
[chromium-blink-merge.git] / tools / telemetry / third_party / modulegraph / setup.py
bloba1a4cb6eb261134d55a76db3b4ff2b11be03e7a0
1 """
2 Shared setup file for simple python packages. Uses a setup.cfg that
3 is the same as the distutils2 project, unless noted otherwise.
5 It exists for two reasons:
6 1) This makes it easier to reuse setup.py code between my own
7 projects
9 2) Easier migration to distutils2 when that catches on.
11 Additional functionality:
13 * Section metadata:
14 requires-test: Same as 'tests_require' option for setuptools.
16 """
18 import sys
19 import os
20 import re
21 import platform
22 from fnmatch import fnmatch
23 import os
24 import sys
25 import time
26 import tempfile
27 import tarfile
28 try:
29 import urllib.request as urllib
30 except ImportError:
31 import urllib
32 from distutils import log
33 try:
34 from hashlib import md5
36 except ImportError:
37 from md5 import md5
39 if sys.version_info[0] == 2:
40 from ConfigParser import RawConfigParser, NoOptionError, NoSectionError
41 else:
42 from configparser import RawConfigParser, NoOptionError, NoSectionError
44 ROOTDIR = os.path.dirname(os.path.abspath(__file__))
50 # Parsing the setup.cfg and converting it to something that can be
51 # used by setuptools.setup()
56 def eval_marker(value):
57 """
58 Evaluate an distutils2 environment marker.
60 This code is unsafe when used with hostile setup.cfg files,
61 but that's not a problem for our own files.
62 """
63 value = value.strip()
65 class M:
66 def __init__(self, **kwds):
67 for k, v in kwds.items():
68 setattr(self, k, v)
70 variables = {
71 'python_version': '%d.%d'%(sys.version_info[0], sys.version_info[1]),
72 'python_full_version': sys.version.split()[0],
73 'os': M(
74 name=os.name,
76 'sys': M(
77 platform=sys.platform,
79 'platform': M(
80 version=platform.version(),
81 machine=platform.machine(),
85 return bool(eval(value, variables, variables))
88 return True
90 def _opt_value(cfg, into, section, key, transform = None):
91 try:
92 v = cfg.get(section, key)
93 if transform != _as_lines and ';' in v:
94 v, marker = v.rsplit(';', 1)
95 if not eval_marker(marker):
96 return
98 v = v.strip()
100 if v:
101 if transform:
102 into[key] = transform(v.strip())
103 else:
104 into[key] = v.strip()
106 except (NoOptionError, NoSectionError):
107 pass
109 def _as_bool(value):
110 if value.lower() in ('y', 'yes', 'on'):
111 return True
112 elif value.lower() in ('n', 'no', 'off'):
113 return False
114 elif value.isdigit():
115 return bool(int(value))
116 else:
117 raise ValueError(value)
119 def _as_list(value):
120 return value.split()
122 def _as_lines(value):
123 result = []
124 for v in value.splitlines():
125 if ';' in v:
126 v, marker = v.rsplit(';', 1)
127 if not eval_marker(marker):
128 continue
130 v = v.strip()
131 if v:
132 result.append(v)
133 else:
134 result.append(v)
135 return result
137 def _map_requirement(value):
138 m = re.search(r'(\S+)\s*(?:\((.*)\))?', value)
139 name = m.group(1)
140 version = m.group(2)
142 if version is None:
143 return name
145 else:
146 mapped = []
147 for v in version.split(','):
148 v = v.strip()
149 if v[0].isdigit():
150 # Checks for a specific version prefix
151 m = v.rsplit('.', 1)
152 mapped.append('>=%s,<%s.%s'%(
153 v, m[0], int(m[1])+1))
155 else:
156 mapped.append(v)
157 return '%s %s'%(name, ','.join(mapped),)
159 def _as_requires(value):
160 requires = []
161 for req in value.splitlines():
162 if ';' in req:
163 req, marker = v.rsplit(';', 1)
164 if not eval_marker(marker):
165 continue
166 req = req.strip()
168 if not req:
169 continue
170 requires.append(_map_requirement(req))
171 return requires
173 def parse_setup_cfg():
174 cfg = RawConfigParser()
175 r = cfg.read([os.path.join(ROOTDIR, 'setup.cfg')])
176 if len(r) != 1:
177 print("Cannot read 'setup.cfg'")
178 sys.exit(1)
180 metadata = dict(
181 name = cfg.get('metadata', 'name'),
182 version = cfg.get('metadata', 'version'),
183 description = cfg.get('metadata', 'description'),
186 _opt_value(cfg, metadata, 'metadata', 'license')
187 _opt_value(cfg, metadata, 'metadata', 'maintainer')
188 _opt_value(cfg, metadata, 'metadata', 'maintainer_email')
189 _opt_value(cfg, metadata, 'metadata', 'author')
190 _opt_value(cfg, metadata, 'metadata', 'author_email')
191 _opt_value(cfg, metadata, 'metadata', 'url')
192 _opt_value(cfg, metadata, 'metadata', 'download_url')
193 _opt_value(cfg, metadata, 'metadata', 'classifiers', _as_lines)
194 _opt_value(cfg, metadata, 'metadata', 'platforms', _as_list)
195 _opt_value(cfg, metadata, 'metadata', 'packages', _as_list)
196 _opt_value(cfg, metadata, 'metadata', 'keywords', _as_list)
198 try:
199 v = cfg.get('metadata', 'requires-dist')
201 except (NoOptionError, NoSectionError):
202 pass
204 else:
205 requires = _as_requires(v)
206 if requires:
207 metadata['install_requires'] = requires
209 try:
210 v = cfg.get('metadata', 'requires-test')
212 except (NoOptionError, NoSectionError):
213 pass
215 else:
216 requires = _as_requires(v)
217 if requires:
218 metadata['tests_require'] = requires
221 try:
222 v = cfg.get('metadata', 'long_description_file')
223 except (NoOptionError, NoSectionError):
224 pass
226 else:
227 parts = []
228 for nm in v.split():
229 fp = open(nm, 'rU')
230 parts.append(fp.read())
231 fp.close()
233 metadata['long_description'] = '\n\n'.join(parts)
236 try:
237 v = cfg.get('metadata', 'zip-safe')
238 except (NoOptionError, NoSectionError):
239 pass
241 else:
242 metadata['zip_safe'] = _as_bool(v)
244 try:
245 v = cfg.get('metadata', 'console_scripts')
246 except (NoOptionError, NoSectionError):
247 pass
249 else:
250 if 'entry_points' not in metadata:
251 metadata['entry_points'] = {}
253 metadata['entry_points']['console_scripts'] = v.splitlines()
255 if sys.version_info[:2] <= (2,6):
256 try:
257 metadata['tests_require'] += ", unittest2"
258 except KeyError:
259 metadata['tests_require'] = "unittest2"
261 return metadata
267 # Bootstrapping setuptools/distribute, based on
268 # a heavily modified version of distribute_setup.py
274 SETUPTOOLS_PACKAGE='setuptools'
277 try:
278 import subprocess
280 def _python_cmd(*args):
281 args = (sys.executable,) + args
282 return subprocess.call(args) == 0
284 except ImportError:
285 def _python_cmd(*args):
286 args = (sys.executable,) + args
287 new_args = []
288 for a in args:
289 new_args.append(a.replace("'", "'\"'\"'"))
290 os.system(' '.join(new_args)) == 0
293 try:
294 import json
296 def get_pypi_src_download(package):
297 url = 'https://pypi.python.org/pypi/%s/json'%(package,)
298 fp = urllib.urlopen(url)
299 try:
300 try:
301 data = fp.read()
303 finally:
304 fp.close()
305 except urllib.error:
306 raise RuntimeError("Cannot determine download link for %s"%(package,))
308 pkgdata = json.loads(data.decode('utf-8'))
309 if 'urls' not in pkgdata:
310 raise RuntimeError("Cannot determine download link for %s"%(package,))
312 for info in pkgdata['urls']:
313 if info['packagetype'] == 'sdist' and info['url'].endswith('tar.gz'):
314 return (info.get('md5_digest'), info['url'])
316 raise RuntimeError("Cannot determine downlink link for %s"%(package,))
318 except ImportError:
319 # Python 2.5 compatibility, no JSON in stdlib but luckily JSON syntax is
320 # simular enough to Python's syntax to be able to abuse the Python compiler
322 import _ast as ast
324 def get_pypi_src_download(package):
325 url = 'https://pypi.python.org/pypi/%s/json'%(package,)
326 fp = urllib.urlopen(url)
327 try:
328 try:
329 data = fp.read()
331 finally:
332 fp.close()
333 except urllib.error:
334 raise RuntimeError("Cannot determine download link for %s"%(package,))
337 a = compile(data, '-', 'eval', ast.PyCF_ONLY_AST)
338 if not isinstance(a, ast.Expression):
339 raise RuntimeError("Cannot determine download link for %s"%(package,))
341 a = a.body
342 if not isinstance(a, ast.Dict):
343 raise RuntimeError("Cannot determine download link for %s"%(package,))
345 for k, v in zip(a.keys, a.values):
346 if not isinstance(k, ast.Str):
347 raise RuntimeError("Cannot determine download link for %s"%(package,))
349 k = k.s
350 if k == 'urls':
351 a = v
352 break
353 else:
354 raise RuntimeError("PyPI JSON for %s doesn't contain URLs section"%(package,))
356 if not isinstance(a, ast.List):
357 raise RuntimeError("Cannot determine download link for %s"%(package,))
359 for info in v.elts:
360 if not isinstance(info, ast.Dict):
361 raise RuntimeError("Cannot determine download link for %s"%(package,))
362 url = None
363 packagetype = None
364 chksum = None
366 for k, v in zip(info.keys, info.values):
367 if not isinstance(k, ast.Str):
368 raise RuntimeError("Cannot determine download link for %s"%(package,))
370 if k.s == 'url':
371 if not isinstance(v, ast.Str):
372 raise RuntimeError("Cannot determine download link for %s"%(package,))
373 url = v.s
375 elif k.s == 'packagetype':
376 if not isinstance(v, ast.Str):
377 raise RuntimeError("Cannot determine download link for %s"%(package,))
378 packagetype = v.s
380 elif k.s == 'md5_digest':
381 if not isinstance(v, ast.Str):
382 raise RuntimeError("Cannot determine download link for %s"%(package,))
383 chksum = v.s
385 if url is not None and packagetype == 'sdist' and url.endswith('.tar.gz'):
386 return (chksum, url)
388 raise RuntimeError("Cannot determine download link for %s"%(package,))
390 def _build_egg(egg, tarball, to_dir):
391 # extracting the tarball
392 tmpdir = tempfile.mkdtemp()
393 log.warn('Extracting in %s', tmpdir)
394 old_wd = os.getcwd()
395 try:
396 os.chdir(tmpdir)
397 tar = tarfile.open(tarball)
398 _extractall(tar)
399 tar.close()
401 # going in the directory
402 subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
403 os.chdir(subdir)
404 log.warn('Now working in %s', subdir)
406 # building an egg
407 log.warn('Building a %s egg in %s', egg, to_dir)
408 _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
410 finally:
411 os.chdir(old_wd)
412 # returning the result
413 log.warn(egg)
414 if not os.path.exists(egg):
415 raise IOError('Could not build the egg.')
418 def _do_download(to_dir, packagename=SETUPTOOLS_PACKAGE):
419 tarball = download_setuptools(packagename, to_dir)
420 version = tarball.split('-')[-1][:-7]
421 egg = os.path.join(to_dir, '%s-%s-py%d.%d.egg'
422 % (packagename, version, sys.version_info[0], sys.version_info[1]))
423 if not os.path.exists(egg):
424 _build_egg(egg, tarball, to_dir)
425 sys.path.insert(0, egg)
426 import setuptools
427 setuptools.bootstrap_install_from = egg
430 def use_setuptools():
431 # making sure we use the absolute path
432 return _do_download(os.path.abspath(os.curdir))
434 def download_setuptools(packagename, to_dir):
435 # making sure we use the absolute path
436 to_dir = os.path.abspath(to_dir)
437 try:
438 from urllib.request import urlopen
439 except ImportError:
440 from urllib2 import urlopen
442 chksum, url = get_pypi_src_download(packagename)
443 tgz_name = os.path.basename(url)
444 saveto = os.path.join(to_dir, tgz_name)
446 src = dst = None
447 if not os.path.exists(saveto): # Avoid repeated downloads
448 try:
449 log.warn("Downloading %s", url)
450 src = urlopen(url)
451 # Read/write all in one block, so we don't create a corrupt file
452 # if the download is interrupted.
453 data = src.read()
455 if chksum is not None:
456 data_sum = md5(data).hexdigest()
457 if data_sum != chksum:
458 raise RuntimeError("Downloading %s failed: corrupt checksum"%(url,))
461 dst = open(saveto, "wb")
462 dst.write(data)
463 finally:
464 if src:
465 src.close()
466 if dst:
467 dst.close()
468 return os.path.realpath(saveto)
472 def _extractall(self, path=".", members=None):
473 """Extract all members from the archive to the current working
474 directory and set owner, modification time and permissions on
475 directories afterwards. `path' specifies a different directory
476 to extract to. `members' is optional and must be a subset of the
477 list returned by getmembers().
479 import copy
480 import operator
481 from tarfile import ExtractError
482 directories = []
484 if members is None:
485 members = self
487 for tarinfo in members:
488 if tarinfo.isdir():
489 # Extract directories with a safe mode.
490 directories.append(tarinfo)
491 tarinfo = copy.copy(tarinfo)
492 tarinfo.mode = 448 # decimal for oct 0700
493 self.extract(tarinfo, path)
495 # Reverse sort directories.
496 if sys.version_info < (2, 4):
497 def sorter(dir1, dir2):
498 return cmp(dir1.name, dir2.name)
499 directories.sort(sorter)
500 directories.reverse()
501 else:
502 directories.sort(key=operator.attrgetter('name'), reverse=True)
504 # Set correct owner, mtime and filemode on directories.
505 for tarinfo in directories:
506 dirpath = os.path.join(path, tarinfo.name)
507 try:
508 self.chown(tarinfo, dirpath)
509 self.utime(tarinfo, dirpath)
510 self.chmod(tarinfo, dirpath)
511 except ExtractError:
512 e = sys.exc_info()[1]
513 if self.errorlevel > 1:
514 raise
515 else:
516 self._dbg(1, "tarfile: %s" % e)
522 # Definitions of custom commands
527 try:
528 import setuptools
530 except ImportError:
531 use_setuptools()
533 from setuptools import setup
535 try:
536 from distutils.core import PyPIRCCommand
537 except ImportError:
538 PyPIRCCommand = None # Ancient python version
540 from distutils.core import Command
541 from distutils.errors import DistutilsError
542 from distutils import log
544 if PyPIRCCommand is None:
545 class upload_docs (Command):
546 description = "upload sphinx documentation"
547 user_options = []
549 def initialize_options(self):
550 pass
552 def finalize_options(self):
553 pass
555 def run(self):
556 raise DistutilsError("not supported on this version of python")
558 else:
559 class upload_docs (PyPIRCCommand):
560 description = "upload sphinx documentation"
561 user_options = PyPIRCCommand.user_options
563 def initialize_options(self):
564 PyPIRCCommand.initialize_options(self)
565 self.username = ''
566 self.password = ''
569 def finalize_options(self):
570 PyPIRCCommand.finalize_options(self)
571 config = self._read_pypirc()
572 if config != {}:
573 self.username = config['username']
574 self.password = config['password']
577 def run(self):
578 import subprocess
579 import shutil
580 import zipfile
581 import os
582 import urllib
583 import StringIO
584 from base64 import standard_b64encode
585 import httplib
586 import urlparse
588 # Extract the package name from distutils metadata
589 meta = self.distribution.metadata
590 name = meta.get_name()
592 # Run sphinx
593 if os.path.exists('doc/_build'):
594 shutil.rmtree('doc/_build')
595 os.mkdir('doc/_build')
597 p = subprocess.Popen(['make', 'html'],
598 cwd='doc')
599 exit = p.wait()
600 if exit != 0:
601 raise DistutilsError("sphinx-build failed")
603 # Collect sphinx output
604 if not os.path.exists('dist'):
605 os.mkdir('dist')
606 zf = zipfile.ZipFile('dist/%s-docs.zip'%(name,), 'w',
607 compression=zipfile.ZIP_DEFLATED)
609 for toplevel, dirs, files in os.walk('doc/_build/html'):
610 for fn in files:
611 fullname = os.path.join(toplevel, fn)
612 relname = os.path.relpath(fullname, 'doc/_build/html')
614 print ("%s -> %s"%(fullname, relname))
616 zf.write(fullname, relname)
618 zf.close()
620 # Upload the results, this code is based on the distutils
621 # 'upload' command.
622 content = open('dist/%s-docs.zip'%(name,), 'rb').read()
624 data = {
625 ':action': 'doc_upload',
626 'name': name,
627 'content': ('%s-docs.zip'%(name,), content),
629 auth = "Basic " + standard_b64encode(self.username + ":" +
630 self.password)
633 boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254'
634 sep_boundary = '\n--' + boundary
635 end_boundary = sep_boundary + '--'
636 body = StringIO.StringIO()
637 for key, value in data.items():
638 if not isinstance(value, list):
639 value = [value]
641 for value in value:
642 if isinstance(value, tuple):
643 fn = ';filename="%s"'%(value[0])
644 value = value[1]
645 else:
646 fn = ''
648 body.write(sep_boundary)
649 body.write('\nContent-Disposition: form-data; name="%s"'%key)
650 body.write(fn)
651 body.write("\n\n")
652 body.write(value)
654 body.write(end_boundary)
655 body.write('\n')
656 body = body.getvalue()
658 self.announce("Uploading documentation to %s"%(self.repository,), log.INFO)
660 schema, netloc, url, params, query, fragments = \
661 urlparse.urlparse(self.repository)
664 if schema == 'http':
665 http = httplib.HTTPConnection(netloc)
666 elif schema == 'https':
667 http = httplib.HTTPSConnection(netloc)
668 else:
669 raise AssertionError("unsupported schema "+schema)
671 data = ''
672 loglevel = log.INFO
673 try:
674 http.connect()
675 http.putrequest("POST", url)
676 http.putheader('Content-type',
677 'multipart/form-data; boundary=%s'%boundary)
678 http.putheader('Content-length', str(len(body)))
679 http.putheader('Authorization', auth)
680 http.endheaders()
681 http.send(body)
682 except socket.error:
683 e = socket.exc_info()[1]
684 self.announce(str(e), log.ERROR)
685 return
687 r = http.getresponse()
688 if r.status in (200, 301):
689 self.announce('Upload succeeded (%s): %s' % (r.status, r.reason),
690 log.INFO)
691 else:
692 self.announce('Upload failed (%s): %s' % (r.status, r.reason),
693 log.ERROR)
695 print ('-'*75)
696 print (r.read())
697 print ('-'*75)
700 def recursiveGlob(root, pathPattern):
702 Recursively look for files matching 'pathPattern'. Return a list
703 of matching files/directories.
705 result = []
707 for rootpath, dirnames, filenames in os.walk(root):
708 for fn in filenames:
709 if fnmatch(fn, pathPattern):
710 result.append(os.path.join(rootpath, fn))
711 return result
714 def importExternalTestCases(unittest,
715 pathPattern="test_*.py", root=".", package=None):
717 Import all unittests in the PyObjC tree starting at 'root'
720 testFiles = recursiveGlob(root, pathPattern)
721 testModules = map(lambda x:x[len(root)+1:-3].replace('/', '.'), testFiles)
722 if package is not None:
723 testModules = [(package + '.' + m) for m in testModules]
725 suites = []
727 for modName in testModules:
728 try:
729 module = __import__(modName)
730 except ImportError:
731 print("SKIP %s: %s"%(modName, sys.exc_info()[1]))
732 continue
734 if '.' in modName:
735 for elem in modName.split('.')[1:]:
736 module = getattr(module, elem)
738 s = unittest.defaultTestLoader.loadTestsFromModule(module)
739 suites.append(s)
741 return unittest.TestSuite(suites)
745 class test (Command):
746 description = "run test suite"
747 user_options = [
748 ('verbosity=', None, "print what tests are run"),
751 def initialize_options(self):
752 self.verbosity='1'
754 def finalize_options(self):
755 if isinstance(self.verbosity, str):
756 self.verbosity = int(self.verbosity)
759 def cleanup_environment(self):
760 ei_cmd = self.get_finalized_command('egg_info')
761 egg_name = ei_cmd.egg_name.replace('-', '_')
763 to_remove = []
764 for dirname in sys.path:
765 bn = os.path.basename(dirname)
766 if bn.startswith(egg_name + "-"):
767 to_remove.append(dirname)
769 for dirname in to_remove:
770 log.info("removing installed %r from sys.path before testing"%(
771 dirname,))
772 sys.path.remove(dirname)
774 def add_project_to_sys_path(self):
775 from pkg_resources import normalize_path, add_activation_listener
776 from pkg_resources import working_set, require
778 self.reinitialize_command('egg_info')
779 self.run_command('egg_info')
780 self.reinitialize_command('build_ext', inplace=1)
781 self.run_command('build_ext')
784 # Check if this distribution is already on sys.path
785 # and remove that version, this ensures that the right
786 # copy of the package gets tested.
788 self.__old_path = sys.path[:]
789 self.__old_modules = sys.modules.copy()
792 ei_cmd = self.get_finalized_command('egg_info')
793 sys.path.insert(0, normalize_path(ei_cmd.egg_base))
794 sys.path.insert(1, os.path.dirname(__file__))
796 # Strip the namespace packages defined in this distribution
797 # from sys.modules, needed to reset the search path for
798 # those modules.
800 nspkgs = getattr(self.distribution, 'namespace_packages')
801 if nspkgs is not None:
802 for nm in nspkgs:
803 del sys.modules[nm]
805 # Reset pkg_resources state:
806 add_activation_listener(lambda dist: dist.activate())
807 working_set.__init__()
808 require('%s==%s'%(ei_cmd.egg_name, ei_cmd.egg_version))
810 def remove_from_sys_path(self):
811 from pkg_resources import working_set
812 sys.path[:] = self.__old_path
813 sys.modules.clear()
814 sys.modules.update(self.__old_modules)
815 working_set.__init__()
818 def run(self):
819 import unittest
821 # Ensure that build directory is on sys.path (py3k)
823 self.cleanup_environment()
824 self.add_project_to_sys_path()
826 try:
827 meta = self.distribution.metadata
828 name = meta.get_name()
829 test_pkg = name + "_tests"
830 suite = importExternalTestCases(unittest,
831 "test_*.py", test_pkg, test_pkg)
833 runner = unittest.TextTestRunner(verbosity=self.verbosity)
834 result = runner.run(suite)
836 # Print out summary. This is a structured format that
837 # should make it easy to use this information in scripts.
838 summary = dict(
839 count=result.testsRun,
840 fails=len(result.failures),
841 errors=len(result.errors),
842 xfails=len(getattr(result, 'expectedFailures', [])),
843 xpass=len(getattr(result, 'expectedSuccesses', [])),
844 skip=len(getattr(result, 'skipped', [])),
846 print("SUMMARY: %s"%(summary,))
848 finally:
849 self.remove_from_sys_path()
854 # And finally run the setuptools main entry point.
859 metadata = parse_setup_cfg()
861 setup(
862 cmdclass=dict(
863 upload_docs=upload_docs,
864 test=test,
866 **metadata