bootstrap/bootstrap.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 """
   5 bootstrap.py - bootstrapping utility for cabal-install.
   6
   7 See bootstrap/README.md for usage instructions.
   8 """
   9
  10 USAGE = """
  11 This utility is only intended for use in building cabal-install
  12 on a new platform. If you already have a functional (if dated) cabal-install
  13 please rather run `cabal install .`.
  14 """
  15
  16 import argparse
  17 from enum import Enum
  18 import hashlib
  19 import json
  20 from pathlib import Path
  21 import platform
  22 import shutil
  23 import subprocess
  24 import sys
  25 import tempfile
  26 import urllib.request
  27 from textwrap import dedent
  28 from typing import Optional, Dict, List, Tuple, \
  29                    NewType, BinaryIO, NamedTuple
  30
  31 #logging.basicConfig(level=logging.INFO)
  32
  33 BUILDDIR    = Path('_build')
  34
  35 BINDIR      = BUILDDIR / 'bin'            # binaries go there (--bindir)
  36 DISTDIR     = BUILDDIR / 'dists'          # --builddir
  37 UNPACKED    = BUILDDIR / 'unpacked'       # where we unpack tarballs
  38 TARBALLS    = BUILDDIR / 'tarballs'       # where we download tarballks
  39 PSEUDOSTORE = BUILDDIR / 'pseudostore'    # where we install packages
  40 ARTIFACTS   = BUILDDIR / 'artifacts'      # Where we put the archive
  41 TMPDIR      = BUILDDIR / 'tmp'            #
  42 PKG_DB      = BUILDDIR / 'packages.conf'  # package db
  43
  44 PackageName = NewType('PackageName', str)
  45 Version = NewType('Version', str)
  46 SHA256Hash = NewType('SHA256Hash', str)
  47
  48 class PackageSource(Enum):
  49     HACKAGE = 'hackage'
  50     LOCAL = 'local'
  51
  52 BuiltinDep = NamedTuple('BuiltinDep', [
  53     ('package', PackageName),
  54     ('version', Version),
  55 ])
  56
  57 BootstrapDep = NamedTuple('BootstrapDep', [
  58     ('package', PackageName),
  59     ('version', Version),
  60     ('source', PackageSource),
  61     # source tarball SHA256
  62     ('src_sha256', Optional[SHA256Hash]),
  63     # `revision` is only valid when source == HACKAGE.
  64     ('revision', Optional[int]),
  65     ('cabal_sha256', Optional[SHA256Hash]),
  66     ('flags', List[str]),
  67     ('component', Optional[str])
  68 ])
  69
  70 BootstrapInfo = NamedTuple('BootstrapInfo', [
  71     ('builtin', List[BuiltinDep]),
  72     ('dependencies', List[BootstrapDep]),
  73 ])
  74
  75 FetchInfo = NamedTuple('FetchInfo', [
  76     ('url', str),
  77     ('sha256', SHA256Hash)
  78 ])
  79
  80 FetchPlan = Dict[Path, FetchInfo]
  81
  82 local_packages: List[PackageName] = [ "Cabal-syntax"
  83                                     , "Cabal"
  84                                     , "Cabal-hooks"
  85                                     , "Cabal-QuickCheck"
  86                                     , "Cabal-described"
  87                                     , "Cabal-tests"
  88                                     , "Cabal-tree-diff"
  89                                     , "cabal-install-solver"
  90                                     , "cabal-install" ]
  91
  92 class Compiler:
  93     def __init__(self, ghc_path: Path):
  94         if not ghc_path.is_file():
  95             raise TypeError(f'GHC {ghc_path} is not a file')
  96
  97         self.ghc_path = ghc_path.resolve()
  98
  99         exe = ''
 100         if platform.system() == 'Windows': exe = '.exe'
 101
 102         info = self._get_ghc_info()
 103         self.version = info['Project version']
 104         #self.lib_dir = Path(info['LibDir'])
 105         #self.ghc_pkg_path = (self.lib_dir / 'bin' / 'ghc-pkg').resolve()
 106         self.ghc_pkg_path = (self.ghc_path.parent / ('ghc-pkg' + exe)).resolve()
 107         if not self.ghc_pkg_path.is_file():
 108             raise TypeError(f'ghc-pkg {self.ghc_pkg_path} is not a file')
 109         self.hsc2hs_path = (self.ghc_path.parent / ('hsc2hs' + exe)).resolve()
 110         if not self.hsc2hs_path.is_file():
 111             raise TypeError(f'hsc2hs {self.hsc2hs_path} is not a file')
 112
 113     def _get_ghc_info(self) -> Dict[str,str]:
 114         from ast import literal_eval
 115         p = subprocess_run([self.ghc_path, '--info'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 116         out = p.stdout.replace('\n', '').strip()
 117         return dict(literal_eval(out))
 118
 119 PackageSpec = Tuple[PackageName, Version]
 120
 121 class BadTarball(Exception):
 122     def __init__(self, path: Path, expected_sha256: SHA256Hash, found_sha256: SHA256Hash):
 123         self.path = path
 124         self.expected_sha256 = expected_sha256
 125         self.found_sha256 = found_sha256
 126
 127     def __str__(self):
 128         return '\n'.join([
 129             f'Bad tarball hash: {str(self.path)}',
 130             f'  expected: {self.expected_sha256}',
 131             f'  found:    {self.found_sha256}',
 132         ])
 133
 134 def package_url(package: PackageName, version: Version) -> str:
 135     return f'http://hackage.haskell.org/package/{package}-{version}/{package}-{version}.tar.gz'
 136
 137 def package_cabal_url(package: PackageName, version: Version, revision: int) -> str:
 138     return f'http://hackage.haskell.org/package/{package}-{version}/revision/{revision}.cabal'
 139
 140 def verify_sha256(expected_hash: SHA256Hash, f: Path):
 141     h = hash_file(hashlib.sha256(), f.open('rb'))
 142     if h != expected_hash:
 143         raise BadTarball(f, expected_hash, h)
 144
 145 def read_bootstrap_info(path: Path) -> BootstrapInfo:
 146     obj = json.load(path.open())
 147
 148     def bi_from_json(o: dict) -> BuiltinDep:
 149         return BuiltinDep(**o)
 150
 151     def dep_from_json(o: dict) -> BootstrapDep:
 152         o['source'] = PackageSource(o['source'])
 153         return BootstrapDep(**o)
 154
 155     builtin = [bi_from_json(dep) for dep in obj['builtin'] ]
 156     deps = [dep_from_json(dep) for dep in obj['dependencies'] ]
 157
 158     return BootstrapInfo(dependencies=deps, builtin=builtin)
 159
 160 def check_builtin(dep: BuiltinDep, ghc: Compiler) -> None:
 161     subprocess_run([str(ghc.ghc_pkg_path), 'describe', f'{dep.package}-{dep.version}'],
 162                    check=True, stdout=subprocess.DEVNULL)
 163     print(f'Using {dep.package}-{dep.version} from GHC...')
 164     return
 165
 166 def resolve_dep(dep : BootstrapDep) -> Path:
 167     if dep.source == PackageSource.HACKAGE:
 168
 169         tarball = TARBALLS / f'{dep.package}-{dep.version}.tar.gz'
 170         verify_sha256(dep.src_sha256, tarball)
 171
 172         cabal_file = TARBALLS / f'{dep.package}.cabal'
 173         verify_sha256(dep.cabal_sha256, cabal_file)
 174
 175         UNPACKED.mkdir(parents=True, exist_ok=True)
 176         shutil.unpack_archive(tarball.resolve(), UNPACKED, 'gztar')
 177         sdist_dir = UNPACKED / f'{dep.package}-{dep.version}'
 178
 179         # Update cabal file with revision
 180         if dep.revision is not None:
 181             shutil.copyfile(cabal_file, sdist_dir / f'{dep.package}.cabal')
 182
 183         # We rely on the presence of Setup.hs
 184         if len(list(sdist_dir.glob('Setup.*hs'))) == 0:
 185             with open(sdist_dir / 'Setup.hs', 'w') as f:
 186                 f.write('import Distribution.Simple\n')
 187                 f.write('main = defaultMain\n')
 188
 189     elif dep.source == PackageSource.LOCAL:
 190         if dep.package in local_packages:
 191             sdist_dir = Path(dep.package).resolve()
 192         else:
 193             raise ValueError(f'Unknown local package {dep.package}')
 194     return sdist_dir
 195
 196 def install_dep(dep: BootstrapDep, ghc: Compiler) -> None:
 197     dist_dir = (DISTDIR / f'{dep.package}-{dep.version}').resolve()
 198
 199     sdist_dir = resolve_dep(dep)
 200
 201     install_sdist(dist_dir, sdist_dir, ghc, dep.flags, dep.component)
 202
 203 def install_sdist(dist_dir: Path, sdist_dir: Path, ghc: Compiler, flags: List[str], component):
 204     prefix = PSEUDOSTORE.resolve()
 205     flags_option = ' '.join(flags)
 206     setup_dist_dir = dist_dir / 'setup'
 207     setup = setup_dist_dir / 'Setup'
 208
 209     build_args = [
 210         f'--builddir={dist_dir}',
 211     ]
 212
 213     configure_args = build_args + [
 214         f'--package-db={PKG_DB.resolve()}',
 215         f'--prefix={prefix}',
 216         f'--bindir={BINDIR.resolve()}',
 217         f'--extra-prog-path={BINDIR.resolve()}',
 218         f'--with-compiler={ghc.ghc_path}',
 219         f'--with-hc-pkg={ghc.ghc_pkg_path}',
 220         f'--with-hsc2hs={ghc.hsc2hs_path}',
 221         f'--flags={flags_option}',
 222         f'{component or ""}'
 223     ]
 224
 225     def check_call(args: List[str]) -> None:
 226         subprocess_run(args, cwd=sdist_dir, check=True)
 227
 228     setup_dist_dir.mkdir(parents=True, exist_ok=True)
 229
 230     # Note: we pass -i so GHC doesn't look for anything else
 231     # This should be fine for cabal-install dependencies.
 232     check_call([str(ghc.ghc_path), '--make', '-package-env=-', '-i', f'-odir={setup_dist_dir}', f'-hidir={setup_dist_dir}', '-o', setup, 'Setup'])
 233     check_call([setup, 'configure'] + configure_args)
 234     check_call([setup, 'build'] + build_args)
 235     check_call([setup, 'install'] + build_args)
 236
 237 def hash_file(h, f: BinaryIO) -> SHA256Hash:
 238     while True:
 239         d = f.read(1024)
 240         if len(d) == 0:
 241             return SHA256Hash(h.hexdigest())
 242
 243         h.update(d)
 244
 245
 246 # Cabal plan.json representation
 247 UnitId = NewType('UnitId', str)
 248 PlanUnit = NewType('PlanUnit', dict)
 249
 250 def bootstrap(info: BootstrapInfo, ghc: Compiler) -> None:
 251     if not PKG_DB.exists():
 252         print(f'Creating package database {PKG_DB}')
 253         PKG_DB.parent.mkdir(parents=True, exist_ok=True)
 254         subprocess_run([ghc.ghc_pkg_path, 'init', PKG_DB])
 255
 256     for dep in info.builtin:
 257         check_builtin(dep, ghc)
 258
 259     for dep in info.dependencies:
 260         install_dep(dep, ghc)
 261
 262 # Steps
 263 #######################################################################
 264
 265 def linuxname(i, r):
 266   i = i.strip() # id
 267   r = r.strip() # release
 268   if i == '': return 'linux'
 269   else: return f"{i}-{r}".lower()
 270
 271 def macname(macver):
 272   # https://en.wikipedia.org/wiki/MacOS_version_history#Releases
 273   if macver.startswith('10.12.'): return 'sierra'
 274   if macver.startswith('10.13.'): return 'high-sierra'
 275   if macver.startswith('10.14.'): return 'mojave'
 276   if macver.startswith('10.15.'): return 'catalina'
 277   if macver.startswith('11.0.'): return 'big-sur'
 278   else: return macver
 279
 280 def archive_name(cabalversion):
 281     # Ask platform information
 282     machine = platform.machine()
 283     if machine == '': machine = "unknown"
 284
 285     system = platform.system().lower()
 286     if system == '': system = "unknown"
 287
 288     version = system
 289     if system == 'linux':
 290         try:
 291             i = subprocess_run(['lsb_release', '-si'], stdout=subprocess.PIPE, encoding='UTF-8')
 292             r = subprocess_run(['lsb_release', '-sr'], stdout=subprocess.PIPE, encoding='UTF-8')
 293             version = linuxname(i.stdout, r.stdout)
 294         except:
 295             try:
 296                 with open('/etc/alpine-release') as f:
 297                     alpinever = f.read().strip()
 298                     return f'alpine-{alpinever}'
 299             except:
 300                 pass
 301     elif system == 'darwin':
 302         version = 'darwin-' + macname(platform.mac_ver()[0])
 303     elif system == 'freebsd':
 304         version = 'freebsd-' + platform.release().lower()
 305
 306     return f'cabal-install-{cabalversion}-{machine}-{version}'
 307
 308 def make_distribution_archive(cabal_path):
 309     import tempfile
 310
 311     print(f'Creating distribution tarball')
 312
 313     # Get bootstrapped cabal version
 314     # This also acts as smoke test
 315     p = subprocess_run([cabal_path, '--numeric-version'], stdout=subprocess.PIPE, check=True, encoding='UTF-8')
 316     cabalversion = p.stdout.replace('\n', '').strip()
 317
 318     # Archive name
 319     basename = ARTIFACTS.resolve() / (archive_name(cabalversion) + '-bootstrapped')
 320
 321     # In temporary directory, create a directory which we will archive
 322     tmpdir = TMPDIR.resolve()
 323     tmpdir.mkdir(parents=True, exist_ok=True)
 324
 325     rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 326     shutil.copy(cabal_path, rootdir / 'cabal')
 327
 328     # Make archive...
 329     fmt = 'xztar'
 330     if platform.system() == 'Windows': fmt = 'zip'
 331     archivename = shutil.make_archive(basename, fmt, rootdir)
 332
 333     return archivename
 334
 335 def fetch_from_plan(plan : FetchPlan, output_dir : Path):
 336   output_dir.resolve()
 337   output_dir.mkdir(parents=True, exist_ok=True)
 338
 339   for path in plan:
 340     output_path = output_dir / path
 341     url = plan[path].url
 342     sha = plan[path].sha256
 343     if not output_path.exists():
 344       print(f'Fetching {url}...')
 345       with urllib.request.urlopen(url, timeout = 10) as resp:
 346         shutil.copyfileobj(resp, output_path.open('wb'))
 347     verify_sha256(sha, output_path)
 348
 349 def gen_fetch_plan(info : BootstrapInfo) -> FetchPlan :
 350     sources_dict = {}
 351     for dep in info.dependencies:
 352       if not(dep.package in local_packages):
 353         sources_dict[f"{dep.package}-{dep.version}.tar.gz"] = FetchInfo(package_url(dep.package, dep.version), dep.src_sha256)
 354         if dep.revision is not None:
 355           sources_dict[f"{dep.package}.cabal"] = FetchInfo(package_cabal_url(dep.package, dep.version, dep.revision), dep.cabal_sha256)
 356     return sources_dict
 357
 358 def find_ghc(compiler) -> Compiler:
 359   if compiler is None:
 360       path = shutil.which('ghc')
 361       if path is None:
 362           raise ValueError("Couldn't find ghc in PATH")
 363       ghc = Compiler(Path(path))
 364   else:
 365       ghc = Compiler(compiler)
 366   return ghc
 367
 368 def main() -> None:
 369     parser = argparse.ArgumentParser(
 370         description="bootstrapping utility for cabal-install.",
 371         epilog = USAGE,
 372         formatter_class = argparse.RawDescriptionHelpFormatter)
 373     parser.add_argument('-d', '--deps', type=Path,
 374                         help='bootstrap dependency file')
 375     parser.add_argument('-w', '--with-compiler', type=Path,
 376                         help='path to GHC')
 377     parser.add_argument('-s', '--bootstrap-sources', type=Path,
 378                         help='path to prefetched bootstrap sources archive')
 379     parser.add_argument('--archive', dest='want_archive', action='store_true')
 380     parser.add_argument('--no-archive', dest='want_archive', action='store_false')
 381     parser.set_defaults(want_archive=True)
 382
 383     subparsers = parser.add_subparsers(dest="command")
 384
 385     parser_fetch = subparsers.add_parser('build', help='build cabal-install (default)')
 386
 387     parser_fetch = subparsers.add_parser('fetch', help='fetch all required sources from Hackage (for offline builds)')
 388     parser_fetch.add_argument('-o','--output', type=Path, default='bootstrap-sources')
 389
 390     args = parser.parse_args()
 391
 392     print(dedent("""
 393         DO NOT use this script if you have another recent cabal-install available.
 394         This script is intended only for bootstrapping cabal-install on new
 395         architectures.
 396     """))
 397
 398     ghc = find_ghc(args.with_compiler)
 399
 400     sources_fmt = 'gztar'
 401     if platform.system() == 'Windows': sources_fmt = 'zip'
 402
 403     if args.deps is None:
 404       # We have a tarball with all the required information, unpack it
 405       if args.bootstrap_sources is not None:
 406         print(f'Unpacking {args.bootstrap_sources} to {TARBALLS}')
 407         shutil.unpack_archive(args.bootstrap_sources.resolve(), TARBALLS, sources_fmt)
 408         args.deps = TARBALLS / 'plan-bootstrap.json'
 409         print(f"using plan-bootstrap.json ({args.deps}) from {args.bootstrap_sources}")
 410       else:
 411         print("The bootstrap script requires a bootstrap plan JSON file.")
 412         print("See bootstrap/README.md for more information.")
 413         sys.exit(1)
 414
 415     info = read_bootstrap_info(args.deps)
 416
 417     if args.command == 'fetch':
 418         plan = gen_fetch_plan(info)
 419
 420         print(f'Fetching sources to bootstrap cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 421
 422         # In temporary directory, create a directory which we will archive
 423         tmpdir = TMPDIR.resolve()
 424         tmpdir.mkdir(parents=True, exist_ok=True)
 425
 426         rootdir = Path(tempfile.mkdtemp(dir=tmpdir))
 427
 428         fetch_from_plan(plan, rootdir)
 429
 430         shutil.copyfile(args.deps, rootdir / 'plan-bootstrap.json')
 431
 432         archivename = shutil.make_archive(args.output, sources_fmt, root_dir=rootdir)
 433
 434         print(dedent(f"""
 435             Bootstrap sources saved to {archivename}
 436
 437             Use these with the command:
 438
 439             bootstrap.py -w {ghc.ghc_path} -s {archivename}
 440             """))
 441
 442     else: # 'build' command (default behaviour)
 443
 444         print(f'Bootstrapping cabal-install with GHC {ghc.version} at {ghc.ghc_path}...')
 445
 446         if args.bootstrap_sources is None:
 447           plan = gen_fetch_plan(info)
 448           fetch_from_plan(plan, TARBALLS)
 449
 450         bootstrap(info, ghc)
 451         cabal_path = (BINDIR / 'cabal').resolve()
 452
 453         print(dedent(f'''
 454             Bootstrapping finished!
 455
 456             The resulting cabal-install executable can be found at
 457
 458               {cabal_path}
 459             '''))
 460
 461         if args.want_archive:
 462             dist_archive = make_distribution_archive(cabal_path)
 463
 464             print(dedent(f'''
 465                 The cabal-install executable has been archived for distribution in
 466
 467                   {dist_archive}
 468                 '''))
 469
 470         print(dedent(f'''
 471             You now should use this to build a full cabal-install distribution
 472             using 'cabal build'.
 473             '''))
 474
 475 def subprocess_run(args, **kwargs):
 476     "Like subprocess.run, but also print what we run"
 477
 478     args_str = ' '.join(map(str, args))
 479     extras = ''
 480     if 'cwd' in kwargs:
 481         extras += f' cwd={kwargs["cwd"]}'
 482     print(f'bootstrap: running{extras} {args_str}')
 483
 484     return subprocess.run(args, **kwargs)
 485
 486 if __name__ == '__main__':
 487     main()