build/toolchain/get_concurrent_links.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 # This script computs the number of concurrent links we want to run in the build
   6 # as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP.
   7
   8 import os
   9 import re
  10 import subprocess
  11 import sys
  12
  13 def GetDefaultConcurrentLinks():
  14   # Inherit the legacy environment variable for people that have set it in GYP.
  15   pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
  16   if pool_size:
  17     return pool_size
  18
  19   if sys.platform in ('win32', 'cygwin'):
  20     import ctypes
  21
  22     class MEMORYSTATUSEX(ctypes.Structure):
  23       _fields_ = [
  24         ("dwLength", ctypes.c_ulong),
  25         ("dwMemoryLoad", ctypes.c_ulong),
  26         ("ullTotalPhys", ctypes.c_ulonglong),
  27         ("ullAvailPhys", ctypes.c_ulonglong),
  28         ("ullTotalPageFile", ctypes.c_ulonglong),
  29         ("ullAvailPageFile", ctypes.c_ulonglong),
  30         ("ullTotalVirtual", ctypes.c_ulonglong),
  31         ("ullAvailVirtual", ctypes.c_ulonglong),
  32         ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
  33       ]
  34
  35     stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
  36     ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
  37
  38     mem_limit = max(1, stat.ullTotalPhys / (4 * (2 ** 30)))  # total / 4GB
  39     hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
  40     return min(mem_limit, hard_cap)
  41   elif sys.platform.startswith('linux'):
  42     if os.path.exists("/proc/meminfo"):
  43       with open("/proc/meminfo") as meminfo:
  44         memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
  45         for line in meminfo:
  46           match = memtotal_re.match(line)
  47           if not match:
  48             continue
  49           # Allow 8Gb per link on Linux because Gold is quite memory hungry
  50           return max(1, int(match.group(1)) / (8 * (2 ** 20)))
  51     return 1
  52   elif sys.platform == 'darwin':
  53     try:
  54       avail_bytes = int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
  55       # A static library debug build of Chromium's unit_tests takes ~2.7GB, so
  56       # 4GB per ld process allows for some more bloat.
  57       return max(1, avail_bytes / (4 * (2 ** 30)))  # total / 4GB
  58     except Exception:
  59       return 1
  60   else:
  61     # TODO(scottmg): Implement this for other platforms.
  62     return 1
  63
  64 print GetDefaultConcurrentLinks()