tools/check_git_config.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Script that attempts to push to a special git repository to verify that git
   7 credentials are configured correctly. It also verifies that gclient solution is
   8 configured to use git checkout.
   9
  10 It will be added as gclient hook shortly before Chromium switches to git and
  11 removed after the switch.
  12
  13 When running as hook in *.corp.google.com network it will also report status
  14 of the push attempt to the server (on appengine), so that chrome-infra team can
  15 collect information about misconfigured Git accounts.
  16 """
  17
  18 import contextlib
  19 import datetime
  20 import errno
  21 import getpass
  22 import json
  23 import logging
  24 import netrc
  25 import optparse
  26 import os
  27 import pprint
  28 import shutil
  29 import socket
  30 import ssl
  31 import subprocess
  32 import sys
  33 import tempfile
  34 import time
  35 import urllib2
  36 import urlparse
  37
  38
  39 # Absolute path to src/ directory.
  40 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  41
  42 # Absolute path to a file with gclient solutions.
  43 GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
  44
  45 # Incremented whenever some changes to scrip logic are made. Change in version
  46 # will cause the check to be rerun on next gclient runhooks invocation.
  47 CHECKER_VERSION = 1
  48
  49 # Do not attempt to upload a report after this date.
  50 UPLOAD_DISABLE_TS = datetime.datetime(2014, 10, 1)
  51
  52 # URL to POST json with results to.
  53 MOTHERSHIP_URL = (
  54     'https://chromium-git-access.appspot.com/'
  55     'git_access/api/v1/reports/access_check')
  56
  57 # Repository to push test commits to.
  58 TEST_REPO_URL = 'https://chromium.googlesource.com/a/playground/access_test'
  59
  60 # Git-compatible gclient solution.
  61 GOOD_GCLIENT_SOLUTION = {
  62   'name': 'src',
  63   'deps_file': 'DEPS',
  64   'managed': False,
  65   'url': 'https://chromium.googlesource.com/chromium/src.git',
  66 }
  67
  68 # Possible chunks of git push response in case .netrc is misconfigured.
  69 BAD_ACL_ERRORS = (
  70   '(prohibited by Gerrit)',
  71   'does not match your user account',
  72   'Git repository not found',
  73   'Invalid user name or password',
  74   'Please make sure you have the correct access rights',
  75 )
  76
  77 # Git executable to call.
  78 GIT_EXE = 'git.bat' if sys.platform == 'win32' else 'git'
  79
  80
  81 def is_on_bot():
  82   """True when running under buildbot."""
  83   return os.environ.get('CHROME_HEADLESS') == '1'
  84
  85
  86 def is_in_google_corp():
  87   """True when running in google corp network."""
  88   try:
  89     return socket.getfqdn().endswith('.corp.google.com')
  90   except socket.error:
  91     logging.exception('Failed to get FQDN')
  92     return False
  93
  94
  95 def is_using_git():
  96   """True if git checkout is used."""
  97   return os.path.exists(os.path.join(REPO_ROOT, '.git', 'objects'))
  98
  99
 100 def is_using_svn():
 101   """True if svn checkout is used."""
 102   return os.path.exists(os.path.join(REPO_ROOT, '.svn'))
 103
 104
 105 def read_git_config(prop):
 106   """Reads git config property of src.git repo.
 107
 108   Returns empty string in case of errors.
 109   """
 110   try:
 111     proc = subprocess.Popen(
 112         [GIT_EXE, 'config', prop], stdout=subprocess.PIPE, cwd=REPO_ROOT)
 113     out, _ = proc.communicate()
 114     return out.strip().decode('utf-8')
 115   except OSError as exc:
 116     if exc.errno != errno.ENOENT:
 117       logging.exception('Unexpected error when calling git')
 118     return ''
 119
 120
 121 def read_netrc_user(netrc_obj, host):
 122   """Reads 'user' field of a host entry in netrc.
 123
 124   Returns empty string if netrc is missing, or host is not there.
 125   """
 126   if not netrc_obj:
 127     return ''
 128   entry = netrc_obj.authenticators(host)
 129   if not entry:
 130     return ''
 131   return entry[0]
 132
 133
 134 def get_git_version():
 135   """Returns version of git or None if git is not available."""
 136   try:
 137     proc = subprocess.Popen([GIT_EXE, '--version'], stdout=subprocess.PIPE)
 138     out, _ = proc.communicate()
 139     return out.strip() if proc.returncode == 0 else ''
 140   except OSError as exc:
 141     if exc.errno != errno.ENOENT:
 142       logging.exception('Unexpected error when calling git')
 143     return ''
 144
 145
 146 def read_gclient_solution():
 147   """Read information about 'src' gclient solution from .gclient file.
 148
 149   Returns tuple:
 150     (url, deps_file, managed)
 151     or
 152     (None, None, None) if no such solution.
 153   """
 154   try:
 155     env = {}
 156     execfile(GCLIENT_CONFIG, env, env)
 157     for sol in (env.get('solutions') or []):
 158       if sol.get('name') == 'src':
 159         return sol.get('url'), sol.get('deps_file'), sol.get('managed')
 160     return None, None, None
 161   except Exception:
 162     logging.exception('Failed to read .gclient solution')
 163     return None, None, None
 164
 165
 166 def read_git_insteadof(host):
 167   """Reads relevant insteadOf config entries."""
 168   try:
 169     proc = subprocess.Popen([GIT_EXE, 'config', '-l'], stdout=subprocess.PIPE)
 170     out, _ = proc.communicate()
 171     lines = []
 172     for line in out.strip().split('\n'):
 173       line = line.lower()
 174       if 'insteadof=' in line and host in line:
 175         lines.append(line)
 176     return '\n'.join(lines)
 177   except OSError as exc:
 178     if exc.errno != errno.ENOENT:
 179       logging.exception('Unexpected error when calling git')
 180     return ''
 181
 182
 183 def scan_configuration():
 184   """Scans local environment for git related configuration values."""
 185   # Git checkout?
 186   is_git = is_using_git()
 187
 188   # On Windows HOME should be set.
 189   if 'HOME' in os.environ:
 190     netrc_path = os.path.join(
 191         os.environ['HOME'],
 192         '_netrc' if sys.platform.startswith('win') else '.netrc')
 193   else:
 194     netrc_path = None
 195
 196   # Netrc exists?
 197   is_using_netrc = netrc_path and os.path.exists(netrc_path)
 198
 199   # Read it.
 200   netrc_obj = None
 201   if is_using_netrc:
 202     try:
 203       netrc_obj = netrc.netrc(netrc_path)
 204     except Exception:
 205       logging.exception('Failed to read netrc from %s', netrc_path)
 206       netrc_obj = None
 207
 208   # Read gclient 'src' solution.
 209   gclient_url, gclient_deps, gclient_managed = read_gclient_solution()
 210
 211   return {
 212     'checker_version': CHECKER_VERSION,
 213     'is_git': is_git,
 214     'is_home_set': 'HOME' in os.environ,
 215     'is_using_netrc': is_using_netrc,
 216     'netrc_file_mode': os.stat(netrc_path).st_mode if is_using_netrc else 0,
 217     'git_version': get_git_version(),
 218     'platform': sys.platform,
 219     'username': getpass.getuser(),
 220     'git_user_email': read_git_config('user.email') if is_git else '',
 221     'git_user_name': read_git_config('user.name') if is_git else '',
 222     'git_insteadof': read_git_insteadof('chromium.googlesource.com'),
 223     'chromium_netrc_email':
 224         read_netrc_user(netrc_obj, 'chromium.googlesource.com'),
 225     'chrome_internal_netrc_email':
 226         read_netrc_user(netrc_obj, 'chrome-internal.googlesource.com'),
 227     'gclient_deps': gclient_deps,
 228     'gclient_managed': gclient_managed,
 229     'gclient_url': gclient_url,
 230   }
 231
 232
 233 def last_configuration_path():
 234   """Path to store last checked configuration."""
 235   if is_using_git():
 236     return os.path.join(REPO_ROOT, '.git', 'check_git_push_access_conf.json')
 237   elif is_using_svn():
 238     return os.path.join(REPO_ROOT, '.svn', 'check_git_push_access_conf.json')
 239   else:
 240     return os.path.join(REPO_ROOT, '.check_git_push_access_conf.json')
 241
 242
 243 def read_last_configuration():
 244   """Reads last checked configuration if it exists."""
 245   try:
 246     with open(last_configuration_path(), 'r') as f:
 247       return json.load(f)
 248   except (IOError, ValueError):
 249     return None
 250
 251
 252 def write_last_configuration(conf):
 253   """Writes last checked configuration to a file."""
 254   try:
 255     with open(last_configuration_path(), 'w') as f:
 256       json.dump(conf, f, indent=2, sort_keys=True)
 257   except IOError:
 258     logging.exception('Failed to write JSON to %s', path)
 259
 260
 261 @contextlib.contextmanager
 262 def temp_directory():
 263   """Creates a temp directory, then nukes it."""
 264   tmp = tempfile.mkdtemp()
 265   try:
 266     yield tmp
 267   finally:
 268     try:
 269       shutil.rmtree(tmp)
 270     except (OSError, IOError):
 271       logging.exception('Failed to remove temp directory %s', tmp)
 272
 273
 274 class Runner(object):
 275   """Runs a bunch of commands in some directory, collects logs from them."""
 276
 277   def __init__(self, cwd, verbose):
 278     self.cwd = cwd
 279     self.verbose = verbose
 280     self.log = []
 281
 282   def run(self, cmd):
 283     self.append_to_log('> ' + ' '.join(cmd))
 284     retcode = -1
 285     try:
 286       proc = subprocess.Popen(
 287           cmd,
 288           stdout=subprocess.PIPE,
 289           stderr=subprocess.STDOUT,
 290           cwd=self.cwd)
 291       out, _ = proc.communicate()
 292       out = out.strip()
 293       retcode = proc.returncode
 294     except OSError as exc:
 295       out = str(exc)
 296     if retcode:
 297       out += '\n(exit code: %d)' % retcode
 298     self.append_to_log(out)
 299     return retcode
 300
 301   def append_to_log(self, text):
 302     if text:
 303       self.log.append(text)
 304       if self.verbose:
 305         logging.warning(text)
 306
 307
 308 def check_git_config(conf, report_url, verbose):
 309   """Attempts to push to a git repository, reports results to a server.
 310
 311   Returns True if the check finished without incidents (push itself may
 312   have failed) and should NOT be retried on next invocation of the hook.
 313   """
 314   # Don't even try to push if netrc is not configured.
 315   if not conf['chromium_netrc_email']:
 316     return upload_report(
 317         conf,
 318         report_url,
 319         verbose,
 320         push_works=False,
 321         push_log='',
 322         push_duration_ms=0)
 323
 324   # Ref to push to, each user has its own ref.
 325   ref = 'refs/push-test/%s' % conf['chromium_netrc_email']
 326
 327   push_works = False
 328   flake = False
 329   started = time.time()
 330   try:
 331     logging.warning('Checking push access to the git repository...')
 332     with temp_directory() as tmp:
 333       # Prepare a simple commit on a new timeline.
 334       runner = Runner(tmp, verbose)
 335       runner.run([GIT_EXE, 'init', '.'])
 336       if conf['git_user_name']:
 337         runner.run([GIT_EXE, 'config', 'user.name', conf['git_user_name']])
 338       if conf['git_user_email']:
 339         runner.run([GIT_EXE, 'config', 'user.email', conf['git_user_email']])
 340       with open(os.path.join(tmp, 'timestamp'), 'w') as f:
 341         f.write(str(int(time.time() * 1000)))
 342       runner.run([GIT_EXE, 'add', 'timestamp'])
 343       runner.run([GIT_EXE, 'commit', '-m', 'Push test.'])
 344       # Try to push multiple times if it fails due to issues other than ACLs.
 345       attempt = 0
 346       while attempt < 5:
 347         attempt += 1
 348         logging.info('Pushing to %s %s', TEST_REPO_URL, ref)
 349         ret = runner.run(
 350             [GIT_EXE, 'push', TEST_REPO_URL, 'HEAD:%s' % ref, '-f'])
 351         if not ret:
 352           push_works = True
 353           break
 354         if any(x in runner.log[-1] for x in BAD_ACL_ERRORS):
 355           push_works = False
 356           break
 357   except Exception:
 358     logging.exception('Unexpected exception when pushing')
 359     flake = True
 360
 361   if push_works:
 362     logging.warning('Git push works!')
 363   else:
 364     logging.warning(
 365         'Git push doesn\'t work, which is fine if you are not a committer.')
 366
 367   uploaded = upload_report(
 368       conf,
 369       report_url,
 370       verbose,
 371       push_works=push_works,
 372       push_log='\n'.join(runner.log),
 373       push_duration_ms=int((time.time() - started) * 1000))
 374   return uploaded and not flake
 375
 376
 377 def check_gclient_config(conf):
 378   """Shows warning if gclient solution is not properly configured for git."""
 379   # Ignore configs that do not have 'src' solution at all.
 380   if not conf['gclient_url']:
 381     return
 382   current = {
 383     'name': 'src',
 384     'deps_file': conf['gclient_deps'] or 'DEPS',
 385     'managed': conf['gclient_managed'] or False,
 386     'url': conf['gclient_url'],
 387   }
 388   # After depot_tools r291592 both DEPS and .DEPS.git are valid.
 389   good = GOOD_GCLIENT_SOLUTION.copy()
 390   good['deps_file'] = current['deps_file']
 391   if current == good:
 392     return
 393   # Show big warning if url or deps_file is wrong.
 394   if current['url'] != good['url'] or current['deps_file'] != good['deps_file']:
 395     print '-' * 80
 396     print 'Your gclient solution is not set to use supported git workflow!'
 397     print
 398     print 'Your \'src\' solution (in %s):' % GCLIENT_CONFIG
 399     print pprint.pformat(current, indent=2)
 400     print
 401     print 'Correct \'src\' solution to use git:'
 402     print pprint.pformat(good, indent=2)
 403     print
 404     print 'Please update your .gclient file ASAP.'
 405     print '-' * 80
 406   # Show smaller (additional) warning about managed workflow.
 407   if current['managed']:
 408     print '-' * 80
 409     print (
 410         'You are using managed gclient mode with git, which was deprecated '
 411         'on 8/22/13:')
 412     print (
 413         'https://groups.google.com/a/chromium.org/'
 414         'forum/#!topic/chromium-dev/n9N5N3JL2_U')
 415     print
 416     print (
 417         'It is strongly advised to switch to unmanaged mode. For more '
 418         'information about managed mode and reasons for its deprecation see:')
 419     print 'http://www.chromium.org/developers/how-tos/get-the-code/gclient-managed-mode'
 420     print
 421     print (
 422         'There\'s also a large suite of tools to assist managing git '
 423         'checkouts.\nSee \'man depot_tools\' (or read '
 424         'depot_tools/man/html/depot_tools.html).')
 425     print '-' * 80
 426
 427
 428 def upload_report(
 429     conf, report_url, verbose, push_works, push_log, push_duration_ms):
 430   """Posts report to the server, returns True if server accepted it.
 431
 432   Uploads the report only if script is running in Google corp network. Otherwise
 433   just prints the report.
 434   """
 435   report = conf.copy()
 436   report.update(
 437       push_works=push_works,
 438       push_log=push_log,
 439       push_duration_ms=push_duration_ms)
 440
 441   as_bytes = json.dumps({'access_check': report}, indent=2, sort_keys=True)
 442   if verbose:
 443     print 'Status of git push attempt:'
 444     print as_bytes
 445
 446   # Do not upload it outside of corp or if server side is already disabled.
 447   if not is_in_google_corp() or datetime.datetime.now() > UPLOAD_DISABLE_TS:
 448     if verbose:
 449       print (
 450           'You can send the above report to chrome-git-migration@google.com '
 451           'if you need help to set up you committer git account.')
 452     return True
 453
 454   req = urllib2.Request(
 455       url=report_url,
 456       data=as_bytes,
 457       headers={'Content-Type': 'application/json; charset=utf-8'})
 458
 459   attempt = 0
 460   success = False
 461   while not success and attempt < 10:
 462     attempt += 1
 463     try:
 464       logging.warning(
 465           'Attempting to upload the report to %s...',
 466           urlparse.urlparse(report_url).netloc)
 467       resp = urllib2.urlopen(req, timeout=5)
 468       report_id = None
 469       try:
 470         report_id = json.load(resp)['report_id']
 471       except (ValueError, TypeError, KeyError):
 472         pass
 473       logging.warning('Report uploaded: %s', report_id)
 474       success = True
 475     except (urllib2.URLError, socket.error, ssl.SSLError) as exc:
 476       logging.warning('Failed to upload the report: %s', exc)
 477   return success
 478
 479
 480 def main(args):
 481   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
 482   parser.add_option(
 483       '--running-as-hook',
 484       action='store_true',
 485       help='Set when invoked from gclient hook')
 486   parser.add_option(
 487       '--report-url',
 488       default=MOTHERSHIP_URL,
 489       help='URL to submit the report to')
 490   parser.add_option(
 491       '--verbose',
 492       action='store_true',
 493       help='More logging')
 494   options, args = parser.parse_args()
 495   if args:
 496     parser.error('Unknown argument %s' % args)
 497   logging.basicConfig(
 498       format='%(message)s',
 499       level=logging.INFO if options.verbose else logging.WARN)
 500
 501   # When invoked not as a hook, always run the check.
 502   if not options.running_as_hook:
 503     config = scan_configuration()
 504     check_gclient_config(config)
 505     check_git_config(config, options.report_url, True)
 506     return 0
 507
 508   # Always do nothing on bots.
 509   if is_on_bot():
 510     return 0
 511
 512   # Read current config, verify gclient solution looks correct.
 513   config = scan_configuration()
 514   check_gclient_config(config)
 515
 516   # Do not attempt to push from non-google owned machines.
 517   if not is_in_google_corp():
 518     logging.info('Skipping git push check: non *.corp.google.com machine.')
 519     return 0
 520
 521   # Skip git push check if current configuration was already checked.
 522   if config == read_last_configuration():
 523     logging.info('Check already performed, skipping.')
 524     return 0
 525
 526   # Run the check. Mark configuration as checked only on success. Ignore any
 527   # exceptions or errors. This check must not break gclient runhooks.
 528   try:
 529     ok = check_git_config(config, options.report_url, False)
 530     if ok:
 531       write_last_configuration(config)
 532     else:
 533       logging.warning('Check failed and will be retried on the next run')
 534   except Exception:
 535     logging.exception('Unexpected exception when performing git access check')
 536   return 0
 537
 538
 539 if __name__ == '__main__':
 540   sys.exit(main(sys.argv[1:]))