tools/telemetry/third_party/gsutilz/gslib/__main__.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Copyright 2013 Google Inc. All Rights Reserved.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 """Main module for Google Cloud Storage command line tool."""
  17
  18 from __future__ import absolute_import
  19
  20 import ConfigParser
  21 import datetime
  22 import errno
  23 import getopt
  24 import logging
  25 import os
  26 import re
  27 import signal
  28 import socket
  29 import sys
  30 import textwrap
  31 import traceback
  32
  33 # Load the gsutil version number and append it to boto.UserAgent so the value is
  34 # set before anything instantiates boto. This has to run after THIRD_PARTY_DIR
  35 # is modified (done in gsutil.py) but before any calls are made that would cause
  36 # boto.s3.Connection to be loaded - otherwise the Connection class would end up
  37 # with a static reference to the pre-modified version of the UserAgent field,
  38 # so boto requests would not include gsutil/version# in the UserAgent string.
  39 import boto
  40 import gslib
  41 # TODO: gsutil-beta: Cloud SDK scans for this string and performs
  42 # substitution; ensure this works with both apitools and boto.
  43 boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform)
  44 if os.environ.get('CLOUDSDK_WRAPPER') == '1':
  45   boto.UserAgent += ' Cloud SDK Command Line Tool'
  46   if os.environ.get('CLOUDSDK_VERSION'):
  47     boto.UserAgent += ' %s' % os.environ.get('CLOUDSDK_VERSION')
  48
  49 # pylint: disable=g-bad-import-order
  50 # pylint: disable=g-import-not-at-top
  51 import httplib2
  52 import oauth2client
  53 from gslib import wildcard_iterator
  54 from gslib.cloud_api import AccessDeniedException
  55 from gslib.cloud_api import ArgumentException
  56 from gslib.cloud_api import BadRequestException
  57 from gslib.cloud_api import ProjectIdException
  58 from gslib.cloud_api import ServiceException
  59 from gslib.command_runner import CommandRunner
  60 import gslib.exception
  61 from gslib.exception import CommandException
  62 import apitools.base.py.exceptions as apitools_exceptions
  63 from gslib.util import CreateLock
  64 from gslib.util import GetBotoConfigFileList
  65 from gslib.util import GetCertsFile
  66 from gslib.util import GetCleanupFiles
  67 from gslib.util import GsutilStreamHandler
  68 from gslib.util import ProxyInfoFromEnvironmentVar
  69 from gslib.sig_handling import GetCaughtSignals
  70 from gslib.sig_handling import InitializeSignalHandling
  71 from gslib.sig_handling import RegisterSignalHandler
  72
  73 GSUTIL_CLIENT_ID = '909320924072.apps.googleusercontent.com'
  74 # Google OAuth2 clients always have a secret, even if the client is an installed
  75 # application/utility such as gsutil.  Of course, in such cases the "secret" is
  76 # actually publicly known; security depends entirely on the secrecy of refresh
  77 # tokens, which effectively become bearer tokens.
  78 GSUTIL_CLIENT_NOTSOSECRET = 'p3RlpR10xMFh9ZXBS/ZNLYUu'
  79 if os.environ.get('CLOUDSDK_WRAPPER') == '1':
  80   # Cloud SDK installs have a separate client ID / secret.
  81   GSUTIL_CLIENT_ID = '32555940559.apps.googleusercontent.com'
  82   GSUTIL_CLIENT_NOTSOSECRET = 'ZmssLNjJy2998hD4CTg2ejr2'
  83
  84 CONFIG_KEYS_TO_REDACT = ['proxy', 'proxy_port', 'proxy_user', 'proxy_pass']
  85
  86
  87 # We don't use the oauth2 authentication plugin directly; importing it here
  88 # ensures that it's loaded and available by default when an operation requiring
  89 # authentication is performed.
  90 try:
  91   # pylint: disable=unused-import,g-import-not-at-top
  92   import gcs_oauth2_boto_plugin
  93 except ImportError:
  94   pass
  95
  96 DEBUG_WARNING = """
  97 ***************************** WARNING *****************************
  98 *** You are running gsutil with debug output enabled.
  99 *** Be aware that debug output includes authentication credentials.
 100 *** Make sure to remove the value of the Authorization header for
 101 *** each HTTP request printed to the console prior to posting to
 102 *** a public medium such as a forum post or Stack Overflow.
 103 ***************************** WARNING *****************************
 104 """.lstrip()
 105
 106 HTTP_WARNING = """
 107 ***************************** WARNING *****************************
 108 *** You are running gsutil with the "https_validate_certificates" config
 109 *** variable set to False. This option should always be set to True in
 110 *** production environments to protect against man-in-the-middle attacks,
 111 *** and leaking of user data.
 112 ***************************** WARNING *****************************
 113 """.lstrip()
 114
 115 debug = 0
 116 test_exception_traces = False
 117
 118
 119 # pylint: disable=unused-argument
 120 def _CleanupSignalHandler(signal_num, cur_stack_frame):
 121   """Cleans up if process is killed with SIGINT, SIGQUIT or SIGTERM."""
 122   _Cleanup()
 123
 124
 125 def _Cleanup():
 126   for fname in GetCleanupFiles():
 127     try:
 128       os.unlink(fname)
 129     except:  # pylint: disable=bare-except
 130       pass
 131
 132
 133 def _OutputAndExit(message):
 134   """Outputs message and exists with code 1."""
 135   from gslib.util import UTF8  # pylint: disable=g-import-not-at-top
 136   if debug >= 2 or test_exception_traces:
 137     stack_trace = traceback.format_exc()
 138     err = ('DEBUG: Exception stack trace:\n    %s\n' %
 139            re.sub('\\n', '\n    ', stack_trace))
 140   else:
 141     err = '%s\n' % message
 142   try:
 143     sys.stderr.write(err.encode(UTF8))
 144   except UnicodeDecodeError:
 145     # Can happen when outputting invalid Unicode filenames.
 146     sys.stderr.write(err)
 147   sys.exit(1)
 148
 149
 150 def _OutputUsageAndExit(command_runner):
 151   command_runner.RunNamedCommand('help')
 152   sys.exit(1)
 153
 154
 155 class GsutilFormatter(logging.Formatter):
 156   """A logging.Formatter that supports logging microseconds (%f)."""
 157
 158   def formatTime(self, record, datefmt=None):
 159     if datefmt:
 160       return datetime.datetime.fromtimestamp(record.created).strftime(datefmt)
 161
 162     # Use default implementation if datefmt is not specified.
 163     return super(GsutilFormatter, self).formatTime(record, datefmt=datefmt)
 164
 165
 166 def _ConfigureLogging(level=logging.INFO):
 167   """Similar to logging.basicConfig() except it always adds a handler."""
 168   log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s'
 169   date_format = '%m%d %H:%M:%S.%f'
 170   formatter = GsutilFormatter(fmt=log_format, datefmt=date_format)
 171   handler = GsutilStreamHandler()
 172   handler.setFormatter(formatter)
 173   root_logger = logging.getLogger()
 174   root_logger.addHandler(handler)
 175   root_logger.setLevel(level)
 176
 177
 178 def main():
 179   InitializeSignalHandling()
 180   # Any modules used in initializing multiprocessing variables must be
 181   # imported after importing gslib.__main__.
 182   # pylint: disable=redefined-outer-name,g-import-not-at-top
 183   import gslib.boto_translation
 184   import gslib.command
 185   import gslib.util
 186   from gslib.util import BOTO_IS_SECURE
 187   from gslib.util import CERTIFICATE_VALIDATION_ENABLED
 188   # pylint: disable=unused-variable
 189   from gcs_oauth2_boto_plugin import oauth2_client
 190   # pylint: enable=unused-variable
 191   from gslib.util import MultiprocessingIsAvailable
 192   if MultiprocessingIsAvailable()[0]:
 193     # These setup methods must be called, and, on Windows, they can only be
 194     # called from within an "if __name__ == '__main__':" block.
 195     gslib.util.InitializeMultiprocessingVariables()
 196     gslib.command.InitializeMultiprocessingVariables()
 197     gslib.boto_translation.InitializeMultiprocessingVariables()
 198
 199   # This needs to be done after gslib.util.InitializeMultiprocessingVariables(),
 200   # since otherwise we can't call gslib.util.CreateLock.
 201   try:
 202     # pylint: disable=unused-import,g-import-not-at-top
 203     import gcs_oauth2_boto_plugin
 204     gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret(
 205         GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET)
 206     gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock())
 207   except ImportError:
 208     pass
 209
 210   global debug
 211   global test_exception_traces
 212
 213   if not (2, 6) <= sys.version_info[:3] < (3,):
 214     raise gslib.exception.CommandException(
 215         'gsutil requires python 2.6 or 2.7.')
 216
 217   # In gsutil 4.0 and beyond, we don't use the boto library for the JSON
 218   # API. However, we still store gsutil configuration data in the .boto
 219   # config file for compatibility with previous versions and user convenience.
 220   # Many users have a .boto configuration file from previous versions, and it
 221   # is useful to have all of the configuration for gsutil stored in one place.
 222   command_runner = CommandRunner()
 223   if not BOTO_IS_SECURE:
 224     raise CommandException('\n'.join(textwrap.wrap(
 225         'Your boto configuration has is_secure = False. Gsutil cannot be '
 226         'run this way, for security reasons.')))
 227
 228   headers = {}
 229   parallel_operations = False
 230   quiet = False
 231   version = False
 232   debug = 0
 233   test_exception_traces = False
 234
 235   # If user enters no commands just print the usage info.
 236   if len(sys.argv) == 1:
 237     sys.argv.append('help')
 238
 239   # Change the default of the 'https_validate_certificates' boto option to
 240   # True (it is currently False in boto).
 241   if not boto.config.has_option('Boto', 'https_validate_certificates'):
 242     if not boto.config.has_section('Boto'):
 243       boto.config.add_section('Boto')
 244     boto.config.setbool('Boto', 'https_validate_certificates', True)
 245
 246   gslib.util.certs_file_lock = CreateLock()
 247   for signal_num in GetCaughtSignals():
 248     RegisterSignalHandler(signal_num, _CleanupSignalHandler)
 249   GetCertsFile()
 250
 251   try:
 252     try:
 253       opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq',
 254                                  ['debug', 'detailedDebug', 'version', 'option',
 255                                   'help', 'header', 'multithreaded', 'quiet',
 256                                   'testexceptiontraces'])
 257     except getopt.GetoptError as e:
 258       _HandleCommandException(gslib.exception.CommandException(e.msg))
 259     for o, a in opts:
 260       if o in ('-d', '--debug'):
 261         # Passing debug=2 causes boto to include httplib header output.
 262         debug = 3
 263       elif o in ('-D', '--detailedDebug'):
 264         # We use debug level 3 to ask gsutil code to output more detailed
 265         # debug output. This is a bit of a hack since it overloads the same
 266         # flag that was originally implemented for boto use. And we use -DD
 267         # to ask for really detailed debugging (i.e., including HTTP payload).
 268         if debug == 3:
 269           debug = 4
 270         else:
 271           debug = 3
 272       elif o in ('-?', '--help'):
 273         _OutputUsageAndExit(command_runner)
 274       elif o in ('-h', '--header'):
 275         (hdr_name, _, hdr_val) = a.partition(':')
 276         if not hdr_name:
 277           _OutputUsageAndExit(command_runner)
 278         headers[hdr_name.lower()] = hdr_val
 279       elif o in ('-m', '--multithreaded'):
 280         parallel_operations = True
 281       elif o in ('-q', '--quiet'):
 282         quiet = True
 283       elif o in ('-v', '--version'):
 284         version = True
 285       elif o == '--testexceptiontraces':  # Hidden flag for integration tests.
 286         test_exception_traces = True
 287       elif o in ('-o', '--option'):
 288         (opt_section_name, _, opt_value) = a.partition('=')
 289         if not opt_section_name:
 290           _OutputUsageAndExit(command_runner)
 291         (opt_section, _, opt_name) = opt_section_name.partition(':')
 292         if not opt_section or not opt_name:
 293           _OutputUsageAndExit(command_runner)
 294         if not boto.config.has_section(opt_section):
 295           boto.config.add_section(opt_section)
 296         boto.config.set(opt_section, opt_name, opt_value)
 297     httplib2.debuglevel = debug
 298     if debug > 1:
 299       sys.stderr.write(DEBUG_WARNING)
 300     if debug >= 2:
 301       _ConfigureLogging(level=logging.DEBUG)
 302       command_runner.RunNamedCommand('ver', ['-l'])
 303       config_items = []
 304       try:
 305         config_items.extend(boto.config.items('Boto'))
 306         config_items.extend(boto.config.items('GSUtil'))
 307       except ConfigParser.NoSectionError:
 308         pass
 309       for i in xrange(len(config_items)):
 310         config_item_key = config_items[i][0]
 311         if config_item_key in CONFIG_KEYS_TO_REDACT:
 312           config_items[i] = (config_item_key, 'REDACTED')
 313       sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv))
 314       sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList())
 315       sys.stderr.write('config: %s\n' % str(config_items))
 316     elif quiet:
 317       _ConfigureLogging(level=logging.WARNING)
 318     else:
 319       _ConfigureLogging(level=logging.INFO)
 320       # oauth2client uses info logging in places that would better
 321       # correspond to gsutil's debug logging (e.g., when refreshing
 322       # access tokens).
 323       oauth2client.client.logger.setLevel(logging.WARNING)
 324
 325     if not CERTIFICATE_VALIDATION_ENABLED:
 326       sys.stderr.write(HTTP_WARNING)
 327
 328     if version:
 329       command_name = 'version'
 330     elif not args:
 331       command_name = 'help'
 332     else:
 333       command_name = args[0]
 334
 335     _CheckAndWarnForProxyDifferences()
 336
 337     if os.environ.get('_ARGCOMPLETE', '0') == '1':
 338       return _PerformTabCompletion(command_runner)
 339
 340     return _RunNamedCommandAndHandleExceptions(
 341         command_runner, command_name, args=args[1:], headers=headers,
 342         debug_level=debug, parallel_operations=parallel_operations)
 343   finally:
 344     _Cleanup()
 345
 346
 347 def _CheckAndWarnForProxyDifferences():
 348   # If there are both boto config and environment variable config present for
 349   # proxies, unset the environment variable and warn if it differs.
 350   boto_port = boto.config.getint('Boto', 'proxy_port', 0)
 351   if boto.config.get('Boto', 'proxy', None) or boto_port:
 352     for proxy_env_var in ['http_proxy', 'https_proxy', 'HTTPS_PROXY']:
 353       if proxy_env_var in os.environ and os.environ[proxy_env_var]:
 354         differing_values = []
 355         proxy_info = ProxyInfoFromEnvironmentVar(proxy_env_var)
 356         if proxy_info.proxy_host != boto.config.get('Boto', 'proxy', None):
 357           differing_values.append(
 358               'Boto proxy host: "%s" differs from %s proxy host: "%s"' %
 359               (boto.config.get('Boto', 'proxy', None), proxy_env_var,
 360                proxy_info.proxy_host))
 361         if (proxy_info.proxy_user !=
 362             boto.config.get('Boto', 'proxy_user', None)):
 363           differing_values.append(
 364               'Boto proxy user: "%s" differs from %s proxy user: "%s"' %
 365               (boto.config.get('Boto', 'proxy_user', None), proxy_env_var,
 366                proxy_info.proxy_user))
 367         if (proxy_info.proxy_pass !=
 368             boto.config.get('Boto', 'proxy_pass', None)):
 369           differing_values.append(
 370               'Boto proxy password differs from %s proxy password' %
 371               proxy_env_var)
 372         # Only compare ports if at least one is present, since the
 373         # boto logic for selecting default ports has not yet executed.
 374         if ((proxy_info.proxy_port or boto_port) and
 375             proxy_info.proxy_port != boto_port):
 376           differing_values.append(
 377               'Boto proxy port: "%s" differs from %s proxy port: "%s"' %
 378               (boto_port, proxy_env_var, proxy_info.proxy_port))
 379         if differing_values:
 380           sys.stderr.write('\n'.join(textwrap.wrap(
 381               'WARNING: Proxy configuration is present in both the %s '
 382               'environment variable and boto configuration, but '
 383               'configuration differs. boto configuration proxy values will '
 384               'be used. Differences detected:' % proxy_env_var)))
 385           sys.stderr.write('\n%s\n' % '\n'.join(differing_values))
 386         # Regardless of whether the proxy configuration values matched,
 387         # delete the environment variable so as not to confuse boto.
 388         del os.environ[proxy_env_var]
 389
 390
 391 def _HandleUnknownFailure(e):
 392   # Called if we fall through all known/handled exceptions. Allows us to
 393   # print a stacktrace if -D option used.
 394   if debug >= 2:
 395     stack_trace = traceback.format_exc()
 396     sys.stderr.write('DEBUG: Exception stack trace:\n    %s\n' %
 397                      re.sub('\\n', '\n    ', stack_trace))
 398   else:
 399     _OutputAndExit('Failure: %s.' % e)
 400
 401
 402 def _HandleCommandException(e):
 403   if e.informational:
 404     _OutputAndExit(e.reason)
 405   else:
 406     _OutputAndExit('CommandException: %s' % e.reason)
 407
 408
 409 # pylint: disable=unused-argument
 410 def _HandleControlC(signal_num, cur_stack_frame):
 411   """Called when user hits ^C.
 412
 413   This function prints a brief message instead of the normal Python stack trace
 414   (unless -D option is used).
 415
 416   Args:
 417     signal_num: Signal that was caught.
 418     cur_stack_frame: Unused.
 419   """
 420   if debug >= 2:
 421     stack_trace = ''.join(traceback.format_list(traceback.extract_stack()))
 422     _OutputAndExit(
 423         'DEBUG: Caught signal %d - Exception stack trace:\n'
 424         '    %s' % (signal_num, re.sub('\\n', '\n    ', stack_trace)))
 425   else:
 426     _OutputAndExit('Caught signal %d - exiting' % signal_num)
 427
 428
 429 def _HandleSigQuit(signal_num, cur_stack_frame):
 430   """Called when user hits ^\\, so we can force breakpoint a running gsutil."""
 431   import pdb  # pylint: disable=g-import-not-at-top
 432   pdb.set_trace()
 433
 434
 435 def _ConstructAccountProblemHelp(reason):
 436   """Constructs a help string for an access control error.
 437
 438   Args:
 439     reason: e.reason string from caught exception.
 440
 441   Returns:
 442     Contructed help text.
 443   """
 444   default_project_id = boto.config.get_value('GSUtil', 'default_project_id')
 445   # pylint: disable=line-too-long, g-inconsistent-quotes
 446   acct_help = (
 447       "Your request resulted in an AccountProblem (403) error. Usually this "
 448       "happens if you attempt to create a bucket without first having "
 449       "enabled billing for the project you are using. Please ensure billing is "
 450       "enabled for your project by following the instructions at "
 451       "`Google Developers Console<https://developers.google.com/console/help/billing>`. ")
 452   if default_project_id:
 453     acct_help += (
 454         "In the project overview, ensure that the Project Number listed for "
 455         "your project matches the project ID (%s) from your boto config file. "
 456         % default_project_id)
 457   acct_help += (
 458       "If the above doesn't resolve your AccountProblem, please send mail to "
 459       "gs-team@google.com requesting assistance, noting the exact command you "
 460       "ran, the fact that you received a 403 AccountProblem error, and your "
 461       "project ID. Please do not post your project ID on StackOverflow. "
 462       "Note: It's possible to use Google Cloud Storage without enabling "
 463       "billing if you're only listing or reading objects for which you're "
 464       "authorized, or if you're uploading objects to a bucket billed to a "
 465       "project that has billing enabled. But if you're attempting to create "
 466       "buckets or upload objects to a bucket owned by your own project, you "
 467       "must first enable billing for that project.")
 468   return acct_help
 469
 470
 471 def _CheckAndHandleCredentialException(e, args):
 472   # Provide detail to users who have no boto config file (who might previously
 473   # have been using gsutil only for accessing publicly readable buckets and
 474   # objects).
 475   # pylint: disable=g-import-not-at-top
 476   from gslib.util import HasConfiguredCredentials
 477   if (not HasConfiguredCredentials() and
 478       not boto.config.get_value('Tests', 'bypass_anonymous_access_warning',
 479                                 False)):
 480     # The check above allows tests to assert that we get a particular,
 481     # expected failure, rather than always encountering this error message
 482     # when there are no configured credentials. This allows tests to
 483     # simulate a second user without permissions, without actually requiring
 484     # two separate configured users.
 485     if os.environ.get('CLOUDSDK_WRAPPER') == '1':
 486       _OutputAndExit('\n'.join(textwrap.wrap(
 487           'You are attempting to access protected data with no configured '
 488           'credentials. Please visit '
 489           'https://cloud.google.com/console#/project and sign up for an '
 490           'account, and then run the "gcloud auth login" command to '
 491           'configure gsutil to use these credentials.')))
 492     else:
 493       _OutputAndExit('\n'.join(textwrap.wrap(
 494           'You are attempting to access protected data with no configured '
 495           'credentials. Please visit '
 496           'https://cloud.google.com/console#/project and sign up for an '
 497           'account, and then run the "gsutil config" command to configure '
 498           'gsutil to use these credentials.')))
 499   elif (e.reason and
 500         (e.reason == 'AccountProblem' or e.reason == 'Account disabled.' or
 501          'account for the specified project has been disabled' in e.reason)
 502         and ','.join(args).find('gs://') != -1):
 503     _OutputAndExit('\n'.join(textwrap.wrap(
 504         _ConstructAccountProblemHelp(e.reason))))
 505
 506
 507 def _RunNamedCommandAndHandleExceptions(command_runner, command_name, args=None,
 508                                         headers=None, debug_level=0,
 509                                         parallel_operations=False):
 510   """Runs the command with the given command runner and arguments."""
 511   # pylint: disable=g-import-not-at-top
 512   from gslib.util import GetConfigFilePath
 513   from gslib.util import IS_WINDOWS
 514   from gslib.util import IsRunningInteractively
 515   try:
 516     # Catch ^C so we can print a brief message instead of the normal Python
 517     # stack trace. Register as a final signal handler because this handler kills
 518     # the main gsutil process (so it must run last).
 519     RegisterSignalHandler(signal.SIGINT, _HandleControlC, is_final_handler=True)
 520     # Catch ^\ so we can force a breakpoint in a running gsutil.
 521     if not IS_WINDOWS:
 522       RegisterSignalHandler(signal.SIGQUIT, _HandleSigQuit)
 523     return command_runner.RunNamedCommand(command_name, args, headers,
 524                                           debug_level, parallel_operations)
 525   except AttributeError as e:
 526     if str(e).find('secret_access_key') != -1:
 527       _OutputAndExit('Missing credentials for the given URI(s). Does your '
 528                      'boto config file contain all needed credentials?')
 529     else:
 530       _OutputAndExit(str(e))
 531   except gslib.exception.CommandException as e:
 532     _HandleCommandException(e)
 533   except getopt.GetoptError as e:
 534     _HandleCommandException(gslib.exception.CommandException(e.msg))
 535   except boto.exception.InvalidUriError as e:
 536     _OutputAndExit('InvalidUriError: %s.' % e.message)
 537   except gslib.exception.InvalidUrlError as e:
 538     _OutputAndExit('InvalidUrlError: %s.' % e.message)
 539   except boto.auth_handler.NotReadyToAuthenticate:
 540     _OutputAndExit('NotReadyToAuthenticate')
 541   except OSError as e:
 542     _OutputAndExit('OSError: %s.' % e.strerror)
 543   except IOError as e:
 544     if (e.errno == errno.EPIPE or (IS_WINDOWS and e.errno == errno.EINVAL)
 545         and not IsRunningInteractively()):
 546       # If we get a pipe error, this just means that the pipe to stdout or
 547       # stderr is broken. This can happen if the user pipes gsutil to a command
 548       # that doesn't use the entire output stream. Instead of raising an error,
 549       # just swallow it up and exit cleanly.
 550       sys.exit(0)
 551     else:
 552       raise
 553   except wildcard_iterator.WildcardException as e:
 554     _OutputAndExit(e.reason)
 555   except ProjectIdException as e:
 556     _OutputAndExit(
 557         'You are attempting to perform an operation that requires a '
 558         'project id, with none configured. Please re-run '
 559         'gsutil config and make sure to follow the instructions for '
 560         'finding and entering your default project id.')
 561   except BadRequestException as e:
 562     if e.reason == 'MissingSecurityHeader':
 563       _CheckAndHandleCredentialException(e, args)
 564     _OutputAndExit(e)
 565   except AccessDeniedException as e:
 566     _CheckAndHandleCredentialException(e, args)
 567     _OutputAndExit(e)
 568   except ArgumentException as e:
 569     _OutputAndExit(e)
 570   except ServiceException as e:
 571     _OutputAndExit(e)
 572   except apitools_exceptions.HttpError as e:
 573     # These should usually be retried by the underlying implementation or
 574     # wrapped by CloudApi ServiceExceptions, but if we do get them,
 575     # print something useful.
 576     _OutputAndExit('HttpError: %s, %s' % (getattr(e.response, 'status', ''),
 577                                           e.content or ''))
 578   except socket.error as e:
 579     if e.args[0] == errno.EPIPE:
 580       # Retrying with a smaller file (per suggestion below) works because
 581       # the library code send loop (in boto/s3/key.py) can get through the
 582       # entire file and then request the HTTP response before the socket
 583       # gets closed and the response lost.
 584       _OutputAndExit(
 585           'Got a "Broken pipe" error. This can happen to clients using Python '
 586           '2.x, when the server sends an error response and then closes the '
 587           'socket (see http://bugs.python.org/issue5542). If you are trying to '
 588           'upload a large object you might retry with a small (say 200k) '
 589           'object, and see if you get a more specific error code.'
 590       )
 591     else:
 592       _HandleUnknownFailure(e)
 593   except Exception as e:
 594     # Check for two types of errors related to service accounts. These errors
 595     # appear to be the same except for their messages, but they are caused by
 596     # different problems and both have unhelpful error messages. Moreover,
 597     # the error type belongs to PyOpenSSL, which is not necessarily installed.
 598     if 'mac verify failure' in str(e):
 599       _OutputAndExit(
 600           'Encountered an error while refreshing access token. '
 601           'If you are using a service account,\nplease verify that the '
 602           'gs_service_key_file_password field in your config file,'
 603           '\n%s, is correct.' % GetConfigFilePath())
 604     elif 'asn1 encoding routines' in str(e):
 605       _OutputAndExit(
 606           'Encountered an error while refreshing access token. '
 607           'If you are using a service account,\nplease verify that the '
 608           'gs_service_key_file field in your config file,\n%s, is correct.'
 609           % GetConfigFilePath())
 610     _HandleUnknownFailure(e)
 611
 612
 613 def _PerformTabCompletion(command_runner):
 614   """Performs gsutil-specific tab completion for the shell."""
 615   # argparse and argcomplete are bundled with the Google Cloud SDK.
 616   # When gsutil is invoked from the Google Cloud SDK, both should be available.
 617   try:
 618     import argcomplete
 619     import argparse
 620   except ImportError as e:
 621     _OutputAndExit('A library required for performing tab completion was'
 622                    ' not found.\nCause: %s' % e)
 623   parser = argparse.ArgumentParser(add_help=False)
 624   subparsers = parser.add_subparsers()
 625   command_runner.ConfigureCommandArgumentParsers(subparsers)
 626   argcomplete.autocomplete(parser, exit_method=sys.exit)
 627
 628   return 0
 629
 630 if __name__ == '__main__':
 631   sys.exit(main())