tools/telemetry/third_party/gsutilz/gslib/commands/signurl.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2014 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Implementation of Url Signing workflow.
  16
  17 see: https://developers.google.com/storage/docs/accesscontrol#Signed-URLs)
  18 """
  19
  20 from __future__ import absolute_import
  21
  22 import base64
  23 import calendar
  24 from datetime import datetime
  25 from datetime import timedelta
  26 import getpass
  27 import re
  28 import time
  29 import urllib
  30
  31 from apitools.base.py.exceptions import HttpError
  32 from apitools.base.py.http_wrapper import MakeRequest
  33 from apitools.base.py.http_wrapper import Request
  34
  35 from gslib.command import Command
  36 from gslib.command_argument import CommandArgument
  37 from gslib.cs_api_map import ApiSelector
  38 from gslib.exception import CommandException
  39 from gslib.storage_url import ContainsWildcard
  40 from gslib.storage_url import StorageUrlFromString
  41 from gslib.util import GetNewHttp
  42 from gslib.util import NO_MAX
  43 from gslib.util import UTF8
  44
  45 try:
  46   # Check for openssl.
  47   # pylint: disable=C6204
  48   from OpenSSL.crypto import load_pkcs12
  49   from OpenSSL.crypto import sign
  50   HAVE_OPENSSL = True
  51 except ImportError:
  52   load_pkcs12 = None
  53   sign = None
  54   HAVE_OPENSSL = False
  55
  56
  57 _SYNOPSIS = """
  58   gsutil signurl [-c] [-d] [-m] [-p] pkcs12-file url...
  59 """
  60
  61 _DETAILED_HELP_TEXT = ("""
  62 <B>SYNOPSIS</B>
  63 """ + _SYNOPSIS + """
  64
  65
  66 <B>DESCRIPTION</B>
  67   The signurl command will generate signed urls that can be used to access
  68   the specified objects without authentication for a specific period of time.
  69
  70   Please see the `Signed URLs documentation
  71   <https://developers.google.com/storage/docs/accesscontrol#Signed-URLs>`_ for
  72   background about signed URLs.
  73
  74   Multiple gs:// urls may be provided and may contain wildcards.  A signed url
  75   will be produced for each provided url, authorized
  76   for the specified HTTP method and valid for the given duration.
  77
  78   Note: Unlike the gsutil ls command, the signurl command does not support
  79   operations on sub-directories. For example, if you run the command:
  80
  81     gsutil signurl <private-key-file> gs://some-bucket/some-object/
  82
  83   The signurl command uses the private key for a  service account (the
  84   '<private-key-file>' argument) to generate the cryptographic
  85   signature for the generated URL.  The private key file must be in PKCS12
  86   format. The signurl command will prompt for the passphrase used to protect
  87   the private key file (default 'notasecret').  For more information
  88   regarding generating a private key for use with the signurl command please
  89   see the `Authentication documentation.
  90   <https://developers.google.com/storage/docs/authentication#generating-a-private-key>`_
  91
  92   gsutil will look up information about the object "some-object/" (with a
  93   trailing slash) inside bucket "some-bucket", as opposed to operating on
  94   objects nested under gs://some-bucket/some-object. Unless you actually
  95   have an object with that name, the operation will fail.
  96
  97 <B>OPTIONS</B>
  98   -m          Specifies the HTTP method to be authorized for use
  99               with the signed url, default is GET.
 100
 101   -d          Specifies the duration that the signed url should be valid
 102               for, default duration is 1 hour.
 103
 104               Times may be specified with no suffix (default hours), or
 105               with s = seconds, m = minutes, h = hours, d = days.
 106
 107               This option may be specified multiple times, in which case
 108               the duration the link remains valid is the sum of all the
 109               duration options.
 110
 111   -c          Specifies the content type for which the signed url is
 112               valid for.
 113
 114   -p          Specify the keystore password instead of prompting.
 115
 116 <B>USAGE</B>
 117
 118   Create a signed url for downloading an object valid for 10 minutes:
 119
 120     gsutil signurl -d 10m <private-key-file> gs://<bucket>/<object>
 121
 122   Create a signed url for uploading a plain text file via HTTP PUT:
 123
 124     gsutil signurl -m PUT -d 1h -c text/plain <private-key-file> \\
 125         gs://<bucket>/<obj>
 126
 127   To construct a signed URL that allows anyone in possession of
 128   the URL to PUT to the specified bucket for one day, creating
 129   any object of Content-Type image/jpg, run:
 130
 131     gsutil signurl -m PUT -d 1d -c image/jpg <private-key-file> \\
 132         gs://<bucket>/<obj>
 133
 134
 135 """)
 136
 137
 138 def _DurationToTimeDelta(duration):
 139   r"""Parses the given duration and returns an equivalent timedelta."""
 140
 141   match = re.match(r'^(\d+)([dDhHmMsS])?$', duration)
 142   if not match:
 143     raise CommandException('Unable to parse duration string')
 144
 145   duration, modifier = match.groups('h')
 146   duration = int(duration)
 147   modifier = modifier.lower()
 148
 149   if modifier == 'd':
 150     ret = timedelta(days=duration)
 151   elif modifier == 'h':
 152     ret = timedelta(hours=duration)
 153   elif modifier == 'm':
 154     ret = timedelta(minutes=duration)
 155   elif modifier == 's':
 156     ret = timedelta(seconds=duration)
 157
 158   return ret
 159
 160
 161 def _GenSignedUrl(key, client_id, method, md5,
 162                   content_type, expiration, gcs_path):
 163   """Construct a string to sign with the provided key and returns \
 164   the complete url."""
 165
 166   tosign = ('{0}\n{1}\n{2}\n{3}\n/{4}'
 167             .format(method, md5, content_type,
 168                     expiration, gcs_path))
 169   signature = base64.b64encode(sign(key, tosign, 'RSA-SHA256'))
 170
 171   final_url = ('https://storage.googleapis.com/{0}?'
 172                'GoogleAccessId={1}&Expires={2}&Signature={3}'
 173                .format(gcs_path, client_id, expiration,
 174                        urllib.quote_plus(str(signature))))
 175
 176   return final_url
 177
 178
 179 def _ReadKeystore(ks_contents, passwd):
 180   ks = load_pkcs12(ks_contents, passwd)
 181   client_id = (ks.get_certificate()
 182                .get_subject()
 183                .CN.replace('.apps.googleusercontent.com',
 184                            '@developer.gserviceaccount.com'))
 185
 186   return ks, client_id
 187
 188
 189 class UrlSignCommand(Command):
 190   """Implementation of gsutil url_sign command."""
 191
 192   # Command specification. See base class for documentation.
 193   command_spec = Command.CreateCommandSpec(
 194       'signurl',
 195       command_name_aliases=['signedurl', 'queryauth'],
 196       usage_synopsis=_SYNOPSIS,
 197       min_args=2,
 198       max_args=NO_MAX,
 199       supported_sub_args='m:d:c:p:',
 200       file_url_ok=False,
 201       provider_url_ok=False,
 202       urls_start_arg=1,
 203       gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
 204       gs_default_api=ApiSelector.JSON,
 205       argparse_arguments=[
 206           CommandArgument.MakeNFileURLsArgument(1),
 207           CommandArgument.MakeZeroOrMoreCloudURLsArgument()
 208       ]
 209   )
 210   # Help specification. See help_provider.py for documentation.
 211   help_spec = Command.HelpSpec(
 212       help_name='signurl',
 213       help_name_aliases=['signedurl', 'queryauth'],
 214       help_type='command_help',
 215       help_one_line_summary='Create a signed url',
 216       help_text=_DETAILED_HELP_TEXT,
 217       subcommand_help_text={},
 218   )
 219
 220   def _ParseAndCheckSubOpts(self):
 221     # Default argument values
 222     delta = None
 223     method = 'GET'
 224     content_type = ''
 225     passwd = None
 226
 227     for o, v in self.sub_opts:
 228       if o == '-d':
 229         if delta is not None:
 230           delta += _DurationToTimeDelta(v)
 231         else:
 232           delta = _DurationToTimeDelta(v)
 233       elif o == '-m':
 234         method = v
 235       elif o == '-c':
 236         content_type = v
 237       elif o == '-p':
 238         passwd = v
 239       else:
 240         self.RaiseInvalidArgumentException()
 241
 242     if delta is None:
 243       delta = timedelta(hours=1)
 244
 245     expiration = calendar.timegm((datetime.utcnow() + delta).utctimetuple())
 246     if method not in ['GET', 'PUT', 'DELETE', 'HEAD']:
 247       raise CommandException('HTTP method must be one of [GET|HEAD|PUT|DELETE]')
 248
 249     return method, expiration, content_type, passwd
 250
 251   def _ProbeObjectAccessWithClient(self, key, client_id, gcs_path):
 252     """Performs a head request against a signed url to check for read access."""
 253
 254     signed_url = _GenSignedUrl(key, client_id, 'HEAD', '', '',
 255                                int(time.time()) + 10, gcs_path)
 256
 257     try:
 258       h = GetNewHttp()
 259       req = Request(signed_url, 'HEAD')
 260       response = MakeRequest(h, req)
 261
 262       if response.status_code not in [200, 403, 404]:
 263         raise HttpError(response)
 264
 265       return response.status_code
 266     except HttpError as e:
 267       raise CommandException('Unexpected response code while querying'
 268                              'object readability ({0})'.format(e.message))
 269
 270   def _EnumerateStorageUrls(self, in_urls):
 271     ret = []
 272
 273     for url_str in in_urls:
 274       if ContainsWildcard(url_str):
 275         ret.extend([blr.storage_url for blr in self.WildcardIterator(url_str)])
 276       else:
 277         ret.append(StorageUrlFromString(url_str))
 278
 279     return ret
 280
 281   def RunCommand(self):
 282     """Command entry point for signurl command."""
 283     if not HAVE_OPENSSL:
 284       raise CommandException(
 285           'The signurl command requires the pyopenssl library (try pip '
 286           'install pyopenssl or easy_install pyopenssl)')
 287
 288     method, expiration, content_type, passwd = self._ParseAndCheckSubOpts()
 289     storage_urls = self._EnumerateStorageUrls(self.args[1:])
 290
 291     if not passwd:
 292       passwd = getpass.getpass('Keystore password:')
 293
 294     ks, client_id = _ReadKeystore(open(self.args[0], 'rb').read(), passwd)
 295
 296     print 'URL\tHTTP Method\tExpiration\tSigned URL'
 297     for url in storage_urls:
 298       if url.scheme != 'gs':
 299         raise CommandException('Can only create signed urls from gs:// urls')
 300       if url.IsBucket():
 301         gcs_path = url.bucket_name
 302       else:
 303         # Need to url encode the object name as Google Cloud Storage does when
 304         # computing the string to sign when checking the signature.
 305         gcs_path = '{0}/{1}'.format(url.bucket_name,
 306                                     urllib.quote(url.object_name.encode(UTF8)))
 307
 308       final_url = _GenSignedUrl(ks.get_privatekey(), client_id,
 309                                 method, '', content_type, expiration,
 310                                 gcs_path)
 311
 312       expiration_dt = datetime.fromtimestamp(expiration)
 313
 314       print '{0}\t{1}\t{2}\t{3}'.format(url.url_string.encode(UTF8), method,
 315                                         (expiration_dt
 316                                          .strftime('%Y-%m-%d %H:%M:%S')),
 317                                         final_url.encode(UTF8))
 318
 319       response_code = self._ProbeObjectAccessWithClient(ks.get_privatekey(),
 320                                                         client_id, gcs_path)
 321
 322       if response_code == 404 and method != 'PUT':
 323         if url.IsBucket():
 324           msg = ('Bucket {0} does not exist. Please create a bucket with '
 325                  'that name before a creating signed URL to access it.'
 326                  .format(url))
 327         else:
 328           msg = ('Object {0} does not exist. Please create/upload an object '
 329                  'with that name before a creating signed URL to access it.'
 330                  .format(url))
 331
 332         raise CommandException(msg)
 333       elif response_code == 403:
 334         self.logger.warn(
 335             '%s does not have permissions on %s, using this link will likely '
 336             'result in a 403 error until at least READ permissions are granted',
 337             client_id, url)
 338
 339     return 0