tools/telemetry/third_party/gsutilz/gslib/commands/notification.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2013 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """This module provides the notification command to gsutil."""
  16
  17 from __future__ import absolute_import
  18
  19 import getopt
  20 import uuid
  21
  22 from gslib.cloud_api import AccessDeniedException
  23 from gslib.command import Command
  24 from gslib.command import NO_MAX
  25 from gslib.command_argument import CommandArgument
  26 from gslib.cs_api_map import ApiSelector
  27 from gslib.exception import CommandException
  28 from gslib.help_provider import CreateHelpText
  29 from gslib.storage_url import StorageUrlFromString
  30
  31
  32 _WATCHBUCKET_SYNOPSIS = """
  33   gsutil notification watchbucket [-i id] [-t token] app_url bucket_url...
  34 """
  35
  36 _STOPCHANNEL_SYNOPSIS = """
  37   gsutil notification stopchannel channel_id resource_id
  38 """
  39
  40 _SYNOPSIS = _WATCHBUCKET_SYNOPSIS + _STOPCHANNEL_SYNOPSIS.lstrip('\n')
  41
  42 _WATCHBUCKET_DESCRIPTION = """
  43 <B>WATCHBUCKET</B>
  44   The watchbucket sub-command can be used to watch a bucket for object changes.
  45   A service account must be used when running this command.
  46
  47   The app_url parameter must be an HTTPS URL to an application that will be
  48   notified of changes to any object in the bucket. The URL endpoint must be
  49   a verified domain on your project. See
  50   `Notification Authorization <https://developers.google.com/storage/docs/object-change-notification#_Authorization>`_
  51   for details.
  52
  53   The optional id parameter can be used to assign a unique identifier to the
  54   created notification channel. If not provided, a random UUID string will be
  55   generated.
  56
  57   The optional token parameter can be used to validate notifications events.
  58   To do this, set this custom token and store it to later verify that
  59   notification events contain the client token you expect.
  60
  61 """
  62
  63 _STOPCHANNEL_DESCRIPTION = """
  64 <B>STOPCHANNEL</B>
  65   The stopchannel sub-command can be used to stop sending change events to a
  66   notification channel.
  67
  68   The channel_id and resource_id parameters should match the values from the
  69   response of a bucket watch request.
  70
  71 """
  72
  73 _DESCRIPTION = """
  74   The notification command can be used to configure notifications.
  75   For more information on the Object Change Notification feature, please see:
  76   https://developers.google.com/storage/docs/object-change-notification
  77
  78   The notification command has two sub-commands:
  79 """ + _WATCHBUCKET_DESCRIPTION + _STOPCHANNEL_DESCRIPTION + """
  80
  81 <B>EXAMPLES</B>
  82
  83   Watch the bucket example-bucket for changes and send notifications to an
  84   application server running at example.com:
  85
  86     gsutil notification watchbucket https://example.com/notify \\
  87       gs://example-bucket
  88
  89   Assign identifier my-channel-id to the created notification channel:
  90
  91     gsutil notification watchbucket -i my-channel-id \\
  92       https://example.com/notify gs://example-bucket
  93
  94   Set a custom client token that will be included with each notification event:
  95
  96     gsutil notification watchbucket -t my-client-token \\
  97       https://example.com/notify gs://example-bucket
  98
  99   Stop the notification event channel with channel identifier channel1 and
 100   resource identifier SoGqan08XDIFWr1Fv_nGpRJBHh8:
 101
 102     gsutil notification stopchannel channel1 SoGqan08XDIFWr1Fv_nGpRJBHh8
 103
 104 <B>NOTIFICATIONS AND PARALLEL COMPOSITE UPLOADS</B>
 105
 106   By default, gsutil enables parallel composite uploads for large files (see
 107   "gsutil help cp"), which means that an upload of a large object can result
 108   in multiple temporary component objects being uploaded before the actual
 109   intended object is created. Any subscriber to notifications for this bucket
 110   will then see a notification for each of these components being created and
 111   deleted. If this is a concern for you, note that parallel composite uploads
 112   can be disabled by setting "parallel_composite_upload_threshold = 0" in your
 113   boto config file.
 114
 115 """
 116
 117 NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE = """
 118 Watch bucket attempt failed:
 119   {watch_error}
 120
 121 You attempted to watch a bucket with an application URL of:
 122
 123   {watch_url}
 124
 125 which is not authorized for your project. Please ensure that you are using
 126 Service Account authentication and that the Service Account's project is
 127 authorized for the application URL. Notification endpoint URLs must also be
 128 whitelisted in your Cloud Console project. To do that, the domain must also be
 129 verified using Google Webmaster Tools. For instructions, please see:
 130
 131   https://developers.google.com/storage/docs/object-change-notification#_Authorization
 132 """
 133
 134 _DETAILED_HELP_TEXT = CreateHelpText(_SYNOPSIS, _DESCRIPTION)
 135
 136 _watchbucket_help_text = (
 137     CreateHelpText(_WATCHBUCKET_SYNOPSIS, _WATCHBUCKET_DESCRIPTION))
 138 _stopchannel_help_text = (
 139     CreateHelpText(_STOPCHANNEL_SYNOPSIS, _STOPCHANNEL_DESCRIPTION))
 140
 141
 142 class NotificationCommand(Command):
 143   """Implementation of gsutil notification command."""
 144
 145   # Command specification. See base class for documentation.
 146   command_spec = Command.CreateCommandSpec(
 147       'notification',
 148       command_name_aliases=[
 149           'notify', 'notifyconfig', 'notifications', 'notif'],
 150       usage_synopsis=_SYNOPSIS,
 151       min_args=3,
 152       max_args=NO_MAX,
 153       supported_sub_args='i:t:',
 154       file_url_ok=False,
 155       provider_url_ok=False,
 156       urls_start_arg=1,
 157       gs_api_support=[ApiSelector.JSON],
 158       gs_default_api=ApiSelector.JSON,
 159       argparse_arguments={
 160           'watchbucket': [
 161               CommandArgument.MakeFreeTextArgument(),
 162               CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
 163           ],
 164           'stopchannel': []
 165       }
 166   )
 167   # Help specification. See help_provider.py for documentation.
 168   help_spec = Command.HelpSpec(
 169       help_name='notification',
 170       help_name_aliases=['watchbucket', 'stopchannel', 'notifyconfig'],
 171       help_type='command_help',
 172       help_one_line_summary='Configure object change notification',
 173       help_text=_DETAILED_HELP_TEXT,
 174       subcommand_help_text={'watchbucket': _watchbucket_help_text,
 175                             'stopchannel': _stopchannel_help_text},
 176   )
 177
 178   def _WatchBucket(self):
 179     """Creates a watch on a bucket given in self.args."""
 180     self.CheckArguments()
 181     identifier = None
 182     client_token = None
 183     if self.sub_opts:
 184       for o, a in self.sub_opts:
 185         if o == '-i':
 186           identifier = a
 187         if o == '-t':
 188           client_token = a
 189
 190     identifier = identifier or str(uuid.uuid4())
 191     watch_url = self.args[0]
 192     bucket_arg = self.args[-1]
 193
 194     if not watch_url.lower().startswith('https://'):
 195       raise CommandException('The application URL must be an https:// URL.')
 196
 197     bucket_url = StorageUrlFromString(bucket_arg)
 198     if not (bucket_url.IsBucket() and bucket_url.scheme == 'gs'):
 199       raise CommandException(
 200           'The %s command can only be used with gs:// bucket URLs.' %
 201           self.command_name)
 202     if not bucket_url.IsBucket():
 203       raise CommandException('URL must name a bucket for the %s command.' %
 204                              self.command_name)
 205
 206     self.logger.info('Watching bucket %s with application URL %s ...',
 207                      bucket_url, watch_url)
 208
 209     try:
 210       channel = self.gsutil_api.WatchBucket(
 211           bucket_url.bucket_name, watch_url, identifier, token=client_token,
 212           provider=bucket_url.scheme)
 213     except AccessDeniedException, e:
 214       self.logger.warn(NOTIFICATION_AUTHORIZATION_FAILED_MESSAGE.format(
 215           watch_error=str(e), watch_url=watch_url))
 216       raise
 217
 218     channel_id = channel.id
 219     resource_id = channel.resourceId
 220     client_token = channel.token
 221     self.logger.info('Successfully created watch notification channel.')
 222     self.logger.info('Watch channel identifier: %s', channel_id)
 223     self.logger.info('Canonicalized resource identifier: %s', resource_id)
 224     self.logger.info('Client state token: %s', client_token)
 225
 226     return 0
 227
 228   def _StopChannel(self):
 229     channel_id = self.args[0]
 230     resource_id = self.args[1]
 231
 232     self.logger.info('Removing channel %s with resource identifier %s ...',
 233                      channel_id, resource_id)
 234     self.gsutil_api.StopChannel(channel_id, resource_id, provider='gs')
 235     self.logger.info('Succesfully removed channel.')
 236
 237     return 0
 238
 239   def _RunSubCommand(self, func):
 240     try:
 241       (self.sub_opts, self.args) = getopt.getopt(
 242           self.args, self.command_spec.supported_sub_args)
 243       return func()
 244     except getopt.GetoptError, e:
 245       self.RaiseInvalidArgumentException()
 246
 247   def RunCommand(self):
 248     """Command entry point for the notification command."""
 249     subcommand = self.args.pop(0)
 250
 251     if subcommand == 'watchbucket':
 252       return self._RunSubCommand(self._WatchBucket)
 253     elif subcommand == 'stopchannel':
 254       return self._RunSubCommand(self._StopChannel)
 255     else:
 256       raise CommandException('Invalid subcommand "%s" for the %s command.' %
 257                              (subcommand, self.command_name))