tools/telemetry/third_party/gsutilz/gslib/commands/mb.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2011 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Implementation of mb command for creating cloud storage buckets."""
  16
  17 from __future__ import absolute_import
  18
  19 import textwrap
  20
  21 from gslib.cloud_api import BadRequestException
  22 from gslib.command import Command
  23 from gslib.command_argument import CommandArgument
  24 from gslib.cs_api_map import ApiSelector
  25 from gslib.exception import CommandException
  26 from gslib.storage_url import StorageUrlFromString
  27 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
  28 from gslib.util import NO_MAX
  29
  30
  31 _SYNOPSIS = """
  32   gsutil mb [-c class] [-l location] [-p proj_id] uri...
  33 """
  34
  35 _DETAILED_HELP_TEXT = ("""
  36 <B>SYNOPSIS</B>
  37 """ + _SYNOPSIS + """
  38
  39
  40 <B>DESCRIPTION</B>
  41   The mb command creates a new bucket. Google Cloud Storage has a single
  42   namespace, so you will not be allowed to create a bucket with a name already
  43   in use by another user. You can, however, carve out parts of the bucket name
  44   space corresponding to your company's domain name (see "gsutil help naming").
  45
  46   If you don't specify a project ID using the -p option, the bucket
  47   will be created using the default project ID specified in your gsutil
  48   configuration file (see "gsutil help config"). For more details about
  49   projects see "gsutil help projects".
  50
  51   The -c and -l options specify the storage class and location, respectively,
  52   for the bucket. Once a bucket is created in a given location and with a
  53   given storage class, it cannot be moved to a different location, and the
  54   storage class cannot be changed. Instead, you would need to create a new
  55   bucket and move the data over and then delete the original bucket.
  56
  57
  58 <B>BUCKET STORAGE CLASSES</B>
  59   If you don't specify a -c option, the bucket will be created with the default
  60   (Standard) storage class.
  61
  62   If you specify -c DURABLE_REDUCED_AVAILABILITY (or -c DRA), it causes the data
  63   stored in the bucket to use Durable Reduced Availability storage. Buckets
  64   created with this storage class have lower availability than Standard storage
  65   class buckets, but durability equal to that of buckets created with Standard
  66   storage class. This option allows users to reduce costs for data for which
  67   lower availability is acceptable. Durable Reduced Availability storage would
  68   not be appropriate for "hot" objects (i.e., objects being accessed frequently)
  69   or for interactive workloads; however, it might be appropriate for other types
  70   of applications. See the online documentation for
  71   `pricing <https://cloud.google.com/storage/pricing>`_
  72   and `SLA <https://cloud.google.com/storage/sla>`_
  73   details.
  74
  75
  76   If you specify -c NEARLINE (or -c NL), it causes the data
  77   stored in the bucket to use Nearline storage. Buckets created with this
  78   storage class have higher latency and lower throughput than Standard storage
  79   class buckets. The availability and durability remains equal to that of
  80   buckets created with the Standard storage class. This option is best for
  81   objects which are accessed rarely and for which slower performance is
  82   acceptable. See the online documentation for
  83   `pricing <https://cloud.google.com/storage/pricing>`_ and
  84   `SLA <https://cloud.google.com/storage/sla>`_ details.
  85
  86
  87 <B>BUCKET LOCATIONS</B>
  88   If you don't specify a -l option, the bucket will be created in the default
  89   location (US). Otherwise, you can specify one of the available continental
  90   locations:
  91
  92   - ASIA (Asia)
  93   - EU (European Union)
  94   - US (United States)
  95
  96   Example:
  97     gsutil mb -l ASIA gs://some-bucket
  98
  99   If you specify the Durable Reduced Availability storage class (-c DRA), you
 100   can specify one of the continental locations above or one of the regional
 101   locations below:
 102
 103   - ASIA-EAST1 (Eastern Asia-Pacific)
 104   - US-EAST1 (Eastern United States) [1]_
 105   - US-EAST2 (Eastern United States) [1]_
 106   - US-EAST3 (Eastern United States) [1]_
 107   - US-CENTRAL1 (Central United States) [1]_
 108   - US-CENTRAL2 (Central United States) [1]_
 109   - US-WEST1 (Western United States) [1]_
 110
 111   Example:
 112     gsutil mb -c DRA -l US-CENTRAL1 gs://some-bucket
 113
 114   .. [1] These locations are for `Regional Buckets <https://developers.google.com/storage/docs/regional-buckets>`_.
 115      Regional Buckets is an experimental feature and data stored in these
 116      locations is not subject to the usual SLA. See the documentation for
 117      additional information.
 118
 119
 120 <B>OPTIONS</B>
 121   -c class          Can be DRA (or DURABLE_REDUCED_AVAILABILITY), NL (or
 122                     NEARLINE), or S (or STANDARD). Default is STANDARD.
 123
 124   -l location       Can be any continental location as described above, or
 125                     for DRA storage class, any regional or continental
 126                     location. Default is US. Locations are case insensitive.
 127
 128   -p proj_id        Specifies the project ID under which to create the bucket.
 129 """)
 130
 131
 132 class MbCommand(Command):
 133   """Implementation of gsutil mb command."""
 134
 135   # Command specification. See base class for documentation.
 136   command_spec = Command.CreateCommandSpec(
 137       'mb',
 138       command_name_aliases=['makebucket', 'createbucket', 'md', 'mkdir'],
 139       usage_synopsis=_SYNOPSIS,
 140       min_args=1,
 141       max_args=NO_MAX,
 142       supported_sub_args='c:l:p:',
 143       file_url_ok=False,
 144       provider_url_ok=False,
 145       urls_start_arg=0,
 146       gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
 147       gs_default_api=ApiSelector.JSON,
 148       argparse_arguments=[
 149           CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument()
 150       ]
 151   )
 152   # Help specification. See help_provider.py for documentation.
 153   help_spec = Command.HelpSpec(
 154       help_name='mb',
 155       help_name_aliases=[
 156           'createbucket', 'makebucket', 'md', 'mkdir', 'location', 'dra',
 157           'dras', 'reduced_availability', 'durable_reduced_availability', 'rr',
 158           'reduced_redundancy', 'standard', 'storage class', 'nearline', 'nl'],
 159       help_type='command_help',
 160       help_one_line_summary='Make buckets',
 161       help_text=_DETAILED_HELP_TEXT,
 162       subcommand_help_text={},
 163   )
 164
 165   def RunCommand(self):
 166     """Command entry point for the mb command."""
 167     location = None
 168     storage_class = None
 169     if self.sub_opts:
 170       for o, a in self.sub_opts:
 171         if o == '-l':
 172           location = a
 173         elif o == '-p':
 174           self.project_id = a
 175         elif o == '-c':
 176           storage_class = self._Normalize_Storage_Class(a)
 177
 178     bucket_metadata = apitools_messages.Bucket(location=location,
 179                                                storageClass=storage_class)
 180
 181     for bucket_uri_str in self.args:
 182       bucket_uri = StorageUrlFromString(bucket_uri_str)
 183       if not bucket_uri.IsBucket():
 184         raise CommandException('The mb command requires a URI that specifies a '
 185                                'bucket.\n"%s" is not valid.' % bucket_uri)
 186
 187       self.logger.info('Creating %s...', bucket_uri)
 188       # Pass storage_class param only if this is a GCS bucket. (In S3 the
 189       # storage class is specified on the key object.)
 190       try:
 191         self.gsutil_api.CreateBucket(
 192             bucket_uri.bucket_name, project_id=self.project_id,
 193             metadata=bucket_metadata, provider=bucket_uri.scheme)
 194       except BadRequestException as e:
 195         if (e.status == 400 and e.reason == 'DotfulBucketNameNotUnderTld' and
 196             bucket_uri.scheme == 'gs'):
 197           bucket_name = bucket_uri.bucket_name
 198           final_comp = bucket_name[bucket_name.rfind('.')+1:]
 199           raise CommandException('\n'.join(textwrap.wrap(
 200               'Buckets with "." in the name must be valid DNS names. The bucket'
 201               ' you are attempting to create (%s) is not a valid DNS name,'
 202               ' because the final component (%s) is not currently a valid part'
 203               ' of the top-level DNS tree.' % (bucket_name, final_comp))))
 204         else:
 205           raise
 206
 207     return 0
 208
 209   def _Normalize_Storage_Class(self, sc):
 210     sc = sc.upper()
 211     if sc in ('DRA', 'DURABLE_REDUCED_AVAILABILITY'):
 212       return 'DURABLE_REDUCED_AVAILABILITY'
 213     if sc in ('S', 'STD', 'STANDARD'):
 214       return 'STANDARD'
 215     if sc in ('NL', 'NEARLINE'):
 216       return 'NEARLINE'
 217     return sc