tools/telemetry/third_party/gsutilz/gslib/translation_helper.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2014 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Utility module for translating XML API objects to/from JSON objects."""
  16
  17 from __future__ import absolute_import
  18
  19 import datetime
  20 import json
  21 import re
  22 import textwrap
  23 import xml.etree.ElementTree
  24
  25 from apitools.base.py import encoding
  26 import boto
  27 from boto.gs.acl import ACL
  28 from boto.gs.acl import ALL_AUTHENTICATED_USERS
  29 from boto.gs.acl import ALL_USERS
  30 from boto.gs.acl import Entries
  31 from boto.gs.acl import Entry
  32 from boto.gs.acl import GROUP_BY_DOMAIN
  33 from boto.gs.acl import GROUP_BY_EMAIL
  34 from boto.gs.acl import GROUP_BY_ID
  35 from boto.gs.acl import USER_BY_EMAIL
  36 from boto.gs.acl import USER_BY_ID
  37
  38 from gslib.cloud_api import ArgumentException
  39 from gslib.cloud_api import NotFoundException
  40 from gslib.cloud_api import Preconditions
  41 from gslib.exception import CommandException
  42 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
  43
  44 # In Python 2.6, ElementTree raises ExpatError instead of ParseError.
  45 # pylint: disable=g-import-not-at-top
  46 try:
  47   from xml.etree.ElementTree import ParseError as XmlParseError
  48 except ImportError:
  49   from xml.parsers.expat import ExpatError as XmlParseError
  50
  51 CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I)
  52 CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I)
  53 CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I)
  54 CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I)
  55 CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I)
  56 CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I)
  57 GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I)
  58 GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I)
  59 GOOG_METAGENERATION_MATCH_REGEX = re.compile(
  60     r'^x-goog-if-metageneration-match', re.I)
  61 CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)',
  62                                         re.I)
  63 CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I)
  64 CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I)
  65
  66 # gsutil-specific GUIDs for marking special metadata for S3 compatibility.
  67 S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker'
  68 S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker'
  69 S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID]
  70 # This distinguishes S3 custom headers from S3 metadata on objects.
  71 S3_HEADER_PREFIX = 'custom-amz-header'
  72
  73 DEFAULT_CONTENT_TYPE = 'application/octet-stream'
  74
  75 # Because CORS is just a list in apitools, we need special handling or blank
  76 # CORS lists will get sent with other configuration commands such as lifecycle,
  77 # commands, which would cause CORS configuration to be unintentionally removed.
  78 # Protorpc defaults list values to an empty list, and won't allow us to set the
  79 # value to None like other configuration fields, so there is no way to
  80 # distinguish the default value from when we actually want to remove the CORS
  81 # configuration.  To work around this, we create a dummy CORS entry that
  82 # signifies that we should nullify the CORS configuration.
  83 # A value of [] means don't modify the CORS configuration.
  84 # A value of REMOVE_CORS_CONFIG means remove the CORS configuration.
  85 REMOVE_CORS_CONFIG = [apitools_messages.Bucket.CorsValueListEntry(
  86     maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG'])]
  87
  88
  89 def ObjectMetadataFromHeaders(headers):
  90   """Creates object metadata according to the provided headers.
  91
  92   gsutil -h allows specifiying various headers (originally intended
  93   to be passed to boto in gsutil v3).  For the JSON API to be compatible with
  94   this option, we need to parse these headers into gsutil_api Object fields.
  95
  96   Args:
  97     headers: Dict of headers passed via gsutil -h
  98
  99   Raises:
 100     ArgumentException if an invalid header is encountered.
 101
 102   Returns:
 103     apitools Object with relevant fields populated from headers.
 104   """
 105   obj_metadata = apitools_messages.Object()
 106   for header, value in headers.items():
 107     if CACHE_CONTROL_REGEX.match(header):
 108       obj_metadata.cacheControl = value.strip()
 109     elif CONTENT_DISPOSITION_REGEX.match(header):
 110       obj_metadata.contentDisposition = value.strip()
 111     elif CONTENT_ENCODING_REGEX.match(header):
 112       obj_metadata.contentEncoding = value.strip()
 113     elif CONTENT_MD5_REGEX.match(header):
 114       obj_metadata.md5Hash = value.strip()
 115     elif CONTENT_LANGUAGE_REGEX.match(header):
 116       obj_metadata.contentLanguage = value.strip()
 117     elif CONTENT_TYPE_REGEX.match(header):
 118       if not value:
 119         obj_metadata.contentType = DEFAULT_CONTENT_TYPE
 120       else:
 121         obj_metadata.contentType = value.strip()
 122     elif GOOG_API_VERSION_REGEX.match(header):
 123       # API version is only relevant for XML, ignore and rely on the XML API
 124       # to add the appropriate version.
 125       continue
 126     elif GOOG_GENERATION_MATCH_REGEX.match(header):
 127       # Preconditions are handled elsewhere, but allow these headers through.
 128       continue
 129     elif GOOG_METAGENERATION_MATCH_REGEX.match(header):
 130       # Preconditions are handled elsewhere, but allow these headers through.
 131       continue
 132     else:
 133       custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header)
 134       custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header)
 135       custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header)
 136       header_key = None
 137       if custom_goog_metadata_match:
 138         header_key = custom_goog_metadata_match.group('header_key')
 139       elif custom_amz_metadata_match:
 140         header_key = custom_amz_metadata_match.group('header_key')
 141       elif custom_amz_header_match:
 142         # If we got here we are guaranteed by the prior statement that this is
 143         # not an x-amz-meta- header.
 144         header_key = (S3_HEADER_PREFIX +
 145                       custom_amz_header_match.group('header_key'))
 146       if header_key:
 147         if header_key.lower() == 'x-goog-content-language':
 148           # Work around content-language being inserted into custom metadata.
 149           continue
 150         if not obj_metadata.metadata:
 151           obj_metadata.metadata = apitools_messages.Object.MetadataValue()
 152         if not obj_metadata.metadata.additionalProperties:
 153           obj_metadata.metadata.additionalProperties = []
 154         obj_metadata.metadata.additionalProperties.append(
 155             apitools_messages.Object.MetadataValue.AdditionalProperty(
 156                 key=header_key, value=value))
 157       else:
 158         raise ArgumentException(
 159             'Invalid header specifed: %s:%s' % (header, value))
 160   return obj_metadata
 161
 162
 163 def HeadersFromObjectMetadata(dst_obj_metadata, provider):
 164   """Creates a header dictionary based on existing object metadata.
 165
 166   Args:
 167     dst_obj_metadata: Object metadata to create the headers from.
 168     provider: Provider string ('gs' or 's3')
 169
 170   Returns:
 171     Headers dictionary.
 172   """
 173   headers = {}
 174   if not dst_obj_metadata:
 175     return
 176   # Metadata values of '' mean suppress/remove this header.
 177   if dst_obj_metadata.cacheControl is not None:
 178     if not dst_obj_metadata.cacheControl:
 179       headers['cache-control'] = None
 180     else:
 181       headers['cache-control'] = dst_obj_metadata.cacheControl.strip()
 182   if dst_obj_metadata.contentDisposition:
 183     if not dst_obj_metadata.contentDisposition:
 184       headers['content-disposition'] = None
 185     else:
 186       headers['content-disposition'] = (
 187           dst_obj_metadata.contentDisposition.strip())
 188   if dst_obj_metadata.contentEncoding:
 189     if not dst_obj_metadata.contentEncoding:
 190       headers['content-encoding'] = None
 191     else:
 192       headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip()
 193   if dst_obj_metadata.contentLanguage:
 194     if not dst_obj_metadata.contentLanguage:
 195       headers['content-language'] = None
 196     else:
 197       headers['content-language'] = dst_obj_metadata.contentLanguage.strip()
 198   if dst_obj_metadata.md5Hash:
 199     if not dst_obj_metadata.md5Hash:
 200       headers['Content-MD5'] = None
 201     else:
 202       headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip()
 203   if dst_obj_metadata.contentType is not None:
 204     if not dst_obj_metadata.contentType:
 205       headers['content-type'] = None
 206     else:
 207       headers['content-type'] = dst_obj_metadata.contentType.strip()
 208   if (dst_obj_metadata.metadata and
 209       dst_obj_metadata.metadata.additionalProperties):
 210     for additional_property in dst_obj_metadata.metadata.additionalProperties:
 211       # Work around content-language being inserted into custom metadata by
 212       # the XML API.
 213       if additional_property.key == 'content-language':
 214         continue
 215       # Don't translate special metadata markers.
 216       if additional_property.key in S3_MARKER_GUIDS:
 217         continue
 218       if provider == 'gs':
 219         header_name = 'x-goog-meta-' + additional_property.key
 220       elif provider == 's3':
 221         if additional_property.key.startswith(S3_HEADER_PREFIX):
 222           header_name = ('x-amz-' +
 223                          additional_property.key[len(S3_HEADER_PREFIX):])
 224         else:
 225           header_name = 'x-amz-meta-' + additional_property.key
 226       else:
 227         raise ArgumentException('Invalid provider specified: %s' % provider)
 228       if (additional_property.value is not None and
 229           not additional_property.value):
 230         headers[header_name] = None
 231       else:
 232         headers[header_name] = additional_property.value
 233   return headers
 234
 235
 236 def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False):
 237   """Copies metadata from src_obj_metadata to dst_obj_metadata.
 238
 239   Args:
 240     src_obj_metadata: Metadata from source object
 241     dst_obj_metadata: Initialized metadata for destination object
 242     override: If true, will overwrite metadata in destination object.
 243               If false, only writes metadata for values that don't already
 244               exist.
 245   """
 246   if override or not dst_obj_metadata.cacheControl:
 247     dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl
 248   if override or not dst_obj_metadata.contentDisposition:
 249     dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition
 250   if override or not dst_obj_metadata.contentEncoding:
 251     dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding
 252   if override or not dst_obj_metadata.contentLanguage:
 253     dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage
 254   if override or not dst_obj_metadata.contentType:
 255     dst_obj_metadata.contentType = src_obj_metadata.contentType
 256   if override or not dst_obj_metadata.md5Hash:
 257     dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash
 258
 259   # TODO: Apitools should ideally treat metadata like a real dictionary instead
 260   # of a list of key/value pairs (with an O(N^2) lookup).  In practice the
 261   # number of values is typically small enough not to matter.
 262   # Work around this by creating our own dictionary.
 263   if (src_obj_metadata.metadata and
 264       src_obj_metadata.metadata.additionalProperties):
 265     if not dst_obj_metadata.metadata:
 266       dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue()
 267     if not dst_obj_metadata.metadata.additionalProperties:
 268       dst_obj_metadata.metadata.additionalProperties = []
 269     dst_metadata_dict = {}
 270     for dst_prop in dst_obj_metadata.metadata.additionalProperties:
 271       dst_metadata_dict[dst_prop.key] = dst_prop.value
 272     for src_prop in src_obj_metadata.metadata.additionalProperties:
 273       if src_prop.key in dst_metadata_dict:
 274         if override:
 275           # Metadata values of '' mean suppress/remove this header.
 276           if src_prop.value is not None and not src_prop.value:
 277             dst_metadata_dict[src_prop.key] = None
 278           else:
 279             dst_metadata_dict[src_prop.key] = src_prop.value
 280       else:
 281         dst_metadata_dict[src_prop.key] = src_prop.value
 282     # Rewrite the list with our updated dict.
 283     dst_obj_metadata.metadata.additionalProperties = []
 284     for k, v in dst_metadata_dict.iteritems():
 285       dst_obj_metadata.metadata.additionalProperties.append(
 286           apitools_messages.Object.MetadataValue.AdditionalProperty(key=k,
 287                                                                     value=v))
 288
 289
 290 def PreconditionsFromHeaders(headers):
 291   """Creates bucket or object preconditions acccording to the provided headers.
 292
 293   Args:
 294     headers: Dict of headers passed via gsutil -h
 295
 296   Returns:
 297     gsutil Cloud API Preconditions object fields populated from headers, or None
 298     if no precondition headers are present.
 299   """
 300   return_preconditions = Preconditions()
 301   try:
 302     for header, value in headers.items():
 303       if GOOG_GENERATION_MATCH_REGEX.match(header):
 304         return_preconditions.gen_match = long(value)
 305       if GOOG_METAGENERATION_MATCH_REGEX.match(header):
 306         return_preconditions.meta_gen_match = long(value)
 307   except ValueError, _:
 308     raise ArgumentException('Invalid precondition header specified. '
 309                             'x-goog-if-generation-match and '
 310                             'x-goog-if-metageneration match must be specified '
 311                             'with a positive integer value.')
 312   return return_preconditions
 313
 314
 315 def CreateBucketNotFoundException(code, provider, bucket_name):
 316   return NotFoundException('%s://%s bucket does not exist.' %
 317                            (provider, bucket_name), status=code)
 318
 319
 320 def CreateObjectNotFoundException(code, provider, bucket_name, object_name,
 321                                   generation=None):
 322   uri_string = '%s://%s/%s' % (provider, bucket_name, object_name)
 323   if generation:
 324     uri_string += '#%s' % str(generation)
 325   return NotFoundException('%s does not exist.' % uri_string, status=code)
 326
 327
 328 def EncodeStringAsLong(string_to_convert):
 329   """Encodes an ASCII string as a python long.
 330
 331   This is used for modeling S3 version_id's as apitools generation.  Because
 332   python longs can be arbitrarily large, this works.
 333
 334   Args:
 335     string_to_convert: ASCII string to convert to a long.
 336
 337   Returns:
 338     Long that represents the input string.
 339   """
 340   return long(string_to_convert.encode('hex'), 16)
 341
 342
 343 def _DecodeLongAsString(long_to_convert):
 344   """Decodes an encoded python long into an ASCII string.
 345
 346   This is used for modeling S3 version_id's as apitools generation.
 347
 348   Args:
 349     long_to_convert: long to convert to ASCII string. If this is already a
 350                      string, it is simply returned.
 351
 352   Returns:
 353     String decoded from the input long.
 354   """
 355   if isinstance(long_to_convert, basestring):
 356     # Already converted.
 357     return long_to_convert
 358   return hex(long_to_convert)[2:-1].decode('hex')
 359
 360
 361 def GenerationFromUrlAndString(url, generation):
 362   """Decodes a generation from a StorageURL and a generation string.
 363
 364   This is used to represent gs and s3 versioning.
 365
 366   Args:
 367     url: StorageUrl representing the object.
 368     generation: Long or string representing the object's generation or
 369                 version.
 370
 371   Returns:
 372     Valid generation string for use in URLs.
 373   """
 374   if url.scheme == 's3' and generation:
 375     return _DecodeLongAsString(generation)
 376   return generation
 377
 378
 379 def CheckForXmlConfigurationAndRaise(config_type_string, json_txt):
 380   """Checks a JSON parse exception for provided XML configuration."""
 381   try:
 382     xml.etree.ElementTree.fromstring(str(json_txt))
 383     raise ArgumentException('\n'.join(textwrap.wrap(
 384         'XML {0} data provided; Google Cloud Storage {0} configuration '
 385         'now uses JSON format. To convert your {0}, set the desired XML '
 386         'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then '
 387         'use \'gsutil {1} get ...\' with gsutil version 4 or greater to '
 388         'get the corresponding JSON {0}.'.format(config_type_string,
 389                                                  config_type_string.lower()))))
 390   except XmlParseError:
 391     pass
 392   raise ArgumentException('JSON %s data could not be loaded '
 393                           'from: %s' % (config_type_string, json_txt))
 394
 395
 396 class LifecycleTranslation(object):
 397   """Functions for converting between various lifecycle formats.
 398
 399     This class handles conversation to and from Boto Cors objects, JSON text,
 400     and apitools Message objects.
 401   """
 402
 403   @classmethod
 404   def BotoLifecycleFromMessage(cls, lifecycle_message):
 405     """Translates an apitools message to a boto lifecycle object."""
 406     boto_lifecycle = boto.gs.lifecycle.LifecycleConfig()
 407     if lifecycle_message:
 408       for rule_message in lifecycle_message.rule:
 409         boto_rule = boto.gs.lifecycle.Rule()
 410         if (rule_message.action and rule_message.action.type and
 411             rule_message.action.type.lower() == 'delete'):
 412           boto_rule.action = boto.gs.lifecycle.DELETE
 413         if rule_message.condition:
 414           if rule_message.condition.age:
 415             boto_rule.conditions[boto.gs.lifecycle.AGE] = (
 416                 str(rule_message.condition.age))
 417           if rule_message.condition.createdBefore:
 418             boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = (
 419                 str(rule_message.condition.createdBefore))
 420           if rule_message.condition.isLive:
 421             boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = (
 422                 str(rule_message.condition.isLive))
 423           if rule_message.condition.numNewerVersions:
 424             boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = (
 425                 str(rule_message.condition.numNewerVersions))
 426         boto_lifecycle.append(boto_rule)
 427     return boto_lifecycle
 428
 429   @classmethod
 430   def BotoLifecycleToMessage(cls, boto_lifecycle):
 431     """Translates a boto lifecycle object to an apitools message."""
 432     lifecycle_message = None
 433     if boto_lifecycle:
 434       lifecycle_message = apitools_messages.Bucket.LifecycleValue()
 435       for boto_rule in boto_lifecycle:
 436         lifecycle_rule = (
 437             apitools_messages.Bucket.LifecycleValue.RuleValueListEntry())
 438         lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue.
 439                                     RuleValueListEntry.ConditionValue())
 440         if boto_rule.action and boto_rule.action == boto.gs.lifecycle.DELETE:
 441           lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue.
 442                                    RuleValueListEntry.ActionValue(
 443                                        type='Delete'))
 444         if boto.gs.lifecycle.AGE in boto_rule.conditions:
 445           lifecycle_rule.condition.age = int(
 446               boto_rule.conditions[boto.gs.lifecycle.AGE])
 447         if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions:
 448           lifecycle_rule.condition.createdBefore = (
 449               LifecycleTranslation.TranslateBotoLifecycleTimestamp(
 450                   boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE]))
 451         if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions:
 452           lifecycle_rule.condition.isLive = bool(
 453               boto_rule.conditions[boto.gs.lifecycle.IS_LIVE])
 454         if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions:
 455           lifecycle_rule.condition.numNewerVersions = int(
 456               boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS])
 457         lifecycle_message.rule.append(lifecycle_rule)
 458     return lifecycle_message
 459
 460   @classmethod
 461   def JsonLifecycleFromMessage(cls, lifecycle_message):
 462     """Translates an apitools message to lifecycle JSON."""
 463     return str(encoding.MessageToJson(lifecycle_message)) + '\n'
 464
 465   @classmethod
 466   def JsonLifecycleToMessage(cls, json_txt):
 467     """Translates lifecycle JSON to an apitools message."""
 468     try:
 469       deserialized_lifecycle = json.loads(json_txt)
 470       # If lifecycle JSON is the in the following format
 471       # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key
 472       # and reduce it to the following format
 473       # {'rule': ...
 474       if 'lifecycle' in deserialized_lifecycle:
 475         deserialized_lifecycle = deserialized_lifecycle['lifecycle']
 476       lifecycle = encoding.DictToMessage(
 477           deserialized_lifecycle, apitools_messages.Bucket.LifecycleValue)
 478       return lifecycle
 479     except ValueError:
 480       CheckForXmlConfigurationAndRaise('lifecycle', json_txt)
 481
 482   @classmethod
 483   def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime):
 484     """Parses the timestamp from the boto lifecycle into a datetime object."""
 485     return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date()
 486
 487
 488 class CorsTranslation(object):
 489   """Functions for converting between various CORS formats.
 490
 491     This class handles conversation to and from Boto Cors objects, JSON text,
 492     and apitools Message objects.
 493   """
 494
 495   @classmethod
 496   def BotoCorsFromMessage(cls, cors_message):
 497     """Translates an apitools message to a boto Cors object."""
 498     cors = boto.gs.cors.Cors()
 499     cors.cors = []
 500     for collection_message in cors_message:
 501       collection_elements = []
 502       if collection_message.maxAgeSeconds:
 503         collection_elements.append((boto.gs.cors.MAXAGESEC,
 504                                     str(collection_message.maxAgeSeconds)))
 505       if collection_message.method:
 506         method_elements = []
 507         for method in collection_message.method:
 508           method_elements.append((boto.gs.cors.METHOD, method))
 509         collection_elements.append((boto.gs.cors.METHODS, method_elements))
 510       if collection_message.origin:
 511         origin_elements = []
 512         for origin in collection_message.origin:
 513           origin_elements.append((boto.gs.cors.ORIGIN, origin))
 514         collection_elements.append((boto.gs.cors.ORIGINS, origin_elements))
 515       if collection_message.responseHeader:
 516         header_elements = []
 517         for header in collection_message.responseHeader:
 518           header_elements.append((boto.gs.cors.HEADER, header))
 519         collection_elements.append((boto.gs.cors.HEADERS, header_elements))
 520       cors.cors.append(collection_elements)
 521     return cors
 522
 523   @classmethod
 524   def BotoCorsToMessage(cls, boto_cors):
 525     """Translates a boto Cors object to an apitools message."""
 526     message_cors = []
 527     if boto_cors.cors:
 528       for cors_collection in boto_cors.cors:
 529         if cors_collection:
 530           collection_message = apitools_messages.Bucket.CorsValueListEntry()
 531           for element_tuple in cors_collection:
 532             if element_tuple[0] == boto.gs.cors.MAXAGESEC:
 533               collection_message.maxAgeSeconds = int(element_tuple[1])
 534             if element_tuple[0] == boto.gs.cors.METHODS:
 535               for method_tuple in element_tuple[1]:
 536                 collection_message.method.append(method_tuple[1])
 537             if element_tuple[0] == boto.gs.cors.ORIGINS:
 538               for origin_tuple in element_tuple[1]:
 539                 collection_message.origin.append(origin_tuple[1])
 540             if element_tuple[0] == boto.gs.cors.HEADERS:
 541               for header_tuple in element_tuple[1]:
 542                 collection_message.responseHeader.append(header_tuple[1])
 543           message_cors.append(collection_message)
 544     return message_cors
 545
 546   @classmethod
 547   def JsonCorsToMessageEntries(cls, json_cors):
 548     """Translates CORS JSON to an apitools message.
 549
 550     Args:
 551       json_cors: JSON string representing CORS configuration.
 552
 553     Returns:
 554       List of apitools Bucket.CorsValueListEntry. An empty list represents
 555       no CORS configuration.
 556     """
 557     try:
 558       deserialized_cors = json.loads(json_cors)
 559       cors = []
 560       for cors_entry in deserialized_cors:
 561         cors.append(encoding.DictToMessage(
 562             cors_entry, apitools_messages.Bucket.CorsValueListEntry))
 563       return cors
 564     except ValueError:
 565       CheckForXmlConfigurationAndRaise('CORS', json_cors)
 566
 567   @classmethod
 568   def MessageEntriesToJson(cls, cors_message):
 569     """Translates an apitools message to CORS JSON."""
 570     json_text = ''
 571     # Because CORS is a MessageField, serialize/deserialize as JSON list.
 572     json_text += '['
 573     printed_one = False
 574     for cors_entry in cors_message:
 575       if printed_one:
 576         json_text += ','
 577       else:
 578         printed_one = True
 579       json_text += encoding.MessageToJson(cors_entry)
 580     json_text += ']\n'
 581     return json_text
 582
 583
 584 def S3MarkerAclFromObjectMetadata(object_metadata):
 585   """Retrieves GUID-marked S3 ACL from object metadata, if present.
 586
 587   Args:
 588     object_metadata: Object metadata to check.
 589
 590   Returns:
 591     S3 ACL text, if present, None otherwise.
 592   """
 593   if (object_metadata and object_metadata.metadata and
 594       object_metadata.metadata.additionalProperties):
 595     for prop in object_metadata.metadata.additionalProperties:
 596       if prop.key == S3_ACL_MARKER_GUID:
 597         return prop.value
 598
 599
 600 def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text):
 601   """Adds a GUID-marked S3 ACL to the object metadata.
 602
 603   Args:
 604     object_metadata: Object metadata to add the acl to.
 605     acl_text: S3 ACL text to add.
 606   """
 607   if not object_metadata.metadata:
 608     object_metadata.metadata = apitools_messages.Object.MetadataValue()
 609   if not object_metadata.metadata.additionalProperties:
 610     object_metadata.metadata.additionalProperties = []
 611
 612   object_metadata.metadata.additionalProperties.append(
 613       apitools_messages.Object.MetadataValue.AdditionalProperty(
 614           key=S3_ACL_MARKER_GUID, value=acl_text))
 615
 616
 617 class AclTranslation(object):
 618   """Functions for converting between various ACL formats.
 619
 620     This class handles conversion to and from Boto ACL objects, JSON text,
 621     and apitools Message objects.
 622   """
 623
 624   JSON_TO_XML_ROLES = {'READER': 'READ', 'WRITER': 'WRITE',
 625                        'OWNER': 'FULL_CONTROL'}
 626   XML_TO_JSON_ROLES = {'READ': 'READER', 'WRITE': 'WRITER',
 627                        'FULL_CONTROL': 'OWNER'}
 628
 629   @classmethod
 630   def BotoAclFromJson(cls, acl_json):
 631     acl = ACL()
 632     acl.parent = None
 633     acl.entries = cls.BotoEntriesFromJson(acl_json, acl)
 634     return acl
 635
 636   @classmethod
 637   # acl_message is a list of messages, either object or bucketaccesscontrol
 638   def BotoAclFromMessage(cls, acl_message):
 639     acl_dicts = []
 640     for message in acl_message:
 641       acl_dicts.append(encoding.MessageToDict(message))
 642     return cls.BotoAclFromJson(acl_dicts)
 643
 644   @classmethod
 645   def BotoAclToJson(cls, acl):
 646     if hasattr(acl, 'entries'):
 647       return cls.BotoEntriesToJson(acl.entries)
 648     return []
 649
 650   @classmethod
 651   def BotoObjectAclToMessage(cls, acl):
 652     for entry in cls.BotoAclToJson(acl):
 653       message = encoding.DictToMessage(entry,
 654                                        apitools_messages.ObjectAccessControl)
 655       message.kind = u'storage#objectAccessControl'
 656       yield message
 657
 658   @classmethod
 659   def BotoBucketAclToMessage(cls, acl):
 660     for entry in cls.BotoAclToJson(acl):
 661       message = encoding.DictToMessage(entry,
 662                                        apitools_messages.BucketAccessControl)
 663       message.kind = u'storage#bucketAccessControl'
 664       yield message
 665
 666   @classmethod
 667   def BotoEntriesFromJson(cls, acl_json, parent):
 668     entries = Entries(parent)
 669     entries.parent = parent
 670     entries.entry_list = [cls.BotoEntryFromJson(entry_json)
 671                           for entry_json in acl_json]
 672     return entries
 673
 674   @classmethod
 675   def BotoEntriesToJson(cls, entries):
 676     return [cls.BotoEntryToJson(entry) for entry in entries.entry_list]
 677
 678   @classmethod
 679   def BotoEntryFromJson(cls, entry_json):
 680     """Converts a JSON entry into a Boto ACL entry."""
 681     entity = entry_json['entity']
 682     permission = cls.JSON_TO_XML_ROLES[entry_json['role']]
 683     if entity.lower() == ALL_USERS.lower():
 684       return Entry(type=ALL_USERS, permission=permission)
 685     elif entity.lower() == ALL_AUTHENTICATED_USERS.lower():
 686       return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission)
 687     elif entity.startswith('project'):
 688       raise CommandException('XML API does not support project scopes, '
 689                              'cannot translate ACL.')
 690     elif 'email' in entry_json:
 691       if entity.startswith('user'):
 692         scope_type = USER_BY_EMAIL
 693       elif entity.startswith('group'):
 694         scope_type = GROUP_BY_EMAIL
 695       return Entry(type=scope_type, email_address=entry_json['email'],
 696                    permission=permission)
 697     elif 'entityId' in entry_json:
 698       if entity.startswith('user'):
 699         scope_type = USER_BY_ID
 700       elif entity.startswith('group'):
 701         scope_type = GROUP_BY_ID
 702       return Entry(type=scope_type, id=entry_json['entityId'],
 703                    permission=permission)
 704     elif 'domain' in entry_json:
 705       if entity.startswith('domain'):
 706         scope_type = GROUP_BY_DOMAIN
 707       return Entry(type=scope_type, domain=entry_json['domain'],
 708                    permission=permission)
 709     raise CommandException('Failed to translate JSON ACL to XML.')
 710
 711   @classmethod
 712   def BotoEntryToJson(cls, entry):
 713     """Converts a Boto ACL entry to a valid JSON dictionary."""
 714     acl_entry_json = {}
 715     # JSON API documentation uses camel case.
 716     scope_type_lower = entry.scope.type.lower()
 717     if scope_type_lower == ALL_USERS.lower():
 718       acl_entry_json['entity'] = 'allUsers'
 719     elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower():
 720       acl_entry_json['entity'] = 'allAuthenticatedUsers'
 721     elif scope_type_lower == USER_BY_EMAIL.lower():
 722       acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address
 723       acl_entry_json['email'] = entry.scope.email_address
 724     elif scope_type_lower == USER_BY_ID.lower():
 725       acl_entry_json['entity'] = 'user-%s' % entry.scope.id
 726       acl_entry_json['entityId'] = entry.scope.id
 727     elif scope_type_lower == GROUP_BY_EMAIL.lower():
 728       acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address
 729       acl_entry_json['email'] = entry.scope.email_address
 730     elif scope_type_lower == GROUP_BY_ID.lower():
 731       acl_entry_json['entity'] = 'group-%s' % entry.scope.id
 732       acl_entry_json['entityId'] = entry.scope.id
 733     elif scope_type_lower == GROUP_BY_DOMAIN.lower():
 734       acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain
 735       acl_entry_json['domain'] = entry.scope.domain
 736     else:
 737       raise ArgumentException('ACL contains invalid scope type: %s' %
 738                               scope_type_lower)
 739
 740     acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission]
 741     return acl_entry_json
 742
 743   @classmethod
 744   def JsonToMessage(cls, json_data, message_type):
 745     """Converts the input JSON data into list of Object/BucketAccessControls.
 746
 747     Args:
 748       json_data: String of JSON to convert.
 749       message_type: Which type of access control entries to return,
 750                     either ObjectAccessControl or BucketAccessControl.
 751
 752     Raises:
 753       ArgumentException on invalid JSON data.
 754
 755     Returns:
 756       List of ObjectAccessControl or BucketAccessControl elements.
 757     """
 758     try:
 759       deserialized_acl = json.loads(json_data)
 760
 761       acl = []
 762       for acl_entry in deserialized_acl:
 763         acl.append(encoding.DictToMessage(acl_entry, message_type))
 764       return acl
 765     except ValueError:
 766       CheckForXmlConfigurationAndRaise('ACL', json_data)
 767
 768   @classmethod
 769   def JsonFromMessage(cls, acl):
 770     """Strips unnecessary fields from an ACL message and returns valid JSON.
 771
 772     Args:
 773       acl: iterable ObjectAccessControl or BucketAccessControl
 774
 775     Returns:
 776       ACL JSON string.
 777     """
 778     serializable_acl = []
 779     if acl is not None:
 780       for acl_entry in acl:
 781         if acl_entry.kind == u'storage#objectAccessControl':
 782           acl_entry.object = None
 783           acl_entry.generation = None
 784         acl_entry.kind = None
 785         acl_entry.bucket = None
 786         acl_entry.id = None
 787         acl_entry.selfLink = None
 788         acl_entry.etag = None
 789         serializable_acl.append(encoding.MessageToDict(acl_entry))
 790     return json.dumps(serializable_acl, sort_keys=True,
 791                       indent=2, separators=(',', ': '))