tools/telemetry/third_party/gsutilz/gslib/addlhelp/crc32c.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2013 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Additional help about CRC32C and installing crcmod."""
  16
  17 from __future__ import absolute_import
  18
  19 from gslib.help_provider import HelpProvider
  20
  21 _DETAILED_HELP_TEXT = ("""
  22 <B>OVERVIEW</B>
  23   Google Cloud Storage provides a cyclic redundancy check (CRC) header that
  24   allows clients to verify the integrity of object contents. For non-composite
  25   objects GCS also provides an MD5 header to allow clients to verify object
  26   integrity, but for composite objects only the CRC is available. gsutil
  27   automatically performs integrity checks on all uploads and downloads.
  28   Additionally, you can use the "gsutil hash" command to calculate a CRC for
  29   any local file.
  30
  31   The CRC variant used by Google Cloud Storage is called CRC32C (Castagnoli),
  32   which is not available in the standard Python distribution. The implementation
  33   of CRC32C used by gsutil is provided by a third-party Python module called
  34   `crcmod <https://pypi.python.org/pypi/crcmod>`_.
  35
  36   The crcmod module contains a pure-Python implementation of CRC32C, but using
  37   it results in very poor performance. A Python C extension is also provided by
  38   crcmod, which requires compiling into a binary module for use. gsutil ships
  39   with a precompiled crcmod C extension for Mac OS X; for other platforms, see
  40   the installation instructions below.
  41
  42   At the end of each copy operation, the gsutil cp and rsync commands validate
  43   that the checksum of the source file/object matches the checksum of the
  44   destination file/object. If the checksums do not match, gsutil will delete
  45   the invalid copy and print a warning message. This very rarely happens, but
  46   if it does, please contact gs-team@google.com.
  47
  48
  49 <B>CONFIGURATION</B>
  50   To determine if the compiled version of crcmod is available in your Python
  51   environment, you can inspect the output of the gsutil version command for the
  52   "compiled crcmod" entry::
  53
  54     $ gsutil version -l
  55     ...
  56     compiled crcmod: True
  57     ...
  58
  59   If your crcmod library is compiled to a native binary, this value will be
  60   True. If using the pure-Python version, the value will be False.
  61
  62   To control gsutil's behavior in response to crcmod's status, you can set the
  63   "check_hashes" configuration variable. For details on this variable, see the
  64   surrounding comments in your gsutil configuration file. If check_hashes is not
  65   present in your configuration file, rerun gsutil config to regenerate the
  66   file.
  67
  68
  69 <B>INSTALLATION</B>
  70   CentOS, RHEL, and Fedora
  71   ------------------------
  72
  73   To compile and install crcmod:
  74
  75     sudo yum install gcc python-devel python-setuptools
  76     sudo easy_install -U pip
  77     sudo pip uninstall crcmod
  78     sudo pip install -U crcmod
  79
  80   Debian and Ubuntu
  81   -----------------
  82
  83   To compile and install crcmod:
  84
  85     sudo apt-get install gcc python-dev python-setuptools
  86     sudo easy_install -U pip
  87     sudo pip uninstall crcmod
  88     sudo pip install -U crcmod
  89
  90   Mac OS X
  91   --------
  92
  93   gsutil distributes a pre-compiled version of crcmod for OS X, so you shouldn't
  94   need to compile and install it yourself. If for some reason the pre-compiled
  95   version is not being detected, please let the Google Cloud Storage team know
  96   (see "gsutil help support").
  97
  98   To compile manually on OS X, you will first need to install
  99   `XCode <https://developer.apple.com/xcode/>`_ and then run:
 100
 101     sudo easy_install -U pip
 102     sudo pip install -U crcmod
 103
 104   Windows
 105   -------
 106
 107   An installer is available for the compiled version of crcmod from the Python
 108   Package Index (PyPi) at the following URL:
 109
 110   https://pypi.python.org/pypi/crcmod/1.7
 111
 112   MSI installers are available for the 32-bit versions of Python 2.6 and 2.7.
 113
 114 """)
 115
 116
 117 class CommandOptions(HelpProvider):
 118   """Additional help about CRC32C and installing crcmod."""
 119
 120   # Help specification. See help_provider.py for documentation.
 121   help_spec = HelpProvider.HelpSpec(
 122       help_name='crc32c',
 123       help_name_aliases=['crc32', 'crc', 'crcmod'],
 124       help_type='additional_help',
 125       help_one_line_summary='CRC32C and Installing crcmod',
 126       help_text=_DETAILED_HELP_TEXT,
 127       subcommand_help_text={},
 128   )