mediagoblin/storage/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 from __future__ import absolute_import
  18
  19 import shutil
  20 import uuid
  21
  22 import six
  23
  24 from werkzeug.utils import secure_filename
  25
  26 from mediagoblin.tools import common
  27
  28 ########
  29 # Errors
  30 ########
  31
  32
  33 class Error(Exception):
  34     pass
  35
  36
  37 class InvalidFilepath(Error):
  38     pass
  39
  40
  41 class NoWebServing(Error):
  42     pass
  43
  44
  45 class NotImplementedError(Error):
  46     pass
  47
  48
  49 ###############################################
  50 # Storage interface & basic file implementation
  51 ###############################################
  52
  53 class StorageInterface(object):
  54     """
  55     Interface for the storage API.
  56
  57     This interface doesn't actually provide behavior, but it defines
  58     what kind of storage patterns subclasses should provide.
  59
  60     It is important to note that the storage API idea of a "filepath"
  61     is actually like ['dir1', 'dir2', 'file.jpg'], so keep that in
  62     mind while reading method documentation.
  63
  64     You should set up your __init__ method with whatever keyword
  65     arguments are appropriate to your storage system, but you should
  66     also passively accept all extraneous keyword arguments like:
  67
  68       def __init__(self, **kwargs):
  69           pass
  70
  71     See BasicFileStorage as a simple implementation of the
  72     StorageInterface.
  73     """
  74
  75     # Whether this file store is on the local filesystem.
  76     local_storage = False
  77
  78     def __raise_not_implemented(self):
  79         """
  80         Raise a warning about some component not implemented by a
  81         subclass of this interface.
  82         """
  83         raise NotImplementedError(
  84             "This feature not implemented in this storage API implementation.")
  85
  86     def file_exists(self, filepath):
  87         """
  88         Return a boolean asserting whether or not file at filepath
  89         exists in our storage system.
  90
  91         Returns:
  92          True / False depending on whether file exists or not.
  93         """
  94         # Subclasses should override this method.
  95         self.__raise_not_implemented()
  96
  97     def get_file(self, filepath, mode='r'):
  98         """
  99         Return a file-like object for reading/writing from this filepath.
 100
 101         Should create directories, buckets, whatever, as necessary.
 102         """
 103         # Subclasses should override this method.
 104         self.__raise_not_implemented()
 105
 106     def delete_file(self, filepath):
 107         """
 108         Delete or dereference the file (not directory) at filepath.
 109         """
 110         # Subclasses should override this method.
 111         self.__raise_not_implemented()
 112
 113     def delete_dir(self, dirpath, recursive=False):
 114         """Delete the directory at dirpath
 115
 116         :param recursive: Usually, a directory must not contain any
 117             files for the delete to succeed. If True, containing files
 118             and subdirectories within dirpath will be recursively
 119             deleted.
 120
 121         :returns: True in case of success, False otherwise.
 122         """
 123         # Subclasses should override this method.
 124         self.__raise_not_implemented()
 125
 126     def file_url(self, filepath):
 127         """
 128         Get the URL for this file.  This assumes our storage has been
 129         mounted with some kind of URL which makes this possible.
 130         """
 131         # Subclasses should override this method.
 132         self.__raise_not_implemented()
 133
 134     def get_unique_filepath(self, filepath):
 135         """
 136         If a filename at filepath already exists, generate a new name.
 137
 138         Eg, if the filename doesn't exist:
 139         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 140         [u'dir1', u'dir2', u'fname.jpg']
 141
 142         But if a file does exist, let's get one back with at uuid tacked on:
 143         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
 144         [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
 145         """
 146         # Make sure we have a clean filepath to start with, since
 147         # we'll be possibly tacking on stuff to the filename.
 148         filepath = clean_listy_filepath(filepath)
 149
 150         if self.file_exists(filepath):
 151             return filepath[:-1] + ["%s-%s" % (uuid.uuid4(), filepath[-1])]
 152         else:
 153             return filepath
 154
 155     def get_local_path(self, filepath):
 156         """
 157         If this is a local_storage implementation, give us a link to
 158         the local filesystem reference to this file.
 159
 160         >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
 161         u'/path/to/mounting/foo/bar/baz.jpg'
 162         """
 163         # Subclasses should override this method, if applicable.
 164         self.__raise_not_implemented()
 165
 166     def copy_locally(self, filepath, dest_path):
 167         """
 168         Copy this file locally.
 169
 170         A basic working method for this is provided that should
 171         function both for local_storage systems and remote storge
 172         systems, but if more efficient systems for copying locally
 173         apply to your system, override this method with something more
 174         appropriate.
 175         """
 176         if self.local_storage:
 177             # Note: this will copy in small chunks
 178             shutil.copy(self.get_local_path(filepath), dest_path)
 179         else:
 180             with self.get_file(filepath, 'rb') as source_file:
 181                 with open(dest_path, 'wb') as dest_file:
 182                     # Copy from remote storage in 4M chunks
 183                     shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 184
 185     def copy_local_to_storage(self, filename, filepath):
 186         """
 187         Copy this file from locally to the storage system.
 188
 189         This is kind of the opposite of copy_locally.  It's likely you
 190         could override this method with something more appropriate to
 191         your storage system.
 192         """
 193         with self.get_file(filepath, 'wb') as dest_file:
 194             with open(filename, 'rb') as source_file:
 195                 # Copy to storage system in 4M chunks
 196                 shutil.copyfileobj(source_file, dest_file, length=4*1048576)
 197
 198     def get_file_size(self, filepath):
 199         """
 200         Return the size of the file in bytes.
 201         """
 202         # Subclasses should override this method.
 203         self.__raise_not_implemented()
 204
 205
 206 ###########
 207 # Utilities
 208 ###########
 209
 210 def clean_listy_filepath(listy_filepath):
 211     """
 212     Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
 213     clean out any nastiness from it.
 214
 215
 216     >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
 217     [u'dir1', u'foo_.._nasty', u'linooks.jpg']
 218
 219     Args:
 220     - listy_filepath: a list of filepath components, mediagoblin
 221       storage API style.
 222
 223     Returns:
 224       A cleaned list of unicode objects.
 225     """
 226     cleaned_filepath = [
 227         six.text_type(secure_filename(filepath))
 228         for filepath in listy_filepath]
 229
 230     if u'' in cleaned_filepath:
 231         raise InvalidFilepath(
 232             "A filename component could not be resolved into a usable name.")
 233
 234     return cleaned_filepath
 235
 236
 237 def storage_system_from_config(config_section):
 238     """
 239     Utility for setting up a storage system from a config section.
 240
 241     Note that a special argument may be passed in to
 242     the config_section which is "storage_class" which will provide an
 243     import path to a storage system.  This defaults to
 244     "mediagoblin.storage:BasicFileStorage" if otherwise undefined.
 245
 246     Arguments:
 247      - config_section: dictionary of config parameters
 248
 249     Returns:
 250       An instantiated storage system.
 251
 252     Example:
 253       storage_system_from_config(
 254         {'base_url': '/media/',
 255          'base_dir': '/var/whatever/media/'})
 256
 257        Will return:
 258          BasicFileStorage(
 259            base_url='/media/',
 260            base_dir='/var/whatever/media')
 261     """
 262     # This construct is needed, because dict(config) does
 263     # not replace the variables in the config items.
 264     config_params = dict(six.iteritems(config_section))
 265
 266     if 'storage_class' in config_params:
 267         storage_class = config_params['storage_class']
 268         config_params.pop('storage_class')
 269     else:
 270         storage_class = 'mediagoblin.storage.filestorage:BasicFileStorage'
 271
 272     storage_class = common.import_component(storage_class)
 273     return storage_class(**config_params)
 274
 275 from . import filestorage