src/applications/files/engine/PhabricatorFileStorageEngine.php

   1 <?php
   2
   3 /**
   4  * Defines a storage engine which can write file data somewhere (like a
   5  * database, local disk, Amazon S3, the A:\ drive, or a custom filer) and
   6  * retrieve it later.
   7  *
   8  * You can extend this class to provide new file storage backends.
   9  *
  10  * For more information, see @{article:File Storage Technical Documentation}.
  11  *
  12  * @task construct Constructing an Engine
  13  * @task meta Engine Metadata
  14  * @task file Managing File Data
  15  * @task load Loading Storage Engines
  16  */
  17 abstract class PhabricatorFileStorageEngine extends Phobject {
  18
  19   const HMAC_INTEGRITY = 'file.integrity';
  20
  21   /**
  22    * Construct a new storage engine.
  23    *
  24    * @task construct
  25    */
  26   final public function __construct() {
  27     // <empty>
  28   }
  29
  30
  31 /* -(  Engine Metadata  )---------------------------------------------------- */
  32
  33
  34   /**
  35    * Return a unique, nonempty string which identifies this storage engine.
  36    * This is used to look up the storage engine when files needs to be read or
  37    * deleted. For instance, if you store files by giving them to a duck for
  38    * safe keeping in his nest down by the pond, you might return 'duck' from
  39    * this method.
  40    *
  41    * @return string Unique string for this engine, max length 32.
  42    * @task meta
  43    */
  44   abstract public function getEngineIdentifier();
  45
  46
  47   /**
  48    * Prioritize this engine relative to other engines.
  49    *
  50    * Engines with a smaller priority number get an opportunity to write files
  51    * first. Generally, lower-latency filestores should have lower priority
  52    * numbers, and higher-latency filestores should have higher priority
  53    * numbers. Setting priority to approximately the number of milliseconds of
  54    * read latency will generally produce reasonable results.
  55    *
  56    * In conjunction with filesize limits, the goal is to store small files like
  57    * profile images, thumbnails, and text snippets in lower-latency engines,
  58    * and store large files in higher-capacity engines.
  59    *
  60    * @return float Engine priority.
  61    * @task meta
  62    */
  63   abstract public function getEnginePriority();
  64
  65
  66   /**
  67    * Return `true` if the engine is currently writable.
  68    *
  69    * Engines that are disabled or missing configuration should return `false`
  70    * to prevent new writes. If writes were made with this engine in the past,
  71    * the application may still try to perform reads.
  72    *
  73    * @return bool True if this engine can support new writes.
  74    * @task meta
  75    */
  76   abstract public function canWriteFiles();
  77
  78
  79   /**
  80    * Return `true` if the engine has a filesize limit on storable files.
  81    *
  82    * The @{method:getFilesizeLimit} method can retrieve the actual limit. This
  83    * method just removes the ambiguity around the meaning of a `0` limit.
  84    *
  85    * @return bool `true` if the engine has a filesize limit.
  86    * @task meta
  87    */
  88   public function hasFilesizeLimit() {
  89     return true;
  90   }
  91
  92
  93   /**
  94    * Return maximum storable file size, in bytes.
  95    *
  96    * Not all engines have a limit; use @{method:getFilesizeLimit} to check if
  97    * an engine has a limit. Engines without a limit can store files of any
  98    * size.
  99    *
 100    * By default, engines define a limit which supports chunked storage of
 101    * large files. In most cases, you should not change this limit, even if an
 102    * engine has vast storage capacity: chunked storage makes large files more
 103    * manageable and enables features like resumable uploads.
 104    *
 105    * @return int Maximum storable file size, in bytes.
 106    * @task meta
 107    */
 108   public function getFilesizeLimit() {
 109     // NOTE: This 8MB limit is selected to be larger than the 4MB chunk size,
 110     // but not much larger. Files between 0MB and 8MB will be stored normally;
 111     // files larger than 8MB will be chunked.
 112     return (1024 * 1024 * 8);
 113   }
 114
 115
 116   /**
 117    * Identifies storage engines that support unit tests.
 118    *
 119    * These engines are not used for production writes.
 120    *
 121    * @return bool True if this is a test engine.
 122    * @task meta
 123    */
 124   public function isTestEngine() {
 125     return false;
 126   }
 127
 128
 129   /**
 130    * Identifies chunking storage engines.
 131    *
 132    * If this is a storage engine which splits files into chunks and stores the
 133    * chunks in other engines, it can return `true` to signal that other
 134    * chunking engines should not try to store data here.
 135    *
 136    * @return bool True if this is a chunk engine.
 137    * @task meta
 138    */
 139   public function isChunkEngine() {
 140     return false;
 141   }
 142
 143
 144 /* -(  Managing File Data  )------------------------------------------------- */
 145
 146
 147   /**
 148    * Write file data to the backing storage and return a handle which can later
 149    * be used to read or delete it. For example, if the backing storage is local
 150    * disk, the handle could be the path to the file.
 151    *
 152    * The caller will provide a $params array, which may be empty or may have
 153    * some metadata keys (like "name" and "author") in it. You should be prepared
 154    * to handle writes which specify no metadata, but might want to optionally
 155    * use some keys in this array for debugging or logging purposes. This is
 156    * the same dictionary passed to @{method:PhabricatorFile::newFromFileData},
 157    * so you could conceivably do custom things with it.
 158    *
 159    * If you are unable to write for whatever reason (e.g., the disk is full),
 160    * throw an exception. If there are other satisfactory but less-preferred
 161    * storage engines available, they will be tried.
 162    *
 163    * @param  string The file data to write.
 164    * @param  array  File metadata (name, author), if available.
 165    * @return string Unique string which identifies the stored file, max length
 166    *                255.
 167    * @task file
 168    */
 169   abstract public function writeFile($data, array $params);
 170
 171
 172   /**
 173    * Read the contents of a file previously written by @{method:writeFile}.
 174    *
 175    * @param   string  The handle returned from @{method:writeFile} when the
 176    *                  file was written.
 177    * @return  string  File contents.
 178    * @task file
 179    */
 180   abstract public function readFile($handle);
 181
 182
 183   /**
 184    * Delete the data for a file previously written by @{method:writeFile}.
 185    *
 186    * @param   string  The handle returned from @{method:writeFile} when the
 187    *                  file was written.
 188    * @return  void
 189    * @task file
 190    */
 191   abstract public function deleteFile($handle);
 192
 193
 194
 195 /* -(  Loading Storage Engines  )-------------------------------------------- */
 196
 197
 198   /**
 199    * Select viable default storage engines according to configuration. We'll
 200    * select the MySQL and Local Disk storage engines if they are configured
 201    * to allow a given file.
 202    *
 203    * @param int File size in bytes.
 204    * @task load
 205    */
 206   public static function loadStorageEngines($length) {
 207     $engines = self::loadWritableEngines();
 208
 209     $writable = array();
 210     foreach ($engines as $key => $engine) {
 211       if ($engine->hasFilesizeLimit()) {
 212         $limit = $engine->getFilesizeLimit();
 213         if ($limit < $length) {
 214           continue;
 215         }
 216       }
 217
 218       $writable[$key] = $engine;
 219     }
 220
 221     return $writable;
 222   }
 223
 224
 225   /**
 226    * @task load
 227    */
 228   public static function loadAllEngines() {
 229     return id(new PhutilClassMapQuery())
 230       ->setAncestorClass(__CLASS__)
 231       ->setUniqueMethod('getEngineIdentifier')
 232       ->setSortMethod('getEnginePriority')
 233       ->execute();
 234   }
 235
 236
 237   /**
 238    * @task load
 239    */
 240   private static function loadProductionEngines() {
 241     $engines = self::loadAllEngines();
 242
 243     $active = array();
 244     foreach ($engines as $key => $engine) {
 245       if ($engine->isTestEngine()) {
 246         continue;
 247       }
 248
 249       $active[$key] = $engine;
 250     }
 251
 252     return $active;
 253   }
 254
 255
 256   /**
 257    * @task load
 258    */
 259   public static function loadWritableEngines() {
 260     $engines = self::loadProductionEngines();
 261
 262     $writable = array();
 263     foreach ($engines as $key => $engine) {
 264       if (!$engine->canWriteFiles()) {
 265         continue;
 266       }
 267
 268       if ($engine->isChunkEngine()) {
 269         // Don't select chunk engines as writable.
 270         continue;
 271       }
 272       $writable[$key] = $engine;
 273     }
 274
 275     return $writable;
 276   }
 277
 278   /**
 279    * @task load
 280    */
 281   public static function loadWritableChunkEngines() {
 282     $engines = self::loadProductionEngines();
 283
 284     $chunk = array();
 285     foreach ($engines as $key => $engine) {
 286       if (!$engine->canWriteFiles()) {
 287         continue;
 288       }
 289       if (!$engine->isChunkEngine()) {
 290         continue;
 291       }
 292       $chunk[$key] = $engine;
 293     }
 294
 295     return $chunk;
 296   }
 297
 298
 299
 300   /**
 301    * Return the largest file size which can not be uploaded in chunks.
 302    *
 303    * Files smaller than this will always upload in one request, so clients
 304    * can safely skip the allocation step.
 305    *
 306    * @return int|null Byte size, or `null` if there is no chunk support.
 307    */
 308   public static function getChunkThreshold() {
 309     $engines = self::loadWritableChunkEngines();
 310
 311     $min = null;
 312     foreach ($engines as $engine) {
 313       if (!$min) {
 314         $min = $engine;
 315         continue;
 316       }
 317
 318       if ($min->getChunkSize() > $engine->getChunkSize()) {
 319         $min = $engine->getChunkSize();
 320       }
 321     }
 322
 323     if (!$min) {
 324       return null;
 325     }
 326
 327     return $engine->getChunkSize();
 328   }
 329
 330   public function getRawFileDataIterator(
 331     PhabricatorFile $file,
 332     $begin,
 333     $end,
 334     PhabricatorFileStorageFormat $format) {
 335
 336     $formatted_data = $this->readFile($file->getStorageHandle());
 337
 338     $known_integrity = $file->getIntegrityHash();
 339     if ($known_integrity !== null) {
 340       $new_integrity = $this->newIntegrityHash($formatted_data, $format);
 341       if (!phutil_hashes_are_identical($known_integrity, $new_integrity)) {
 342         throw new PhabricatorFileIntegrityException(
 343           pht(
 344             'File data integrity check failed. Dark forces have corrupted '.
 345             'or tampered with this file. The file data can not be read.'));
 346       }
 347     }
 348
 349     $formatted_data = array($formatted_data);
 350
 351     $data = '';
 352     $format_iterator = $format->newReadIterator($formatted_data);
 353     foreach ($format_iterator as $raw_chunk) {
 354       $data .= $raw_chunk;
 355     }
 356
 357     if ($begin !== null && $end !== null) {
 358       $data = substr($data, $begin, ($end - $begin));
 359     } else if ($begin !== null) {
 360       $data = substr($data, $begin);
 361     } else if ($end !== null) {
 362       $data = substr($data, 0, $end);
 363     }
 364
 365     return array($data);
 366   }
 367
 368   public function newIntegrityHash(
 369     $data,
 370     PhabricatorFileStorageFormat $format) {
 371
 372     $hmac_name = self::HMAC_INTEGRITY;
 373
 374     $data_hash = PhabricatorHash::digestWithNamedKey($data, $hmac_name);
 375     $format_hash = $format->newFormatIntegrityHash();
 376
 377     $full_hash = "{$data_hash}/{$format_hash}";
 378
 379     return PhabricatorHash::digestWithNamedKey($full_hash, $hmac_name);
 380   }
 381
 382 }