src/applications/files/uploadsource/PhabricatorFileUploadSource.php

   1 <?php
   2
   3 abstract class PhabricatorFileUploadSource
   4   extends Phobject {
   5
   6   private $name;
   7   private $relativeTTL;
   8   private $viewPolicy;
   9   private $mimeType;
  10   private $authorPHID;
  11
  12   private $rope;
  13   private $data;
  14   private $shouldChunk;
  15   private $didRewind;
  16   private $totalBytesWritten = 0;
  17   private $totalBytesRead = 0;
  18   private $byteLimit = 0;
  19
  20   public function setName($name) {
  21     $this->name = $name;
  22     return $this;
  23   }
  24
  25   public function getName() {
  26     return $this->name;
  27   }
  28
  29   public function setRelativeTTL($relative_ttl) {
  30     $this->relativeTTL = $relative_ttl;
  31     return $this;
  32   }
  33
  34   public function getRelativeTTL() {
  35     return $this->relativeTTL;
  36   }
  37
  38   public function setViewPolicy($view_policy) {
  39     $this->viewPolicy = $view_policy;
  40     return $this;
  41   }
  42
  43   public function getViewPolicy() {
  44     return $this->viewPolicy;
  45   }
  46
  47   public function setByteLimit($byte_limit) {
  48     $this->byteLimit = $byte_limit;
  49     return $this;
  50   }
  51
  52   public function getByteLimit() {
  53     return $this->byteLimit;
  54   }
  55
  56   public function setMIMEType($mime_type) {
  57     $this->mimeType = $mime_type;
  58     return $this;
  59   }
  60
  61   public function getMIMEType() {
  62     return $this->mimeType;
  63   }
  64
  65   public function setAuthorPHID($author_phid) {
  66     $this->authorPHID = $author_phid;
  67     return $this;
  68   }
  69
  70   public function getAuthorPHID() {
  71     return $this->authorPHID;
  72   }
  73
  74   public function uploadFile() {
  75     if (!$this->shouldChunkFile()) {
  76       return $this->writeSingleFile();
  77     } else {
  78       return $this->writeChunkedFile();
  79     }
  80   }
  81
  82   private function getDataIterator() {
  83     if (!$this->data) {
  84       $this->data = $this->newDataIterator();
  85     }
  86     return $this->data;
  87   }
  88
  89   private function getRope() {
  90     if (!$this->rope) {
  91       $this->rope = new PhutilRope();
  92     }
  93     return $this->rope;
  94   }
  95
  96   abstract protected function newDataIterator();
  97   abstract protected function getDataLength();
  98
  99   private function readFileData() {
 100     $data = $this->getDataIterator();
 101
 102     if (!$this->didRewind) {
 103       $data->rewind();
 104       $this->didRewind = true;
 105     } else {
 106       if ($data->valid()) {
 107         $data->next();
 108       }
 109     }
 110
 111     if (!$data->valid()) {
 112       return false;
 113     }
 114
 115     $read_bytes = $data->current();
 116     $this->totalBytesRead += strlen($read_bytes);
 117
 118     if ($this->byteLimit && ($this->totalBytesRead > $this->byteLimit)) {
 119       throw new PhabricatorFileUploadSourceByteLimitException();
 120     }
 121
 122     $rope = $this->getRope();
 123     $rope->append($read_bytes);
 124
 125     return true;
 126   }
 127
 128   private function shouldChunkFile() {
 129     if ($this->shouldChunk !== null) {
 130       return $this->shouldChunk;
 131     }
 132
 133     $threshold = PhabricatorFileStorageEngine::getChunkThreshold();
 134
 135     if ($threshold === null) {
 136       // If there are no chunk engines available, we clearly can't chunk the
 137       // file.
 138       $this->shouldChunk = false;
 139     } else {
 140       // If we don't know how large the file is, we're going to read some data
 141       // from it until we know whether it's a small file or not. This will give
 142       // us enough information to make a decision about chunking.
 143       $length = $this->getDataLength();
 144       if ($length === null) {
 145         $rope = $this->getRope();
 146         while ($this->readFileData()) {
 147           $length = $rope->getByteLength();
 148           if ($length > $threshold) {
 149             break;
 150           }
 151         }
 152       }
 153
 154       $this->shouldChunk = ($length > $threshold);
 155     }
 156
 157     return $this->shouldChunk;
 158   }
 159
 160   private function writeSingleFile() {
 161     while ($this->readFileData()) {
 162       // Read the entire file.
 163     }
 164
 165     $rope = $this->getRope();
 166     $data = $rope->getAsString();
 167
 168     $parameters = $this->getNewFileParameters();
 169
 170     return PhabricatorFile::newFromFileData($data, $parameters);
 171   }
 172
 173   private function writeChunkedFile() {
 174     $engine = $this->getChunkEngine();
 175
 176     $parameters = $this->getNewFileParameters();
 177
 178     $data_length = $this->getDataLength();
 179     if ($data_length !== null) {
 180       $length = $data_length;
 181     } else {
 182       $length = 0;
 183     }
 184
 185     $file = PhabricatorFile::newChunkedFile($engine, $length, $parameters);
 186     $file->saveAndIndex();
 187
 188     $rope = $this->getRope();
 189
 190     // Read the source, writing chunks as we get enough data.
 191     while ($this->readFileData()) {
 192       while (true) {
 193         $rope_length = $rope->getByteLength();
 194         if ($rope_length < $engine->getChunkSize()) {
 195           break;
 196         }
 197         $this->writeChunk($file, $engine);
 198       }
 199     }
 200
 201     // If we have extra bytes at the end, write them. Note that it's possible
 202     // that we have more than one chunk of bytes left if the read was very
 203     // fast.
 204     while ($rope->getByteLength()) {
 205       $this->writeChunk($file, $engine);
 206     }
 207
 208     $file->setIsPartial(0);
 209     if ($data_length === null) {
 210       $file->setByteSize($this->getTotalBytesWritten());
 211     }
 212     $file->save();
 213
 214     return $file;
 215   }
 216
 217   private function writeChunk(
 218     PhabricatorFile $file,
 219     PhabricatorFileStorageEngine $engine) {
 220
 221     $offset = $this->getTotalBytesWritten();
 222     $max_length = $engine->getChunkSize();
 223     $rope = $this->getRope();
 224
 225     $data = $rope->getPrefixBytes($max_length);
 226     $actual_length = strlen($data);
 227     $rope->removeBytesFromHead($actual_length);
 228
 229     $params = array(
 230       'name' => $file->getMonogram().'.chunk-'.$offset,
 231       'viewPolicy' => PhabricatorPolicies::POLICY_NOONE,
 232       'chunk' => true,
 233     );
 234
 235     // If this isn't the initial chunk, provide a dummy MIME type so we do not
 236     // try to detect it. See T12857.
 237     if ($offset > 0) {
 238       $params['mime-type'] = 'application/octet-stream';
 239     }
 240
 241     $chunk_data = PhabricatorFile::newFromFileData($data, $params);
 242
 243     $chunk = PhabricatorFileChunk::initializeNewChunk(
 244       $file->getStorageHandle(),
 245       $offset,
 246       $offset + $actual_length);
 247
 248     $chunk
 249       ->setDataFilePHID($chunk_data->getPHID())
 250       ->save();
 251
 252     $this->setTotalBytesWritten($offset + $actual_length);
 253
 254     return $chunk;
 255   }
 256
 257   private function getNewFileParameters() {
 258     $parameters = array(
 259       'name' => $this->getName(),
 260       'viewPolicy' => $this->getViewPolicy(),
 261     );
 262
 263     $ttl = $this->getRelativeTTL();
 264     if ($ttl !== null) {
 265       $parameters['ttl.relative'] = $ttl;
 266     }
 267
 268     $mime_type = $this->getMimeType();
 269     if ($mime_type !== null) {
 270       $parameters['mime-type'] = $mime_type;
 271     }
 272
 273     $author_phid = $this->getAuthorPHID();
 274     if ($author_phid !== null) {
 275       $parameters['authorPHID'] = $author_phid;
 276     }
 277
 278     return $parameters;
 279   }
 280
 281   private function getChunkEngine() {
 282     $chunk_engines = PhabricatorFileStorageEngine::loadWritableChunkEngines();
 283     if (!$chunk_engines) {
 284       throw new Exception(
 285         pht(
 286           'Unable to upload file: this server is not configured with any '.
 287           'storage engine which can store large files.'));
 288     }
 289
 290     return head($chunk_engines);
 291   }
 292
 293   private function setTotalBytesWritten($total_bytes_written) {
 294     $this->totalBytesWritten = $total_bytes_written;
 295     return $this;
 296   }
 297
 298   private function getTotalBytesWritten() {
 299     return $this->totalBytesWritten;
 300   }
 301
 302 }