src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php

   1 <?php
   2
   3 final class PhabricatorFilesManagementMigrateWorkflow
   4   extends PhabricatorFilesManagementWorkflow {
   5
   6   protected function didConstruct() {
   7     $arguments = $this->newIteratorArguments();
   8
   9     $arguments[] = array(
  10       'name' => 'engine',
  11       'param' => 'storage-engine',
  12       'help' => pht('Migrate to the named storage engine.'),
  13     );
  14
  15     $arguments[] = array(
  16       'name' => 'dry-run',
  17       'help' => pht('Show what would be migrated.'),
  18     );
  19
  20     $arguments[] = array(
  21       'name' => 'min-size',
  22       'param' => 'bytes',
  23       'help' => pht(
  24         'Do not migrate data for files which are smaller than a given '.
  25         'filesize.'),
  26     );
  27
  28     $arguments[] = array(
  29       'name' => 'max-size',
  30       'param' => 'bytes',
  31       'help' => pht(
  32         'Do not migrate data for files which are larger than a given '.
  33         'filesize.'),
  34     );
  35
  36     $arguments[] = array(
  37       'name' => 'copy',
  38       'help' => pht(
  39         'Copy file data instead of moving it: after migrating, do not '.
  40         'remove the old data even if it is no longer referenced.'),
  41     );
  42
  43     $arguments[] = array(
  44       'name' => 'local-disk-source',
  45       'param' => 'path',
  46       'help' => pht(
  47         'When migrating from a local disk source, use the specified '.
  48         'path as the root directory.'),
  49     );
  50
  51     $this
  52       ->setName('migrate')
  53       ->setSynopsis(pht('Migrate files between storage engines.'))
  54       ->setArguments($arguments);
  55   }
  56
  57   public function execute(PhutilArgumentParser $args) {
  58
  59     // See T13306. This flag allows you to import files from a backup of
  60     // local disk storage into some other engine. When the caller provides
  61     // the flag, we override the local disk engine configuration and treat
  62     // it as though it is configured to use the specified location.
  63
  64     $local_disk_source = $args->getArg('local-disk-source');
  65     if (strlen($local_disk_source)) {
  66       $path = Filesystem::resolvePath($local_disk_source);
  67       try {
  68         Filesystem::assertIsDirectory($path);
  69       } catch (FilesystemException $ex) {
  70         throw new PhutilArgumentUsageException(
  71           pht(
  72             'The "--local-disk-source" argument must point to a valid, '.
  73             'readable directory on local disk.'));
  74       }
  75
  76       $env = PhabricatorEnv::beginScopedEnv();
  77       $env->overrideEnvConfig('storage.local-disk.path', $path);
  78     }
  79
  80     $target_key = $args->getArg('engine');
  81     if (!$target_key) {
  82       throw new PhutilArgumentUsageException(
  83         pht(
  84           'Specify an engine to migrate to with `%s`. '.
  85           'Use `%s` to get a list of engines.',
  86           '--engine',
  87           'files engines'));
  88     }
  89
  90     $target_engine = PhabricatorFile::buildEngine($target_key);
  91
  92     $iterator = $this->buildIterator($args);
  93     $is_dry_run = $args->getArg('dry-run');
  94
  95     $min_size = (int)$args->getArg('min-size');
  96     $max_size = (int)$args->getArg('max-size');
  97
  98     $is_copy = $args->getArg('copy');
  99
 100     $failed = array();
 101     $engines = PhabricatorFileStorageEngine::loadAllEngines();
 102     $total_bytes = 0;
 103     $total_files = 0;
 104     foreach ($iterator as $file) {
 105       $monogram = $file->getMonogram();
 106
 107       // See T7148. When we export data for an instance, we copy all the data
 108       // for Files from S3 into the database dump so that the database dump is
 109       // a complete, standalone archive of all the data. In the general case,
 110       // installs may have a similar process using "--copy" to create a more
 111       // complete backup.
 112
 113       // When doing this, we may run into temporary files which have been
 114       // deleted between the time we took the original dump and the current
 115       // timestamp. These files can't be copied since the data no longer
 116       // exists: the daemons on the live install already deleted it.
 117
 118       // Simply avoid this whole mess by declining to migrate expired temporary
 119       // files. They're as good as dead anyway.
 120
 121       $ttl = $file->getTTL();
 122       if ($ttl) {
 123         if ($ttl < PhabricatorTime::getNow()) {
 124           echo tsprintf(
 125             "%s\n",
 126             pht(
 127               '%s: Skipping expired temporary file.',
 128               $monogram));
 129           continue;
 130         }
 131       }
 132
 133       $engine_key = $file->getStorageEngine();
 134       $engine = idx($engines, $engine_key);
 135
 136       if (!$engine) {
 137         echo tsprintf(
 138           "%s\n",
 139           pht(
 140             '%s: Uses unknown storage engine "%s".',
 141             $monogram,
 142             $engine_key));
 143         $failed[] = $file;
 144         continue;
 145       }
 146
 147       if ($engine->isChunkEngine()) {
 148         echo tsprintf(
 149           "%s\n",
 150           pht(
 151             '%s: Stored as chunks, no data to migrate directly.',
 152             $monogram));
 153         continue;
 154       }
 155
 156       if ($engine_key === $target_key) {
 157         echo tsprintf(
 158           "%s\n",
 159           pht(
 160             '%s: Already stored in engine "%s".',
 161             $monogram,
 162             $target_key));
 163         continue;
 164       }
 165
 166       $byte_size = $file->getByteSize();
 167
 168       if ($min_size && ($byte_size < $min_size)) {
 169         echo tsprintf(
 170           "%s\n",
 171           pht(
 172             '%s: File size (%s) is smaller than minimum size (%s).',
 173             $monogram,
 174             phutil_format_bytes($byte_size),
 175             phutil_format_bytes($min_size)));
 176         continue;
 177       }
 178
 179       if ($max_size && ($byte_size > $max_size)) {
 180         echo tsprintf(
 181           "%s\n",
 182           pht(
 183             '%s: File size (%s) is larger than maximum size (%s).',
 184             $monogram,
 185             phutil_format_bytes($byte_size),
 186             phutil_format_bytes($max_size)));
 187         continue;
 188       }
 189
 190       if ($is_dry_run) {
 191         echo tsprintf(
 192           "%s\n",
 193           pht(
 194             '%s: (%s) Would migrate from "%s" to "%s" (dry run)...',
 195             $monogram,
 196             phutil_format_bytes($byte_size),
 197             $engine_key,
 198             $target_key));
 199       } else {
 200         echo tsprintf(
 201           "%s\n",
 202           pht(
 203             '%s: (%s) Migrating from "%s" to "%s"...',
 204             $monogram,
 205             phutil_format_bytes($byte_size),
 206             $engine_key,
 207             $target_key));
 208       }
 209
 210       try {
 211         if ($is_dry_run) {
 212           // Do nothing, this is a dry run.
 213         } else {
 214           $file->migrateToEngine($target_engine, $is_copy);
 215         }
 216
 217         $total_files += 1;
 218         $total_bytes += $byte_size;
 219
 220         echo tsprintf(
 221           "%s\n",
 222           pht('Done.'));
 223
 224       } catch (Exception $ex) {
 225         echo tsprintf(
 226           "%s\n",
 227           pht('Failed! %s', (string)$ex));
 228         $failed[] = $file;
 229
 230         throw $ex;
 231       }
 232     }
 233
 234     echo tsprintf(
 235       "%s\n",
 236       pht(
 237         'Total Migrated Files: %s',
 238         new PhutilNumber($total_files)));
 239
 240     echo tsprintf(
 241       "%s\n",
 242       pht(
 243         'Total Migrated Bytes: %s',
 244         phutil_format_bytes($total_bytes)));
 245
 246     if ($is_dry_run) {
 247       echo tsprintf(
 248         "%s\n",
 249         pht(
 250           'This was a dry run, so no real migrations were performed.'));
 251     }
 252
 253     if ($failed) {
 254       $monograms = mpull($failed, 'getMonogram');
 255
 256       echo tsprintf(
 257         "%s\n",
 258         pht('Failures: %s.', implode(', ', $monograms)));
 259
 260       return 1;
 261     }
 262
 263     return 0;
 264   }
 265
 266 }