src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php

   1 <?php
   2
   3 final class DrydockWorkingCopyBlueprintImplementation
   4   extends DrydockBlueprintImplementation {
   5
   6   const PHASE_SQUASHMERGE = 'squashmerge';
   7   const PHASE_REMOTEFETCH = 'blueprint.workingcopy.fetch.remote';
   8   const PHASE_MERGEFETCH = 'blueprint.workingcopy.fetch.staging';
   9
  10   public function isEnabled() {
  11     return true;
  12   }
  13
  14   public function getBlueprintName() {
  15     return pht('Working Copy');
  16   }
  17
  18   public function getBlueprintIcon() {
  19     return 'fa-folder-open';
  20   }
  21
  22   public function getDescription() {
  23     return pht('Allows Drydock to check out working copies of repositories.');
  24   }
  25
  26   public function canAnyBlueprintEverAllocateResourceForLease(
  27     DrydockLease $lease) {
  28     return true;
  29   }
  30
  31   public function canEverAllocateResourceForLease(
  32     DrydockBlueprint $blueprint,
  33     DrydockLease $lease) {
  34     return true;
  35   }
  36
  37   public function canAllocateResourceForLease(
  38     DrydockBlueprint $blueprint,
  39     DrydockLease $lease) {
  40     $viewer = $this->getViewer();
  41
  42     if ($this->shouldLimitAllocatingPoolSize($blueprint)) {
  43       return false;
  44     }
  45
  46     return true;
  47   }
  48
  49   public function canAcquireLeaseOnResource(
  50     DrydockBlueprint $blueprint,
  51     DrydockResource $resource,
  52     DrydockLease $lease) {
  53
  54     // Don't hand out leases on working copies which have not activated, since
  55     // it may take an arbitrarily long time for them to acquire a host.
  56     if (!$resource->isActive()) {
  57       return false;
  58     }
  59
  60     $need_map = $lease->getAttribute('repositories.map');
  61     if (!is_array($need_map)) {
  62       return false;
  63     }
  64
  65     $have_map = $resource->getAttribute('repositories.map');
  66     if (!is_array($have_map)) {
  67       return false;
  68     }
  69
  70     $have_as = ipull($have_map, 'phid');
  71     $need_as = ipull($need_map, 'phid');
  72
  73     foreach ($need_as as $need_directory => $need_phid) {
  74       if (empty($have_as[$need_directory])) {
  75         // This resource is missing a required working copy.
  76         return false;
  77       }
  78
  79       if ($have_as[$need_directory] != $need_phid) {
  80         // This resource has a required working copy, but it contains
  81         // the wrong repository.
  82         return false;
  83       }
  84
  85       unset($have_as[$need_directory]);
  86     }
  87
  88     if ($have_as && $lease->getAttribute('repositories.strict')) {
  89       // This resource has extra repositories, but the lease is strict about
  90       // which repositories are allowed to exist.
  91       return false;
  92     }
  93
  94     if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) {
  95       return false;
  96     }
  97
  98     return true;
  99   }
 100
 101   public function acquireLease(
 102     DrydockBlueprint $blueprint,
 103     DrydockResource $resource,
 104     DrydockLease $lease) {
 105
 106     $lease
 107       ->needSlotLock($this->getLeaseSlotLock($resource))
 108       ->acquireOnResource($resource);
 109   }
 110
 111   private function getLeaseSlotLock(DrydockResource $resource) {
 112     $resource_phid = $resource->getPHID();
 113     return "workingcopy.lease({$resource_phid})";
 114   }
 115
 116   public function allocateResource(
 117     DrydockBlueprint $blueprint,
 118     DrydockLease $lease) {
 119
 120     $resource = $this->newResourceTemplate($blueprint);
 121
 122     $resource_phid = $resource->getPHID();
 123
 124     $blueprint_phids = $blueprint->getFieldValue('blueprintPHIDs');
 125
 126     $host_lease = $this->newLease($blueprint)
 127       ->setResourceType('host')
 128       ->setOwnerPHID($resource_phid)
 129       ->setAttribute('workingcopy.resourcePHID', $resource_phid)
 130       ->setAllowedBlueprintPHIDs($blueprint_phids);
 131     $resource->setAttribute('host.leasePHID', $host_lease->getPHID());
 132
 133     $map = $this->getWorkingCopyRepositoryMap($lease);
 134     $resource->setAttribute('repositories.map', $map);
 135
 136     $slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint);
 137     if ($slot_lock !== null) {
 138       $resource->needSlotLock($slot_lock);
 139     }
 140
 141     $resource->allocateResource();
 142
 143     $host_lease->queueForActivation();
 144
 145     return $resource;
 146   }
 147
 148   private function getWorkingCopyRepositoryMap(DrydockLease $lease) {
 149     $attribute = 'repositories.map';
 150     $map = $lease->getAttribute($attribute);
 151
 152     // TODO: Leases should validate their attributes more formally.
 153
 154     if (!is_array($map) || !$map) {
 155       $message = array();
 156       if ($map === null) {
 157         $message[] = pht(
 158           'Working copy lease is missing required attribute "%s".',
 159           $attribute);
 160       } else {
 161         $message[] = pht(
 162           'Working copy lease has invalid attribute "%s".',
 163           $attribute);
 164       }
 165
 166       $message[] = pht(
 167         'Attribute "repositories.map" should be a map of repository '.
 168         'specifications.');
 169
 170       $message = implode("\n\n", $message);
 171
 172       throw new Exception($message);
 173     }
 174
 175     foreach ($map as $key => $value) {
 176       $map[$key] = array_select_keys(
 177         $value,
 178         array(
 179           'phid',
 180         ));
 181     }
 182
 183     return $map;
 184   }
 185
 186   public function activateResource(
 187     DrydockBlueprint $blueprint,
 188     DrydockResource $resource) {
 189
 190     $lease = $this->loadHostLease($resource);
 191     $this->requireActiveLease($lease);
 192
 193     $command_type = DrydockCommandInterface::INTERFACE_TYPE;
 194     $interface = $lease->getInterface($command_type);
 195
 196     // TODO: Make this configurable.
 197     $resource_id = $resource->getID();
 198     $root = "/var/drydock/workingcopy-{$resource_id}";
 199
 200     $map = $resource->getAttribute('repositories.map');
 201
 202     $futures = array();
 203     $repositories = $this->loadRepositories(ipull($map, 'phid'));
 204     foreach ($map as $directory => $spec) {
 205       // TODO: Validate directory isn't goofy like "/etc" or "../../lol"
 206       // somewhere?
 207
 208       $repository = $repositories[$spec['phid']];
 209       $path = "{$root}/repo/{$directory}/";
 210
 211       $future = $interface->getExecFuture(
 212         'git clone -- %s %s',
 213         (string)$repository->getCloneURIObject(),
 214         $path);
 215
 216       $future->setTimeout($repository->getEffectiveCopyTimeLimit());
 217
 218       $futures[$directory] = $future;
 219     }
 220
 221     foreach (new FutureIterator($futures) as $key => $future) {
 222       $future->resolvex();
 223     }
 224
 225     $resource
 226       ->setAttribute('workingcopy.root', $root)
 227       ->activateResource();
 228   }
 229
 230   public function destroyResource(
 231     DrydockBlueprint $blueprint,
 232     DrydockResource $resource) {
 233
 234     try {
 235       $lease = $this->loadHostLease($resource);
 236     } catch (Exception $ex) {
 237       // If we can't load the lease, assume we don't need to take any actions
 238       // to destroy it.
 239       return;
 240     }
 241
 242     // Destroy the lease on the host.
 243     $lease->setReleaseOnDestruction(true);
 244
 245     if ($lease->isActive()) {
 246       // Destroy the working copy on disk.
 247       $command_type = DrydockCommandInterface::INTERFACE_TYPE;
 248       $interface = $lease->getInterface($command_type);
 249
 250       $root_key = 'workingcopy.root';
 251       $root = $resource->getAttribute($root_key);
 252       if (strlen($root)) {
 253         $interface->execx('rm -rf -- %s', $root);
 254       }
 255     }
 256   }
 257
 258   public function getResourceName(
 259     DrydockBlueprint $blueprint,
 260     DrydockResource $resource) {
 261     return pht('Working Copy');
 262   }
 263
 264
 265   public function activateLease(
 266     DrydockBlueprint $blueprint,
 267     DrydockResource $resource,
 268     DrydockLease $lease) {
 269
 270     $host_lease = $this->loadHostLease($resource);
 271     $command_type = DrydockCommandInterface::INTERFACE_TYPE;
 272     $interface = $host_lease->getInterface($command_type);
 273
 274     $map = $lease->getAttribute('repositories.map');
 275     $root = $resource->getAttribute('workingcopy.root');
 276
 277     $repositories = $this->loadRepositories(ipull($map, 'phid'));
 278
 279     $default = null;
 280     foreach ($map as $directory => $spec) {
 281       $repository = $repositories[$spec['phid']];
 282
 283       $interface->pushWorkingDirectory("{$root}/repo/{$directory}/");
 284
 285       $cmd = array();
 286       $arg = array();
 287
 288       $cmd[] = 'git clean -d --force';
 289       $cmd[] = 'git fetch';
 290
 291       $commit = idx($spec, 'commit');
 292       $branch = idx($spec, 'branch');
 293
 294       $ref = idx($spec, 'ref');
 295
 296       // Reset things first, in case previous builds left anything staged or
 297       // dirty. Note that we don't reset to "HEAD" because that does not work
 298       // in empty repositories.
 299       $cmd[] = 'git reset --hard';
 300
 301       if ($commit !== null) {
 302         $cmd[] = 'git checkout %s --';
 303         $arg[] = $commit;
 304       } else if ($branch !== null) {
 305         $cmd[] = 'git checkout %s --';
 306         $arg[] = $branch;
 307
 308         $cmd[] = 'git reset --hard origin/%s';
 309         $arg[] = $branch;
 310       }
 311
 312       $this->newExecvFuture($interface, $cmd, $arg)
 313         ->setTimeout($repository->getEffectiveCopyTimeLimit())
 314         ->resolvex();
 315
 316       if (idx($spec, 'default')) {
 317         $default = $directory;
 318       }
 319
 320       // If we're fetching a ref from a remote, do that separately so we can
 321       // raise a more tailored error.
 322       if ($ref) {
 323         $cmd = array();
 324         $arg = array();
 325
 326         $ref_uri = $ref['uri'];
 327         $ref_ref = $ref['ref'];
 328
 329         $cmd[] = 'git fetch --no-tags -- %s +%s:%s';
 330         $arg[] = $ref_uri;
 331         $arg[] = $ref_ref;
 332         $arg[] = $ref_ref;
 333
 334         $cmd[] = 'git checkout %s --';
 335         $arg[] = $ref_ref;
 336
 337         try {
 338           $this->newExecvFuture($interface, $cmd, $arg)
 339             ->setTimeout($repository->getEffectiveCopyTimeLimit())
 340             ->resolvex();
 341         } catch (CommandException $ex) {
 342           $display_command = csprintf(
 343             'git fetch %R %R',
 344             $ref_uri,
 345             $ref_ref);
 346
 347           $error = DrydockCommandError::newFromCommandException($ex)
 348             ->setPhase(self::PHASE_REMOTEFETCH)
 349             ->setDisplayCommand($display_command);
 350
 351           $lease->setAttribute(
 352             'workingcopy.vcs.error',
 353             $error->toDictionary());
 354
 355           throw $ex;
 356         }
 357       }
 358
 359       $merges = idx($spec, 'merges');
 360       if ($merges) {
 361         foreach ($merges as $merge) {
 362           $this->applyMerge($lease, $interface, $merge);
 363         }
 364       }
 365
 366       $interface->popWorkingDirectory();
 367     }
 368
 369     if ($default === null) {
 370       $default = head_key($map);
 371     }
 372
 373     // TODO: Use working storage?
 374     $lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/");
 375
 376     $lease->activateOnResource($resource);
 377   }
 378
 379   public function didReleaseLease(
 380     DrydockBlueprint $blueprint,
 381     DrydockResource $resource,
 382     DrydockLease $lease) {
 383     // We leave working copies around even if there are no leases on them,
 384     // since the cost to maintain them is nearly zero but rebuilding them is
 385     // moderately expensive and it's likely that they'll be reused.
 386     return;
 387   }
 388
 389   public function destroyLease(
 390     DrydockBlueprint $blueprint,
 391     DrydockResource $resource,
 392     DrydockLease $lease) {
 393     // When we activate a lease we just reset the working copy state and do
 394     // not create any new state, so we don't need to do anything special when
 395     // destroying a lease.
 396     return;
 397   }
 398
 399   public function getType() {
 400     return 'working-copy';
 401   }
 402
 403   public function getInterface(
 404     DrydockBlueprint $blueprint,
 405     DrydockResource $resource,
 406     DrydockLease $lease,
 407     $type) {
 408
 409     switch ($type) {
 410       case DrydockCommandInterface::INTERFACE_TYPE:
 411         $host_lease = $this->loadHostLease($resource);
 412         $command_interface = $host_lease->getInterface($type);
 413
 414         $path = $lease->getAttribute('workingcopy.default');
 415         $command_interface->pushWorkingDirectory($path);
 416
 417         return $command_interface;
 418     }
 419   }
 420
 421   private function loadRepositories(array $phids) {
 422     $viewer = $this->getViewer();
 423
 424     $repositories = id(new PhabricatorRepositoryQuery())
 425       ->setViewer($viewer)
 426       ->withPHIDs($phids)
 427       ->execute();
 428     $repositories = mpull($repositories, null, 'getPHID');
 429
 430     foreach ($phids as $phid) {
 431       if (empty($repositories[$phid])) {
 432         throw new Exception(
 433           pht(
 434             'Repository PHID "%s" does not exist.',
 435             $phid));
 436       }
 437     }
 438
 439     foreach ($repositories as $repository) {
 440       $repository_vcs = $repository->getVersionControlSystem();
 441       switch ($repository_vcs) {
 442         case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
 443           break;
 444         default:
 445           throw new Exception(
 446             pht(
 447               'Repository ("%s") has unsupported VCS ("%s").',
 448               $repository->getPHID(),
 449               $repository_vcs));
 450       }
 451     }
 452
 453     return $repositories;
 454   }
 455
 456   private function loadHostLease(DrydockResource $resource) {
 457     $viewer = $this->getViewer();
 458
 459     $lease_phid = $resource->getAttribute('host.leasePHID');
 460
 461     $lease = id(new DrydockLeaseQuery())
 462       ->setViewer($viewer)
 463       ->withPHIDs(array($lease_phid))
 464       ->executeOne();
 465     if (!$lease) {
 466       throw new Exception(
 467         pht(
 468           'Unable to load lease ("%s").',
 469           $lease_phid));
 470     }
 471
 472     return $lease;
 473   }
 474
 475   protected function getCustomFieldSpecifications() {
 476     return array(
 477       'blueprintPHIDs' => array(
 478         'name' => pht('Use Blueprints'),
 479         'type' => 'blueprints',
 480         'required' => true,
 481       ),
 482     );
 483   }
 484
 485   protected function shouldUseConcurrentResourceLimit() {
 486     return true;
 487   }
 488
 489   private function applyMerge(
 490     DrydockLease $lease,
 491     DrydockCommandInterface $interface,
 492     array $merge) {
 493
 494     $src_uri = $merge['src.uri'];
 495     $src_ref = $merge['src.ref'];
 496
 497
 498     try {
 499       $interface->execx(
 500         'git fetch --no-tags -- %s +%s:%s',
 501         $src_uri,
 502         $src_ref,
 503         $src_ref);
 504     } catch (CommandException $ex) {
 505       $display_command = csprintf(
 506         'git fetch %R +%R:%R',
 507         $src_uri,
 508         $src_ref,
 509         $src_ref);
 510
 511       $error = DrydockCommandError::newFromCommandException($ex)
 512         ->setPhase(self::PHASE_MERGEFETCH)
 513         ->setDisplayCommand($display_command);
 514
 515       $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());
 516
 517       throw $ex;
 518     }
 519
 520
 521     // NOTE: This can never actually generate a commit because we pass
 522     // "--squash", but git sometimes runs code to check that a username and
 523     // email are configured anyway.
 524     $real_command = csprintf(
 525       'git -c user.name=%s -c user.email=%s merge --no-stat --squash -- %R',
 526       'drydock',
 527       'drydock@phabricator',
 528       $src_ref);
 529
 530     try {
 531       $interface->execx('%C', $real_command);
 532     } catch (CommandException $ex) {
 533       $display_command = csprintf(
 534         'git merge --squash %R',
 535         $src_ref);
 536
 537       $error = DrydockCommandError::newFromCommandException($ex)
 538         ->setPhase(self::PHASE_SQUASHMERGE)
 539         ->setDisplayCommand($display_command);
 540
 541       $lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());
 542       throw $ex;
 543     }
 544   }
 545
 546   public function getCommandError(DrydockLease $lease) {
 547     return $lease->getAttribute('workingcopy.vcs.error');
 548   }
 549
 550   private function execxv(
 551     DrydockCommandInterface $interface,
 552     array $commands,
 553     array $arguments) {
 554     return $this->newExecvFuture($interface, $commands, $arguments)->resolvex();
 555   }
 556
 557   private function newExecvFuture(
 558     DrydockCommandInterface $interface,
 559     array $commands,
 560     array $arguments) {
 561
 562     $commands = implode(' && ', $commands);
 563     $argv = array_merge(array($commands), $arguments);
 564
 565     return call_user_func_array(array($interface, 'getExecFuture'), $argv);
 566   }
 567
 568 }