src/applications/harbormaster/engine/HarbormasterBuildEngine.php

   1 <?php
   2
   3 /**
   4  * Moves a build forward by queuing build tasks, canceling or restarting the
   5  * build, or failing it in response to task failures.
   6  */
   7 final class HarbormasterBuildEngine extends Phobject {
   8
   9   private $build;
  10   private $viewer;
  11   private $newBuildTargets = array();
  12   private $artifactReleaseQueue = array();
  13   private $forceBuildableUpdate;
  14
  15   public function setForceBuildableUpdate($force_buildable_update) {
  16     $this->forceBuildableUpdate = $force_buildable_update;
  17     return $this;
  18   }
  19
  20   public function shouldForceBuildableUpdate() {
  21     return $this->forceBuildableUpdate;
  22   }
  23
  24   public function queueNewBuildTarget(HarbormasterBuildTarget $target) {
  25     $this->newBuildTargets[] = $target;
  26     return $this;
  27   }
  28
  29   public function getNewBuildTargets() {
  30     return $this->newBuildTargets;
  31   }
  32
  33   public function setViewer(PhabricatorUser $viewer) {
  34     $this->viewer = $viewer;
  35     return $this;
  36   }
  37
  38   public function getViewer() {
  39     return $this->viewer;
  40   }
  41
  42   public function setBuild(HarbormasterBuild $build) {
  43     $this->build = $build;
  44     return $this;
  45   }
  46
  47   public function getBuild() {
  48     return $this->build;
  49   }
  50
  51   public function continueBuild() {
  52     $viewer = $this->getViewer();
  53     $build = $this->getBuild();
  54
  55     $lock_key = 'harbormaster.build:'.$build->getID();
  56     $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
  57
  58     $build->reload();
  59     $old_status = $build->getBuildStatus();
  60
  61     try {
  62       $this->updateBuild($build);
  63     } catch (Exception $ex) {
  64       // If any exception is raised, the build is marked as a failure and the
  65       // exception is re-thrown (this ensures we don't leave builds in an
  66       // inconsistent state).
  67       $build->setBuildStatus(HarbormasterBuildStatus::STATUS_ERROR);
  68       $build->save();
  69
  70       $lock->unlock();
  71
  72       $build->releaseAllArtifacts($viewer);
  73
  74       throw $ex;
  75     }
  76
  77     $lock->unlock();
  78
  79     // NOTE: We queue new targets after releasing the lock so that in-process
  80     // execution via `bin/harbormaster` does not reenter the locked region.
  81     foreach ($this->getNewBuildTargets() as $target) {
  82       $task = PhabricatorWorker::scheduleTask(
  83         'HarbormasterTargetWorker',
  84         array(
  85           'targetID' => $target->getID(),
  86         ),
  87         array(
  88           'objectPHID' => $target->getPHID(),
  89         ));
  90     }
  91
  92     // If the build changed status, we might need to update the overall status
  93     // on the buildable.
  94     $new_status = $build->getBuildStatus();
  95     if ($new_status != $old_status || $this->shouldForceBuildableUpdate()) {
  96       $this->updateBuildable($build->getBuildable());
  97     }
  98
  99     $this->releaseQueuedArtifacts();
 100
 101     // If we are no longer building for any reason, release all artifacts.
 102     if (!$build->isBuilding()) {
 103       $build->releaseAllArtifacts($viewer);
 104     }
 105   }
 106
 107   private function updateBuild(HarbormasterBuild $build) {
 108     $viewer = $this->getViewer();
 109
 110     $content_source = PhabricatorContentSource::newForSource(
 111       PhabricatorDaemonContentSource::SOURCECONST);
 112
 113     $acting_phid = $viewer->getPHID();
 114     if (!$acting_phid) {
 115       $acting_phid = id(new PhabricatorHarbormasterApplication())->getPHID();
 116     }
 117
 118     $editor = $build->getApplicationTransactionEditor()
 119       ->setActor($viewer)
 120       ->setActingAsPHID($acting_phid)
 121       ->setContentSource($content_source)
 122       ->setContinueOnNoEffect(true)
 123       ->setContinueOnMissingFields(true);
 124
 125     $xactions = array();
 126
 127     $messages = $build->getUnprocessedMessagesForApply();
 128     foreach ($messages as $message) {
 129       $message_type = $message->getType();
 130
 131       $message_xaction =
 132         HarbormasterBuildMessageTransaction::getTransactionTypeForMessageType(
 133           $message_type);
 134
 135       if (!$message_xaction) {
 136         continue;
 137       }
 138
 139       $xactions[] = $build->getApplicationTransactionTemplate()
 140         ->setAuthorPHID($message->getAuthorPHID())
 141         ->setTransactionType($message_xaction)
 142         ->setNewValue($message_type);
 143     }
 144
 145     if (!$xactions) {
 146       if ($build->isPending()) {
 147         // TODO: This should be a transaction.
 148
 149         $build->restartBuild($viewer);
 150         $build->setBuildStatus(HarbormasterBuildStatus::STATUS_BUILDING);
 151         $build->save();
 152       }
 153     }
 154
 155     if ($xactions) {
 156       $editor->applyTransactions($build, $xactions);
 157       $build->markUnprocessedMessagesAsProcessed();
 158     }
 159
 160     if ($build->getBuildStatus() == HarbormasterBuildStatus::STATUS_BUILDING) {
 161       $this->updateBuildSteps($build);
 162     }
 163   }
 164
 165   private function updateBuildSteps(HarbormasterBuild $build) {
 166     $all_targets = id(new HarbormasterBuildTargetQuery())
 167       ->setViewer($this->getViewer())
 168       ->withBuildPHIDs(array($build->getPHID()))
 169       ->withBuildGenerations(array($build->getBuildGeneration()))
 170       ->execute();
 171
 172     $this->updateWaitingTargets($all_targets);
 173
 174     $targets = mgroup($all_targets, 'getBuildStepPHID');
 175
 176     $steps = id(new HarbormasterBuildStepQuery())
 177       ->setViewer($this->getViewer())
 178       ->withBuildPlanPHIDs(array($build->getBuildPlan()->getPHID()))
 179       ->execute();
 180     $steps = mpull($steps, null, 'getPHID');
 181
 182     // Identify steps which are in various states.
 183
 184     $queued = array();
 185     $underway = array();
 186     $waiting = array();
 187     $complete = array();
 188     $failed = array();
 189     foreach ($steps as $step) {
 190       $step_targets = idx($targets, $step->getPHID(), array());
 191
 192       if ($step_targets) {
 193         $is_queued = false;
 194
 195         $is_underway = false;
 196         foreach ($step_targets as $target) {
 197           if ($target->isUnderway()) {
 198             $is_underway = true;
 199             break;
 200           }
 201         }
 202
 203         $is_waiting = false;
 204         foreach ($step_targets as $target) {
 205           if ($target->isWaiting()) {
 206             $is_waiting = true;
 207             break;
 208           }
 209         }
 210
 211         $is_complete = true;
 212         foreach ($step_targets as $target) {
 213           if (!$target->isComplete()) {
 214             $is_complete = false;
 215             break;
 216           }
 217         }
 218
 219         $is_failed = false;
 220         foreach ($step_targets as $target) {
 221           if ($target->isFailed()) {
 222             $is_failed = true;
 223             break;
 224           }
 225         }
 226       } else {
 227         $is_queued = true;
 228         $is_underway = false;
 229         $is_waiting = false;
 230         $is_complete = false;
 231         $is_failed = false;
 232       }
 233
 234       if ($is_queued) {
 235         $queued[$step->getPHID()] = true;
 236       }
 237
 238       if ($is_underway) {
 239         $underway[$step->getPHID()] = true;
 240       }
 241
 242       if ($is_waiting) {
 243         $waiting[$step->getPHID()] = true;
 244       }
 245
 246       if ($is_complete) {
 247         $complete[$step->getPHID()] = true;
 248       }
 249
 250       if ($is_failed) {
 251         $failed[$step->getPHID()] = true;
 252       }
 253     }
 254
 255     // If any step failed, fail the whole build, then bail.
 256     if (count($failed)) {
 257       $build->setBuildStatus(HarbormasterBuildStatus::STATUS_FAILED);
 258       $build->save();
 259       return;
 260     }
 261
 262     // If every step is complete, we're done with this build. Mark it passed
 263     // and bail.
 264     if (count($complete) == count($steps)) {
 265       $build->setBuildStatus(HarbormasterBuildStatus::STATUS_PASSED);
 266       $build->save();
 267       return;
 268     }
 269
 270     // Release any artifacts which are not inputs to any remaining build
 271     // step. We're done with these, so something else is free to use them.
 272     $ongoing_phids = array_keys($queued + $waiting + $underway);
 273     $ongoing_steps = array_select_keys($steps, $ongoing_phids);
 274     $this->releaseUnusedArtifacts($all_targets, $ongoing_steps);
 275
 276     // Identify all the steps which are ready to run (because all their
 277     // dependencies are complete).
 278
 279     $runnable = array();
 280     foreach ($steps as $step) {
 281       $dependencies = $step->getStepImplementation()->getDependencies($step);
 282
 283       if (isset($queued[$step->getPHID()])) {
 284         $can_run = true;
 285         foreach ($dependencies as $dependency) {
 286           if (empty($complete[$dependency])) {
 287             $can_run = false;
 288             break;
 289           }
 290         }
 291
 292         if ($can_run) {
 293           $runnable[] = $step;
 294         }
 295       }
 296     }
 297
 298     if (!$runnable && !$waiting && !$underway) {
 299       // This means the build is deadlocked, and the user has configured
 300       // circular dependencies.
 301       $build->setBuildStatus(HarbormasterBuildStatus::STATUS_DEADLOCKED);
 302       $build->save();
 303       return;
 304     }
 305
 306     foreach ($runnable as $runnable_step) {
 307       $target = HarbormasterBuildTarget::initializeNewBuildTarget(
 308         $build,
 309         $runnable_step,
 310         $build->retrieveVariablesFromBuild());
 311       $target->save();
 312
 313       $this->queueNewBuildTarget($target);
 314     }
 315   }
 316
 317
 318   /**
 319    * Release any artifacts which aren't used by any running or waiting steps.
 320    *
 321    * This releases artifacts as soon as they're no longer used. This can be
 322    * particularly relevant when a build uses multiple hosts since it returns
 323    * hosts to the pool more quickly.
 324    *
 325    * @param list<HarbormasterBuildTarget> Targets in the build.
 326    * @param list<HarbormasterBuildStep> List of running and waiting steps.
 327    * @return void
 328    */
 329   private function releaseUnusedArtifacts(array $targets, array $steps) {
 330     assert_instances_of($targets, 'HarbormasterBuildTarget');
 331     assert_instances_of($steps, 'HarbormasterBuildStep');
 332
 333     if (!$targets || !$steps) {
 334       return;
 335     }
 336
 337     $target_phids = mpull($targets, 'getPHID');
 338
 339     $artifacts = id(new HarbormasterBuildArtifactQuery())
 340       ->setViewer($this->getViewer())
 341       ->withBuildTargetPHIDs($target_phids)
 342       ->withIsReleased(false)
 343       ->execute();
 344     if (!$artifacts) {
 345       return;
 346     }
 347
 348     // Collect all the artifacts that remaining build steps accept as inputs.
 349     $must_keep = array();
 350     foreach ($steps as $step) {
 351       $inputs = $step->getStepImplementation()->getArtifactInputs();
 352       foreach ($inputs as $input) {
 353         $artifact_key = $input['key'];
 354         $must_keep[$artifact_key] = true;
 355       }
 356     }
 357
 358     // Queue unreleased artifacts which no remaining step uses for immediate
 359     // release.
 360     foreach ($artifacts as $artifact) {
 361       $key = $artifact->getArtifactKey();
 362       if (isset($must_keep[$key])) {
 363         continue;
 364       }
 365
 366       $this->artifactReleaseQueue[] = $artifact;
 367     }
 368   }
 369
 370
 371   /**
 372    * Process messages which were sent to these targets, kicking applicable
 373    * targets out of "Waiting" and into either "Passed" or "Failed".
 374    *
 375    * @param list<HarbormasterBuildTarget> List of targets to process.
 376    * @return void
 377    */
 378   private function updateWaitingTargets(array $targets) {
 379     assert_instances_of($targets, 'HarbormasterBuildTarget');
 380
 381     // We only care about messages for targets which are actually in a waiting
 382     // state.
 383     $waiting_targets = array();
 384     foreach ($targets as $target) {
 385       if ($target->isWaiting()) {
 386         $waiting_targets[$target->getPHID()] = $target;
 387       }
 388     }
 389
 390     if (!$waiting_targets) {
 391       return;
 392     }
 393
 394     $messages = id(new HarbormasterBuildMessageQuery())
 395       ->setViewer($this->getViewer())
 396       ->withReceiverPHIDs(array_keys($waiting_targets))
 397       ->withConsumed(false)
 398       ->execute();
 399
 400     foreach ($messages as $message) {
 401       $target = $waiting_targets[$message->getReceiverPHID()];
 402
 403       switch ($message->getType()) {
 404         case HarbormasterMessageType::MESSAGE_PASS:
 405           $new_status = HarbormasterBuildTarget::STATUS_PASSED;
 406           break;
 407         case HarbormasterMessageType::MESSAGE_FAIL:
 408           $new_status = HarbormasterBuildTarget::STATUS_FAILED;
 409           break;
 410         case HarbormasterMessageType::MESSAGE_WORK:
 411         default:
 412           $new_status = null;
 413           break;
 414       }
 415
 416       if ($new_status !== null) {
 417         $message->setIsConsumed(true);
 418         $message->save();
 419
 420         $target->setTargetStatus($new_status);
 421
 422         if ($target->isComplete()) {
 423           $target->setDateCompleted(PhabricatorTime::getNow());
 424         }
 425
 426         $target->save();
 427       }
 428     }
 429   }
 430
 431
 432   /**
 433    * Update the overall status of the buildable this build is attached to.
 434    *
 435    * After a build changes state (for example, passes or fails) it may affect
 436    * the overall state of the associated buildable. Compute the new aggregate
 437    * state and save it on the buildable.
 438    *
 439    * @param   HarbormasterBuild The buildable to update.
 440    * @return  void
 441    */
 442    public function updateBuildable(HarbormasterBuildable $buildable) {
 443     $viewer = $this->getViewer();
 444
 445     $lock_key = 'harbormaster.buildable:'.$buildable->getID();
 446     $lock = PhabricatorGlobalLock::newLock($lock_key)->lock(15);
 447
 448     $buildable = id(new HarbormasterBuildableQuery())
 449       ->setViewer($viewer)
 450       ->withIDs(array($buildable->getID()))
 451       ->needBuilds(true)
 452       ->executeOne();
 453
 454     $messages = id(new HarbormasterBuildMessageQuery())
 455       ->setViewer($viewer)
 456       ->withReceiverPHIDs(array($buildable->getPHID()))
 457       ->withConsumed(false)
 458       ->execute();
 459
 460     $done_preparing = false;
 461     $update_container = false;
 462     foreach ($messages as $message) {
 463       switch ($message->getType()) {
 464         case HarbormasterMessageType::BUILDABLE_BUILD:
 465           $done_preparing = true;
 466           break;
 467         case HarbormasterMessageType::BUILDABLE_CONTAINER:
 468           $update_container = true;
 469           break;
 470         default:
 471           break;
 472       }
 473
 474       $message
 475         ->setIsConsumed(true)
 476         ->save();
 477     }
 478
 479     // If we received a "build" command, all builds are scheduled and we can
 480     // move out of "preparing" into "building".
 481     if ($done_preparing) {
 482       if ($buildable->isPreparing()) {
 483         $buildable
 484           ->setBuildableStatus(HarbormasterBuildableStatus::STATUS_BUILDING)
 485           ->save();
 486       }
 487     }
 488
 489     // If we've been informed that the container for the buildable has
 490     // changed, update it.
 491     if ($update_container) {
 492       $object = id(new PhabricatorObjectQuery())
 493         ->setViewer($viewer)
 494         ->withPHIDs(array($buildable->getBuildablePHID()))
 495         ->executeOne();
 496       if ($object) {
 497         $buildable
 498           ->setContainerPHID($object->getHarbormasterContainerPHID())
 499           ->save();
 500       }
 501     }
 502
 503     $old = clone $buildable;
 504
 505     // Don't update the buildable status if we're still preparing builds: more
 506     // builds may still be scheduled shortly, so even if every build we know
 507     // about so far has passed, that doesn't mean the buildable has actually
 508     // passed everything it needs to.
 509
 510     if (!$buildable->isPreparing()) {
 511       $behavior_key = HarbormasterBuildPlanBehavior::BEHAVIOR_BUILDABLE;
 512       $behavior = HarbormasterBuildPlanBehavior::getBehavior($behavior_key);
 513
 514       $key_never = HarbormasterBuildPlanBehavior::BUILDABLE_NEVER;
 515       $key_building = HarbormasterBuildPlanBehavior::BUILDABLE_IF_BUILDING;
 516
 517       $all_pass = true;
 518       $any_fail = false;
 519       foreach ($buildable->getBuilds() as $build) {
 520         $plan = $build->getBuildPlan();
 521         $option = $behavior->getPlanOption($plan);
 522         $option_key = $option->getKey();
 523
 524         $is_never = ($option_key === $key_never);
 525         $is_building = ($option_key === $key_building);
 526
 527         // If this build "Never" affects the buildable, ignore it.
 528         if ($is_never) {
 529           continue;
 530         }
 531
 532         // If this build affects the buildable "If Building", but is already
 533         // complete, ignore it.
 534         if ($is_building && $build->isComplete()) {
 535           continue;
 536         }
 537
 538         if (!$build->isPassed()) {
 539           $all_pass = false;
 540         }
 541
 542         if ($build->isComplete() && !$build->isPassed()) {
 543           $any_fail = true;
 544         }
 545       }
 546
 547       if ($any_fail) {
 548         $new_status = HarbormasterBuildableStatus::STATUS_FAILED;
 549       } else if ($all_pass) {
 550         $new_status = HarbormasterBuildableStatus::STATUS_PASSED;
 551       } else {
 552         $new_status = HarbormasterBuildableStatus::STATUS_BUILDING;
 553       }
 554
 555       $did_update = ($old->getBuildableStatus() !== $new_status);
 556       if ($did_update) {
 557         $buildable->setBuildableStatus($new_status);
 558         $buildable->save();
 559       }
 560     }
 561
 562     $lock->unlock();
 563
 564     // Don't publish anything if we're still preparing builds.
 565     if ($buildable->isPreparing()) {
 566       return;
 567     }
 568
 569     $this->publishBuildable($old, $buildable);
 570   }
 571
 572   public function publishBuildable(
 573     HarbormasterBuildable $old,
 574     HarbormasterBuildable $new) {
 575
 576     $viewer = $this->getViewer();
 577
 578     // Publish the buildable. We publish buildables even if they haven't
 579     // changed status in Harbormaster because applications may care about
 580     // different things than Harbormaster does. For example, Differential
 581     // does not care about local lint and unit tests when deciding whether
 582     // a revision should move out of draft or not.
 583
 584     // NOTE: We're publishing both automatic and manual buildables. Buildable
 585     // objects should generally ignore manual buildables, but it's up to them
 586     // to decide.
 587
 588     $object = id(new PhabricatorObjectQuery())
 589       ->setViewer($viewer)
 590       ->withPHIDs(array($new->getBuildablePHID()))
 591       ->executeOne();
 592     if (!$object) {
 593       return;
 594     }
 595
 596     $engine = HarbormasterBuildableEngine::newForObject($object, $viewer);
 597
 598     $daemon_source = PhabricatorContentSource::newForSource(
 599       PhabricatorDaemonContentSource::SOURCECONST);
 600
 601     $harbormaster_phid = id(new PhabricatorHarbormasterApplication())
 602       ->getPHID();
 603
 604     $engine
 605       ->setActingAsPHID($harbormaster_phid)
 606       ->setContentSource($daemon_source)
 607       ->publishBuildable($old, $new);
 608   }
 609
 610   private function releaseQueuedArtifacts() {
 611     foreach ($this->artifactReleaseQueue as $key => $artifact) {
 612       $artifact->releaseArtifact();
 613       unset($this->artifactReleaseQueue[$key]);
 614     }
 615   }
 616
 617 }