4 * Update the ref cursors for a repository, which track the positions of
5 * branches, bookmarks, and tags.
7 final class PhabricatorRepositoryRefEngine
8 extends PhabricatorRepositoryEngine
{
10 private $newPositions = array();
11 private $deadPositions = array();
12 private $permanentCommits = array();
15 public function setRebuild($rebuild) {
16 $this->rebuild
= $rebuild;
20 public function getRebuild() {
21 return $this->rebuild
;
24 public function updateRefs() {
25 $this->newPositions
= array();
26 $this->deadPositions
= array();
27 $this->permanentCommits
= array();
29 $repository = $this->getRepository();
30 $viewer = $this->getViewer();
32 $branches_may_close = false;
34 $vcs = $repository->getVersionControlSystem();
36 case PhabricatorRepositoryType
::REPOSITORY_TYPE_SVN
:
37 // No meaningful refs of any type in Subversion.
40 case PhabricatorRepositoryType
::REPOSITORY_TYPE_MERCURIAL
:
41 $branches = $this->loadMercurialBranchPositions($repository);
42 $bookmarks = $this->loadMercurialBookmarkPositions($repository);
44 PhabricatorRepositoryRefCursor
::TYPE_BRANCH
=> $branches,
45 PhabricatorRepositoryRefCursor
::TYPE_BOOKMARK
=> $bookmarks,
48 $branches_may_close = true;
50 case PhabricatorRepositoryType
::REPOSITORY_TYPE_GIT
:
51 $maps = $this->loadGitRefPositions($repository);
54 throw new Exception(pht('Unknown VCS "%s"!', $vcs));
57 // Fill in any missing types with empty lists.
58 $maps = $maps +
array(
59 PhabricatorRepositoryRefCursor
::TYPE_BRANCH
=> array(),
60 PhabricatorRepositoryRefCursor
::TYPE_TAG
=> array(),
61 PhabricatorRepositoryRefCursor
::TYPE_BOOKMARK
=> array(),
62 PhabricatorRepositoryRefCursor
::TYPE_REF
=> array(),
65 $all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
67 ->withRepositoryPHIDs(array($repository->getPHID()))
70 $cursor_groups = mgroup($all_cursors, 'getRefType');
72 // Find all the heads of permanent refs.
73 $all_closing_heads = array();
74 foreach ($all_cursors as $cursor) {
76 // See T13284. Note that we're considering whether this ref was a
77 // permanent ref or not the last time we updated refs for this
78 // repository. This allows us to handle things properly when a ref
79 // is reconfigured from non-permanent to permanent.
81 $was_permanent = $cursor->getIsPermanent();
82 if (!$was_permanent) {
86 foreach ($cursor->getPositionIdentifiers() as $identifier) {
87 $all_closing_heads[] = $identifier;
91 $all_closing_heads = array_unique($all_closing_heads);
92 $all_closing_heads = $this->removeMissingCommits($all_closing_heads);
94 foreach ($maps as $type => $refs) {
95 $cursor_group = idx($cursor_groups, $type, array());
96 $this->updateCursors($cursor_group, $refs, $type, $all_closing_heads);
99 if ($this->permanentCommits
) {
100 $this->setPermanentFlagOnCommits($this->permanentCommits
);
103 $save_cursors = $this->getCursorsForUpdate($repository, $all_cursors);
105 if ($this->newPositions ||
$this->deadPositions ||
$save_cursors) {
106 $repository->openTransaction();
108 $this->saveNewPositions();
109 $this->deleteDeadPositions();
111 foreach ($save_cursors as $cursor) {
115 $repository->saveTransaction();
118 $branches = $maps[PhabricatorRepositoryRefCursor
::TYPE_BRANCH
];
119 if ($branches && $branches_may_close) {
120 $this->updateBranchStates($repository, $branches);
124 private function getCursorsForUpdate(
125 PhabricatorRepository
$repository,
127 assert_instances_of($cursors, 'PhabricatorRepositoryRefCursor');
129 $publisher = $repository->newPublisher();
133 foreach ($cursors as $cursor) {
134 $diffusion_ref = $cursor->newDiffusionRepositoryRef();
136 $is_permanent = $publisher->isPermanentRef($diffusion_ref);
137 if ($is_permanent == $cursor->getIsPermanent()) {
141 $cursor->setIsPermanent((int)$is_permanent);
142 $results[] = $cursor;
148 private function updateBranchStates(
149 PhabricatorRepository
$repository,
152 assert_instances_of($branches, 'DiffusionRepositoryRef');
153 $viewer = $this->getViewer();
155 $all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
157 ->withRepositoryPHIDs(array($repository->getPHID()))
158 ->needPositions(true)
161 $state_map = array();
162 $type_branch = PhabricatorRepositoryRefCursor
::TYPE_BRANCH
;
163 foreach ($all_cursors as $cursor) {
164 if ($cursor->getRefType() !== $type_branch) {
167 $raw_name = $cursor->getRefNameRaw();
169 foreach ($cursor->getPositions() as $position) {
170 $hash = $position->getCommitIdentifier();
171 $state_map[$raw_name][$hash] = $position;
176 foreach ($branches as $branch) {
177 $position = idx($state_map, $branch->getShortName(), array());
178 $position = idx($position, $branch->getCommitIdentifier());
183 $fields = $branch->getRawFields();
185 $position_state = (bool)$position->getIsClosed();
186 $branch_state = (bool)idx($fields, 'closed');
188 if ($position_state != $branch_state) {
189 $updates[$position->getID()] = (int)$branch_state;
194 $position_table = id(new PhabricatorRepositoryRefPosition());
195 $conn = $position_table->establishConnection('w');
197 $position_table->openTransaction();
198 foreach ($updates as $position_id => $branch_state) {
201 'UPDATE %T SET isClosed = %d WHERE id = %d',
202 $position_table->getTableName(),
206 $position_table->saveTransaction();
210 private function markPositionNew(
211 PhabricatorRepositoryRefPosition
$position) {
212 $this->newPositions
[] = $position;
216 private function markPositionDead(
217 PhabricatorRepositoryRefPosition
$position) {
218 $this->deadPositions
[] = $position;
222 private function markPermanentCommits(array $identifiers) {
223 foreach ($identifiers as $identifier) {
224 $this->permanentCommits
[$identifier] = $identifier;
230 * Remove commits which no longer exist in the repository from a list.
232 * After a force push and garbage collection, we may have branch cursors which
233 * point at commits which no longer exist. This can make commands issued later
234 * fail. See T5839 for discussion.
236 * @param list<string> List of commit identifiers.
237 * @return list<string> List with nonexistent identifiers removed.
239 private function removeMissingCommits(array $identifiers) {
244 $resolved = id(new DiffusionLowLevelResolveRefsQuery())
245 ->setRepository($this->getRepository())
246 ->withRefs($identifiers)
249 foreach ($identifiers as $key => $identifier) {
250 if (empty($resolved[$identifier])) {
251 unset($identifiers[$key]);
258 private function updateCursors(
262 array $all_closing_heads) {
263 $repository = $this->getRepository();
264 $publisher = $repository->newPublisher();
266 // NOTE: Mercurial branches may have multiple branch heads; this logic
267 // is complex primarily to account for that.
269 $cursors = mpull($cursors, null, 'getRefNameRaw');
271 // Group all the new ref values by their name. As above, these groups may
272 // have multiple members in Mercurial.
273 $ref_groups = mgroup($new_refs, 'getShortName');
275 foreach ($ref_groups as $name => $refs) {
276 $new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
278 $ref_cursor = idx($cursors, $name);
280 $old_positions = $ref_cursor->getPositions();
282 $old_positions = array();
285 // We're going to delete all the cursors pointing at commits which are
286 // no longer associated with the refs. This primarily makes the Mercurial
287 // multiple head case easier, and means that when we update a ref we
288 // delete the old one and write a new one.
289 foreach ($old_positions as $old_position) {
290 $hash = $old_position->getCommitIdentifier();
291 if (isset($new_commits[$hash])) {
292 // This ref previously pointed at this commit, and still does.
295 'Ref %s "%s" still points at %s.',
302 // This ref previously pointed at this commit, but no longer does.
305 'Ref %s "%s" no longer points at %s.',
310 // Nuke the obsolete cursor.
311 $this->markPositionDead($old_position);
314 // Now, we're going to insert new cursors for all the commits which are
315 // associated with this ref that don't currently have cursors.
316 $old_commits = mpull($old_positions, 'getCommitIdentifier');
317 $old_commits = array_fuse($old_commits);
319 $added_commits = array_diff_key($new_commits, $old_commits);
320 foreach ($added_commits as $identifier) {
323 'Ref %s "%s" now points at %s.',
329 // If this is the first time we've seen a particular ref (for
330 // example, a new branch) we need to insert a RefCursor record
331 // for it before we can insert a RefPosition.
333 $ref_cursor = $this->newRefCursor(
339 $new_position = id(new PhabricatorRepositoryRefPosition())
340 ->setCursorID($ref_cursor->getID())
341 ->setCommitIdentifier($identifier)
344 $this->markPositionNew($new_position);
347 if ($publisher->isPermanentRef(head($refs))) {
349 // See T13284. If this cursor was already marked as permanent, we
350 // only need to publish the newly created ref positions. However, if
351 // this cursor was not previously permanent but has become permanent,
352 // we need to publish all the ref positions.
354 // This corresponds to users reconfiguring a branch to make it
355 // permanent without pushing any new commits to it.
357 $is_rebuild = $this->getRebuild();
358 $was_permanent = $ref_cursor->getIsPermanent();
360 if ($is_rebuild ||
!$was_permanent) {
367 $update_commits = $new_commits;
369 $update_commits = $added_commits;
375 $exclude = $all_closing_heads;
378 foreach ($update_commits as $identifier) {
379 $new_identifiers = $this->loadNewCommitIdentifiers(
383 $this->markPermanentCommits($new_identifiers);
388 // Find any cursors for refs which no longer exist. This happens when a
389 // branch, tag or bookmark is deleted.
391 foreach ($cursors as $name => $cursor) {
392 if (!empty($ref_groups[$name])) {
393 // This ref still has some positions, so we don't need to wipe it
394 // out. Try the next one.
398 foreach ($cursor->getPositions() as $position) {
401 'Ref %s "%s" no longer exists.',
402 $cursor->getRefType(),
403 $cursor->getRefName()));
405 $this->markPositionDead($position);
411 * Find all ancestors of a new closing branch head which are not ancestors
412 * of any old closing branch head.
414 private function loadNewCommitIdentifiers(
416 array $all_closing_heads) {
418 $repository = $this->getRepository();
419 $vcs = $repository->getVersionControlSystem();
421 case PhabricatorRepositoryType
::REPOSITORY_TYPE_MERCURIAL
:
422 if ($all_closing_heads) {
424 foreach ($all_closing_heads as $head) {
425 $parts[] = hgsprintf('%s', $head);
428 // See T5896. Mercurial can not parse an "X or Y or ..." rev list
429 // with more than about 300 items, because it exceeds the maximum
430 // allowed recursion depth. Split all the heads into chunks of
431 // 256, and build a query like this:
433 // ((1 or 2 or ... or 255) or (256 or 257 or ... 511))
435 // If we have more than 65535 heads, we'll do that again:
437 // (((1 or ...) or ...) or ((65536 or ...) or ...))
440 while (count($parts) > $chunk_size) {
441 $chunks = array_chunk($parts, $chunk_size);
442 foreach ($chunks as $key => $chunk) {
443 $chunks[$key] = '('.implode(' or ', $chunk).')';
445 $parts = array_values($chunks);
447 $parts = '('.implode(' or ', $parts).')';
449 list($stdout) = $this->getRepository()->execxLocalCommand(
450 'log --template %s --rev %s',
452 hgsprintf('%s', $new_head).' - '.$parts);
454 list($stdout) = $this->getRepository()->execxLocalCommand(
455 'log --template %s --rev %s',
457 hgsprintf('%s', $new_head));
460 $stdout = trim($stdout);
461 if (!strlen($stdout)) {
464 return phutil_split_lines($stdout, $retain_newlines = false);
465 case PhabricatorRepositoryType
::REPOSITORY_TYPE_GIT
:
466 if ($all_closing_heads) {
468 // See PHI1474. This length of list may exceed the maximum size of
469 // a command line argument list, so pipe the list in using "--stdin"
473 $ref_list[] = $new_head;
474 foreach ($all_closing_heads as $old_head) {
475 $ref_list[] = '^'.$old_head;
478 $ref_list = implode("\n", $ref_list)."\n";
480 $future = $this->getRepository()->getLocalCommandFuture(
484 list($stdout) = $future
488 list($stdout) = $this->getRepository()->execxLocalCommand(
491 gitsprintf('%s', $new_head));
494 $stdout = trim($stdout);
495 if (!strlen($stdout)) {
498 return phutil_split_lines($stdout, $retain_newlines = false);
500 throw new Exception(pht('Unsupported VCS "%s"!', $vcs));
505 * Mark a list of commits as permanent, and queue workers for those commits
506 * which don't already have the flag.
508 private function setPermanentFlagOnCommits(array $identifiers) {
509 $repository = $this->getRepository();
510 $commit_table = new PhabricatorRepositoryCommit();
511 $conn = $commit_table->establishConnection('w');
513 $vcs = $repository->getVersionControlSystem();
515 case PhabricatorRepositoryType
::REPOSITORY_TYPE_GIT
:
516 $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
518 case PhabricatorRepositoryType
::REPOSITORY_TYPE_SVN
:
519 $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
521 case PhabricatorRepositoryType
::REPOSITORY_TYPE_MERCURIAL
:
522 $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
525 throw new Exception(pht("Unknown repository type '%s'!", $vcs));
528 $identifier_tokens = array();
529 foreach ($identifiers as $identifier) {
530 $identifier_tokens[] = qsprintf(
536 $all_commits = array();
537 foreach (PhabricatorLiskDAO
::chunkSQL($identifier_tokens) as $chunk) {
540 'SELECT id, phid, commitIdentifier, importStatus FROM %T
541 WHERE repositoryID = %d AND commitIdentifier IN (%LQ)',
542 $commit_table->getTableName(),
543 $repository->getID(),
545 foreach ($rows as $row) {
546 $all_commits[] = $row;
550 $commit_refs = array();
551 foreach ($identifiers as $identifier) {
553 // See T13591. This construction is a bit ad-hoc, but the priority
554 // function currently only cares about the number of refs we have
555 // discovered, so we'll get the right result even without filling
556 // these records out in detail.
558 $commit_refs[] = id(new PhabricatorRepositoryCommitRef())
559 ->setIdentifier($identifier);
562 $task_priority = $this->getImportTaskPriority(
566 $permanent_flag = PhabricatorRepositoryCommit
::IMPORTED_PERMANENT
;
567 $published_flag = PhabricatorRepositoryCommit
::IMPORTED_PUBLISH
;
569 $all_commits = ipull($all_commits, null, 'commitIdentifier');
570 foreach ($identifiers as $identifier) {
571 $row = idx($all_commits, $identifier);
576 'Commit "%s" has not been discovered yet! Run discovery before '.
581 $import_status = $row['importStatus'];
582 if (!($import_status & $permanent_flag)) {
583 // Set the "permanent" flag.
584 $import_status = ($import_status |
$permanent_flag);
586 // See T13580. Clear the "published" flag, so publishing executes
587 // again. We may have previously performed a no-op "publish" on the
588 // commit to make sure it has all bits in the "IMPORTED_ALL" bitmask.
589 $import_status = ($import_status & ~
$published_flag);
593 'UPDATE %T SET importStatus = %d WHERE id = %d',
594 $commit_table->getTableName(),
598 $this->queueCommitImportTask(
609 private function newRefCursor(
610 PhabricatorRepository
$repository,
614 $cursor = id(new PhabricatorRepositoryRefCursor())
615 ->setRepositoryPHID($repository->getPHID())
616 ->setRefType($ref_type)
617 ->setRefName($ref_name);
619 $publisher = $repository->newPublisher();
621 $diffusion_ref = $cursor->newDiffusionRepositoryRef();
622 $is_permanent = $publisher->isPermanentRef($diffusion_ref);
624 $cursor->setIsPermanent((int)$is_permanent);
627 return $cursor->save();
628 } catch (AphrontDuplicateKeyQueryException
$ex) {
629 // If we raced another daemon to create this position and lost the race,
630 // load the cursor the other daemon created instead.
633 $viewer = $this->getViewer();
635 $cursor = id(new PhabricatorRepositoryRefCursorQuery())
637 ->withRepositoryPHIDs(array($repository->getPHID()))
638 ->withRefTypes(array($ref_type))
639 ->withRefNames(array($ref_name))
640 ->needPositions(true)
645 'Failed to create a new ref cursor (for "%s", of type "%s", in '.
646 'repository "%s") because it collided with an existing cursor, '.
647 'but then failed to load that cursor.',
650 $repository->getDisplayName()));
656 private function saveNewPositions() {
657 $positions = $this->newPositions
;
659 foreach ($positions as $position) {
662 } catch (AphrontDuplicateKeyQueryException
$ex) {
663 // We may race another daemon to create this position. If we do, and
664 // we lose the race, that's fine: the other daemon did our work for
665 // us and we can continue.
669 $this->newPositions
= array();
672 private function deleteDeadPositions() {
673 $positions = $this->deadPositions
;
674 $repository = $this->getRepository();
676 foreach ($positions as $position) {
677 // Shove this ref into the old refs table so the discovery engine
678 // can check if any commits have been rendered unreachable.
679 id(new PhabricatorRepositoryOldRef())
680 ->setRepositoryPHID($repository->getPHID())
681 ->setCommitIdentifier($position->getCommitIdentifier())
687 $this->deadPositions
= array();
692 /* -( Updating Git Refs )-------------------------------------------------- */
698 private function loadGitRefPositions(PhabricatorRepository
$repository) {
699 $refs = id(new DiffusionLowLevelGitRefQuery())
700 ->setRepository($repository)
703 return mgroup($refs, 'getRefType');
707 /* -( Updating Mercurial Refs )-------------------------------------------- */
713 private function loadMercurialBranchPositions(
714 PhabricatorRepository
$repository) {
715 return id(new DiffusionLowLevelMercurialBranchesQuery())
716 ->setRepository($repository)
724 private function loadMercurialBookmarkPositions(
725 PhabricatorRepository
$repository) {
726 // TODO: Implement support for Mercurial bookmarks.