Correct Aphlict websocket URI construction after PHP8 compatibility changes
[phabricator.git] / src / applications / repository / engine / PhabricatorRepositoryRefEngine.php
blob60a96578a3ed00854b4c84f697408a5761d3ca19
1 <?php
3 /**
4 * Update the ref cursors for a repository, which track the positions of
5 * branches, bookmarks, and tags.
6 */
7 final class PhabricatorRepositoryRefEngine
8 extends PhabricatorRepositoryEngine {
10 private $newPositions = array();
11 private $deadPositions = array();
12 private $permanentCommits = array();
13 private $rebuild;
15 public function setRebuild($rebuild) {
16 $this->rebuild = $rebuild;
17 return $this;
20 public function getRebuild() {
21 return $this->rebuild;
24 public function updateRefs() {
25 $this->newPositions = array();
26 $this->deadPositions = array();
27 $this->permanentCommits = array();
29 $repository = $this->getRepository();
30 $viewer = $this->getViewer();
32 $branches_may_close = false;
34 $vcs = $repository->getVersionControlSystem();
35 switch ($vcs) {
36 case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
37 // No meaningful refs of any type in Subversion.
38 $maps = array();
39 break;
40 case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
41 $branches = $this->loadMercurialBranchPositions($repository);
42 $bookmarks = $this->loadMercurialBookmarkPositions($repository);
43 $maps = array(
44 PhabricatorRepositoryRefCursor::TYPE_BRANCH => $branches,
45 PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => $bookmarks,
48 $branches_may_close = true;
49 break;
50 case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
51 $maps = $this->loadGitRefPositions($repository);
52 break;
53 default:
54 throw new Exception(pht('Unknown VCS "%s"!', $vcs));
57 // Fill in any missing types with empty lists.
58 $maps = $maps + array(
59 PhabricatorRepositoryRefCursor::TYPE_BRANCH => array(),
60 PhabricatorRepositoryRefCursor::TYPE_TAG => array(),
61 PhabricatorRepositoryRefCursor::TYPE_BOOKMARK => array(),
62 PhabricatorRepositoryRefCursor::TYPE_REF => array(),
65 $all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
66 ->setViewer($viewer)
67 ->withRepositoryPHIDs(array($repository->getPHID()))
68 ->needPositions(true)
69 ->execute();
70 $cursor_groups = mgroup($all_cursors, 'getRefType');
72 // Find all the heads of permanent refs.
73 $all_closing_heads = array();
74 foreach ($all_cursors as $cursor) {
76 // See T13284. Note that we're considering whether this ref was a
77 // permanent ref or not the last time we updated refs for this
78 // repository. This allows us to handle things properly when a ref
79 // is reconfigured from non-permanent to permanent.
81 $was_permanent = $cursor->getIsPermanent();
82 if (!$was_permanent) {
83 continue;
86 foreach ($cursor->getPositionIdentifiers() as $identifier) {
87 $all_closing_heads[] = $identifier;
91 $all_closing_heads = array_unique($all_closing_heads);
92 $all_closing_heads = $this->removeMissingCommits($all_closing_heads);
94 foreach ($maps as $type => $refs) {
95 $cursor_group = idx($cursor_groups, $type, array());
96 $this->updateCursors($cursor_group, $refs, $type, $all_closing_heads);
99 if ($this->permanentCommits) {
100 $this->setPermanentFlagOnCommits($this->permanentCommits);
103 $save_cursors = $this->getCursorsForUpdate($repository, $all_cursors);
105 if ($this->newPositions || $this->deadPositions || $save_cursors) {
106 $repository->openTransaction();
108 $this->saveNewPositions();
109 $this->deleteDeadPositions();
111 foreach ($save_cursors as $cursor) {
112 $cursor->save();
115 $repository->saveTransaction();
118 $branches = $maps[PhabricatorRepositoryRefCursor::TYPE_BRANCH];
119 if ($branches && $branches_may_close) {
120 $this->updateBranchStates($repository, $branches);
124 private function getCursorsForUpdate(
125 PhabricatorRepository $repository,
126 array $cursors) {
127 assert_instances_of($cursors, 'PhabricatorRepositoryRefCursor');
129 $publisher = $repository->newPublisher();
131 $results = array();
133 foreach ($cursors as $cursor) {
134 $diffusion_ref = $cursor->newDiffusionRepositoryRef();
136 $is_permanent = $publisher->isPermanentRef($diffusion_ref);
137 if ($is_permanent == $cursor->getIsPermanent()) {
138 continue;
141 $cursor->setIsPermanent((int)$is_permanent);
142 $results[] = $cursor;
145 return $results;
148 private function updateBranchStates(
149 PhabricatorRepository $repository,
150 array $branches) {
152 assert_instances_of($branches, 'DiffusionRepositoryRef');
153 $viewer = $this->getViewer();
155 $all_cursors = id(new PhabricatorRepositoryRefCursorQuery())
156 ->setViewer($viewer)
157 ->withRepositoryPHIDs(array($repository->getPHID()))
158 ->needPositions(true)
159 ->execute();
161 $state_map = array();
162 $type_branch = PhabricatorRepositoryRefCursor::TYPE_BRANCH;
163 foreach ($all_cursors as $cursor) {
164 if ($cursor->getRefType() !== $type_branch) {
165 continue;
167 $raw_name = $cursor->getRefNameRaw();
169 foreach ($cursor->getPositions() as $position) {
170 $hash = $position->getCommitIdentifier();
171 $state_map[$raw_name][$hash] = $position;
175 $updates = array();
176 foreach ($branches as $branch) {
177 $position = idx($state_map, $branch->getShortName(), array());
178 $position = idx($position, $branch->getCommitIdentifier());
179 if (!$position) {
180 continue;
183 $fields = $branch->getRawFields();
185 $position_state = (bool)$position->getIsClosed();
186 $branch_state = (bool)idx($fields, 'closed');
188 if ($position_state != $branch_state) {
189 $updates[$position->getID()] = (int)$branch_state;
193 if ($updates) {
194 $position_table = id(new PhabricatorRepositoryRefPosition());
195 $conn = $position_table->establishConnection('w');
197 $position_table->openTransaction();
198 foreach ($updates as $position_id => $branch_state) {
199 queryfx(
200 $conn,
201 'UPDATE %T SET isClosed = %d WHERE id = %d',
202 $position_table->getTableName(),
203 $branch_state,
204 $position_id);
206 $position_table->saveTransaction();
210 private function markPositionNew(
211 PhabricatorRepositoryRefPosition $position) {
212 $this->newPositions[] = $position;
213 return $this;
216 private function markPositionDead(
217 PhabricatorRepositoryRefPosition $position) {
218 $this->deadPositions[] = $position;
219 return $this;
222 private function markPermanentCommits(array $identifiers) {
223 foreach ($identifiers as $identifier) {
224 $this->permanentCommits[$identifier] = $identifier;
226 return $this;
230 * Remove commits which no longer exist in the repository from a list.
232 * After a force push and garbage collection, we may have branch cursors which
233 * point at commits which no longer exist. This can make commands issued later
234 * fail. See T5839 for discussion.
236 * @param list<string> List of commit identifiers.
237 * @return list<string> List with nonexistent identifiers removed.
239 private function removeMissingCommits(array $identifiers) {
240 if (!$identifiers) {
241 return array();
244 $resolved = id(new DiffusionLowLevelResolveRefsQuery())
245 ->setRepository($this->getRepository())
246 ->withRefs($identifiers)
247 ->execute();
249 foreach ($identifiers as $key => $identifier) {
250 if (empty($resolved[$identifier])) {
251 unset($identifiers[$key]);
255 return $identifiers;
258 private function updateCursors(
259 array $cursors,
260 array $new_refs,
261 $ref_type,
262 array $all_closing_heads) {
263 $repository = $this->getRepository();
264 $publisher = $repository->newPublisher();
266 // NOTE: Mercurial branches may have multiple branch heads; this logic
267 // is complex primarily to account for that.
269 $cursors = mpull($cursors, null, 'getRefNameRaw');
271 // Group all the new ref values by their name. As above, these groups may
272 // have multiple members in Mercurial.
273 $ref_groups = mgroup($new_refs, 'getShortName');
275 foreach ($ref_groups as $name => $refs) {
276 $new_commits = mpull($refs, 'getCommitIdentifier', 'getCommitIdentifier');
278 $ref_cursor = idx($cursors, $name);
279 if ($ref_cursor) {
280 $old_positions = $ref_cursor->getPositions();
281 } else {
282 $old_positions = array();
285 // We're going to delete all the cursors pointing at commits which are
286 // no longer associated with the refs. This primarily makes the Mercurial
287 // multiple head case easier, and means that when we update a ref we
288 // delete the old one and write a new one.
289 foreach ($old_positions as $old_position) {
290 $hash = $old_position->getCommitIdentifier();
291 if (isset($new_commits[$hash])) {
292 // This ref previously pointed at this commit, and still does.
293 $this->log(
294 pht(
295 'Ref %s "%s" still points at %s.',
296 $ref_type,
297 $name,
298 $hash));
299 continue;
302 // This ref previously pointed at this commit, but no longer does.
303 $this->log(
304 pht(
305 'Ref %s "%s" no longer points at %s.',
306 $ref_type,
307 $name,
308 $hash));
310 // Nuke the obsolete cursor.
311 $this->markPositionDead($old_position);
314 // Now, we're going to insert new cursors for all the commits which are
315 // associated with this ref that don't currently have cursors.
316 $old_commits = mpull($old_positions, 'getCommitIdentifier');
317 $old_commits = array_fuse($old_commits);
319 $added_commits = array_diff_key($new_commits, $old_commits);
320 foreach ($added_commits as $identifier) {
321 $this->log(
322 pht(
323 'Ref %s "%s" now points at %s.',
324 $ref_type,
325 $name,
326 $identifier));
328 if (!$ref_cursor) {
329 // If this is the first time we've seen a particular ref (for
330 // example, a new branch) we need to insert a RefCursor record
331 // for it before we can insert a RefPosition.
333 $ref_cursor = $this->newRefCursor(
334 $repository,
335 $ref_type,
336 $name);
339 $new_position = id(new PhabricatorRepositoryRefPosition())
340 ->setCursorID($ref_cursor->getID())
341 ->setCommitIdentifier($identifier)
342 ->setIsClosed(0);
344 $this->markPositionNew($new_position);
347 if ($publisher->isPermanentRef(head($refs))) {
349 // See T13284. If this cursor was already marked as permanent, we
350 // only need to publish the newly created ref positions. However, if
351 // this cursor was not previously permanent but has become permanent,
352 // we need to publish all the ref positions.
354 // This corresponds to users reconfiguring a branch to make it
355 // permanent without pushing any new commits to it.
357 $is_rebuild = $this->getRebuild();
358 $was_permanent = $ref_cursor->getIsPermanent();
360 if ($is_rebuild || !$was_permanent) {
361 $update_all = true;
362 } else {
363 $update_all = false;
366 if ($update_all) {
367 $update_commits = $new_commits;
368 } else {
369 $update_commits = $added_commits;
372 if ($is_rebuild) {
373 $exclude = array();
374 } else {
375 $exclude = $all_closing_heads;
378 foreach ($update_commits as $identifier) {
379 $new_identifiers = $this->loadNewCommitIdentifiers(
380 $identifier,
381 $exclude);
383 $this->markPermanentCommits($new_identifiers);
388 // Find any cursors for refs which no longer exist. This happens when a
389 // branch, tag or bookmark is deleted.
391 foreach ($cursors as $name => $cursor) {
392 if (!empty($ref_groups[$name])) {
393 // This ref still has some positions, so we don't need to wipe it
394 // out. Try the next one.
395 continue;
398 foreach ($cursor->getPositions() as $position) {
399 $this->log(
400 pht(
401 'Ref %s "%s" no longer exists.',
402 $cursor->getRefType(),
403 $cursor->getRefName()));
405 $this->markPositionDead($position);
411 * Find all ancestors of a new closing branch head which are not ancestors
412 * of any old closing branch head.
414 private function loadNewCommitIdentifiers(
415 $new_head,
416 array $all_closing_heads) {
418 $repository = $this->getRepository();
419 $vcs = $repository->getVersionControlSystem();
420 switch ($vcs) {
421 case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
422 if ($all_closing_heads) {
423 $parts = array();
424 foreach ($all_closing_heads as $head) {
425 $parts[] = hgsprintf('%s', $head);
428 // See T5896. Mercurial can not parse an "X or Y or ..." rev list
429 // with more than about 300 items, because it exceeds the maximum
430 // allowed recursion depth. Split all the heads into chunks of
431 // 256, and build a query like this:
433 // ((1 or 2 or ... or 255) or (256 or 257 or ... 511))
435 // If we have more than 65535 heads, we'll do that again:
437 // (((1 or ...) or ...) or ((65536 or ...) or ...))
439 $chunk_size = 256;
440 while (count($parts) > $chunk_size) {
441 $chunks = array_chunk($parts, $chunk_size);
442 foreach ($chunks as $key => $chunk) {
443 $chunks[$key] = '('.implode(' or ', $chunk).')';
445 $parts = array_values($chunks);
447 $parts = '('.implode(' or ', $parts).')';
449 list($stdout) = $this->getRepository()->execxLocalCommand(
450 'log --template %s --rev %s',
451 '{node}\n',
452 hgsprintf('%s', $new_head).' - '.$parts);
453 } else {
454 list($stdout) = $this->getRepository()->execxLocalCommand(
455 'log --template %s --rev %s',
456 '{node}\n',
457 hgsprintf('%s', $new_head));
460 $stdout = trim($stdout);
461 if (!strlen($stdout)) {
462 return array();
464 return phutil_split_lines($stdout, $retain_newlines = false);
465 case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
466 if ($all_closing_heads) {
468 // See PHI1474. This length of list may exceed the maximum size of
469 // a command line argument list, so pipe the list in using "--stdin"
470 // instead.
472 $ref_list = array();
473 $ref_list[] = $new_head;
474 foreach ($all_closing_heads as $old_head) {
475 $ref_list[] = '^'.$old_head;
477 $ref_list[] = '--';
478 $ref_list = implode("\n", $ref_list)."\n";
480 $future = $this->getRepository()->getLocalCommandFuture(
481 'log %s --stdin --',
482 '--format=%H');
484 list($stdout) = $future
485 ->write($ref_list)
486 ->resolvex();
487 } else {
488 list($stdout) = $this->getRepository()->execxLocalCommand(
489 'log %s %s --',
490 '--format=%H',
491 gitsprintf('%s', $new_head));
494 $stdout = trim($stdout);
495 if (!strlen($stdout)) {
496 return array();
498 return phutil_split_lines($stdout, $retain_newlines = false);
499 default:
500 throw new Exception(pht('Unsupported VCS "%s"!', $vcs));
505 * Mark a list of commits as permanent, and queue workers for those commits
506 * which don't already have the flag.
508 private function setPermanentFlagOnCommits(array $identifiers) {
509 $repository = $this->getRepository();
510 $commit_table = new PhabricatorRepositoryCommit();
511 $conn = $commit_table->establishConnection('w');
513 $vcs = $repository->getVersionControlSystem();
514 switch ($vcs) {
515 case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
516 $class = 'PhabricatorRepositoryGitCommitMessageParserWorker';
517 break;
518 case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN:
519 $class = 'PhabricatorRepositorySvnCommitMessageParserWorker';
520 break;
521 case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL:
522 $class = 'PhabricatorRepositoryMercurialCommitMessageParserWorker';
523 break;
524 default:
525 throw new Exception(pht("Unknown repository type '%s'!", $vcs));
528 $identifier_tokens = array();
529 foreach ($identifiers as $identifier) {
530 $identifier_tokens[] = qsprintf(
531 $conn,
532 '%s',
533 $identifier);
536 $all_commits = array();
537 foreach (PhabricatorLiskDAO::chunkSQL($identifier_tokens) as $chunk) {
538 $rows = queryfx_all(
539 $conn,
540 'SELECT id, phid, commitIdentifier, importStatus FROM %T
541 WHERE repositoryID = %d AND commitIdentifier IN (%LQ)',
542 $commit_table->getTableName(),
543 $repository->getID(),
544 $chunk);
545 foreach ($rows as $row) {
546 $all_commits[] = $row;
550 $commit_refs = array();
551 foreach ($identifiers as $identifier) {
553 // See T13591. This construction is a bit ad-hoc, but the priority
554 // function currently only cares about the number of refs we have
555 // discovered, so we'll get the right result even without filling
556 // these records out in detail.
558 $commit_refs[] = id(new PhabricatorRepositoryCommitRef())
559 ->setIdentifier($identifier);
562 $task_priority = $this->getImportTaskPriority(
563 $repository,
564 $commit_refs);
566 $permanent_flag = PhabricatorRepositoryCommit::IMPORTED_PERMANENT;
567 $published_flag = PhabricatorRepositoryCommit::IMPORTED_PUBLISH;
569 $all_commits = ipull($all_commits, null, 'commitIdentifier');
570 foreach ($identifiers as $identifier) {
571 $row = idx($all_commits, $identifier);
573 if (!$row) {
574 throw new Exception(
575 pht(
576 'Commit "%s" has not been discovered yet! Run discovery before '.
577 'updating refs.',
578 $identifier));
581 $import_status = $row['importStatus'];
582 if (!($import_status & $permanent_flag)) {
583 // Set the "permanent" flag.
584 $import_status = ($import_status | $permanent_flag);
586 // See T13580. Clear the "published" flag, so publishing executes
587 // again. We may have previously performed a no-op "publish" on the
588 // commit to make sure it has all bits in the "IMPORTED_ALL" bitmask.
589 $import_status = ($import_status & ~$published_flag);
591 queryfx(
592 $conn,
593 'UPDATE %T SET importStatus = %d WHERE id = %d',
594 $commit_table->getTableName(),
595 $import_status,
596 $row['id']);
598 $this->queueCommitImportTask(
599 $repository,
600 $row['phid'],
601 $task_priority,
602 $via = 'ref');
606 return $this;
609 private function newRefCursor(
610 PhabricatorRepository $repository,
611 $ref_type,
612 $ref_name) {
614 $cursor = id(new PhabricatorRepositoryRefCursor())
615 ->setRepositoryPHID($repository->getPHID())
616 ->setRefType($ref_type)
617 ->setRefName($ref_name);
619 $publisher = $repository->newPublisher();
621 $diffusion_ref = $cursor->newDiffusionRepositoryRef();
622 $is_permanent = $publisher->isPermanentRef($diffusion_ref);
624 $cursor->setIsPermanent((int)$is_permanent);
626 try {
627 return $cursor->save();
628 } catch (AphrontDuplicateKeyQueryException $ex) {
629 // If we raced another daemon to create this position and lost the race,
630 // load the cursor the other daemon created instead.
633 $viewer = $this->getViewer();
635 $cursor = id(new PhabricatorRepositoryRefCursorQuery())
636 ->setViewer($viewer)
637 ->withRepositoryPHIDs(array($repository->getPHID()))
638 ->withRefTypes(array($ref_type))
639 ->withRefNames(array($ref_name))
640 ->needPositions(true)
641 ->executeOne();
642 if (!$cursor) {
643 throw new Exception(
644 pht(
645 'Failed to create a new ref cursor (for "%s", of type "%s", in '.
646 'repository "%s") because it collided with an existing cursor, '.
647 'but then failed to load that cursor.',
648 $ref_name,
649 $ref_type,
650 $repository->getDisplayName()));
653 return $cursor;
656 private function saveNewPositions() {
657 $positions = $this->newPositions;
659 foreach ($positions as $position) {
660 try {
661 $position->save();
662 } catch (AphrontDuplicateKeyQueryException $ex) {
663 // We may race another daemon to create this position. If we do, and
664 // we lose the race, that's fine: the other daemon did our work for
665 // us and we can continue.
669 $this->newPositions = array();
672 private function deleteDeadPositions() {
673 $positions = $this->deadPositions;
674 $repository = $this->getRepository();
676 foreach ($positions as $position) {
677 // Shove this ref into the old refs table so the discovery engine
678 // can check if any commits have been rendered unreachable.
679 id(new PhabricatorRepositoryOldRef())
680 ->setRepositoryPHID($repository->getPHID())
681 ->setCommitIdentifier($position->getCommitIdentifier())
682 ->save();
684 $position->delete();
687 $this->deadPositions = array();
692 /* -( Updating Git Refs )-------------------------------------------------- */
696 * @task git
698 private function loadGitRefPositions(PhabricatorRepository $repository) {
699 $refs = id(new DiffusionLowLevelGitRefQuery())
700 ->setRepository($repository)
701 ->execute();
703 return mgroup($refs, 'getRefType');
707 /* -( Updating Mercurial Refs )-------------------------------------------- */
711 * @task hg
713 private function loadMercurialBranchPositions(
714 PhabricatorRepository $repository) {
715 return id(new DiffusionLowLevelMercurialBranchesQuery())
716 ->setRepository($repository)
717 ->execute();
722 * @task hg
724 private function loadMercurialBookmarkPositions(
725 PhabricatorRepository $repository) {
726 // TODO: Implement support for Mercurial bookmarks.
727 return array();