src/applications/differential/parser/DifferentialChangesetParser.php

   1 <?php
   2
   3 final class DifferentialChangesetParser extends Phobject {
   4
   5   const HIGHLIGHT_BYTE_LIMIT = 262144;
   6
   7   protected $visible      = array();
   8   protected $new          = array();
   9   protected $old          = array();
  10   protected $intra        = array();
  11   protected $depthOnlyLines = array();
  12   protected $newRender    = null;
  13   protected $oldRender    = null;
  14
  15   protected $filename     = null;
  16   protected $hunkStartLines = array();
  17
  18   protected $comments     = array();
  19   protected $specialAttributes = array();
  20
  21   protected $changeset;
  22
  23   protected $renderCacheKey = null;
  24
  25   private $handles = array();
  26   private $user;
  27
  28   private $leftSideChangesetID;
  29   private $leftSideAttachesToNewFile;
  30
  31   private $rightSideChangesetID;
  32   private $rightSideAttachesToNewFile;
  33
  34   private $originalLeft;
  35   private $originalRight;
  36
  37   private $renderingReference;
  38   private $isSubparser;
  39
  40   private $isTopLevel;
  41
  42   private $coverage;
  43   private $markupEngine;
  44   private $highlightErrors;
  45   private $disableCache;
  46   private $renderer;
  47   private $highlightingDisabled;
  48   private $showEditAndReplyLinks = true;
  49   private $canMarkDone;
  50   private $objectOwnerPHID;
  51   private $offsetMode;
  52
  53   private $rangeStart;
  54   private $rangeEnd;
  55   private $mask;
  56   private $linesOfContext = 8;
  57
  58   private $highlightEngine;
  59   private $viewer;
  60
  61   private $viewState;
  62   private $availableDocumentEngines;
  63
  64   public function setRange($start, $end) {
  65     $this->rangeStart = $start;
  66     $this->rangeEnd = $end;
  67     return $this;
  68   }
  69
  70   public function setMask(array $mask) {
  71     $this->mask = $mask;
  72     return $this;
  73   }
  74
  75   public function renderChangeset() {
  76     return $this->render($this->rangeStart, $this->rangeEnd, $this->mask);
  77   }
  78
  79   public function setShowEditAndReplyLinks($bool) {
  80     $this->showEditAndReplyLinks = $bool;
  81     return $this;
  82   }
  83
  84   public function getShowEditAndReplyLinks() {
  85     return $this->showEditAndReplyLinks;
  86   }
  87
  88   public function setViewState(PhabricatorChangesetViewState $view_state) {
  89     $this->viewState = $view_state;
  90     return $this;
  91   }
  92
  93   public function getViewState() {
  94     return $this->viewState;
  95   }
  96
  97   public function setRenderer(DifferentialChangesetRenderer $renderer) {
  98     $this->renderer = $renderer;
  99     return $this;
 100   }
 101
 102   public function getRenderer() {
 103     return $this->renderer;
 104   }
 105
 106   public function setDisableCache($disable_cache) {
 107     $this->disableCache = $disable_cache;
 108     return $this;
 109   }
 110
 111   public function getDisableCache() {
 112     return $this->disableCache;
 113   }
 114
 115   public function setCanMarkDone($can_mark_done) {
 116     $this->canMarkDone = $can_mark_done;
 117     return $this;
 118   }
 119
 120   public function getCanMarkDone() {
 121     return $this->canMarkDone;
 122   }
 123
 124   public function setObjectOwnerPHID($phid) {
 125     $this->objectOwnerPHID = $phid;
 126     return $this;
 127   }
 128
 129   public function getObjectOwnerPHID() {
 130     return $this->objectOwnerPHID;
 131   }
 132
 133   public function setOffsetMode($offset_mode) {
 134     $this->offsetMode = $offset_mode;
 135     return $this;
 136   }
 137
 138   public function getOffsetMode() {
 139     return $this->offsetMode;
 140   }
 141
 142   public function setViewer(PhabricatorUser $viewer) {
 143     $this->viewer = $viewer;
 144     return $this;
 145   }
 146
 147   public function getViewer() {
 148     return $this->viewer;
 149   }
 150
 151   private function newRenderer() {
 152     $viewer = $this->getViewer();
 153     $viewstate = $this->getViewstate();
 154
 155     $renderer_key = $viewstate->getRendererKey();
 156
 157     if ($renderer_key === null) {
 158       $is_unified = $viewer->compareUserSetting(
 159         PhabricatorUnifiedDiffsSetting::SETTINGKEY,
 160         PhabricatorUnifiedDiffsSetting::VALUE_ALWAYS_UNIFIED);
 161
 162       if ($is_unified) {
 163         $renderer_key = '1up';
 164       } else {
 165         $renderer_key = $viewstate->getDefaultDeviceRendererKey();
 166       }
 167     }
 168
 169     switch ($renderer_key) {
 170       case '1up':
 171         $renderer = new DifferentialChangesetOneUpRenderer();
 172         break;
 173       default:
 174         $renderer = new DifferentialChangesetTwoUpRenderer();
 175         break;
 176     }
 177
 178     return $renderer;
 179   }
 180
 181   const CACHE_VERSION = 14;
 182   const CACHE_MAX_SIZE = 8e6;
 183
 184   const ATTR_GENERATED  = 'attr:generated';
 185   const ATTR_DELETED    = 'attr:deleted';
 186   const ATTR_UNCHANGED  = 'attr:unchanged';
 187   const ATTR_MOVEAWAY   = 'attr:moveaway';
 188
 189   public function setOldLines(array $lines) {
 190     $this->old = $lines;
 191     return $this;
 192   }
 193
 194   public function setNewLines(array $lines) {
 195     $this->new = $lines;
 196     return $this;
 197   }
 198
 199   public function setSpecialAttributes(array $attributes) {
 200     $this->specialAttributes = $attributes;
 201     return $this;
 202   }
 203
 204   public function setIntraLineDiffs(array $diffs) {
 205     $this->intra = $diffs;
 206     return $this;
 207   }
 208
 209   public function setDepthOnlyLines(array $lines) {
 210     $this->depthOnlyLines = $lines;
 211     return $this;
 212   }
 213
 214   public function getDepthOnlyLines() {
 215     return $this->depthOnlyLines;
 216   }
 217
 218   public function setVisibleLinesMask(array $mask) {
 219     $this->visible = $mask;
 220     return $this;
 221   }
 222
 223   public function setLinesOfContext($lines_of_context) {
 224     $this->linesOfContext = $lines_of_context;
 225     return $this;
 226   }
 227
 228   public function getLinesOfContext() {
 229     return $this->linesOfContext;
 230   }
 231
 232
 233   /**
 234    * Configure which Changeset comments added to the right side of the visible
 235    * diff will be attached to. The ID must be the ID of a real Differential
 236    * Changeset.
 237    *
 238    * The complexity here is that we may show an arbitrary side of an arbitrary
 239    * changeset as either the left or right part of a diff. This method allows
 240    * the left and right halves of the displayed diff to be correctly mapped to
 241    * storage changesets.
 242    *
 243    * @param id    The Differential Changeset ID that comments added to the right
 244    *              side of the visible diff should be attached to.
 245    * @param bool  If true, attach new comments to the right side of the storage
 246    *              changeset. Note that this may be false, if the left side of
 247    *              some storage changeset is being shown as the right side of
 248    *              a display diff.
 249    * @return this
 250    */
 251   public function setRightSideCommentMapping($id, $is_new) {
 252     $this->rightSideChangesetID = $id;
 253     $this->rightSideAttachesToNewFile = $is_new;
 254     return $this;
 255   }
 256
 257   /**
 258    * See setRightSideCommentMapping(), but this sets information for the left
 259    * side of the display diff.
 260    */
 261   public function setLeftSideCommentMapping($id, $is_new) {
 262     $this->leftSideChangesetID = $id;
 263     $this->leftSideAttachesToNewFile = $is_new;
 264     return $this;
 265   }
 266
 267   public function setOriginals(
 268     DifferentialChangeset $left,
 269     DifferentialChangeset $right) {
 270
 271     $this->originalLeft = $left;
 272     $this->originalRight = $right;
 273     return $this;
 274   }
 275
 276   public function diffOriginals() {
 277     $engine = new PhabricatorDifferenceEngine();
 278     $changeset = $engine->generateChangesetFromFileContent(
 279       implode('', mpull($this->originalLeft->getHunks(), 'getChanges')),
 280       implode('', mpull($this->originalRight->getHunks(), 'getChanges')));
 281
 282     $parser = new DifferentialHunkParser();
 283
 284     return $parser->parseHunksForHighlightMasks(
 285       $changeset->getHunks(),
 286       $this->originalLeft->getHunks(),
 287       $this->originalRight->getHunks());
 288   }
 289
 290   /**
 291    * Set a key for identifying this changeset in the render cache. If set, the
 292    * parser will attempt to use the changeset render cache, which can improve
 293    * performance for frequently-viewed changesets.
 294    *
 295    * By default, there is no render cache key and parsers do not use the cache.
 296    * This is appropriate for rarely-viewed changesets.
 297    *
 298    * @param   string  Key for identifying this changeset in the render cache.
 299    * @return  this
 300    */
 301   public function setRenderCacheKey($key) {
 302     $this->renderCacheKey = $key;
 303     return $this;
 304   }
 305
 306   private function getRenderCacheKey() {
 307     return $this->renderCacheKey;
 308   }
 309
 310   public function setChangeset(DifferentialChangeset $changeset) {
 311     $this->changeset = $changeset;
 312
 313     $this->setFilename($changeset->getFilename());
 314
 315     return $this;
 316   }
 317
 318   public function setRenderingReference($ref) {
 319     $this->renderingReference = $ref;
 320     return $this;
 321   }
 322
 323   private function getRenderingReference() {
 324     return $this->renderingReference;
 325   }
 326
 327   public function getChangeset() {
 328     return $this->changeset;
 329   }
 330
 331   public function setFilename($filename) {
 332     $this->filename = $filename;
 333     return $this;
 334   }
 335
 336   public function setHandles(array $handles) {
 337     assert_instances_of($handles, 'PhabricatorObjectHandle');
 338     $this->handles = $handles;
 339     return $this;
 340   }
 341
 342   public function setMarkupEngine(PhabricatorMarkupEngine $engine) {
 343     $this->markupEngine = $engine;
 344     return $this;
 345   }
 346
 347   public function setCoverage($coverage) {
 348     $this->coverage = $coverage;
 349     return $this;
 350   }
 351   private function getCoverage() {
 352     return $this->coverage;
 353   }
 354
 355   public function parseInlineComment(
 356     PhabricatorInlineComment $comment) {
 357
 358     // Parse only comments which are actually visible.
 359     if ($this->isCommentVisibleOnRenderedDiff($comment)) {
 360       $this->comments[] = $comment;
 361     }
 362     return $this;
 363   }
 364
 365   private function loadCache() {
 366     $render_cache_key = $this->getRenderCacheKey();
 367     if (!$render_cache_key) {
 368       return false;
 369     }
 370
 371     $data = null;
 372
 373     $changeset = new DifferentialChangeset();
 374     $conn_r = $changeset->establishConnection('r');
 375     $data = queryfx_one(
 376       $conn_r,
 377       'SELECT * FROM %T WHERE cacheIndex = %s',
 378       DifferentialChangeset::TABLE_CACHE,
 379       PhabricatorHash::digestForIndex($render_cache_key));
 380
 381     if (!$data) {
 382       return false;
 383     }
 384
 385     if ($data['cache'][0] == '{') {
 386       // This is likely an old-style JSON cache which we will not be able to
 387       // deserialize.
 388       return false;
 389     }
 390
 391     $data = unserialize($data['cache']);
 392     if (!is_array($data) || !$data) {
 393       return false;
 394     }
 395
 396     foreach (self::getCacheableProperties() as $cache_key) {
 397       if (!array_key_exists($cache_key, $data)) {
 398         // If we're missing a cache key, assume we're looking at an old cache
 399         // and ignore it.
 400         return false;
 401       }
 402     }
 403
 404     if ($data['cacheVersion'] !== self::CACHE_VERSION) {
 405       return false;
 406     }
 407
 408     // Someone displays contents of a partially cached shielded file.
 409     if (!isset($data['newRender']) && (!$this->isTopLevel || $this->comments)) {
 410       return false;
 411     }
 412
 413     unset($data['cacheVersion'], $data['cacheHost']);
 414     $cache_prop = array_select_keys($data, self::getCacheableProperties());
 415     foreach ($cache_prop as $cache_key => $v) {
 416       $this->$cache_key = $v;
 417     }
 418
 419     return true;
 420   }
 421
 422   protected static function getCacheableProperties() {
 423     return array(
 424       'visible',
 425       'new',
 426       'old',
 427       'intra',
 428       'depthOnlyLines',
 429       'newRender',
 430       'oldRender',
 431       'specialAttributes',
 432       'hunkStartLines',
 433       'cacheVersion',
 434       'cacheHost',
 435       'highlightingDisabled',
 436     );
 437   }
 438
 439   public function saveCache() {
 440     if (PhabricatorEnv::isReadOnly()) {
 441       return false;
 442     }
 443
 444     if ($this->highlightErrors) {
 445       return false;
 446     }
 447
 448     $render_cache_key = $this->getRenderCacheKey();
 449     if (!$render_cache_key) {
 450       return false;
 451     }
 452
 453     $cache = array();
 454     foreach (self::getCacheableProperties() as $cache_key) {
 455       switch ($cache_key) {
 456         case 'cacheVersion':
 457           $cache[$cache_key] = self::CACHE_VERSION;
 458           break;
 459         case 'cacheHost':
 460           $cache[$cache_key] = php_uname('n');
 461           break;
 462         default:
 463           $cache[$cache_key] = $this->$cache_key;
 464           break;
 465       }
 466     }
 467     $cache = serialize($cache);
 468
 469     // We don't want to waste too much space by a single changeset.
 470     if (strlen($cache) > self::CACHE_MAX_SIZE) {
 471       return;
 472     }
 473
 474     $changeset = new DifferentialChangeset();
 475     $conn_w = $changeset->establishConnection('w');
 476
 477     $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites();
 478       try {
 479         queryfx(
 480           $conn_w,
 481           'INSERT INTO %T (cacheIndex, cache, dateCreated) VALUES (%s, %B, %d)
 482             ON DUPLICATE KEY UPDATE cache = VALUES(cache)',
 483           DifferentialChangeset::TABLE_CACHE,
 484           PhabricatorHash::digestForIndex($render_cache_key),
 485           $cache,
 486           PhabricatorTime::getNow());
 487       } catch (AphrontQueryException $ex) {
 488         // Ignore these exceptions. A common cause is that the cache is
 489         // larger than 'max_allowed_packet', in which case we're better off
 490         // not writing it.
 491
 492         // TODO: It would be nice to tailor this more narrowly.
 493       }
 494     unset($unguarded);
 495   }
 496
 497   private function markGenerated($new_corpus_block = '') {
 498     $generated_guess = (strpos($new_corpus_block, '@'.'generated') !== false);
 499
 500     if (!$generated_guess) {
 501       $generated_path_regexps = PhabricatorEnv::getEnvConfig(
 502         'differential.generated-paths');
 503       foreach ($generated_path_regexps as $regexp) {
 504         if (preg_match($regexp, $this->changeset->getFilename())) {
 505           $generated_guess = true;
 506           break;
 507         }
 508       }
 509     }
 510
 511     $event = new PhabricatorEvent(
 512       PhabricatorEventType::TYPE_DIFFERENTIAL_WILLMARKGENERATED,
 513       array(
 514         'corpus' => $new_corpus_block,
 515         'is_generated' => $generated_guess,
 516       )
 517     );
 518     PhutilEventEngine::dispatchEvent($event);
 519
 520     $generated = $event->getValue('is_generated');
 521
 522     $attribute = $this->changeset->isGeneratedChangeset();
 523     if ($attribute) {
 524       $generated = true;
 525     }
 526
 527     $this->specialAttributes[self::ATTR_GENERATED] = $generated;
 528   }
 529
 530   public function isGenerated() {
 531     return idx($this->specialAttributes, self::ATTR_GENERATED, false);
 532   }
 533
 534   public function isDeleted() {
 535     return idx($this->specialAttributes, self::ATTR_DELETED, false);
 536   }
 537
 538   public function isUnchanged() {
 539     return idx($this->specialAttributes, self::ATTR_UNCHANGED, false);
 540   }
 541
 542   public function isMoveAway() {
 543     return idx($this->specialAttributes, self::ATTR_MOVEAWAY, false);
 544   }
 545
 546   private function applyIntraline(&$render, $intra, $corpus) {
 547
 548     foreach ($render as $key => $text) {
 549       $result = $text;
 550
 551       if (isset($intra[$key])) {
 552         $result = PhabricatorDifferenceEngine::applyIntralineDiff(
 553           $result,
 554           $intra[$key]);
 555       }
 556
 557       $result = $this->adjustRenderedLineForDisplay($result);
 558
 559       $render[$key] = $result;
 560     }
 561   }
 562
 563   private function getHighlightFuture($corpus) {
 564     $language = $this->getViewState()->getHighlightLanguage();
 565
 566     if (!$language) {
 567       $language = $this->highlightEngine->getLanguageFromFilename(
 568         $this->filename);
 569
 570       if (($language != 'txt') &&
 571           (strlen($corpus) > self::HIGHLIGHT_BYTE_LIMIT)) {
 572         $this->highlightingDisabled = true;
 573         $language = 'txt';
 574       }
 575     }
 576
 577     return $this->highlightEngine->getHighlightFuture(
 578       $language,
 579       $corpus);
 580   }
 581
 582   protected function processHighlightedSource($data, $result) {
 583
 584     $result_lines = phutil_split_lines($result);
 585     foreach ($data as $key => $info) {
 586       if (!$info) {
 587         unset($result_lines[$key]);
 588       }
 589     }
 590     return $result_lines;
 591   }
 592
 593   private function tryCacheStuff() {
 594     $changeset = $this->getChangeset();
 595     if (!$changeset->hasSourceTextBody()) {
 596
 597       // TODO: This isn't really correct (the change is not "generated"), the
 598       // intent is just to not render a text body for Subversion directory
 599       // changes, etc.
 600       $this->markGenerated();
 601
 602       return;
 603     }
 604
 605     $viewstate = $this->getViewState();
 606
 607     $skip_cache = false;
 608
 609     if ($this->disableCache) {
 610       $skip_cache = true;
 611     }
 612
 613     $character_encoding = $viewstate->getCharacterEncoding();
 614     if ($character_encoding !== null) {
 615       $skip_cache = true;
 616     }
 617
 618     $highlight_language = $viewstate->getHighlightLanguage();
 619     if ($highlight_language !== null) {
 620       $skip_cache = true;
 621     }
 622
 623     if ($skip_cache || !$this->loadCache()) {
 624       $this->process();
 625       if (!$skip_cache) {
 626         $this->saveCache();
 627       }
 628     }
 629   }
 630
 631   private function process() {
 632     $changeset = $this->changeset;
 633
 634     $hunk_parser = new DifferentialHunkParser();
 635     $hunk_parser->parseHunksForLineData($changeset->getHunks());
 636
 637     $this->realignDiff($changeset, $hunk_parser);
 638
 639     $hunk_parser->reparseHunksForSpecialAttributes();
 640
 641     $unchanged = false;
 642     if (!$hunk_parser->getHasAnyChanges()) {
 643       $filetype = $this->changeset->getFileType();
 644       if ($filetype == DifferentialChangeType::FILE_TEXT ||
 645           $filetype == DifferentialChangeType::FILE_SYMLINK) {
 646         $unchanged = true;
 647       }
 648     }
 649
 650     $moveaway = false;
 651     $changetype = $this->changeset->getChangeType();
 652     if ($changetype == DifferentialChangeType::TYPE_MOVE_AWAY) {
 653       $moveaway = true;
 654     }
 655
 656     $this->setSpecialAttributes(array(
 657       self::ATTR_UNCHANGED  => $unchanged,
 658       self::ATTR_DELETED    => $hunk_parser->getIsDeleted(),
 659       self::ATTR_MOVEAWAY   => $moveaway,
 660     ));
 661
 662     $lines_context = $this->getLinesOfContext();
 663
 664     $hunk_parser->generateIntraLineDiffs();
 665     $hunk_parser->generateVisibleLinesMask($lines_context);
 666
 667     $this->setOldLines($hunk_parser->getOldLines());
 668     $this->setNewLines($hunk_parser->getNewLines());
 669     $this->setIntraLineDiffs($hunk_parser->getIntraLineDiffs());
 670     $this->setDepthOnlyLines($hunk_parser->getDepthOnlyLines());
 671     $this->setVisibleLinesMask($hunk_parser->getVisibleLinesMask());
 672     $this->hunkStartLines = $hunk_parser->getHunkStartLines(
 673       $changeset->getHunks());
 674
 675     $new_corpus = $hunk_parser->getNewCorpus();
 676     $new_corpus_block = implode('', $new_corpus);
 677     $this->markGenerated($new_corpus_block);
 678
 679     if ($this->isTopLevel &&
 680         !$this->comments &&
 681           ($this->isGenerated() ||
 682            $this->isUnchanged() ||
 683            $this->isDeleted())) {
 684       return;
 685     }
 686
 687     $old_corpus = $hunk_parser->getOldCorpus();
 688     $old_corpus_block = implode('', $old_corpus);
 689     $old_future = $this->getHighlightFuture($old_corpus_block);
 690     $new_future = $this->getHighlightFuture($new_corpus_block);
 691     $futures = array(
 692       'old' => $old_future,
 693       'new' => $new_future,
 694     );
 695     $corpus_blocks = array(
 696       'old' => $old_corpus_block,
 697       'new' => $new_corpus_block,
 698     );
 699
 700     $this->highlightErrors = false;
 701     foreach (new FutureIterator($futures) as $key => $future) {
 702       try {
 703         try {
 704           $highlighted = $future->resolve();
 705         } catch (PhutilSyntaxHighlighterException $ex) {
 706           $this->highlightErrors = true;
 707           $highlighted = id(new PhutilDefaultSyntaxHighlighter())
 708             ->getHighlightFuture($corpus_blocks[$key])
 709             ->resolve();
 710         }
 711         switch ($key) {
 712         case 'old':
 713           $this->oldRender = $this->processHighlightedSource(
 714             $this->old,
 715             $highlighted);
 716           break;
 717         case 'new':
 718           $this->newRender = $this->processHighlightedSource(
 719             $this->new,
 720             $highlighted);
 721           break;
 722         }
 723       } catch (Exception $ex) {
 724         phlog($ex);
 725         throw $ex;
 726       }
 727     }
 728
 729     $this->applyIntraline(
 730       $this->oldRender,
 731       ipull($this->intra, 0),
 732       $old_corpus);
 733     $this->applyIntraline(
 734       $this->newRender,
 735       ipull($this->intra, 1),
 736       $new_corpus);
 737   }
 738
 739   private function shouldRenderPropertyChangeHeader($changeset) {
 740     if (!$this->isTopLevel) {
 741       // We render properties only at top level; otherwise we get multiple
 742       // copies of them when a user clicks "Show More".
 743       return false;
 744     }
 745
 746     return true;
 747   }
 748
 749   public function render(
 750     $range_start  = null,
 751     $range_len    = null,
 752     $mask_force   = array()) {
 753
 754     $viewer = $this->getViewer();
 755
 756     $renderer = $this->getRenderer();
 757     if (!$renderer) {
 758       $renderer = $this->newRenderer();
 759       $this->setRenderer($renderer);
 760     }
 761
 762     // "Top level" renders are initial requests for the whole file, versus
 763     // requests for a specific range generated by clicking "show more". We
 764     // generate property changes and "shield" UI elements only for toplevel
 765     // requests.
 766     $this->isTopLevel = (($range_start === null) && ($range_len === null));
 767     $this->highlightEngine = PhabricatorSyntaxHighlighter::newEngine();
 768
 769     $viewstate = $this->getViewState();
 770
 771     $encoding = null;
 772
 773     $character_encoding = $viewstate->getCharacterEncoding();
 774     if ($character_encoding) {
 775       // We are forcing this changeset to be interpreted with a specific
 776       // character encoding, so force all the hunks into that encoding and
 777       // propagate it to the renderer.
 778       $encoding = $character_encoding;
 779       foreach ($this->changeset->getHunks() as $hunk) {
 780         $hunk->forceEncoding($character_encoding);
 781       }
 782     } else {
 783       // We're just using the default, so tell the renderer what that is
 784       // (by reading the encoding from the first hunk).
 785       foreach ($this->changeset->getHunks() as $hunk) {
 786         $encoding = $hunk->getDataEncoding();
 787         break;
 788       }
 789     }
 790
 791     $this->tryCacheStuff();
 792
 793     // If we're rendering in an offset mode, treat the range numbers as line
 794     // numbers instead of rendering offsets.
 795     $offset_mode = $this->getOffsetMode();
 796     if ($offset_mode) {
 797       if ($offset_mode == 'new') {
 798         $offset_map = $this->new;
 799       } else {
 800         $offset_map = $this->old;
 801       }
 802
 803       // NOTE: Inline comments use zero-based lengths. For example, a comment
 804       // that starts and ends on line 123 has length 0. Rendering considers
 805       // this range to have length 1. Probably both should agree, but that
 806       // ship likely sailed long ago. Tweak things here to get the two systems
 807       // to agree. See PHI985, where this affected mail rendering of inline
 808       // comments left on the final line of a file.
 809
 810       $range_end = $this->getOffset($offset_map, $range_start + $range_len);
 811       $range_start = $this->getOffset($offset_map, $range_start);
 812       $range_len = ($range_end - $range_start) + 1;
 813     }
 814
 815     $render_pch = $this->shouldRenderPropertyChangeHeader($this->changeset);
 816
 817     $rows = max(
 818       count($this->old),
 819       count($this->new));
 820
 821     $renderer = $this->getRenderer()
 822       ->setUser($this->getViewer())
 823       ->setChangeset($this->changeset)
 824       ->setRenderPropertyChangeHeader($render_pch)
 825       ->setIsTopLevel($this->isTopLevel)
 826       ->setOldRender($this->oldRender)
 827       ->setNewRender($this->newRender)
 828       ->setHunkStartLines($this->hunkStartLines)
 829       ->setOldChangesetID($this->leftSideChangesetID)
 830       ->setNewChangesetID($this->rightSideChangesetID)
 831       ->setOldAttachesToNewFile($this->leftSideAttachesToNewFile)
 832       ->setNewAttachesToNewFile($this->rightSideAttachesToNewFile)
 833       ->setCodeCoverage($this->getCoverage())
 834       ->setRenderingReference($this->getRenderingReference())
 835       ->setHandles($this->handles)
 836       ->setOldLines($this->old)
 837       ->setNewLines($this->new)
 838       ->setOriginalCharacterEncoding($encoding)
 839       ->setShowEditAndReplyLinks($this->getShowEditAndReplyLinks())
 840       ->setCanMarkDone($this->getCanMarkDone())
 841       ->setObjectOwnerPHID($this->getObjectOwnerPHID())
 842       ->setHighlightingDisabled($this->highlightingDisabled)
 843       ->setDepthOnlyLines($this->getDepthOnlyLines());
 844
 845     if ($this->markupEngine) {
 846       $renderer->setMarkupEngine($this->markupEngine);
 847     }
 848
 849     list($engine, $old_ref, $new_ref) = $this->newDocumentEngine();
 850     if ($engine) {
 851       $engine_blocks = $engine->newEngineBlocks(
 852         $old_ref,
 853         $new_ref);
 854     } else {
 855       $engine_blocks = null;
 856     }
 857
 858     $has_document_engine = ($engine_blocks !== null);
 859
 860     // Remove empty comments that don't have any unsaved draft data.
 861     PhabricatorInlineComment::loadAndAttachVersionedDrafts(
 862       $viewer,
 863       $this->comments);
 864     foreach ($this->comments as $key => $comment) {
 865       if ($comment->isVoidComment($viewer)) {
 866         unset($this->comments[$key]);
 867       }
 868     }
 869
 870     // See T13515. Sometimes, we collapse file content by default: for
 871     // example, if the file is marked as containing generated code.
 872
 873     // If a file has inline comments, that normally means we never collapse
 874     // it. However, if the viewer has already collapsed all of the inlines,
 875     // it's fine to collapse the file.
 876
 877     $expanded_comments = array();
 878     foreach ($this->comments as $comment) {
 879       if ($comment->isHidden()) {
 880         continue;
 881       }
 882       $expanded_comments[] = $comment;
 883     }
 884
 885     $collapsed_count = (count($this->comments) - count($expanded_comments));
 886
 887     $shield_raw = null;
 888     $shield_text = null;
 889     $shield_type = null;
 890     if ($this->isTopLevel && !$expanded_comments && !$has_document_engine) {
 891       if ($this->isGenerated()) {
 892         $shield_text = pht(
 893           'This file contains generated code, which does not normally '.
 894           'need to be reviewed.');
 895       } else if ($this->isMoveAway()) {
 896         // We put an empty shield on these files. Normally, they do not have
 897         // any diff content anyway. However, if they come through `arc`, they
 898         // may have content. We don't want to show it (it's not useful) and
 899         // we bailed out of fully processing it earlier anyway.
 900
 901         // We could show a message like "this file was moved", but we show
 902         // that as a change header anyway, so it would be redundant. Instead,
 903         // just render an empty shield to skip rendering the diff body.
 904         $shield_raw = '';
 905       } else if ($this->isUnchanged()) {
 906         $type = 'text';
 907         if (!$rows) {
 908           // NOTE: Normally, diffs which don't change files do not include
 909           // file content (for example, if you "chmod +x" a file and then
 910           // run "git show", the file content is not available). Similarly,
 911           // if you move a file from A to B without changing it, diffs normally
 912           // do not show the file content. In some cases `arc` is able to
 913           // synthetically generate content for these diffs, but for raw diffs
 914           // we'll never have it so we need to be prepared to not render a link.
 915           $type = 'none';
 916         }
 917
 918         $shield_type = $type;
 919
 920         $type_add = DifferentialChangeType::TYPE_ADD;
 921         if ($this->changeset->getChangeType() == $type_add) {
 922           // Although the generic message is sort of accurate in a technical
 923           // sense, this more-tailored message is less confusing.
 924           $shield_text = pht('This is an empty file.');
 925         } else {
 926           $shield_text = pht('The contents of this file were not changed.');
 927         }
 928       } else if ($this->isDeleted()) {
 929         $shield_text = pht('This file was completely deleted.');
 930       } else if ($this->changeset->getAffectedLineCount() > 2500) {
 931         $shield_text = pht(
 932           'This file has a very large number of changes (%s lines).',
 933           new PhutilNumber($this->changeset->getAffectedLineCount()));
 934       }
 935     }
 936
 937     $shield = null;
 938     if ($shield_raw !== null) {
 939       $shield = $shield_raw;
 940     } else if ($shield_text !== null) {
 941       if ($shield_type === null) {
 942         $shield_type = 'default';
 943       }
 944
 945       // If we have inlines and the shield would normally show the whole file,
 946       // downgrade it to show only text around the inlines.
 947       if ($collapsed_count) {
 948         if ($shield_type === 'text') {
 949           $shield_type = 'default';
 950         }
 951
 952         $shield_text = array(
 953           $shield_text,
 954           ' ',
 955           pht(
 956             'This file has %d collapsed inline comment(s).',
 957             new PhutilNumber($collapsed_count)),
 958         );
 959       }
 960
 961       $shield = $renderer->renderShield($shield_text, $shield_type);
 962     }
 963
 964     if ($shield !== null) {
 965       return $renderer->renderChangesetTable($shield);
 966     }
 967
 968     // This request should render the "undershield" headers if it's a top-level
 969     // request which made it this far (indicating the changeset has no shield)
 970     // or it's a request with no mask information (indicating it's the request
 971     // that removes the rendering shield). Possibly, this second class of
 972     // request might need to be made more explicit.
 973     $is_undershield = (empty($mask_force) || $this->isTopLevel);
 974     $renderer->setIsUndershield($is_undershield);
 975
 976     $old_comments = array();
 977     $new_comments = array();
 978     $old_mask = array();
 979     $new_mask = array();
 980     $feedback_mask = array();
 981     $lines_context = $this->getLinesOfContext();
 982
 983     if ($this->comments) {
 984       // If there are any comments which appear in sections of the file which
 985       // we don't have, we're going to move them backwards to the closest
 986       // earlier line. Two cases where this may happen are:
 987       //
 988       //   - Porting ghost comments forward into a file which was mostly
 989       //     deleted.
 990       //   - Porting ghost comments forward from a full-context diff to a
 991       //     partial-context diff.
 992
 993       list($old_backmap, $new_backmap) = $this->buildLineBackmaps();
 994
 995       foreach ($this->comments as $comment) {
 996         $new_side = $this->isCommentOnRightSideWhenDisplayed($comment);
 997
 998         $line = $comment->getLineNumber();
 999
1000         // See T13524. Lint inlines from Harbormaster may not have a line
1001         // number.
1002         if ($line === null) {
1003           $back_line = null;
1004         } else if ($new_side) {
1005           $back_line = idx($new_backmap, $line);
1006         } else {
1007           $back_line = idx($old_backmap, $line);
1008         }
1009
1010         if ($back_line != $line) {
1011           // TODO: This should probably be cleaner, but just be simple and
1012           // obvious for now.
1013           $ghost = $comment->getIsGhost();
1014           if ($ghost) {
1015             $moved = pht(
1016               'This comment originally appeared on line %s, but that line '.
1017               'does not exist in this version of the diff. It has been '.
1018               'moved backward to the nearest line.',
1019               new PhutilNumber($line));
1020             $ghost['reason'] = $ghost['reason']."\n\n".$moved;
1021             $comment->setIsGhost($ghost);
1022           }
1023
1024           $comment->setLineNumber($back_line);
1025           $comment->setLineLength(0);
1026         }
1027
1028         $start = max($comment->getLineNumber() - $lines_context, 0);
1029         $end = $comment->getLineNumber() +
1030           $comment->getLineLength() +
1031           $lines_context;
1032         for ($ii = $start; $ii <= $end; $ii++) {
1033           if ($new_side) {
1034             $new_mask[$ii] = true;
1035           } else {
1036             $old_mask[$ii] = true;
1037           }
1038         }
1039       }
1040
1041       foreach ($this->old as $ii => $old) {
1042         if (isset($old['line']) && isset($old_mask[$old['line']])) {
1043           $feedback_mask[$ii] = true;
1044         }
1045       }
1046
1047       foreach ($this->new as $ii => $new) {
1048         if (isset($new['line']) && isset($new_mask[$new['line']])) {
1049           $feedback_mask[$ii] = true;
1050         }
1051       }
1052
1053       $this->comments = id(new PHUIDiffInlineThreader())
1054         ->reorderAndThreadCommments($this->comments);
1055
1056       $old_max_display = 1;
1057       foreach ($this->old as $old) {
1058         if (isset($old['line'])) {
1059           $old_max_display = $old['line'];
1060         }
1061       }
1062
1063       $new_max_display = 1;
1064       foreach ($this->new as $new) {
1065         if (isset($new['line'])) {
1066           $new_max_display = $new['line'];
1067         }
1068       }
1069
1070       foreach ($this->comments as $comment) {
1071         $display_line = $comment->getLineNumber() + $comment->getLineLength();
1072         $display_line = max(1, $display_line);
1073
1074         if ($this->isCommentOnRightSideWhenDisplayed($comment)) {
1075           $display_line = min($new_max_display, $display_line);
1076           $new_comments[$display_line][] = $comment;
1077         } else {
1078           $display_line = min($old_max_display, $display_line);
1079           $old_comments[$display_line][] = $comment;
1080         }
1081       }
1082     }
1083
1084     $renderer
1085       ->setOldComments($old_comments)
1086       ->setNewComments($new_comments);
1087
1088     if ($engine_blocks !== null) {
1089       $reference = $this->getRenderingReference();
1090       $parts = explode('/', $reference);
1091       if (count($parts) == 2) {
1092         list($id, $vs) = $parts;
1093       } else {
1094         $id = $parts[0];
1095         $vs = 0;
1096       }
1097
1098       // If we don't have an explicit "vs" changeset, it's the left side of
1099       // the "id" changeset.
1100       if (!$vs) {
1101         $vs = $id;
1102       }
1103
1104       if ($mask_force) {
1105         $engine_blocks->setRevealedIndexes(array_keys($mask_force));
1106       }
1107
1108       if ($range_start !== null || $range_len !== null) {
1109         $range_min = $range_start;
1110
1111         if ($range_len === null) {
1112           $range_max = null;
1113         } else {
1114           $range_max = (int)$range_start + (int)$range_len;
1115         }
1116
1117         $engine_blocks->setRange($range_min, $range_max);
1118       }
1119
1120       $renderer
1121         ->setDocumentEngine($engine)
1122         ->setDocumentEngineBlocks($engine_blocks);
1123
1124       return $renderer->renderDocumentEngineBlocks(
1125         $engine_blocks,
1126         (string)$id,
1127         (string)$vs);
1128     }
1129
1130     // If we've made it here with a type of file we don't know how to render,
1131     // bail out with a default empty rendering. Normally, we'd expect a
1132     // document engine to catch these changes before we make it this far.
1133     switch ($this->changeset->getFileType()) {
1134       case DifferentialChangeType::FILE_DIRECTORY:
1135       case DifferentialChangeType::FILE_BINARY:
1136       case DifferentialChangeType::FILE_IMAGE:
1137         $output = $renderer->renderChangesetTable(null);
1138         return $output;
1139     }
1140
1141     if ($this->originalLeft && $this->originalRight) {
1142       list($highlight_old, $highlight_new) = $this->diffOriginals();
1143       $highlight_old = array_flip($highlight_old);
1144       $highlight_new = array_flip($highlight_new);
1145       $renderer
1146         ->setHighlightOld($highlight_old)
1147         ->setHighlightNew($highlight_new);
1148     }
1149     $renderer
1150       ->setOriginalOld($this->originalLeft)
1151       ->setOriginalNew($this->originalRight);
1152
1153     if ($range_start === null) {
1154       $range_start = 0;
1155     }
1156     if ($range_len === null) {
1157       $range_len = $rows;
1158     }
1159     $range_len = min($range_len, $rows - $range_start);
1160
1161     list($gaps, $mask) = $this->calculateGapsAndMask(
1162       $mask_force,
1163       $feedback_mask,
1164       $range_start,
1165       $range_len);
1166
1167     $renderer
1168       ->setGaps($gaps)
1169       ->setMask($mask);
1170
1171     $html = $renderer->renderTextChange(
1172       $range_start,
1173       $range_len,
1174       $rows);
1175
1176     return $renderer->renderChangesetTable($html);
1177   }
1178
1179   /**
1180    * This function calculates a lot of stuff we need to know to display
1181    * the diff:
1182    *
1183    * Gaps - compute gaps in the visible display diff, where we will render
1184    * "Show more context" spacers. If a gap is smaller than the context size,
1185    * we just display it. Otherwise, we record it into $gaps and will render a
1186    * "show more context" element instead of diff text below. A given $gap
1187    * is a tuple of $gap_line_number_start and $gap_length.
1188    *
1189    * Mask - compute the actual lines that need to be shown (because they
1190    * are near changes lines, near inline comments, or the request has
1191    * explicitly asked for them, i.e. resulting from the user clicking
1192    * "show more"). The $mask returned is a sparsely populated dictionary
1193    * of $visible_line_number => true.
1194    *
1195    * @return array($gaps, $mask)
1196    */
1197   private function calculateGapsAndMask(
1198     $mask_force,
1199     $feedback_mask,
1200     $range_start,
1201     $range_len) {
1202
1203     $lines_context = $this->getLinesOfContext();
1204
1205     $gaps = array();
1206     $gap_start = 0;
1207     $in_gap = false;
1208     $base_mask = $this->visible + $mask_force + $feedback_mask;
1209     $base_mask[$range_start + $range_len] = true;
1210     for ($ii = $range_start; $ii <= $range_start + $range_len; $ii++) {
1211       if (isset($base_mask[$ii])) {
1212         if ($in_gap) {
1213           $gap_length = $ii - $gap_start;
1214           if ($gap_length <= $lines_context) {
1215             for ($jj = $gap_start; $jj <= $gap_start + $gap_length; $jj++) {
1216               $base_mask[$jj] = true;
1217             }
1218           } else {
1219             $gaps[] = array($gap_start, $gap_length);
1220           }
1221           $in_gap = false;
1222         }
1223       } else {
1224         if (!$in_gap) {
1225           $gap_start = $ii;
1226           $in_gap = true;
1227         }
1228       }
1229     }
1230     $gaps = array_reverse($gaps);
1231     $mask = $base_mask;
1232
1233     return array($gaps, $mask);
1234   }
1235
1236   /**
1237    * Determine if an inline comment will appear on the rendered diff,
1238    * taking into consideration which halves of which changesets will actually
1239    * be shown.
1240    *
1241    * @param PhabricatorInlineComment Comment to test for visibility.
1242    * @return bool True if the comment is visible on the rendered diff.
1243    */
1244   private function isCommentVisibleOnRenderedDiff(
1245     PhabricatorInlineComment $comment) {
1246
1247     $changeset_id = $comment->getChangesetID();
1248     $is_new = $comment->getIsNewFile();
1249
1250     if ($changeset_id == $this->rightSideChangesetID &&
1251         $is_new == $this->rightSideAttachesToNewFile) {
1252         return true;
1253     }
1254
1255     if ($changeset_id == $this->leftSideChangesetID &&
1256         $is_new == $this->leftSideAttachesToNewFile) {
1257         return true;
1258     }
1259
1260     return false;
1261   }
1262
1263
1264   /**
1265    * Determine if a comment will appear on the right side of the display diff.
1266    * Note that the comment must appear somewhere on the rendered changeset, as
1267    * per isCommentVisibleOnRenderedDiff().
1268    *
1269    * @param PhabricatorInlineComment Comment to test for display
1270    *              location.
1271    * @return bool True for right, false for left.
1272    */
1273   private function isCommentOnRightSideWhenDisplayed(
1274     PhabricatorInlineComment $comment) {
1275
1276     if (!$this->isCommentVisibleOnRenderedDiff($comment)) {
1277       throw new Exception(pht('Comment is not visible on changeset!'));
1278     }
1279
1280     $changeset_id = $comment->getChangesetID();
1281     $is_new = $comment->getIsNewFile();
1282
1283     if ($changeset_id == $this->rightSideChangesetID &&
1284         $is_new == $this->rightSideAttachesToNewFile) {
1285       return true;
1286     }
1287
1288     return false;
1289   }
1290
1291   /**
1292    * Parse the 'range' specification that this class and the client-side JS
1293    * emit to indicate that a user clicked "Show more..." on a diff. Generally,
1294    * use is something like this:
1295    *
1296    *   $spec = $request->getStr('range');
1297    *   $parsed = DifferentialChangesetParser::parseRangeSpecification($spec);
1298    *   list($start, $end, $mask) = $parsed;
1299    *   $parser->render($start, $end, $mask);
1300    *
1301    * @param string Range specification, indicating the range of the diff that
1302    *               should be rendered.
1303    * @return tuple List of <start, end, mask> suitable for passing to
1304    *               @{method:render}.
1305    */
1306   public static function parseRangeSpecification($spec) {
1307     $range_s = null;
1308     $range_e = null;
1309     $mask = array();
1310
1311     if ($spec) {
1312       $match = null;
1313       if (preg_match('@^(\d+)-(\d+)(?:/(\d+)-(\d+))?$@', $spec, $match)) {
1314         $range_s = (int)$match[1];
1315         $range_e = (int)$match[2];
1316         if (count($match) > 3) {
1317           $start = (int)$match[3];
1318           $len = (int)$match[4];
1319           for ($ii = $start; $ii < $start + $len; $ii++) {
1320             $mask[$ii] = true;
1321           }
1322         }
1323       }
1324     }
1325
1326     return array($range_s, $range_e, $mask);
1327   }
1328
1329   /**
1330    * Render "modified coverage" information; test coverage on modified lines.
1331    * This synthesizes diff information with unit test information into a useful
1332    * indicator of how well tested a change is.
1333    */
1334   public function renderModifiedCoverage() {
1335     $na = phutil_tag('em', array(), '-');
1336
1337     $coverage = $this->getCoverage();
1338     if (!$coverage) {
1339       return $na;
1340     }
1341
1342     $covered = 0;
1343     $not_covered = 0;
1344
1345     foreach ($this->new as $k => $new) {
1346       if ($new === null) {
1347         continue;
1348       }
1349
1350       if (!$new['line']) {
1351         continue;
1352       }
1353
1354       if (!$new['type']) {
1355         continue;
1356       }
1357
1358       if (empty($coverage[$new['line'] - 1])) {
1359         continue;
1360       }
1361
1362       switch ($coverage[$new['line'] - 1]) {
1363         case 'C':
1364           $covered++;
1365           break;
1366         case 'U':
1367           $not_covered++;
1368           break;
1369       }
1370     }
1371
1372     if (!$covered && !$not_covered) {
1373       return $na;
1374     }
1375
1376     return sprintf('%d%%', 100 * ($covered / ($covered + $not_covered)));
1377   }
1378
1379   /**
1380    * Build maps from lines comments appear on to actual lines.
1381    */
1382   private function buildLineBackmaps() {
1383     $old_back = array();
1384     $new_back = array();
1385     foreach ($this->old as $ii => $old) {
1386       if ($old === null) {
1387         continue;
1388       }
1389       $old_back[$old['line']] = $old['line'];
1390     }
1391     foreach ($this->new as $ii => $new) {
1392       if ($new === null) {
1393         continue;
1394       }
1395       $new_back[$new['line']] = $new['line'];
1396     }
1397
1398     $max_old_line = 0;
1399     $max_new_line = 0;
1400     foreach ($this->comments as $comment) {
1401       if ($this->isCommentOnRightSideWhenDisplayed($comment)) {
1402         $max_new_line = max($max_new_line, $comment->getLineNumber());
1403       } else {
1404         $max_old_line = max($max_old_line, $comment->getLineNumber());
1405       }
1406     }
1407
1408     $cursor = 1;
1409     for ($ii = 1; $ii <= $max_old_line; $ii++) {
1410       if (empty($old_back[$ii])) {
1411         $old_back[$ii] = $cursor;
1412       } else {
1413         $cursor = $old_back[$ii];
1414       }
1415     }
1416
1417     $cursor = 1;
1418     for ($ii = 1; $ii <= $max_new_line; $ii++) {
1419       if (empty($new_back[$ii])) {
1420         $new_back[$ii] = $cursor;
1421       } else {
1422         $cursor = $new_back[$ii];
1423       }
1424     }
1425
1426     return array($old_back, $new_back);
1427   }
1428
1429   private function getOffset(array $map, $line) {
1430     if (!$map) {
1431       return null;
1432     }
1433
1434     $line = (int)$line;
1435     foreach ($map as $key => $spec) {
1436       if ($spec && isset($spec['line'])) {
1437         if ((int)$spec['line'] >= $line) {
1438           return $key;
1439         }
1440       }
1441     }
1442
1443     return $key;
1444   }
1445
1446   private function realignDiff(
1447     DifferentialChangeset $changeset,
1448     DifferentialHunkParser $hunk_parser) {
1449     // Normalizing and realigning the diff depends on rediffing the files, and
1450     // we currently need complete representations of both files to do anything
1451     // reasonable. If we only have parts of the files, skip realignment.
1452
1453     // We have more than one hunk, so we're definitely missing part of the file.
1454     $hunks = $changeset->getHunks();
1455     if (count($hunks) !== 1) {
1456       return null;
1457     }
1458
1459     // The first hunk doesn't start at the beginning of the file, so we're
1460     // missing some context.
1461     $first_hunk = head($hunks);
1462     if ($first_hunk->getOldOffset() != 1 || $first_hunk->getNewOffset() != 1) {
1463       return null;
1464     }
1465
1466     $old_file = $changeset->makeOldFile();
1467     $new_file = $changeset->makeNewFile();
1468     if ($old_file === $new_file) {
1469       // If the old and new files are exactly identical, the synthetic
1470       // diff below will give us nonsense and whitespace modes are
1471       // irrelevant anyway. This occurs when you, e.g., copy a file onto
1472       // itself in Subversion (see T271).
1473       return null;
1474     }
1475
1476
1477     $engine = id(new PhabricatorDifferenceEngine())
1478       ->setNormalize(true);
1479
1480     $normalized_changeset = $engine->generateChangesetFromFileContent(
1481       $old_file,
1482       $new_file);
1483
1484     $type_parser = new DifferentialHunkParser();
1485     $type_parser->parseHunksForLineData($normalized_changeset->getHunks());
1486
1487     $hunk_parser->setNormalized(true);
1488     $hunk_parser->setOldLineTypeMap($type_parser->getOldLineTypeMap());
1489     $hunk_parser->setNewLineTypeMap($type_parser->getNewLineTypeMap());
1490   }
1491
1492   private function adjustRenderedLineForDisplay($line) {
1493     // IMPORTANT: We're using "str_replace()" against raw HTML here, which can
1494     // easily become unsafe. The input HTML has already had syntax highlighting
1495     // and intraline diff highlighting applied, so it's full of "<span />" tags.
1496
1497     static $search;
1498     static $replace;
1499     if ($search === null) {
1500       $rules = $this->newSuspiciousCharacterRules();
1501
1502       $map = array();
1503       foreach ($rules as $key => $spec) {
1504         $tag = phutil_tag(
1505           'span',
1506           array(
1507             'data-copy-text' => $key,
1508             'class' => $spec['class'],
1509             'title' => $spec['title'],
1510           ),
1511           $spec['replacement']);
1512         $map[$key] = phutil_string_cast($tag);
1513       }
1514
1515       $search = array_keys($map);
1516       $replace = array_values($map);
1517     }
1518
1519     $is_html = false;
1520     if ($line instanceof PhutilSafeHTML) {
1521       $is_html = true;
1522       $line = hsprintf('%s', $line);
1523     }
1524
1525     $line = phutil_string_cast($line);
1526
1527     // TODO: This should be flexible, eventually.
1528     $tab_width = 2;
1529
1530     $line = self::replaceTabsWithSpaces($line, $tab_width);
1531     $line = str_replace($search, $replace, $line);
1532
1533     if ($is_html) {
1534       $line = phutil_safe_html($line);
1535     }
1536
1537     return $line;
1538   }
1539
1540   private function newSuspiciousCharacterRules() {
1541     // The "title" attributes are cached in the database, so they're
1542     // intentionally not wrapped in "pht(...)".
1543
1544     $rules = array(
1545       "\xE2\x80\x8B" => array(
1546         'title' => 'ZWS',
1547         'class' => 'suspicious-character',
1548         'replacement' => '!',
1549       ),
1550       "\xC2\xA0" => array(
1551         'title' => 'NBSP',
1552         'class' => 'suspicious-character',
1553         'replacement' => '!',
1554       ),
1555       "\x7F" => array(
1556         'title' => 'DEL (0x7F)',
1557         'class' => 'suspicious-character',
1558         'replacement' => "\xE2\x90\xA1",
1559       ),
1560     );
1561
1562     // Unicode defines special pictures for the control characters in the
1563     // range between "0x00" and "0x1F".
1564
1565     $control = array(
1566       'NULL',
1567       'SOH',
1568       'STX',
1569       'ETX',
1570       'EOT',
1571       'ENQ',
1572       'ACK',
1573       'BEL',
1574       'BS',
1575       null, // "\t" Tab
1576       null, // "\n" New Line
1577       'VT',
1578       'FF',
1579       null, // "\r" Carriage Return,
1580       'SO',
1581       'SI',
1582       'DLE',
1583       'DC1',
1584       'DC2',
1585       'DC3',
1586       'DC4',
1587       'NAK',
1588       'SYN',
1589       'ETB',
1590       'CAN',
1591       'EM',
1592       'SUB',
1593       'ESC',
1594       'FS',
1595       'GS',
1596       'RS',
1597       'US',
1598     );
1599
1600     foreach ($control as $idx => $label) {
1601       if ($label === null) {
1602         continue;
1603       }
1604
1605       $rules[chr($idx)] = array(
1606         'title' => sprintf('%s (0x%02X)', $label, $idx),
1607         'class' => 'suspicious-character',
1608         'replacement' => "\xE2\x90".chr(0x80 + $idx),
1609       );
1610     }
1611
1612     return $rules;
1613   }
1614
1615   public static function replaceTabsWithSpaces($line, $tab_width) {
1616     static $tags = array();
1617     if (empty($tags[$tab_width])) {
1618       for ($ii = 1; $ii <= $tab_width; $ii++) {
1619         $tag = phutil_tag(
1620           'span',
1621           array(
1622             'data-copy-text' => "\t",
1623           ),
1624           str_repeat(' ', $ii));
1625         $tag = phutil_string_cast($tag);
1626         $tags[$ii] = $tag;
1627       }
1628     }
1629
1630     // Expand all prefix tabs until we encounter any non-tab character. This
1631     // is cheap and often immediately produces the correct result with no
1632     // further work (and, particularly, no need to handle any unicode cases).
1633
1634     $len = strlen($line);
1635
1636     $head = 0;
1637     for ($head = 0; $head < $len; $head++) {
1638       $char = $line[$head];
1639       if ($char !== "\t") {
1640         break;
1641       }
1642     }
1643
1644     if ($head) {
1645       if (empty($tags[$tab_width * $head])) {
1646         $tags[$tab_width * $head] = str_repeat($tags[$tab_width], $head);
1647       }
1648       $prefix = $tags[$tab_width * $head];
1649       $line = substr($line, $head);
1650     } else {
1651       $prefix = '';
1652     }
1653
1654     // If we have no remaining tabs elsewhere in the string after taking care
1655     // of all the prefix tabs, we're done.
1656     if (strpos($line, "\t") === false) {
1657       return $prefix.$line;
1658     }
1659
1660     $len = strlen($line);
1661
1662     // If the line is particularly long, don't try to do anything special with
1663     // it. Use a faster approximation of the correct tabstop expansion instead.
1664     // This usually still arrives at the right result.
1665     if ($len > 256) {
1666       return $prefix.str_replace("\t", $tags[$tab_width], $line);
1667     }
1668
1669     $in_tag = false;
1670     $pos = 0;
1671
1672     // See PHI1210. If the line only has single-byte characters, we don't need
1673     // to vectorize it and can avoid an expensive UTF8 call.
1674
1675     $fast_path = preg_match('/^[\x01-\x7F]*\z/', $line);
1676     if ($fast_path) {
1677       $replace = array();
1678       for ($ii = 0; $ii < $len; $ii++) {
1679         $char = $line[$ii];
1680         if ($char === '>') {
1681           $in_tag = false;
1682           continue;
1683         }
1684
1685         if ($in_tag) {
1686           continue;
1687         }
1688
1689         if ($char === '<') {
1690           $in_tag = true;
1691           continue;
1692         }
1693
1694         if ($char === "\t") {
1695           $count = $tab_width - ($pos % $tab_width);
1696           $pos += $count;
1697           $replace[$ii] = $tags[$count];
1698           continue;
1699         }
1700
1701         $pos++;
1702       }
1703
1704       if ($replace) {
1705         // Apply replacements starting at the end of the string so they
1706         // don't mess up the offsets for following replacements.
1707         $replace = array_reverse($replace, true);
1708
1709         foreach ($replace as $replace_pos => $replacement) {
1710           $line = substr_replace($line, $replacement, $replace_pos, 1);
1711         }
1712       }
1713     } else {
1714       $line = phutil_utf8v_combined($line);
1715       foreach ($line as $key => $char) {
1716         if ($char === '>') {
1717           $in_tag = false;
1718           continue;
1719         }
1720
1721         if ($in_tag) {
1722           continue;
1723         }
1724
1725         if ($char === '<') {
1726           $in_tag = true;
1727           continue;
1728         }
1729
1730         if ($char === "\t") {
1731           $count = $tab_width - ($pos % $tab_width);
1732           $pos += $count;
1733           $line[$key] = $tags[$count];
1734           continue;
1735         }
1736
1737         $pos++;
1738       }
1739
1740       $line = implode('', $line);
1741     }
1742
1743     return $prefix.$line;
1744   }
1745
1746   private function newDocumentEngine() {
1747     $changeset = $this->changeset;
1748     $viewer = $this->getViewer();
1749
1750     list($old_file, $new_file) = $this->loadFileObjectsForChangeset();
1751
1752     $no_old = !$changeset->hasOldState();
1753     $no_new = !$changeset->hasNewState();
1754
1755     if ($no_old) {
1756       $old_ref = null;
1757     } else {
1758       $old_ref = id(new PhabricatorDocumentRef())
1759         ->setName($changeset->getOldFile());
1760       if ($old_file) {
1761         $old_ref->setFile($old_file);
1762       } else {
1763         $old_data = $this->getRawDocumentEngineData($this->old);
1764         $old_ref->setData($old_data);
1765       }
1766     }
1767
1768     if ($no_new) {
1769       $new_ref = null;
1770     } else {
1771       $new_ref = id(new PhabricatorDocumentRef())
1772         ->setName($changeset->getFilename());
1773       if ($new_file) {
1774         $new_ref->setFile($new_file);
1775       } else {
1776         $new_data = $this->getRawDocumentEngineData($this->new);
1777         $new_ref->setData($new_data);
1778       }
1779     }
1780
1781     $old_engines = null;
1782     if ($old_ref) {
1783       $old_engines = PhabricatorDocumentEngine::getEnginesForRef(
1784         $viewer,
1785         $old_ref);
1786     }
1787
1788     $new_engines = null;
1789     if ($new_ref) {
1790       $new_engines = PhabricatorDocumentEngine::getEnginesForRef(
1791         $viewer,
1792         $new_ref);
1793     }
1794
1795     if ($new_engines !== null && $old_engines !== null) {
1796       $shared_engines = array_intersect_key($new_engines, $old_engines);
1797       $default_engine = head_key($new_engines);
1798     } else if ($new_engines !== null) {
1799       $shared_engines = $new_engines;
1800       $default_engine = head_key($shared_engines);
1801     } else if ($old_engines !== null) {
1802       $shared_engines = $old_engines;
1803       $default_engine = head_key($shared_engines);
1804     } else {
1805       return null;
1806     }
1807
1808     foreach ($shared_engines as $key => $shared_engine) {
1809       if (!$shared_engine->canDiffDocuments($old_ref, $new_ref)) {
1810         unset($shared_engines[$key]);
1811       }
1812     }
1813
1814     $this->availableDocumentEngines = $shared_engines;
1815
1816     $viewstate = $this->getViewState();
1817
1818     $engine_key = $viewstate->getDocumentEngineKey();
1819     if (phutil_nonempty_string($engine_key)) {
1820       if (isset($shared_engines[$engine_key])) {
1821         $document_engine = $shared_engines[$engine_key];
1822       } else {
1823         $document_engine = null;
1824       }
1825     } else {
1826       // If we aren't rendering with a specific engine, only use a default
1827       // engine if the best engine for the new file is a shared engine which
1828       // can diff files. If we're less picky (for example, by accepting any
1829       // shared engine) we can end up with silly behavior (like ".json" files
1830       // rendering as Jupyter documents).
1831
1832       if (isset($shared_engines[$default_engine])) {
1833         $document_engine = $shared_engines[$default_engine];
1834       } else {
1835         $document_engine = null;
1836       }
1837     }
1838
1839     if ($document_engine) {
1840       return array(
1841         $document_engine,
1842         $old_ref,
1843         $new_ref);
1844     }
1845
1846     return null;
1847   }
1848
1849   private function loadFileObjectsForChangeset() {
1850     $changeset = $this->changeset;
1851     $viewer = $this->getViewer();
1852
1853     $old_phid = $changeset->getOldFileObjectPHID();
1854     $new_phid = $changeset->getNewFileObjectPHID();
1855
1856     $old_file = null;
1857     $new_file = null;
1858
1859     if ($old_phid || $new_phid) {
1860       $file_phids = array();
1861       if ($old_phid) {
1862         $file_phids[] = $old_phid;
1863       }
1864       if ($new_phid) {
1865         $file_phids[] = $new_phid;
1866       }
1867
1868       $files = id(new PhabricatorFileQuery())
1869         ->setViewer($viewer)
1870         ->withPHIDs($file_phids)
1871         ->execute();
1872       $files = mpull($files, null, 'getPHID');
1873
1874       if ($old_phid) {
1875         $old_file = idx($files, $old_phid);
1876         if (!$old_file) {
1877           throw new Exception(
1878             pht(
1879               'Failed to load file data for changeset ("%s").',
1880               $old_phid));
1881         }
1882         $changeset->attachOldFileObject($old_file);
1883       }
1884
1885       if ($new_phid) {
1886         $new_file = idx($files, $new_phid);
1887         if (!$new_file) {
1888           throw new Exception(
1889             pht(
1890               'Failed to load file data for changeset ("%s").',
1891               $new_phid));
1892         }
1893         $changeset->attachNewFileObject($new_file);
1894       }
1895     }
1896
1897     return array($old_file, $new_file);
1898   }
1899
1900   public function newChangesetResponse() {
1901     // NOTE: This has to happen first because it has side effects. Yuck.
1902     $rendered_changeset = $this->renderChangeset();
1903
1904     $renderer = $this->getRenderer();
1905     $renderer_key = $renderer->getRendererKey();
1906
1907     $viewstate = $this->getViewState();
1908
1909     $undo_templates = $renderer->renderUndoTemplates();
1910     foreach ($undo_templates as $key => $undo_template) {
1911       $undo_templates[$key] = hsprintf('%s', $undo_template);
1912     }
1913
1914     $document_engine = $renderer->getDocumentEngine();
1915     if ($document_engine) {
1916       $document_engine_key = $document_engine->getDocumentEngineKey();
1917     } else {
1918       $document_engine_key = null;
1919     }
1920
1921     $available_keys = array();
1922     $engines = $this->availableDocumentEngines;
1923     if (!$engines) {
1924       $engines = array();
1925     }
1926
1927     $available_keys = mpull($engines, 'getDocumentEngineKey');
1928
1929     // TODO: Always include "source" as a usable engine to default to
1930     // the buitin rendering. This is kind of a hack and does not actually
1931     // use the source engine. The source engine isn't a diff engine, so
1932     // selecting it causes us to fall through and render with builtin
1933     // behavior. For now, overall behavir is reasonable.
1934
1935     $available_keys[] = PhabricatorSourceDocumentEngine::ENGINEKEY;
1936     $available_keys = array_fuse($available_keys);
1937     $available_keys = array_values($available_keys);
1938
1939     $state = array(
1940       'undoTemplates' => $undo_templates,
1941       'rendererKey' => $renderer_key,
1942       'highlight' => $viewstate->getHighlightLanguage(),
1943       'characterEncoding' => $viewstate->getCharacterEncoding(),
1944       'requestDocumentEngineKey' => $viewstate->getDocumentEngineKey(),
1945       'responseDocumentEngineKey' => $document_engine_key,
1946       'availableDocumentEngineKeys' => $available_keys,
1947       'isHidden' => $viewstate->getHidden(),
1948     );
1949
1950     return id(new PhabricatorChangesetResponse())
1951       ->setRenderedChangeset($rendered_changeset)
1952       ->setChangesetState($state);
1953   }
1954
1955   private function getRawDocumentEngineData(array $lines) {
1956     $text = array();
1957
1958     foreach ($lines as $line) {
1959       if ($line === null) {
1960         continue;
1961       }
1962
1963       // If this is a "No newline at end of file." annotation, don't hand it
1964       // off to the DocumentEngine.
1965       if ($line['type'] === '\\') {
1966         continue;
1967       }
1968
1969       $text[] = $line['text'];
1970     }
1971
1972     return implode('', $text);
1973   }
1974
1975 }