Remove all "FileHasObject" edge reads and writes
[phabricator.git] / src / applications / harbormaster / controller / HarbormasterBuildLogRenderController.php
blob0f1f2bc028e931b21537ed3ce801e6fc9b383eb6
1 <?php
3 final class HarbormasterBuildLogRenderController
4 extends HarbormasterController {
6 public function shouldAllowPublic() {
7 return true;
10 public function handleRequest(AphrontRequest $request) {
11 $viewer = $this->getViewer();
13 $id = $request->getURIData('id');
15 $log = id(new HarbormasterBuildLogQuery())
16 ->setViewer($viewer)
17 ->withIDs(array($id))
18 ->executeOne();
19 if (!$log) {
20 return new Aphront404Response();
23 $highlight_range = $request->getURILineRange('lines', 1000);
25 $log_size = $this->getTotalByteLength($log);
27 $head_lines = $request->getInt('head');
28 if ($head_lines === null) {
29 $head_lines = 8;
31 $head_lines = min($head_lines, 1024);
32 $head_lines = max($head_lines, 0);
34 $tail_lines = $request->getInt('tail');
35 if ($tail_lines === null) {
36 $tail_lines = 16;
38 $tail_lines = min($tail_lines, 1024);
39 $tail_lines = max($tail_lines, 0);
41 $head_offset = $request->getInt('headOffset');
42 if ($head_offset === null) {
43 $head_offset = 0;
46 $tail_offset = $request->getInt('tailOffset');
47 if ($tail_offset === null) {
48 $tail_offset = $log_size;
51 // Figure out which ranges we're actually going to read. We'll read either
52 // one range (either just at the head, or just at the tail) or two ranges
53 // (one at the head and one at the tail).
55 // This gets a little bit tricky because: the ranges may overlap; we just
56 // want to do one big read if there is only a little bit of text left
57 // between the ranges; we may not know where the tail range ends; and we
58 // can only read forward from line map markers, not from any arbitrary
59 // position in the file.
61 $bytes_per_line = 140;
62 $body_lines = 8;
64 $views = array();
65 if ($head_lines > 0) {
66 $views[] = array(
67 'offset' => $head_offset,
68 'lines' => $head_lines,
69 'direction' => 1,
70 'limit' => $tail_offset,
74 if ($highlight_range) {
75 $highlight_views = $this->getHighlightViews(
76 $log,
77 $highlight_range,
78 $log_size);
79 foreach ($highlight_views as $highlight_view) {
80 $views[] = $highlight_view;
84 if ($tail_lines > 0) {
85 $views[] = array(
86 'offset' => $tail_offset,
87 'lines' => $tail_lines,
88 'direction' => -1,
89 'limit' => $head_offset,
93 $reads = $views;
94 foreach ($reads as $key => $read) {
95 $offset = $read['offset'];
97 $lines = $read['lines'];
99 $read_length = 0;
100 $read_length += ($lines * $bytes_per_line);
101 $read_length += ($body_lines * $bytes_per_line);
103 $direction = $read['direction'];
104 if ($direction < 0) {
105 if ($offset > $read_length) {
106 $offset -= $read_length;
107 } else {
108 $read_length = $offset;
109 $offset = 0;
113 $position = $log->getReadPosition($offset);
114 list($position_offset, $position_line) = $position;
115 $read_length += ($offset - $position_offset);
117 $reads[$key]['fetchOffset'] = $position_offset;
118 $reads[$key]['fetchLength'] = $read_length;
119 $reads[$key]['fetchLine'] = $position_line;
122 $reads = $this->mergeOverlappingReads($reads);
124 foreach ($reads as $key => $read) {
125 $fetch_offset = $read['fetchOffset'];
126 $fetch_length = $read['fetchLength'];
127 if ($fetch_offset + $fetch_length > $log_size) {
128 $fetch_length = $log_size - $fetch_offset;
131 $data = $log->loadData($fetch_offset, $fetch_length);
133 $offset = $read['fetchOffset'];
134 $line = $read['fetchLine'];
135 $lines = $this->getLines($data);
136 $line_data = array();
137 foreach ($lines as $line_text) {
138 $length = strlen($line_text);
139 $line_data[] = array(
140 'offset' => $offset,
141 'length' => $length,
142 'line' => $line,
143 'data' => $line_text,
145 $line += 1;
146 $offset += $length;
149 $reads[$key]['data'] = $data;
150 $reads[$key]['lines'] = $line_data;
153 foreach ($views as $view_key => $view) {
154 $anchor_byte = $view['offset'];
156 if ($view['direction'] < 0) {
157 $anchor_byte = $anchor_byte - 1;
160 $data_key = null;
161 foreach ($reads as $read_key => $read) {
162 $s = $read['fetchOffset'];
163 $e = $s + $read['fetchLength'];
165 if (($s <= $anchor_byte) && ($e >= $anchor_byte)) {
166 $data_key = $read_key;
167 break;
171 if ($data_key === null) {
172 throw new Exception(
173 pht('Unable to find fetch!'));
176 $anchor_key = null;
177 foreach ($reads[$data_key]['lines'] as $line_key => $line) {
178 $s = $line['offset'];
179 $e = $s + $line['length'];
181 if (($s <= $anchor_byte) && ($e > $anchor_byte)) {
182 $anchor_key = $line_key;
183 break;
187 if ($anchor_key === null) {
188 throw new Exception(
189 pht(
190 'Unable to find lines.'));
193 if ($view['direction'] > 0) {
194 $slice_offset = $anchor_key;
195 } else {
196 $slice_offset = max(0, $anchor_key - ($view['lines'] - 1));
198 $slice_length = $view['lines'];
200 $views[$view_key] += array(
201 'sliceKey' => $data_key,
202 'sliceOffset' => $slice_offset,
203 'sliceLength' => $slice_length,
207 foreach ($views as $view_key => $view) {
208 $slice_key = $view['sliceKey'];
209 $lines = array_slice(
210 $reads[$slice_key]['lines'],
211 $view['sliceOffset'],
212 $view['sliceLength']);
214 $data_offset = null;
215 $data_length = null;
216 foreach ($lines as $line) {
217 if ($data_offset === null) {
218 $data_offset = $line['offset'];
220 $data_length += $line['length'];
223 // If the view cursor starts in the middle of a line, we're going to
224 // strip part of the line.
225 $direction = $view['direction'];
226 if ($direction > 0) {
227 $view_offset = $view['offset'];
228 $view_length = $data_length;
229 if ($data_offset < $view_offset) {
230 $trim = ($view_offset - $data_offset);
231 $view_length -= $trim;
234 $limit = $view['limit'];
235 if ($limit !== null) {
236 if ($limit < ($view_offset + $view_length)) {
237 $view_length = ($limit - $view_offset);
240 } else {
241 $view_offset = $data_offset;
242 $view_length = $data_length;
243 if ($data_offset + $data_length > $view['offset']) {
244 $view_length -= (($data_offset + $data_length) - $view['offset']);
247 $limit = $view['limit'];
248 if ($limit !== null) {
249 if ($limit > $view_offset) {
250 $view_length -= ($limit - $view_offset);
251 $view_offset = $limit;
256 $views[$view_key] += array(
257 'viewOffset' => $view_offset,
258 'viewLength' => $view_length,
262 $views = $this->mergeOverlappingViews($views);
264 foreach ($views as $view_key => $view) {
265 $slice_key = $view['sliceKey'];
266 $lines = array_slice(
267 $reads[$slice_key]['lines'],
268 $view['sliceOffset'],
269 $view['sliceLength']);
271 $view_offset = $view['viewOffset'];
272 foreach ($lines as $line_key => $line) {
273 $line_offset = $line['offset'];
275 if ($line_offset >= $view_offset) {
276 break;
279 $trim = ($view_offset - $line_offset);
280 if ($trim && ($trim >= strlen($line['data']))) {
281 unset($lines[$line_key]);
282 continue;
285 $line_data = substr($line['data'], $trim);
286 $lines[$line_key]['data'] = $line_data;
287 $lines[$line_key]['length'] = strlen($line_data);
288 $lines[$line_key]['offset'] += $trim;
289 break;
292 $view_end = $view['viewOffset'] + $view['viewLength'];
293 foreach ($lines as $line_key => $line) {
294 $line_end = $line['offset'] + $line['length'];
295 if ($line_end <= $view_end) {
296 continue;
299 $trim = ($line_end - $view_end);
300 if ($trim && ($trim >= strlen($line['data']))) {
301 unset($lines[$line_key]);
302 continue;
305 $line_data = substr($line['data'], -$trim);
306 $lines[$line_key]['data'] = $line_data;
307 $lines[$line_key]['length'] = strlen($line_data);
310 $views[$view_key]['viewData'] = $lines;
313 $spacer = null;
314 $render = array();
316 $head_view = head($views);
317 if ($head_view['viewOffset'] > $head_offset) {
318 $render[] = array(
319 'spacer' => true,
320 'head' => $head_offset,
321 'tail' => $head_view['viewOffset'],
325 foreach ($views as $view) {
326 if ($spacer) {
327 $spacer['tail'] = $view['viewOffset'];
328 $render[] = $spacer;
331 $render[] = $view;
333 $spacer = array(
334 'spacer' => true,
335 'head' => ($view['viewOffset'] + $view['viewLength']),
339 $tail_view = last($views);
340 if ($tail_view['viewOffset'] + $tail_view['viewLength'] < $tail_offset) {
341 $render[] = array(
342 'spacer' => true,
343 'head' => $tail_view['viewOffset'] + $tail_view['viewLength'],
344 'tail' => $tail_offset,
348 $uri = $log->getURI();
350 $rows = array();
351 foreach ($render as $range) {
352 if (isset($range['spacer'])) {
353 $rows[] = $this->renderExpandRow($range);
354 continue;
357 $lines = $range['viewData'];
358 foreach ($lines as $line) {
359 $display_line = ($line['line'] + 1);
360 $display_text = ($line['data']);
362 $row_attr = array();
363 if ($highlight_range) {
364 if (($display_line >= $highlight_range[0]) &&
365 ($display_line <= $highlight_range[1])) {
366 $row_attr = array(
367 'class' => 'phabricator-source-highlight',
372 $display_line = phutil_tag(
373 'a',
374 array(
375 'href' => $uri.'$'.$display_line,
376 'data-n' => $display_line,
378 '');
380 $line_cell = phutil_tag('th', array(), $display_line);
381 $text_cell = phutil_tag('td', array(), $display_text);
383 $rows[] = phutil_tag(
384 'tr',
385 $row_attr,
386 array(
387 $line_cell,
388 $text_cell,
393 if ($log->getLive()) {
394 $last_view = last($views);
395 $last_line = last($last_view['viewData']);
396 if ($last_line) {
397 $last_offset = $last_line['offset'];
398 } else {
399 $last_offset = 0;
402 $last_tail = $last_view['viewOffset'] + $last_view['viewLength'];
403 $show_live = ($last_tail === $log_size);
404 if ($show_live) {
405 $rows[] = $this->renderLiveRow($last_offset);
409 $table = javelin_tag(
410 'table',
411 array(
412 'class' => 'harbormaster-log-table PhabricatorMonospaced',
413 'sigil' => 'phabricator-source',
414 'meta' => array(
415 'uri' => $log->getURI(),
418 $rows);
420 // When this is a normal AJAX request, return the rendered log fragment
421 // in an AJAX payload.
422 if ($request->isAjax()) {
423 return id(new AphrontAjaxResponse())
424 ->setContent(
425 array(
426 'markup' => hsprintf('%s', $table),
430 // If the page is being accessed as a standalone page, present a
431 // readable version of the fragment for debugging.
433 require_celerity_resource('harbormaster-css');
435 $header = pht('Standalone Log Fragment');
437 $render_view = id(new PHUIObjectBoxView())
438 ->setBackground(PHUIObjectBoxView::BLUE_PROPERTY)
439 ->setHeaderText($header)
440 ->appendChild($table);
442 $page_view = id(new PHUITwoColumnView())
443 ->setFooter($render_view);
445 $crumbs = $this->buildApplicationCrumbs()
446 ->addTextCrumb(pht('Build Log %d', $log->getID()), $log->getURI())
447 ->addTextCrumb(pht('Fragment'))
448 ->setBorder(true);
450 return $this->newPage()
451 ->setTitle(
452 array(
453 pht('Build Log %d', $log->getID()),
454 pht('Standalone Fragment'),
456 ->setCrumbs($crumbs)
457 ->appendChild($page_view);
460 private function getTotalByteLength(HarbormasterBuildLog $log) {
461 $total_bytes = $log->getByteLength();
462 if ($total_bytes) {
463 return (int)$total_bytes;
466 // TODO: Remove this after enough time has passed for installs to run
467 // log rebuilds or decide they don't care about older logs.
469 // Older logs don't have this data denormalized onto the log record unless
470 // an administrator has run `bin/harbormaster rebuild-log --all` or
471 // similar. Try to figure it out by summing up the size of each chunk.
473 // Note that the log may also be legitimately empty and have actual size
474 // zero.
475 $chunk = new HarbormasterBuildLogChunk();
476 $conn = $chunk->establishConnection('r');
478 $row = queryfx_one(
479 $conn,
480 'SELECT SUM(size) total FROM %T WHERE logID = %d',
481 $chunk->getTableName(),
482 $log->getID());
484 return (int)$row['total'];
487 private function getLines($data) {
488 $parts = preg_split("/(\r\n|\r|\n)/", $data, 0, PREG_SPLIT_DELIM_CAPTURE);
490 if (last($parts) === '') {
491 array_pop($parts);
494 $lines = array();
495 for ($ii = 0; $ii < count($parts); $ii += 2) {
496 $line = $parts[$ii];
497 if (isset($parts[$ii + 1])) {
498 $line .= $parts[$ii + 1];
500 $lines[] = $line;
503 return $lines;
507 private function mergeOverlappingReads(array $reads) {
508 // Find planned reads which will overlap and merge them into a single
509 // larger read.
511 $uk = array_keys($reads);
512 $vk = array_keys($reads);
514 foreach ($uk as $ukey) {
515 foreach ($vk as $vkey) {
516 // Don't merge a range into itself, even though they do technically
517 // overlap.
518 if ($ukey === $vkey) {
519 continue;
522 $uread = idx($reads, $ukey);
523 if ($uread === null) {
524 continue;
527 $vread = idx($reads, $vkey);
528 if ($vread === null) {
529 continue;
532 $us = $uread['fetchOffset'];
533 $ue = $us + $uread['fetchLength'];
535 $vs = $vread['fetchOffset'];
536 $ve = $vs + $vread['fetchLength'];
538 if (($vs > $ue) || ($ve < $us)) {
539 continue;
542 $min = min($us, $vs);
543 $max = max($ue, $ve);
545 $reads[$ukey]['fetchOffset'] = $min;
546 $reads[$ukey]['fetchLength'] = ($max - $min);
547 $reads[$ukey]['fetchLine'] = min(
548 $uread['fetchLine'],
549 $vread['fetchLine']);
551 unset($reads[$vkey]);
555 return $reads;
558 private function mergeOverlappingViews(array $views) {
559 $uk = array_keys($views);
560 $vk = array_keys($views);
562 $body_lines = 8;
563 $body_bytes = ($body_lines * 140);
565 foreach ($uk as $ukey) {
566 foreach ($vk as $vkey) {
567 if ($ukey === $vkey) {
568 continue;
571 $uview = idx($views, $ukey);
572 if ($uview === null) {
573 continue;
576 $vview = idx($views, $vkey);
577 if ($vview === null) {
578 continue;
581 // If these views don't use the same line data, don't try to
582 // merge them.
583 if ($uview['sliceKey'] != $vview['sliceKey']) {
584 continue;
587 // If these views are overlapping or separated by only a few bytes,
588 // merge them into a single view.
589 $us = $uview['viewOffset'];
590 $ue = $us + $uview['viewLength'];
592 $vs = $vview['viewOffset'];
593 $ve = $vs + $vview['viewLength'];
595 // Don't merge if one of the slices starts at a byte offset
596 // significantly after the other ends.
597 if (($vs > $ue + $body_bytes) || ($us > $ve + $body_bytes)) {
598 continue;
601 $uss = $uview['sliceOffset'];
602 $use = $uss + $uview['sliceLength'];
604 $vss = $vview['sliceOffset'];
605 $vse = $vss + $vview['sliceLength'];
607 // Don't merge if one of the slices starts at a line offset
608 // significantly after the other ends.
609 if ($uss > ($vse + $body_lines) || $vss > ($use + $body_lines)) {
610 continue;
613 // These views are overlapping or nearly overlapping, so we merge
614 // them. We merge views even if they aren't exactly adjacent since
615 // it's silly to render an "expand more" which only expands a couple
616 // of lines.
618 $offset = min($us, $vs);
619 $length = max($ue, $ve) - $offset;
621 $slice_offset = min($uss, $vss);
622 $slice_length = max($use, $vse) - $slice_offset;
624 $views[$ukey] = array(
625 'viewOffset' => $offset,
626 'viewLength' => $length,
627 'sliceOffset' => $slice_offset,
628 'sliceLength' => $slice_length,
629 ) + $views[$ukey];
631 unset($views[$vkey]);
635 return $views;
638 private function renderExpandRow($range) {
640 $icon_up = id(new PHUIIconView())
641 ->setIcon('fa-chevron-up');
643 $icon_down = id(new PHUIIconView())
644 ->setIcon('fa-chevron-down');
646 $up_text = array(
647 pht('Show More Above'),
648 ' ',
649 $icon_up,
652 $expand_up = javelin_tag(
653 'a',
654 array(
655 'sigil' => 'harbormaster-log-expand',
656 'meta' => array(
657 'headOffset' => $range['head'],
658 'tailOffset' => $range['tail'],
659 'head' => 128,
660 'tail' => 0,
663 $up_text);
665 $mid_text = pht(
666 'Show More (%s Bytes)',
667 new PhutilNumber($range['tail'] - $range['head']));
669 $expand_mid = javelin_tag(
670 'a',
671 array(
672 'sigil' => 'harbormaster-log-expand',
673 'meta' => array(
674 'headOffset' => $range['head'],
675 'tailOffset' => $range['tail'],
676 'head' => 128,
677 'tail' => 128,
680 $mid_text);
682 $down_text = array(
683 $icon_down,
684 ' ',
685 pht('Show More Below'),
688 $expand_down = javelin_tag(
689 'a',
690 array(
691 'sigil' => 'harbormaster-log-expand',
692 'meta' => array(
693 'headOffset' => $range['head'],
694 'tailOffset' => $range['tail'],
695 'head' => 0,
696 'tail' => 128,
699 $down_text);
701 $expand_cells = array(
702 phutil_tag(
703 'td',
704 array(
705 'class' => 'harbormaster-log-expand-up',
707 $expand_up),
708 phutil_tag(
709 'td',
710 array(
711 'class' => 'harbormaster-log-expand-mid',
713 $expand_mid),
714 phutil_tag(
715 'td',
716 array(
717 'class' => 'harbormaster-log-expand-down',
719 $expand_down),
722 return $this->renderActionTable($expand_cells);
725 private function renderLiveRow($log_size) {
726 $icon_down = id(new PHUIIconView())
727 ->setIcon('fa-angle-double-down');
729 $icon_pause = id(new PHUIIconView())
730 ->setIcon('fa-pause');
732 $follow = javelin_tag(
733 'a',
734 array(
735 'sigil' => 'harbormaster-log-expand harbormaster-log-live',
736 'class' => 'harbormaster-log-follow-start',
737 'meta' => array(
738 'headOffset' => $log_size,
739 'head' => 0,
740 'tail' => 1024,
741 'live' => true,
744 array(
745 $icon_down,
746 ' ',
747 pht('Follow Log'),
750 $stop_following = javelin_tag(
751 'a',
752 array(
753 'sigil' => 'harbormaster-log-expand',
754 'class' => 'harbormaster-log-follow-stop',
755 'meta' => array(
756 'stop' => true,
759 array(
760 $icon_pause,
761 ' ',
762 pht('Stop Following Log'),
765 $expand_cells = array(
766 phutil_tag(
767 'td',
768 array(
769 'class' => 'harbormaster-log-follow',
771 array(
772 $follow,
773 $stop_following,
777 return $this->renderActionTable($expand_cells);
780 private function renderActionTable(array $action_cells) {
781 $action_row = phutil_tag('tr', array(), $action_cells);
783 $action_table = phutil_tag(
784 'table',
785 array(
786 'class' => 'harbormaster-log-expand-table',
788 $action_row);
790 $format_cells = array(
791 phutil_tag('th', array()),
792 phutil_tag(
793 'td',
794 array(
795 'class' => 'harbormaster-log-expand-cell',
797 $action_table),
800 return phutil_tag('tr', array(), $format_cells);
803 private function getHighlightViews(
804 HarbormasterBuildLog $log,
805 array $range,
806 $log_size) {
807 // If we're highlighting a line range in the file, we first need to figure
808 // out the offsets for the lines we care about.
809 list($range_min, $range_max) = $range;
811 // Read the markers to find a range we can load which includes both lines.
812 $read_range = $log->getLineSpanningRange($range_min, $range_max);
813 list($min_pos, $max_pos, $min_line) = $read_range;
815 $length = ($max_pos - $min_pos);
817 // Reject to do the read if it requires us to examine a huge amount of
818 // data. For example, the user may request lines "$1-1000" of a file where
819 // each line has 100MB of text.
820 $limit = (1024 * 1024 * 16);
821 if ($length > $limit) {
822 return array();
825 $data = $log->loadData($min_pos, $length);
827 $offset = $min_pos;
828 $min_offset = null;
829 $max_offset = null;
831 $lines = $this->getLines($data);
832 $number = ($min_line + 1);
834 foreach ($lines as $line) {
835 if ($min_offset === null) {
836 if ($number === $range_min) {
837 $min_offset = $offset;
841 $offset += strlen($line);
843 if ($max_offset === null) {
844 if ($number === $range_max) {
845 $max_offset = $offset;
846 break;
850 $number += 1;
853 $context_lines = 8;
855 // Build views around the beginning and ends of the respective lines. We
856 // expect these views to overlap significantly in normal circumstances
857 // and be merged later.
858 $views = array();
860 if ($min_offset !== null) {
861 $views[] = array(
862 'offset' => $min_offset,
863 'lines' => $context_lines + ($range_max - $range_min) - 1,
864 'direction' => 1,
865 'limit' => null,
867 if ($min_offset > 0) {
868 $views[] = array(
869 'offset' => $min_offset,
870 'lines' => $context_lines,
871 'direction' => -1,
872 'limit' => null,
877 if ($max_offset !== null) {
878 $views[] = array(
879 'offset' => $max_offset,
880 'lines' => $context_lines + ($range_max - $range_min),
881 'direction' => -1,
882 'limit' => null,
884 if ($max_offset < $log_size) {
885 $views[] = array(
886 'offset' => $max_offset,
887 'lines' => $context_lines,
888 'direction' => 1,
889 'limit' => null,
894 return $views;