Added coverage stats and report generation.
[chromium-blink-merge.git] / build / android / emma_coverage_stats_test.py
blobd12c3252dae416cdc04ff66f193c83961a5fbf88
1 #!/usr/bin/python
2 # Copyright 2015 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 import json
7 import os
8 import sys
9 import unittest
10 from xml.etree import ElementTree
12 import emma_coverage_stats
13 from pylib import constants
15 sys.path.append(os.path.join(
16 constants.DIR_SOURCE_ROOT, 'third_party', 'pymock'))
17 import mock # pylint: disable=F0401
19 EMPTY_COVERAGE_STATS_DICT = {
20 'files': {},
21 'patch': {
22 'incremental': {
23 'covered': 0, 'total': 0
29 class _EmmaHtmlParserTest(unittest.TestCase):
30 """Tests for _EmmaHtmlParser.
32 Uses modified EMMA report HTML that contains only the subset of tags needed
33 for test verification.
34 """
36 def setUp(self):
37 self.emma_dir = 'fake/dir/'
38 self.parser = emma_coverage_stats._EmmaHtmlParser(self.emma_dir)
39 self.simple_html = '<TR><TD CLASS="p">Test HTML</TD></TR>'
40 self.index_html = (
41 '<HTML>'
42 '<BODY>'
43 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
44 '</TABLE>'
45 '<TABLE CELLSPACING="0" WIDTH="100%">'
46 '</TABLE>'
47 '<TABLE CLASS="it" CELLSPACING="0">'
48 '</TABLE>'
49 '<TABLE CELLSPACING="0" WIDTH="100%">'
50 '<TR>'
51 '<TH CLASS="f">name</TH>'
52 '<TH>class, %</TH>'
53 '<TH>method, %</TH>'
54 '<TH>block, %</TH>'
55 '<TH>line, %</TH>'
56 '</TR>'
57 '<TR CLASS="o">'
58 '<TD><A HREF="_files/0.html"'
59 '>org.chromium.chrome.browser</A></TD>'
60 '<TD CLASS="h">0% (0/3)</TD>'
61 '</TR>'
62 '<TR>'
63 '<TD><A HREF="_files/1.html"'
64 '>org.chromium.chrome.browser.tabmodel</A></TD>'
65 '<TD CLASS="h">0% (0/8)</TD>'
66 '</TR>'
67 '</TABLE>'
68 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
69 '</TABLE>'
70 '</BODY>'
71 '</HTML>'
73 self.package_1_class_list_html = (
74 '<HTML>'
75 '<BODY>'
76 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
77 '</TABLE>'
78 '<TABLE CELLSPACING="0" WIDTH="100%">'
79 '</TABLE>'
80 '<TABLE CELLSPACING="0" WIDTH="100%">'
81 '<TR>'
82 '<TH CLASS="f">name</TH>'
83 '<TH>class, %</TH>'
84 '<TH>method, %</TH>'
85 '<TH>block, %</TH>'
86 '<TH>line, %</TH>'
87 '</TR>'
88 '<TR CLASS="o">'
89 '<TD><A HREF="1e.html">IntentHelper.java</A></TD>'
90 '<TD CLASS="h">0% (0/3)</TD>'
91 '<TD CLASS="h">0% (0/9)</TD>'
92 '<TD CLASS="h">0% (0/97)</TD>'
93 '<TD CLASS="h">0% (0/26)</TD>'
94 '</TR>'
95 '</TABLE>'
96 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
97 '</TABLE>'
98 '</BODY>'
99 '</HTML>'
101 self.package_2_class_list_html = (
102 '<HTML>'
103 '<BODY>'
104 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
105 '</TABLE>'
106 '<TABLE CELLSPACING="0" WIDTH="100%">'
107 '</TABLE>'
108 '<TABLE CELLSPACING="0" WIDTH="100%">'
109 '<TR>'
110 '<TH CLASS="f">name</TH>'
111 '<TH>class, %</TH>'
112 '<TH>method, %</TH>'
113 '<TH>block, %</TH>'
114 '<TH>line, %</TH>'
115 '</TR>'
116 '<TR CLASS="o">'
117 '<TD><A HREF="1f.html">ContentSetting.java</A></TD>'
118 '<TD CLASS="h">0% (0/1)</TD>'
119 '</TR>'
120 '<TR>'
121 '<TD><A HREF="20.html">DevToolsServer.java</A></TD>'
122 '</TR>'
123 '<TR CLASS="o">'
124 '<TD><A HREF="21.html">FileProviderHelper.java</A></TD>'
125 '</TR>'
126 '<TR>'
127 '<TD><A HREF="22.html">ContextualMenuBar.java</A></TD>'
128 '</TR>'
129 '<TR CLASS="o">'
130 '<TD><A HREF="23.html">AccessibilityUtil.java</A></TD>'
131 '</TR>'
132 '<TR>'
133 '<TD><A HREF="24.html">NavigationPopup.java</A></TD>'
134 '</TR>'
135 '</TABLE>'
136 '<TABLE CLASS="hdft" CELLSPACING="0" WIDTH="100%">'
137 '</TABLE>'
138 '</BODY>'
139 '</HTML>'
141 self.partially_covered_tr_html = (
142 '<TR CLASS="p">'
143 '<TD CLASS="l" TITLE="78% line coverage (7 out of 9)">108</TD>'
144 '<TD TITLE="78% line coverage (7 out of 9 instructions)">'
145 'if (index &lt; 0 || index = mSelectors.size()) index = 0;</TD>'
146 '</TR>'
148 self.covered_tr_html = (
149 '<TR CLASS="c">'
150 '<TD CLASS="l">110</TD>'
151 '<TD> if (mSelectors.get(index) != null) {</TD>'
152 '</TR>'
154 self.not_executable_tr_html = (
155 '<TR>'
156 '<TD CLASS="l">109</TD>'
157 '<TD> </TD>'
158 '</TR>'
160 self.tr_with_extra_a_tag = (
161 '<TR CLASS="z">'
162 '<TD CLASS="l">'
163 '<A name="1f">54</A>'
164 '</TD>'
165 '<TD> }</TD>'
166 '</TR>'
169 def testInit(self):
170 emma_dir = self.emma_dir
171 parser = emma_coverage_stats._EmmaHtmlParser(emma_dir)
172 self.assertEqual(parser._base_dir, emma_dir)
173 self.assertEqual(parser._emma_files_path, 'fake/dir/_files')
174 self.assertEqual(parser._index_path, 'fake/dir/index.html')
176 def testFindElements_basic(self):
177 read_values = [self.simple_html]
178 found, _ = MockOpenForFunction(self.parser._FindElements, read_values,
179 file_path='fake', xpath_selector='.//TD')
180 self.assertIs(type(found), list)
181 self.assertIs(type(found[0]), ElementTree.Element)
182 self.assertEqual(found[0].text, 'Test HTML')
184 def testFindElements_multipleElements(self):
185 multiple_trs = self.not_executable_tr_html + self.covered_tr_html
186 read_values = ['<div>' + multiple_trs + '</div>']
187 found, _ = MockOpenForFunction(self.parser._FindElements, read_values,
188 file_path='fake', xpath_selector='.//TR')
189 self.assertEquals(2, len(found))
191 def testFindElements_noMatch(self):
192 read_values = [self.simple_html]
193 found, _ = MockOpenForFunction(self.parser._FindElements, read_values,
194 file_path='fake', xpath_selector='.//TR')
195 self.assertEqual(found, [])
197 def testFindElements_badFilePath(self):
198 with self.assertRaises(IOError):
199 with mock.patch('os.path.exists', return_value=False):
200 self.parser._FindElements('fake', xpath_selector='//tr')
202 def testGetPackageNameToEmmaFileDict_basic(self):
203 expected_dict = {
204 'org.chromium.chrome.browser.AccessibilityUtil.java':
205 'fake/dir/_files/23.html',
206 'org.chromium.chrome.browser.ContextualMenuBar.java':
207 'fake/dir/_files/22.html',
208 'org.chromium.chrome.browser.tabmodel.IntentHelper.java':
209 'fake/dir/_files/1e.html',
210 'org.chromium.chrome.browser.ContentSetting.java':
211 'fake/dir/_files/1f.html',
212 'org.chromium.chrome.browser.DevToolsServer.java':
213 'fake/dir/_files/20.html',
214 'org.chromium.chrome.browser.NavigationPopup.java':
215 'fake/dir/_files/24.html',
216 'org.chromium.chrome.browser.FileProviderHelper.java':
217 'fake/dir/_files/21.html'}
219 read_values = [self.index_html, self.package_1_class_list_html,
220 self.package_2_class_list_html]
221 return_dict, mock_open = MockOpenForFunction(
222 self.parser.GetPackageNameToEmmaFileDict, read_values)
224 self.assertDictEqual(return_dict, expected_dict)
225 self.assertEqual(mock_open.call_count, 3)
226 calls = [mock.call('fake/dir/index.html'),
227 mock.call('fake/dir/_files/1.html'),
228 mock.call('fake/dir/_files/0.html')]
229 mock_open.assert_has_calls(calls)
231 def testGetPackageNameToEmmaFileDict_noPackageElements(self):
232 self.parser._FindElements = mock.Mock(return_value=[])
233 return_dict = self.parser.GetPackageNameToEmmaFileDict()
234 self.assertDictEqual({}, return_dict)
236 def testGetPackageNameToEmmaFileDict_badFilePath(self):
237 self.parser._FindElements = mock.Mock(return_value=[])
238 return_dict = self.parser.GetPackageNameToEmmaFileDict()
239 self.assertEqual(return_dict, {})
241 def testGetLineCoverage_status_basic(self):
242 line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html])
243 self.assertEqual(line_coverage[0].covered_status,
244 emma_coverage_stats.COVERED)
246 def testGetLineCoverage_status_statusMissing(self):
247 line_coverage = self.GetLineCoverageWithFakeElements(
248 [self.not_executable_tr_html])
249 self.assertEqual(line_coverage[0].covered_status,
250 emma_coverage_stats.NOT_EXECUTABLE)
252 def testGetLineCoverage_fractionalCoverage_basic(self):
253 line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html])
254 self.assertEqual(line_coverage[0].fractional_line_coverage, 1.0)
256 def testGetLineCoverage_fractionalCoverage_partial(self):
257 line_coverage = self.GetLineCoverageWithFakeElements(
258 [self.partially_covered_tr_html])
259 self.assertEqual(line_coverage[0].fractional_line_coverage, 0.78)
261 def testGetLineCoverage_lineno_basic(self):
262 line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html])
263 self.assertEqual(line_coverage[0].lineno, 110)
265 def testGetLineCoverage_lineno_withAlternativeHtml(self):
266 line_coverage = self.GetLineCoverageWithFakeElements(
267 [self.tr_with_extra_a_tag])
268 self.assertEqual(line_coverage[0].lineno, 54)
270 def testGetLineCoverage_source(self):
271 self.parser._FindElements = mock.Mock(
272 return_value=[ElementTree.fromstring(self.covered_tr_html)])
273 line_coverage = self.parser.GetLineCoverage('fake_path')
274 self.assertEqual(line_coverage[0].source,
275 ' if (mSelectors.get(index) != null) {')
277 def testGetLineCoverage_multipleElements(self):
278 line_coverage = self.GetLineCoverageWithFakeElements(
279 [self.covered_tr_html, self.partially_covered_tr_html,
280 self.tr_with_extra_a_tag])
281 self.assertEqual(len(line_coverage), 3)
283 def GetLineCoverageWithFakeElements(self, html_elements):
284 """Wraps GetLineCoverage to work with extra whitespace characters.
286 The test HTML strings include extra whitespace characters to make the HTML
287 human readable. This isn't the case with EMMA HTML files, so we need to
288 remove all the unnecessary whitespace.
290 Args:
291 html_elements: List of strings each representing an HTML element.
293 Returns:
294 A list of LineCoverage objects.
296 elements = [ElementTree.fromstring(string) for string in html_elements]
297 with mock.patch('emma_coverage_stats._EmmaHtmlParser._FindElements',
298 return_value=elements):
299 return self.parser.GetLineCoverage('fake_path')
302 class _EmmaCoverageStatsTest(unittest.TestCase):
303 """Tests for _EmmaCoverageStats."""
305 def setUp(self):
306 self.good_source_to_emma = {
307 '/path/to/1/File1.java': '/emma/1.html',
308 '/path/2/File2.java': '/emma/2.html',
309 '/path/2/File3.java': '/emma/3.html'
311 self.line_coverage = [
312 emma_coverage_stats.LineCoverage(
313 1, '', emma_coverage_stats.COVERED, 1.0),
314 emma_coverage_stats.LineCoverage(
315 2, '', emma_coverage_stats.COVERED, 1.0),
316 emma_coverage_stats.LineCoverage(
317 3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0),
318 emma_coverage_stats.LineCoverage(
319 4, '', emma_coverage_stats.NOT_COVERED, 1.0),
320 emma_coverage_stats.LineCoverage(
321 5, '', emma_coverage_stats.PARTIALLY_COVERED, 0.85),
322 emma_coverage_stats.LineCoverage(
323 6, '', emma_coverage_stats.PARTIALLY_COVERED, 0.20)
325 self.lines_for_coverage = [1, 3, 5, 6]
326 with mock.patch('emma_coverage_stats._EmmaHtmlParser._FindElements',
327 return_value=[]):
328 self.simple_coverage = emma_coverage_stats._EmmaCoverageStats(
329 'fake_dir', {})
331 def testInit(self):
332 coverage_stats = self.simple_coverage
333 self.assertIsInstance(coverage_stats._emma_parser,
334 emma_coverage_stats._EmmaHtmlParser)
335 self.assertIsInstance(coverage_stats._source_to_emma, dict)
337 def testNeedsCoverage_withExistingJavaFile(self):
338 test_file = '/path/to/file/File.java'
339 with mock.patch('os.path.exists', return_value=True):
340 self.assertTrue(
341 emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
343 def testNeedsCoverage_withNonJavaFile(self):
344 test_file = '/path/to/file/File.c'
345 with mock.patch('os.path.exists', return_value=True):
346 self.assertFalse(
347 emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
349 def testNeedsCoverage_fileDoesNotExist(self):
350 test_file = '/path/to/file/File.java'
351 with mock.patch('os.path.exists', return_value=False):
352 self.assertFalse(
353 emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
355 def testGetPackageNameFromFile_basic(self):
356 test_file_text = """// Test Copyright
357 package org.chromium.chrome.browser;
358 import android.graphics.RectF;"""
359 result_package, _ = MockOpenForFunction(
360 emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile,
361 [test_file_text], file_path='/path/to/file/File.java')
362 self.assertEqual(result_package, 'org.chromium.chrome.browser.File.java')
364 def testGetPackageNameFromFile_noPackageStatement(self):
365 result_package, _ = MockOpenForFunction(
366 emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile,
367 ['not a package statement'], file_path='/path/to/file/File.java')
368 self.assertIsNone(result_package)
370 def testGetSummaryStatsForLines_basic(self):
371 covered, total = self.simple_coverage.GetSummaryStatsForLines(
372 self.line_coverage)
373 self.assertEqual(covered, 3.05)
374 self.assertEqual(total, 5)
376 def testGetSourceFileToEmmaFileDict(self):
377 package_names = {
378 '/path/to/1/File1.java': 'org.fake.one.File1.java',
379 '/path/2/File2.java': 'org.fake.File2.java',
380 '/path/2/File3.java': 'org.fake.File3.java'
382 package_to_emma = {
383 'org.fake.one.File1.java': '/emma/1.html',
384 'org.fake.File2.java': '/emma/2.html',
385 'org.fake.File3.java': '/emma/3.html'
387 with mock.patch('os.path.exists', return_value=True):
388 coverage_stats = self.simple_coverage
389 coverage_stats._emma_parser.GetPackageNameToEmmaFileDict = mock.MagicMock(
390 return_value=package_to_emma)
391 coverage_stats.GetPackageNameFromFile = lambda x: package_names[x]
392 result_dict = coverage_stats._GetSourceFileToEmmaFileDict(
393 package_names.keys())
394 self.assertDictEqual(result_dict, self.good_source_to_emma)
396 def testGetLineCoverageForFile_basic(self):
397 java_file_path = '/path/to/1/File1.java'
398 line_coverage = emma_coverage_stats.LineCoverage(
399 1, '', emma_coverage_stats.COVERED, 1.0)
400 expected_line_coverage = list(line_coverage)
401 coverage_stats = self.simple_coverage
402 coverage_stats._source_to_emma = self.good_source_to_emma
403 coverage_stats._emma_parser.GetLineCoverage = mock.MagicMock(
404 return_value=expected_line_coverage)
405 coverage_info = coverage_stats._GetLineCoverageForFile(java_file_path)
406 self.assertListEqual(coverage_info, expected_line_coverage)
408 def testGetLineCoverageForFile_noInfo(self):
409 with mock.patch('os.path.exists', return_value=False):
410 coverage_info = self.simple_coverage._GetLineCoverageForFile('fake_path')
411 self.assertIsNone(coverage_info)
413 def testGetCoverageDictForFile(self):
414 line_coverage = self.line_coverage
415 self.simple_coverage._GetLineCoverageForFile = mock.Mock(
416 return_value=line_coverage)
417 lines = self.lines_for_coverage
418 expected_dict = {
419 'absolute': {
420 'covered': 3.05,
421 'total': 5
423 'incremental': {
424 'covered': 2.05,
425 'total': 3
427 'source': [
429 'line': line_coverage[0].source,
430 'coverage': line_coverage[0].covered_status,
431 'changed': True
434 'line': line_coverage[1].source,
435 'coverage': line_coverage[1].covered_status,
436 'changed': False
439 'line': line_coverage[2].source,
440 'coverage': line_coverage[2].covered_status,
441 'changed': True
444 'line': line_coverage[3].source,
445 'coverage': line_coverage[3].covered_status,
446 'changed': False
449 'line': line_coverage[4].source,
450 'coverage': line_coverage[4].covered_status,
451 'changed': True
454 'line': line_coverage[5].source,
455 'coverage': line_coverage[5].covered_status,
456 'changed': True
460 result_dict = self.simple_coverage.GetCoverageDictForFile(
461 line_coverage, lines)
462 self.assertDictEqual(result_dict, expected_dict)
464 def testGetCoverageDictForFile_emptyCoverage(self):
465 expected_dict = {
466 'absolute': {'covered': 0, 'total': 0},
467 'incremental': {'covered': 0, 'total': 0},
468 'source': []
470 self.simple_coverage._GetLineCoverageForFile = mock.Mock(return_value=[])
471 result_dict = self.simple_coverage.GetCoverageDictForFile('fake_dir', {})
472 self.assertDictEqual(result_dict, expected_dict)
474 def testGetCoverageDictFor_basic(self):
475 files_for_coverage = {
476 '/path/to/1/File1.java': [1, 3, 4],
477 '/path/2/File2.java': [1, 2]
479 coverage_info = {
480 '/path/to/1/File1.java': [
481 emma_coverage_stats.LineCoverage(
482 1, '', emma_coverage_stats.COVERED, 1.0),
483 emma_coverage_stats.LineCoverage(
484 2, '', emma_coverage_stats.PARTIALLY_COVERED, 0.5),
485 emma_coverage_stats.LineCoverage(
486 3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0),
487 emma_coverage_stats.LineCoverage(
488 4, '', emma_coverage_stats.COVERED, 1.0)
490 '/path/2/File2.java': [
491 emma_coverage_stats.LineCoverage(
492 1, '', emma_coverage_stats.NOT_COVERED, 1.0),
493 emma_coverage_stats.LineCoverage(
494 2, '', emma_coverage_stats.COVERED, 1.0)
497 expected_dict = {
498 'files': {
499 '/path/2/File2.java': {
500 'absolute': {'covered': 1, 'total': 2},
501 'incremental': {'covered': 1, 'total': 2},
502 'source': [{'changed': True, 'coverage': 0, 'line': ''},
503 {'changed': True, 'coverage': 1, 'line': ''}]
505 '/path/to/1/File1.java': {
506 'absolute': {'covered': 2.5, 'total': 3},
507 'incremental': {'covered': 2, 'total': 2},
508 'source': [{'changed': True, 'coverage': 1, 'line': ''},
509 {'changed': False, 'coverage': 2, 'line': ''},
510 {'changed': True, 'coverage': -1, 'line': ''},
511 {'changed': True, 'coverage': 1, 'line': ''}]
514 'patch': {'incremental': {'covered': 3, 'total': 4}}
516 # Return the relevant coverage info for each file. We aren't testing
517 # _GetCoverageStatusForFile here.
518 self.simple_coverage._GetLineCoverageForFile = lambda x: coverage_info[x]
519 result_dict = self.simple_coverage.GetCoverageDict(
520 files_for_coverage)
521 self.assertDictEqual(result_dict, expected_dict)
523 def testGetCoverageDict_noCoverage(self):
524 result_dict = self.simple_coverage.GetCoverageDict({})
525 self.assertDictEqual(result_dict, EMPTY_COVERAGE_STATS_DICT)
528 class EmmaCoverageStatsGenerateCoverageReport(unittest.TestCase):
529 """Tests for GenerateCoverageReport."""
531 def testGenerateCoverageReport_missingJsonFile(self):
532 with self.assertRaises(IOError):
533 with mock.patch('os.path.exists', return_value=False):
534 emma_coverage_stats.GenerateCoverageReport('', '', '')
536 def testGenerateCoverageReport_invalidJsonFile(self):
537 with self.assertRaises(ValueError):
538 with mock.patch('os.path.exists', return_value=True):
539 MockOpenForFunction(emma_coverage_stats.GenerateCoverageReport, [''],
540 line_coverage_file='', out_file_path='',
541 coverage_dir='')
544 def MockOpenForFunction(func, side_effects, **kwargs):
545 """Allows easy mock open and read for callables that open multiple files.
547 Will mock the python open function in a way such that each time read() is
548 called on an open file, the next element in |side_effects| is returned. This
549 makes it easier to test functions that call open() multiple times.
551 Args:
552 func: The callable to invoke once mock files are setup.
553 side_effects: A list of return values for each file to return once read.
554 Length of list should be equal to the number calls to open in |func|.
555 **kwargs: Keyword arguments to be passed to |func|.
557 Returns:
558 A tuple containing the return value of |func| and the MagicMock object used
559 to mock all calls to open respectively.
561 mock_open = mock.mock_open()
562 mock_open.side_effect = [mock.mock_open(read_data=side_effect).return_value
563 for side_effect in side_effects]
564 with mock.patch('__builtin__.open', mock_open):
565 return func(**kwargs), mock_open
568 if __name__ == '__main__':
569 # Suppress logging messages.
570 unittest.main(buffer=True)