coverage: Replace color terminal tests with HTML output tests

rust-lang · Mar 20, 2024 · 2f20ff0 · 2f20ff0
1 parent b7dcabe
commit 2f20ff0
Show file tree

Hide file tree

Showing 10 changed files with 434 additions and 87 deletions.
diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs
@@ -726,6 +726,7 @@ pub const UI_EXTENSIONS: &[&str] = &[
     UI_STDERR_32,
     UI_STDERR_16,
     UI_COVERAGE,
+    UI_COVERAGE_HTML,
     UI_COVERAGE_MAP,
 ];
 pub const UI_STDERR: &str = "stderr";
@@ -739,6 +740,7 @@ pub const UI_STDERR_64: &str = "64bit.stderr";
 pub const UI_STDERR_32: &str = "32bit.stderr";
 pub const UI_STDERR_16: &str = "16bit.stderr";
 pub const UI_COVERAGE: &str = "coverage";
+pub const UI_COVERAGE_HTML: &str = "coverage.html";
 pub const UI_COVERAGE_MAP: &str = "cov-map";
 
 /// Absolute path to the directory where all output for all tests in the given

diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs
@@ -9,7 +9,7 @@ use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc};
 use crate::common::{CompareMode, FailMode, PassMode};
 use crate::common::{Config, TestPaths};
 use crate::common::{CoverageMap, CoverageRun, Pretty, RunPassValgrind};
-use crate::common::{UI_COVERAGE, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
+use crate::common::{UI_COVERAGE, UI_COVERAGE_HTML, UI_COVERAGE_MAP, UI_RUN_STDERR, UI_RUN_STDOUT};
 use crate::compute_diff::{write_diff, write_filtered_diff};
 use crate::errors::{self, Error, ErrorKind};
 use crate::header::TestProps;
@@ -579,13 +579,18 @@ impl<'test> TestCx<'test> {
             self.fatal_proc_rec("llvm-cov show failed!", &proc_res);
         }
 
-        let kind = UI_COVERAGE;
+        let is_html = self.props.llvm_cov_flags.iter().any(|s| s.contains("-format=html"));
+        let kind = if is_html { UI_COVERAGE_HTML } else { UI_COVERAGE };
 
         let expected_coverage = self.load_expected_output(kind);
-        let normalized_actual_coverage =
-            self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| {
+
+        let normalized_actual_coverage = if is_html {
+            self.normalize_coverage_html(&proc_res.stdout)
+        } else {
+            self.normalize_coverage_text(&proc_res.stdout).unwrap_or_else(|err| {
                 self.fatal_proc_rec(&err, &proc_res);
-            });
+            })
+        };
 
         let coverage_errors =
             self.compare_output(kind, &normalized_actual_coverage, &expected_coverage);
@@ -703,17 +708,52 @@ impl<'test> TestCx<'test> {
         proc_res
     }
 
-    fn normalize_coverage_output(&self, coverage: &str) -> Result<String, String> {
-        let normalized = self.normalize_output(coverage, &[]);
-        let normalized = Self::anonymize_coverage_line_numbers(&normalized);
+    fn normalize_coverage_text(&self, coverage: &str) -> Result<String, String> {
+        let coverage = self.normalize_output(coverage, &[]);
+        let coverage = Self::anonymize_coverage_line_numbers(&coverage);
 
-        let mut lines = normalized.lines().collect::<Vec<_>>();
+        let mut lines = coverage.lines().collect::<Vec<_>>();
 
         Self::sort_coverage_file_sections(&mut lines)?;
         Self::sort_coverage_subviews(&mut lines)?;
 
-        let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
-        Ok(joined_lines)
+        let coverage = lines.iter().flat_map(|line| [line, "\n"]).collect::<String>();
+        Ok(coverage)
+    }
+
+    fn normalize_coverage_html(&self, coverage: &str) -> String {
+        let coverage = self.normalize_output(&coverage, &[]);
+
+        // HTML coverage reports are produced by a known tool from input we control,
+        // so we can get away with using simple regexes and string replacement.
+
+        // Replace line number hyperlinks with the placeholder `LL`,
+        // so that the tests are less sensitive to lines being added/removed.
+        // (We match a lot of context so that we don't scrub execution counts.)
+        static LINE_NUMBER: Lazy<Regex> = Lazy::new(|| {
+            let re = r"(<td class='line-number'><a) name='L\d+' href='#L\d+'(><pre>)\d+(</pre></a></td>)";
+            //          ^^^^^^^^^^^^^^^^^^^^^^^^^^                           ^^^^^^  LL ^^^^^^^^^^^^^^^
+            Regex::new(re).unwrap()
+        });
+        let coverage = LINE_NUMBER.replace_all(&coverage, "${1}${2}LL${3}");
+
+        // Also remove the line link from "jump to first uncovered line".
+        static JUMP_TO: Lazy<Regex> = Lazy::new(|| {
+            let re = r"(<a) href='#L\d+'(>jump to first)";
+            //          ^^               ^^^^^^^^^^^^^^
+            Regex::new(re).unwrap()
+        });
+        let coverage = JUMP_TO.replace_all(&coverage, "${1}${2}");
+
+        // FIXME(Zalathar): Figure out how to sort file sections, if we ever
+        // need an HTML coverage test with multiple files.
+
+        // Add a line break after `</tr>`, for ease of reading and nicer diffs.
+        let mut coverage = coverage.replace("</tr>", "</tr>\n");
+
+        // Add a final line break so that git doesn't bug us about it.
+        coverage.push('\n');
+        coverage
     }
 
     /// Replace line numbers in coverage reports with the placeholder `LL`,

diff --git a/tests/coverage/color.coverage b/tests/coverage/color.coverage
diff --git a/tests/coverage/color.rs b/tests/coverage/color.rs
diff --git a/tests/coverage/html.coverage.html b/tests/coverage/html.coverage.html
@@ -0,0 +1,171 @@
+<!doctype html><html><head><meta name='viewport' content='width=device-width,initial-scale=1'><meta charset='UTF-8'><style>.red {
+  background-color: #ffd0d0;
+}
+.cyan {
+  background-color: cyan;
+}
+body {
+  font-family: -apple-system, sans-serif;
+}
+pre {
+  margin-top: 0px !important;
+  margin-bottom: 0px !important;
+}
+.source-name-title {
+  padding: 5px 10px;
+  border-bottom: 1px solid #dbdbdb;
+  background-color: #eee;
+  line-height: 35px;
+}
+.centered {
+  display: table;
+  margin-left: left;
+  margin-right: auto;
+  border: 1px solid #dbdbdb;
+  border-radius: 3px;
+}
+.expansion-view {
+  background-color: rgba(0, 0, 0, 0);
+  margin-left: 0px;
+  margin-top: 5px;
+  margin-right: 5px;
+  margin-bottom: 5px;
+  border: 1px solid #dbdbdb;
+  border-radius: 3px;
+}
+table {
+  border-collapse: collapse;
+}
+.light-row {
+  background: #ffffff;
+  border: 1px solid #dbdbdb;
+  border-left: none;
+  border-right: none;
+}
+.light-row-bold {
+  background: #ffffff;
+  border: 1px solid #dbdbdb;
+  border-left: none;
+  border-right: none;
+  font-weight: bold;
+}
+.column-entry {
+  text-align: left;
+}
+.column-entry-bold {
+  font-weight: bold;
+  text-align: left;
+}
+.column-entry-yellow {
+  text-align: left;
+  background-color: #ffffd0;
+}
+.column-entry-yellow:hover, tr:hover .column-entry-yellow {
+  background-color: #fffff0;
+}
+.column-entry-red {
+  text-align: left;
+  background-color: #ffd0d0;
+}
+.column-entry-red:hover, tr:hover .column-entry-red {
+  background-color: #fff0f0;
+}
+.column-entry-gray {
+  text-align: left;
+  background-color: #fbfbfb;
+}
+.column-entry-gray:hover, tr:hover .column-entry-gray {
+  background-color: #f0f0f0;
+}
+.column-entry-green {
+  text-align: left;
+  background-color: #d0ffd0;
+}
+.column-entry-green:hover, tr:hover .column-entry-green {
+  background-color: #f0fff0;
+}
+.line-number {
+  text-align: right;
+  color: #aaa;
+}
+.covered-line {
+  text-align: right;
+  color: #0080ff;
+}
+.uncovered-line {
+  text-align: right;
+  color: #ff3300;
+}
+.tooltip {
+  position: relative;
+  display: inline;
+  background-color: #b3e6ff;
+  text-decoration: none;
+}
+.tooltip span.tooltip-content {
+  position: absolute;
+  width: 100px;
+  margin-left: -50px;
+  color: #FFFFFF;
+  background: #000000;
+  height: 30px;
+  line-height: 30px;
+  text-align: center;
+  visibility: hidden;
+  border-radius: 6px;
+}
+.tooltip span.tooltip-content:after {
+  content: '';
+  position: absolute;
+  top: 100%;
+  left: 50%;
+  margin-left: -8px;
+  width: 0; height: 0;
+  border-top: 8px solid #000000;
+  border-right: 8px solid transparent;
+  border-left: 8px solid transparent;
+}
+:hover.tooltip span.tooltip-content {
+  visibility: visible;
+  opacity: 0.8;
+  bottom: 30px;
+  left: 50%;
+  z-index: 999;
+}
+th, td {
+  vertical-align: top;
+  padding: 2px 8px;
+  border-collapse: collapse;
+  border-right: solid 1px #eee;
+  border-left: solid 1px #eee;
+  text-align: left;
+}
+td pre {
+  display: inline-block;
+}
+td:first-child {
+  border-left: none;
+}
+td:last-child {
+  border-right: none;
+}
+tr:hover {
+  background-color: #f0f0f0;
+}
+tr:last-child {
+  border-bottom: none;
+}
+tr:has(> td >a:target) > td.code > pre {
+  background-color: #ffa;
+}
+</style></head><body><div class='centered'><table><div class='source-name-title'><pre>$DIR/html.rs</pre></div><tr><td><pre>Line</pre></td><td><pre>Count</pre></td><td><pre>Source (<a>jump to first uncovered line</a>)</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ edition: 2021</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ ignore-mode-coverage-map</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>//@ llvm-cov-flags: --format=html</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre>// Verify that telling `llvm-cov` to emit HTML actually works.</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='uncovered-line'></td><td class='code'><pre></pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>fn main() {</pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>    for <div class='tooltip'><span class='red'>_i</span><span class='tooltip-content'>0</span></div> in 0..0 <div class='tooltip'><span class='red'>{}</span><span class='tooltip-content'>0</span></div></pre></td></tr>
+<tr><td class='line-number'><a><pre>LL</pre></a></td><td class='covered-line'><pre>1</pre></td><td class='code'><pre>}</pre></td></tr>
+</table></div></body></html>
diff --git a/tests/coverage/html.rs b/tests/coverage/html.rs
@@ -0,0 +1,9 @@
+//@ edition: 2021
+//@ ignore-mode-coverage-map
+//@ llvm-cov-flags: --format=html
+
+// Verify that telling `llvm-cov` to emit HTML actually works.
+
+fn main() {
+    for _i in 0..0 {}
+}
diff --git a/tests/coverage/unicode.cov-map b/tests/coverage/unicode.cov-map
@@ -1,5 +1,5 @@
 Function name: unicode::main
-Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 0e, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
+Raw bytes (67): 0x[01, 01, 09, 01, 05, 03, 05, 1e, 0d, 22, 09, 03, 05, 11, 1b, 1e, 0d, 22, 09, 03, 05, 09, 01, 09, 01, 00, 0b, 05, 01, 09, 00, 0c, 03, 00, 10, 00, 1b, 05, 00, 1c, 00, 28, 22, 02, 08, 00, 25, 09, 00, 29, 00, 46, 11, 00, 47, 02, 06, 1b, 02, 06, 00, 07, 17, 02, 05, 01, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 9
@@ -13,7 +13,7 @@ Number of expressions: 9
 - expression 7 operands: lhs = Expression(8, Sub), rhs = Counter(2)
 - expression 8 operands: lhs = Expression(0, Add), rhs = Counter(1)
 Number of file 0 mappings: 9
-- Code(Counter(0)) at (prev + 14, 1) to (start + 0, 11)
+- Code(Counter(0)) at (prev + 9, 1) to (start + 0, 11)
 - Code(Counter(1)) at (prev + 1, 9) to (start + 0, 12)
 - Code(Expression(0, Add)) at (prev + 0, 16) to (start + 0, 27)
     = (c0 + c1)
@@ -28,18 +28,18 @@ Number of file 0 mappings: 9
     = (c4 + ((((c0 + c1) - c1) - c2) + c3))
 
 Function name: unicode::他 (unused)
-Raw bytes (9): 0x[01, 01, 00, 01, 00, 1e, 19, 00, 25]
+Raw bytes (9): 0x[01, 01, 00, 01, 00, 19, 19, 00, 25]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 1
-- Code(Zero) at (prev + 30, 25) to (start + 0, 37)
+- Code(Zero) at (prev + 25, 25) to (start + 0, 37)
 
 Function name: unicode::申し訳ございません
-Raw bytes (9): 0x[01, 01, 00, 01, 01, 18, 01, 02, 02]
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 13, 01, 02, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 0
 Number of file 0 mappings: 1
-- Code(Counter(0)) at (prev + 24, 1) to (start + 2, 2)
+- Code(Counter(0)) at (prev + 19, 1) to (start + 2, 2)
 
diff --git a/tests/coverage/unicode.coverage b/tests/coverage/unicode.coverage