Allow <failure> elements to contain details of a test failure, while …

…allowing <system-err> to catch others

internal/command/junit/junit.go

@@ -18,6 +18,10 @@ import (
 	"github.com/hashicorp/terraform/internal/tfdiags"
 )
 
+var (
+	failedTestSummary = "Test assertion failed"
+)
+
 // TestJUnitXMLFile produces a JUnit XML file at the conclusion of a test
 // run, summarizing the outcome of the test in a form that can then be
 // interpreted by tools which render JUnit XML result reports.
@@ -197,6 +201,14 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source
 		for i, run := range file.Runs {
 			// Each run is a "test case".
 
+			// By creating a map of diags we can delete them as they're used below
+			// This helps to identify diags that are only appropriate to include in
+			// the "system-err" element
+			diagsMap := make(map[int]tfdiags.Diagnostic, len(run.Diagnostics))
+			for i, diag := range run.Diagnostics {
+				diagsMap[i] = diag
+			}
+
 			testCase := testCase{
 				Name: run.Name,
 
@@ -219,31 +231,39 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source
 					Body:    body,
 				}
 			case moduletest.Fail:
+				var diagsStr strings.Builder
+				for key, diag := range diagsMap {
+					// Select for diags resulting from failed assertions
+					if diag.Description().Summary == failedTestSummary {
+						diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80))
+						delete(diagsMap, key)
+					}
+				}
 				testCase.Failure = &withMessage{
 					Message: "Test run failed",
-					// FIXME: What's a useful thing to report in the body
-					// here? A summary of the statuses from all of the
-					// checkable objects in the configuration?
+					Body:    diagsStr.String(),
 				}
 			case moduletest.Error:
 				var diagsStr strings.Builder
-				for _, diag := range run.Diagnostics {
+				for key, diag := range diagsMap {
 					diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80))
+					delete(diagsMap, key)
 				}
 				testCase.Error = &withMessage{
 					Message: "Encountered an error",
 					Body:    diagsStr.String(),
 				}
 			}
-			if len(run.Diagnostics) != 0 && testCase.Error == nil {
+			if len(diagsMap) != 0 && testCase.Error == nil {
 				// If we have diagnostics but the outcome wasn't an error
 				// then we're presumably holding diagnostics that didn't
 				// cause the test to error, such as warnings. We'll place
 				// those into the "system-err" element instead, so that
 				// they'll be reported _somewhere_ at least.
 				var diagsStr strings.Builder
-				for _, diag := range run.Diagnostics {
+				for key, diag := range diagsMap {
 					diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80))
+					delete(diagsMap, key)
 				}
 				testCase.Stderr = &withMessage{
 					Body: diagsStr.String(),