From b70005c3890f169a1ac7b9e05cb5d4377d64d55e Mon Sep 17 00:00:00 2001
From: Esme Povirk <esme@codeweavers.com>
Date: Tue, 18 Jun 2024 19:27:45 +0000
Subject: [PATCH] run-tests: Group all tests for a single dll together.

---
 tools/run-tests/run-tests.cs | 90 ++++++++++++++++++------------------
 1 file changed, 46 insertions(+), 44 deletions(-)

diff --git a/tools/run-tests/run-tests.cs b/tools/run-tests/run-tests.cs
index 0d2371cf..75e9f90b 100644
--- a/tools/run-tests/run-tests.cs
+++ b/tools/run-tests/run-tests.cs
@@ -306,9 +306,9 @@ bool should_run_fixture(string fixture, string arch, bool run_all)
 		return result;
 	}
 
-	void run_clr_test_list(string filename, string fixture, string arch, List<string> to_run, ref bool any_passed, ref bool any_failed, ref bool any_skipped)
+	void run_clr_test_list(string testname, string filename, string arch, List<string> to_run, ref bool any_passed, ref bool any_failed, ref bool any_skipped)
 	{
-		string fullfixture = String.Format("{0}.{1}", arch, fixture);
+		string fulltest = String.Format("{0}.{1}", arch, testname);
 		string outputfile = Path.GetTempFileName();
 
 		try
@@ -319,7 +319,7 @@ void run_clr_test_list(string filename, string fixture, string arch, List<string
 				p.StartInfo.Arguments = String.Format("{0} -labels -format:nunit3 \"-result:{1}\"", Path.GetFileName(filename), outputfile);
 				foreach (string test in to_run)
 				{
-					p.StartInfo.Arguments += String.Format(" -test:{0}.{1}", fixture, test);
+					p.StartInfo.Arguments += String.Format(" -test:{0}", test);
 				}
 				foreach (string cat in skip_categories)
 				{
@@ -332,8 +332,8 @@ void run_clr_test_list(string filename, string fixture, string arch, List<string
 				if (!p.HasExited)
 				{
 					p.Kill();
-					Console.WriteLine("Test timed out: {0}", fullfixture);
-					failing_tests.Add(fullfixture);
+					Console.WriteLine("Test timed out: {0}", fulltest);
+					failing_tests.Add(fulltest);
 					any_failed = true;
 					return;
 				}
@@ -343,9 +343,15 @@ void run_clr_test_list(string filename, string fixture, string arch, List<string
 					using (var reader = XmlReader.Create(outputfile))
 					{
 						bool in_failure = false;
+						string fullfixture = null;
 						while (reader.Read())
 						{
-							if (reader.NodeType == XmlNodeType.Element &&
+							if (reader.NodeType == XmlNodeType.Element && 
+								reader.Name == "test-suite")
+							{
+								fullfixture = string.Format("{0}.{1}", arch, reader["fullname"]);
+							}
+							else if (reader.NodeType == XmlNodeType.Element &&
 								reader.Name == "test-case")
 							{
 								if (reader["result"] == "Passed")
@@ -379,16 +385,16 @@ void run_clr_test_list(string filename, string fixture, string arch, List<string
 				}
 				catch (XmlException)
 				{
-					failing_tests.Add(fullfixture);
-					Console.WriteLine("Test failed(couldn't read test results): {0}", fullfixture);
+					failing_tests.Add(fulltest);
+					Console.WriteLine("Test failed(couldn't read test results): {0}", fulltest);
 					return;
 				}
 				if (num_tests_run < to_run.Count)
 					any_skipped = true;
 				if (!any_failed && p.ExitCode != 0)
 				{
-					failing_tests.Add(fullfixture);
-					Console.WriteLine("Test failed({0}): {1}", p.ExitCode, fullfixture);
+					failing_tests.Add(fulltest);
+					Console.WriteLine("Test failed({0}): {1}", p.ExitCode, fulltest);
 					any_failed = true;
 				}
 			}
@@ -399,12 +405,9 @@ void run_clr_test_list(string filename, string fixture, string arch, List<string
 		}
 	}
 
-	void run_clr_test_fixture(string filename, string fixture, string arch, List<string> testlist, bool run_all)
+	void collect_clr_tests(string filename, string fixture, string arch, List<string> testlist, bool run_all, List<string> to_run)
 	{
 		string fullfixture = String.Format("{0}.{1}", arch, fixture);
-		List<string> to_run = new List<string> ();
-
-		Console.WriteLine("Running {0}", fullfixture);
 
 		List<string> runs = new List<string> ();
 		if (run_list.ContainsKey(fixture) && run_list[fixture] != null)
@@ -425,15 +428,16 @@ void run_clr_test_fixture(string filename, string fixture, string arch, List<str
 		{
 			if ((run_all || runs.Contains(test)) && !skips.Contains(test))
 			{
-				to_run.Add(test);
+				to_run.Add(String.Format("{0}.{1}", fixture, test));
 			}
 		}
+	}
 
-		if (to_run.Count == 0)
-		{
-			Console.WriteLine("All tests skipped: {0}", fullfixture);
-			return;
-		}
+	void run_clr_tests(string testname, string filename, string arch, List<string> to_run)
+	{
+		string fulltest = String.Format("{0}.{1}", arch, filename);
+
+		Console.WriteLine("Running {0}", fulltest);
 
 		int batch_size = 100;
 		bool any_passed = false;
@@ -442,36 +446,28 @@ void run_clr_test_fixture(string filename, string fixture, string arch, List<str
 
 		for (int i=0; i < to_run.Count; i += batch_size)
 		{
-			run_clr_test_list(filename, fixture, arch, to_run.GetRange(i, Math.Min(batch_size, to_run.Count - i)), ref any_passed, ref any_failed, ref any_skipped);
+			run_clr_test_list(testname, filename, arch, to_run.GetRange(i, Math.Min(batch_size, to_run.Count - i)), ref any_passed, ref any_failed, ref any_skipped);
 		}
 
 		if (any_passed)
 		{
 			if (!any_failed && !any_skipped)
 			{
-				passing_tests.Add(fullfixture);
-				Console.WriteLine("Test succeeded: {0}", fullfixture);
+				passing_tests.Add(fulltest);
+				Console.WriteLine("Test succeeded: {0}", fulltest);
 			}
 			else
 			{
-				Console.WriteLine("Some tests succeeded: {0}", fullfixture);
+				Console.WriteLine("Some tests succeeded: {0}", fulltest);
 			}
 		}
 		else if (any_failed)
 		{
-			if (!any_skipped)
-			{
-				failing_tests.Add(fullfixture);
-				Console.WriteLine("Test failed: {0}", fullfixture);
-			}
-			else
-			{
-				Console.WriteLine("Some tests failed: {0}", fullfixture);
-			}
+			Console.WriteLine("Some tests failed: {0}", fulltest);
 		}
 		else
 		{
-			Console.WriteLine("All tests skipped: {0}", fullfixture);
+			Console.WriteLine("All tests skipped: {0}", fulltest);
 		}
 	}
 
@@ -481,6 +477,7 @@ void run_clr_test_dll(string filename, string arch)
 		string testname = basename.Substring(8, basename.Length - 13);
 		string fulltestname = String.Format("{0}.{1}", arch, testname);
 		bool run_all;
+		List<string> tests_to_run = new List<string>();
 		
 		if (skip_list.ContainsKey(testname) ||
 			skip_list.ContainsKey(fulltestname))
@@ -496,8 +493,6 @@ void run_clr_test_dll(string filename, string arch)
 		if (fixtures == null)
 			return;
 
-		bool copied_config = false;
-
 		string nunitlite_config = filename+".nunitlite.config";
 		string exe_config = null;
 
@@ -507,17 +502,24 @@ void run_clr_test_dll(string filename, string arch)
 			var testlist = t.Item2;
 
 			if (should_run_fixture(testfixture, arch, run_all))
-			{
-				if (!copied_config && File.Exists(nunitlite_config)) {
-					exe_config = get_nunit_lite_console(arch)+".config";
-					File.Copy(nunitlite_config, exe_config, true);
-					copied_config = true;
-				}
+				collect_clr_tests(filename, testfixture, arch, testlist, run_all, tests_to_run);
+		}
 
-				run_clr_test_fixture(filename, testfixture, arch, testlist, run_all);
-			}
+		if (tests_to_run.Count == 0) {
+			return;
 		}
 
+		bool copied_config = false;
+
+		if (File.Exists(nunitlite_config))
+		{
+			exe_config = get_nunit_lite_console(arch)+".config";
+			File.Copy(nunitlite_config, exe_config, true);
+			copied_config = true;
+		}
+
+		run_clr_tests(testname, filename, arch, tests_to_run);
+
 		if (copied_config) {
 			File.Delete(exe_config);
 		}