From 932a3411af98f7aada7100b66c905045f8ca4119 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emysl=20Karbula?= Date: Sun, 7 Jan 2024 16:39:27 +0100 Subject: [PATCH] Implemented `console-lines` mode which prints each test on separate line. (#443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is handy for environments with non-standard (buffered) handling of standard output, for example Github Actions (where a progress of tests cannot be seen until until end-of-line appears, which in standard `console` mode happens only when all tests finish) or Docker Compose logging output, where in standard `console` mode each finished test's dot is printed alone on separate line. Or the console-lines mode can be handy just to see a more detailed progress of tests in all environments, because it outputs something like this: ``` 路 1/85 Framework/Assert.contains.phpt OK in 0.14 s 路 2/85 CodeCoverage/PhpParser.parse.edge.phpt OK in 0.17 s 路 3/85 CodeCoverage/PhpParser.parse.lines-of-code.phpt SKIPPED in 0.18 s 路 4/85 CodeCoverage/PhpParser.parse.lines.phpt FAILED in 0.19 s ... ``` Also, "cider mode" now shows a lemon emoji for skipped tests. Co-authored-by: David Grudl --- phpstan.neon | 2 + readme.md | 3 +- src/Runner/CliTester.php | 33 +++-- src/Runner/Output/ConsolePrinter.php | 127 ++++++++++++++---- src/Runner/Test.php | 19 ++- .../OutputHandlers.expect.consoleLines.txt | 73 ++++++++++ tests/RunnerOutput/OutputHandlers.phpt | 3 +- 7 files changed, 225 insertions(+), 35 deletions(-) create mode 100644 tests/RunnerOutput/OutputHandlers.expect.consoleLines.txt diff --git a/phpstan.neon b/phpstan.neon index da7b95c1..550d4542 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -4,3 +4,5 @@ parameters: paths: - src + typeAliases: + Alias_TestResultState: 'Tester\Runner\Test::Passed|Tester\Runner\Test::Skipped|Tester\Runner\Test::Failed|Tester\Runner\Test::Prepared' diff --git a/readme.md b/readme.md index 441047e4..62ea436b 100644 --- a/readme.md +++ b/readme.md @@ -223,7 +223,8 @@ Options: -s Show information about skipped tests. --stop-on-fail Stop execution upon the first failure. -j Run jobs in parallel (default: 8). - -o Specify output format. + -o + Specify output format. -w | --watch Watch directory. -i | --info Show tests environment info and exit. --setup Script for runner setup. diff --git a/src/Runner/CliTester.php b/src/Runner/CliTester.php index d1cfb5cd..34487e59 100644 --- a/src/Runner/CliTester.php +++ b/src/Runner/CliTester.php @@ -13,7 +13,7 @@ use Tester\Dumper; use Tester\Environment; use Tester\Helpers; - +use Tester\Runner\Output\ConsolePrinter; /** * CLI Tester. @@ -112,7 +112,7 @@ private function loadOptions(): CommandLine -s Show information about skipped tests. --stop-on-fail Stop execution upon the first failure. -j Run jobs in parallel (default: 8). - -o (e.g. -o junit:output.xml) + -o (e.g. -o junit:output.xml) Specify one or more output formats with optional file name. -w | --watch Watch directory. -i | --info Show tests environment info and exit. @@ -219,18 +219,14 @@ private function createRunner(): Runner $runner->setTempDirectory($this->options['--temp']); if ($this->stdoutFormat === null) { - $runner->outputHandlers[] = new Output\ConsolePrinter( - $runner, - (bool) $this->options['-s'], - 'php://output', - (bool) $this->options['--cider'], - ); + $runner->outputHandlers[] = $this->buildConsolePrinter($runner, 'php://output', false); } foreach ($this->options['-o'] as $output) { [$format, $file] = $output; match ($format) { - 'console' => $runner->outputHandlers[] = new Output\ConsolePrinter($runner, (bool) $this->options['-s'], $file, (bool) $this->options['--cider']), + 'console' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, false), + 'console-lines' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, true), 'tap' => $runner->outputHandlers[] = new Output\TapPrinter($file), 'junit' => $runner->outputHandlers[] = new Output\JUnitPrinter($file), 'log' => $runner->outputHandlers[] = new Output\Logger($runner, $file), @@ -248,6 +244,25 @@ private function createRunner(): Runner return $runner; } + /** + * Builds and returns a new `ConsolePrinter`. + * @param bool $lineMode If `true`, reports each finished test on separate line. + */ + private function buildConsolePrinter( + Runner $runner, + ?string $file, + bool $lineMode, + ): ConsolePrinter + { + return new Output\ConsolePrinter( + $runner, + (bool) $this->options['-s'], + $file, + (bool) $this->options['--cider'], + $lineMode, + ); + } + private function prepareCodeCoverage(Runner $runner): string { diff --git a/src/Runner/Output/ConsolePrinter.php b/src/Runner/Output/ConsolePrinter.php index 772697e4..646ff59f 100644 --- a/src/Runner/Output/ConsolePrinter.php +++ b/src/Runner/Output/ConsolePrinter.php @@ -20,46 +20,66 @@ */ class ConsolePrinter implements Tester\Runner\OutputHandler { - private Runner $runner; - /** @var resource */ private $file; - private bool $displaySkipped = false; - private string $buffer; + + /** @var list */ + private array $buffer; + + /** + * @phpstan-var array + * @var array + */ + private array $symbols; + + /** + * @phpstan-var array + * @var array + */ + private array $results = [ + Test::Passed => 0, + Test::Skipped => 0, + Test::Failed => 0, + ]; + private float $time; private int $count; - private array $results; private ?string $baseDir; - private array $symbols; - + private int $resultsCount = 0; + /** + * @param bool $lineMode If `true`, reports each finished test on separate line. + */ public function __construct( - Runner $runner, - bool $displaySkipped = false, + private Runner $runner, + private bool $displaySkipped = false, ?string $file = null, bool $ciderMode = false, + private bool $lineMode = false, ) { $this->runner = $runner; $this->displaySkipped = $displaySkipped; $this->file = fopen($file ?: 'php://output', 'w'); + $this->symbols = [ - Test::Passed => $ciderMode ? Dumper::color('green', '馃崕') : '.', - Test::Skipped => 's', - Test::Failed => $ciderMode ? Dumper::color('red', '馃崕') : Dumper::color('white/red', 'F'), + Test::Passed => $this->lineMode ? Dumper::color('lime', 'OK') : '.', + Test::Skipped => $this->lineMode ? Dumper::color('yellow', 'SKIP') : 's', + Test::Failed => $this->lineMode ? Dumper::color('white/red', 'FAIL') : Dumper::color('white/red', 'F'), ]; + + if ($ciderMode) { + $this->symbols[Test::Passed] = '馃崗'; + $this->symbols[Test::Skipped] = '馃崑'; + $this->symbols[Test::Failed] = '馃崕'; + } } public function begin(): void { $this->count = 0; - $this->buffer = ''; + $this->buffer = []; $this->baseDir = null; - $this->results = [ - Test::Passed => 0, - Test::Skipped => 0, - Test::Failed => 0, - ]; $this->time = -microtime(true); fwrite($this->file, $this->runner->getInterpreter()->getShortInfo() . ' | ' . $this->runner->getInterpreter()->getCommandLine() @@ -94,15 +114,17 @@ public function prepare(Test $test): void public function finish(Test $test): void { $this->results[$test->getResult()]++; - fwrite($this->file, $this->symbols[$test->getResult()]); + $this->lineMode + ? $this->printFinishedTestLine($test) + : $this->printFinishedTestDot($test); $title = ($test->title ? "$test->title | " : '') . substr($test->getSignature(), strlen($this->baseDir)); $message = ' ' . str_replace("\n", "\n ", trim((string) $test->message)) . "\n\n"; $message = preg_replace('/^ $/m', '', $message); if ($test->getResult() === Test::Failed) { - $this->buffer .= Dumper::color('red', "-- FAILED: $title") . "\n$message"; + $this->buffer[] = Dumper::color('red', "-- FAILED: $title") . "\n$message"; } elseif ($test->getResult() === Test::Skipped && $this->displaySkipped) { - $this->buffer .= "-- Skipped: $title\n$message"; + $this->buffer[] = "-- Skipped: $title\n$message"; } } @@ -111,7 +133,7 @@ public function end(): void { $run = array_sum($this->results); fwrite($this->file, !$this->count ? "No tests found\n" : - "\n\n" . $this->buffer . "\n" + "\n\n" . implode('', $this->buffer) . "\n" . ($this->results[Test::Failed] ? Dumper::color('white/red') . 'FAILURES!' : Dumper::color('white/green') . 'OK') . " ($this->count test" . ($this->count > 1 ? 's' : '') . ', ' . ($this->results[Test::Failed] ? $this->results[Test::Failed] . ' failure' . ($this->results[Test::Failed] > 1 ? 's' : '') . ', ' : '') @@ -119,6 +141,65 @@ public function end(): void . ($this->count !== $run ? ($this->count - $run) . ' not run, ' : '') . sprintf('%0.1f', $this->time + microtime(true)) . ' seconds)' . Dumper::color() . "\n"); - $this->buffer = ''; + $this->buffer = []; + $this->resultsCount = 0; + } + + + private function printFinishedTestDot(Test $test): void + { + fwrite($this->file, $this->symbols[$test->getResult()]); + } + + + private function printFinishedTestLine(Test $test): void + { + $this->resultsCount++; + $result = $test->getResult(); + + $shortFilePath = str_replace($this->baseDir, '', $test->getFile()); + $shortDirPath = dirname($shortFilePath) . DIRECTORY_SEPARATOR; + $basename = basename($shortFilePath); + + // Filename. + if ($result === Test::Failed) { + $fileText = Dumper::color('red', $shortDirPath) . Dumper::color('white/red', $basename); + } else { + $fileText = Dumper::color('gray', $shortDirPath) . Dumper::color('silver', $basename); + } + + // Line header. + $header = "路 "; + // Test's title. + $titleText = $test->title + ? Dumper::color('fuchsia', " [$test->title]") + : ''; + + // Print test's message only if it's not failed (those will be printed + // after all tests are finished and will contain detailed info about the + // failed test). + $message = ''; + if ($result !== Test::Failed && $test->message) { + $message = $test->message; + $indent = str_repeat(' ', mb_strlen($header)); + + if (preg_match('#\n#', $message)) { + $message = "\n$indent" . preg_replace('#\r?\n#', '\0' . $indent, $message); + } else { + $message = Dumper::color('olive', "[$message]"); + } + } + + $output = sprintf( + "%s%s %s %s %s %s\n", + $header, + "{$this->resultsCount}/{$this->count}", + "$fileText{$titleText}", + $this->symbols[$result], + Dumper::color('gray', sprintf("in %.2f s", $test->getDuration())), + $message, + ); + + fwrite($this->file, $output); } } diff --git a/src/Runner/Test.php b/src/Runner/Test.php index 43687801..95fa23a7 100644 --- a/src/Runner/Test.php +++ b/src/Runner/Test.php @@ -28,14 +28,23 @@ class Test PASSED = self::Passed, SKIPPED = self::Skipped; + private const PossibleResults = [ + self::Prepared, + self::Failed, + self::Passed, + self::Skipped, + ]; + public ?string $title; public ?string $message = null; public string $stdout = ''; public string $stderr = ''; private string $file; - private int $result = self::Prepared; private ?float $duration = null; + /** @phpstan-var Alias_TestResultState */ + private int $result = self::Prepared; + /** @var string[]|string[][] */ private $args = []; @@ -70,6 +79,9 @@ public function getSignature(): string } + /** + * @phpstan-return Alias_TestResultState + */ public function getResult(): int { return $this->result; @@ -123,6 +135,7 @@ public function withArguments(array $args): self /** + * @phpstan-param Alias_TestResultState $result * @return static */ public function withResult(int $result, ?string $message, ?float $duration = null): self @@ -131,6 +144,10 @@ public function withResult(int $result, ?string $message, ?float $duration = nul throw new \LogicException("Result of test is already set to $this->result with message '$this->message'."); } + if (!in_array($result, self::PossibleResults, true)) { + throw new \LogicException("Invalid test result $result"); + } + $me = clone $this; $me->result = $result; $me->message = $message; diff --git a/tests/RunnerOutput/OutputHandlers.expect.consoleLines.txt b/tests/RunnerOutput/OutputHandlers.expect.consoleLines.txt new file mode 100644 index 00000000..6181bb2b --- /dev/null +++ b/tests/RunnerOutput/OutputHandlers.expect.consoleLines.txt @@ -0,0 +1,73 @@ +%a% | %a% | 1 thread + +路 1/%d% ./01-basic.fail.phptx FAIL in %f% s +路 2/%d% ./01-basic.pass.phptx OK in %f% s +路 3/%d% ./01-basic.skip.phptx SKIP in %f% s +路 4/%d% ./02-title.fail.phptx [Title for output handlers] FAIL in %f% s +路 5/%d% ./02-title.pass.phptx [Title for output handlers] OK in %f% s +路 6/%d% ./02-title.skip.phptx [Title for output handlers] SKIP in %f% s +路 7/%d% ./03-message.fail.phptx FAIL in %f% s +路 8/%d% ./03-message.skip.phptx SKIP in %f% s + Multi + line + message. +路 9/%d% ./04-args.fail.phptx FAIL in %f% s +路 10/%d% ./04-args.pass.phptx OK in %f% s +路 11/%d% ./04-args.skip.phptx SKIP in %f% s + Multi + line + message. + + +-- FAILED: 01-basic.fail.phptx + Multi + line + stdout.Failed: + + in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail(''); + + STDERR: + Multi + line + stderr. + +-- FAILED: Title for output handlers | 02-title.fail.phptx + Multi + line + stdout.Failed: + + in %a%02-title.fail.phptx(%d%) Tester\Assert::fail(''); + + STDERR: + Multi + line + stderr. + +-- FAILED: 03-message.fail.phptx + Multi + line + stdout.Failed: Multi + line + message. + + in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage."); + + STDERR: + Multi + line + stderr. + +-- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini + Multi + line + stdout.Failed: + + in %a%04-args.fail.phptx(%d%) Tester\Assert::fail(''); + + STDERR: + Multi + line + stderr. + + +FAILURES! (11 tests, 4 failures, 4 skipped, %a% seconds) diff --git a/tests/RunnerOutput/OutputHandlers.phpt b/tests/RunnerOutput/OutputHandlers.phpt index c32a4cbb..a467aaa0 100644 --- a/tests/RunnerOutput/OutputHandlers.phpt +++ b/tests/RunnerOutput/OutputHandlers.phpt @@ -32,14 +32,15 @@ $runner->setEnvironmentVariable(Tester\Environment::VariableColors, '0'); $runner->paths[] = __DIR__ . '/cases/*.phptx'; $runner->outputHandlers[] = new Output\ConsolePrinter($runner, false, $console = FileMock::create('')); $runner->outputHandlers[] = new Output\ConsolePrinter($runner, true, $consoleWithSkipped = FileMock::create('')); +$runner->outputHandlers[] = new Output\ConsolePrinter($runner, false, $consoleLines = FileMock::create(''), false, true); $runner->outputHandlers[] = new Output\JUnitPrinter($jUnit = FileMock::create('')); $runner->outputHandlers[] = new Output\Logger($runner, $logger = FileMock::create('')); $runner->outputHandlers[] = new Output\TapPrinter($tap = FileMock::create('')); $runner->run(); - Assert::matchFile(__DIR__ . '/OutputHandlers.expect.console.txt', Dumper::removeColors(file_get_contents($console))); Assert::matchFile(__DIR__ . '/OutputHandlers.expect.consoleWithSkip.txt', Dumper::removeColors(file_get_contents($consoleWithSkipped))); +Assert::matchFile(__DIR__ . '/OutputHandlers.expect.consoleLines.txt', Dumper::removeColors(file_get_contents($consoleLines))); Assert::matchFile(__DIR__ . '/OutputHandlers.expect.jUnit.xml', file_get_contents($jUnit)); Assert::matchFile(__DIR__ . '/OutputHandlers.expect.logger.txt', file_get_contents($logger)); Assert::matchFile(__DIR__ . '/OutputHandlers.expect.tap.txt', file_get_contents($tap));