From cf1386db3dd15c4c6650295ec393ae9c07ce4d1a Mon Sep 17 00:00:00 2001 From: Daniel Cazzulino Date: Thu, 4 Jul 2024 02:53:55 -0300 Subject: [PATCH] Sort test results by name, improve duration calculation By sorting tests by name we make it easier to find visually those that belong logically together. While doing this change, we switch to adding the duration of each individual test counted test run, which is more precise than adding the overal test results file duration which can include a full run from a previous trx (i.e. a later one run partially only failed tests, for example). --- src/dotnet-trx/TrxCommand.cs | 171 +++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 79 deletions(-) diff --git a/src/dotnet-trx/TrxCommand.cs b/src/dotnet-trx/TrxCommand.cs index dde899c..88907cf 100644 --- a/src/dotnet-trx/TrxCommand.cs +++ b/src/dotnet-trx/TrxCommand.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using System.Threading.Tasks; using System.Xml.Linq; using Devlooped.Web; using Humanizer; @@ -53,7 +54,7 @@ public override int Execute(CommandContext context, TrxSettings settings) path = Path.Combine(Directory.GetCurrentDirectory(), path); if (File.Exists(path)) - path = new FileInfo(path).DirectoryName; + path = new FileInfo(path).DirectoryName!; else path = Path.GetFullPath(path); @@ -74,99 +75,105 @@ public override int Execute(CommandContext context, TrxSettings settings) """); - // Process from newest files to oldest - foreach (var trx in Directory.EnumerateFiles(path, "*.trx", search).OrderByDescending(File.GetLastWriteTime)) - { - using var file = File.OpenRead(trx); - // Clears namespaces - var doc = HtmlDocument.Load(file, new HtmlReaderSettings { CaseFolding = Sgml.CaseFolding.None }); + var results = new List(); - foreach (var result in doc.CssSelectElements("UnitTestResult").OrderBy(x => x.Attribute("testName")?.Value)) + Status().Start("Discovering test results...", ctx => + { + // Process from newest files to oldest so that newest result we find (by test id) is the one we keep + foreach (var trx in Directory.EnumerateFiles(path, "*.trx", search).OrderByDescending(File.GetLastWriteTime)) { - var id = result.Attribute("testId")!.Value; - // Process only once per test id, this avoids duplicates when multiple trx files are processed - if (!testIds.Add(id)) - continue; + ctx.Status($"Discovering test results in {Path.GetFileName(trx)}..."); + using var file = File.OpenRead(trx); + // Clears namespaces + var doc = HtmlDocument.Load(file, new HtmlReaderSettings { CaseFolding = Sgml.CaseFolding.None }); + foreach (var result in doc.CssSelectElements("UnitTestResult")) + { + var id = result.Attribute("testId")!.Value; + // Process only once per test id, this avoids duplicates when multiple trx files are processed + if (testIds.Add(id)) + results.Add(result); + } + } - var test = result.Attribute("testName")!.Value; - string? output = settings.Output ? result.CssSelectElement("StdOut")?.Value : default; + ctx.Status("Sorting tests by name..."); + results.Sort(new Comparison((x, y) => x.Attribute("testName")!.Value.CompareTo(y.Attribute("testName")!.Value))); + }); - switch (result.Attribute("outcome")?.Value) - { - case "Passed": - passed++; - MarkupLine($":check_mark_button: {test}"); - if (output == null) - details.AppendLine($":white_check_mark: {test}"); - else - details.AppendLine( - $""" + foreach (var result in results) + { + var test = result.Attribute("testName")!.Value; + var elapsed = TimeSpan.Parse(result.Attribute("duration")!.Value); + var output = settings.Output ? result.CssSelectElement("StdOut")?.Value : default; + + switch (result.Attribute("outcome")?.Value) + { + case "Passed": + passed++; + duration += elapsed; + MarkupLine($":check_mark_button: {test}"); + if (output == null) + details.AppendLine($":white_check_mark: {test}"); + else + details.AppendLine( + $"""
:white_check_mark: {test} """) - .AppendLineIndented(output, "> > ") - .AppendLine( - """ + .AppendLineIndented(output, "> > ") + .AppendLine( + """
"""); - break; - case "Failed": - failed++; - MarkupLine($":cross_mark: {test}"); - details.AppendLine( - $""" + break; + case "Failed": + failed++; + duration += elapsed; + MarkupLine($":cross_mark: {test}"); + details.AppendLine( + $"""
:x: {test} """); - WriteError(path, failures, result, details); - if (output != null) - details.AppendLineIndented(output, "> > "); - details.AppendLine().AppendLine("
").AppendLine(); - break; - case "NotExecuted": - if (!settings.Skipped) - break; - - skipped++; - var reason = result.CssSelectElement("Output > ErrorInfo > Message")?.Value; - Markup($"[dim]:white_question_mark: {test}[/]"); - details.Append($":grey_question: {test}"); - - if (reason != null) - { - Markup($"[dim] => {reason}[/]"); - details.Append($" => {reason}"); - } - - WriteLine(); - details.AppendLine(); + WriteError(path, failures, result, details); + if (output != null) + details.AppendLineIndented(output, "> > "); + details.AppendLine().AppendLine("").AppendLine(); + break; + case "NotExecuted": + if (!settings.Skipped) break; - default: - break; - } - if (output != null) - { - Write(new Panel($"[dim]{output.ReplaceLineEndings()}[/]") + skipped++; + var reason = result.CssSelectElement("Output > ErrorInfo > Message")?.Value; + Markup($"[dim]:white_question_mark: {test}[/]"); + details.Append($":grey_question: {test}"); + + if (reason != null) { - Border = BoxBorder.None, - Padding = new Padding(5, 0, 0, 0), - }); - } + Markup($"[dim] => {reason}[/]"); + details.Append($" => {reason}"); + } + + WriteLine(); + details.AppendLine(); + break; + default: + break; } - var times = doc.CssSelectElement("Times"); - if (times == null) - continue; - - var start = DateTime.Parse(times.Attribute("start")!.Value); - var finish = DateTime.Parse(times.Attribute("finish")!.Value); - duration += finish - start; + if (output != null) + { + Write(new Panel($"[dim]{output.ReplaceLineEndings()}[/]") + { + Border = BoxBorder.None, + Padding = new Padding(5, 0, 0, 0), + }); + } } details.AppendLine().AppendLine(""); @@ -175,13 +182,16 @@ public override int Execute(CommandContext context, TrxSettings settings) WriteLine(); MarkupSummary(summary); WriteLine(); - GitHubReport(summary, details); - if (failures.Count > 0 && Environment.GetEnvironmentVariable("CI") == "true") + if (Environment.GetEnvironmentVariable("CI") == "true") { - // Send workflow commands for each failure to be annotated in GH CI - foreach (var failure in failures) - WriteLine($"::error file={failure.File},line={failure.Line},title={failure.Message}::{failure.Message}"); + GitHubReport(summary, details); + if (failures.Count > 0 && Environment.GetEnvironmentVariable("CI") == "true") + { + // Send workflow commands for each failure to be annotated in GH CI + foreach (var failure in failures) + WriteLine($"::error file={failure.File},line={failure.Line},title={failure.Message}::{failure.Message}"); + } } return 0; @@ -231,8 +241,11 @@ static void GitHubReport(Summary summary, StringBuilder details) sb.AppendLine($"     :grey_question: {summary.Skipped} skipped"); sb.AppendLine(); - sb.Append(details); - sb.AppendLine(); + if (summary.Total > 0) + { + sb.Append(details); + sb.AppendLine(); + } sb.AppendLine( $"from [dotnet-trx](https://github.com/devlooped/dotnet-trx) with [:purple_heart:](https://github.com/sponsors/devlooped)");