Improve ExUnit failure/success output (#15168)

lib/ex_unit/lib/ex_unit/cli_formatter.ex

@@ -27,6 +27,7 @@ defmodule ExUnit.CLIFormatter do
       test_counter: %{},
       test_timings: [],
       failure_counter: 0,
+      failure_type_counter: %{},
       skipped_counter: 0,
       excluded_counter: 0,
       invalid_counter: 0
@@ -139,7 +140,14 @@ defmodule ExUnit.CLIFormatter do
 
     test_counter = update_test_counter(config.test_counter, test)
     failure_counter = config.failure_counter + 1
-    config = %{config | test_counter: test_counter, failure_counter: failure_counter}
+    failure_type_counter = update_test_counter(config.failure_type_counter, test)
+
+    config = %{
+      config
+      | test_counter: test_counter,
+        failure_counter: failure_counter,
+        failure_type_counter: failure_type_counter
+    }
 
     {:noreply, update_test_timings(config, test)}
   end
@@ -176,8 +184,16 @@ defmodule ExUnit.CLIFormatter do
     # The failed tests have already contributed to the counter,
     # so we should only add the successful tests to the count
     config =
-      update_in(config.failure_counter, fn counter ->
-        counter + Enum.count(test_module.tests, &is_nil(&1.state))
+      Enum.reduce(test_module.tests, config, fn
+        %{state: nil} = test, acc ->
+          %{
+            acc
+            | failure_counter: acc.failure_counter + 1,
+              failure_type_counter: update_test_counter(acc.failure_type_counter, test)
+          }
+
+        _test, acc ->
+          acc
       end)
 
     formatted =
@@ -352,11 +368,36 @@ defmodule ExUnit.CLIFormatter do
   defp print_summary(config, force_failures?) do
     test_type_counts = collect_test_type_counts(config)
     test_counter = test_counter_or_default(config, test_type_counts)
-    formatted_test_type_counts = format_test_type_counts(test_counter)
-    failure_pl = pluralize(config.failure_counter, "failure", "failures")
+
+    passed_total =
+      test_type_counts - config.failure_counter - config.skipped_counter - config.invalid_counter
+
+    # Passed line: "Passed: 447/455 (53/54 doctests, 393/403 tests)" or
+    # "Passed: 455 (70 tests, 14 properties)" when all pass
+    all_passed? = passed_total == test_type_counts
+
+    passed_breakdown =
+      format_passed_breakdown(test_counter, config.failure_type_counter, all_passed?)
+
+    passed_line =
+      if all_passed? do
+        "Passed: #{passed_total}"
+      else
+        "Passed: #{passed_total}/#{test_type_counts}"
+      end
+      |> if_true(passed_breakdown != "", &(&1 <> " (#{passed_breakdown})"))
+
+    # Failed line: "Failed: 8 tests, 1 property"
+    failed_line =
+      if config.failure_counter > 0 do
+        failed_breakdown = format_type_counts(config.failure_type_counter)
+        "\n" <> failure("Failed: #{failed_breakdown}", config)
+      else
+        ""
+      end
 
     message =
-      "#{formatted_test_type_counts}#{config.failure_counter} #{failure_pl}"
+      ("\n" <> passed_line)
       |> if_true(
         config.invalid_counter > 0,
         &(&1 <> ", #{config.invalid_counter} invalid")
@@ -372,7 +413,7 @@ defmodule ExUnit.CLIFormatter do
 
     cond do
       config.failure_counter > 0 or force_failures? ->
-        IO.puts(failure(message, config))
+        IO.puts(message <> failed_line)
 
       config.invalid_counter > 0 ->
         IO.puts(invalid(message, config))
@@ -404,14 +445,40 @@ defmodule ExUnit.CLIFormatter do
     IO.puts(formatted)
   end
 
-  defp format_test_type_counts(test_counter) do
-    test_counter
+  defp format_type_counts(type_counter) do
+    type_counter
     |> Enum.sort()
     |> Enum.map(fn {test_type, count} ->
-      type_pluralized = pluralize(count, test_type, ExUnit.plural_rule(test_type |> to_string()))
-
-      "#{count} #{type_pluralized}, "
+      "#{count} #{pluralize_type(count, test_type)}"
     end)
+    |> Enum.join(", ")
+  end
+
+  defp format_passed_breakdown(test_counter, failure_type_counter, all_passed?) do
+    # If there are no different test types, we just print "Passed: N/N"
+    # without the type.
+    if map_size(test_counter) in 0..1 do
+      ""
+    else
+      test_counter
+      |> Map.keys()
+      |> Enum.sort()
+      |> Enum.map_join(", ", fn type ->
+        total = Map.fetch!(test_counter, type)
+
+        if all_passed? do
+          "#{total} #{pluralize_type(total, type)}"
+        else
+          failed = Map.get(failure_type_counter, type, 0)
+          passed = total - failed
+          "#{passed}/#{total} #{pluralize_type(total, type)}"
+        end
+      end)
+    end
+  end
+
+  defp pluralize_type(count, type) do
+    pluralize(count, type, ExUnit.plural_rule(to_string(type)))
   end
 
   defp test_counter_or_default(_config, 0) do

lib/ex_unit/test/ex_unit/callbacks_test.exs

@@ -36,7 +36,7 @@ defmodule ExUnit.CallbacksTest do
       end
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "1 test, 0 failures"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 1"
   end
 
   test "named callbacks run custom code in order" do
@@ -66,7 +66,7 @@ defmodule ExUnit.CallbacksTest do
       defp store_5(context), do: store(context, 5)
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "1 test, 0 failures"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 1"
   end
 
   test "named callbacks support {module, function} tuples" do
@@ -87,7 +87,7 @@ defmodule ExUnit.CallbacksTest do
       def setup_3(_), do: [setup_3: true]
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "1 test, 0 failures"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 1"
   end
 
   test "doesn't choke on setup errors" do
@@ -141,7 +141,7 @@ defmodule ExUnit.CallbacksTest do
       end
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "1 test, 0 failures, 1 invalid"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 0/1"
   end
 
   test "doesn't choke on dead supervisor" do
@@ -209,7 +209,7 @@ defmodule ExUnit.CallbacksTest do
       end
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "2 tests, 2 failures"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Failed: 2 tests"
   end
 
   defp no_formatters! do
@@ -253,7 +253,7 @@ defmodule ExUnit.CallbacksTest do
 
     output = capture_io(fn -> ExUnit.run() end)
     assert output =~ "on_exit run"
-    assert output =~ "1 test, 0 failures"
+    assert output =~ "Passed: 1"
   end
 
   test "runs multiple on_exit exits and overrides by ref" do

lib/ex_unit/test/ex_unit/doc_test_test.exs

@@ -616,7 +616,7 @@ defmodule ExUnit.DocTestTest do
       doctest ExUnit.DocTestTest.SomewhatGoodModuleWithOnly, only: [one: 0, two: 0], import: true
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "2 doctests, 1 failure"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Failed: 1 doctest"
   end
 
   test "empty :only" do
@@ -627,7 +627,7 @@ defmodule ExUnit.DocTestTest do
 
     output = capture_io(fn -> ExUnit.run() end)
 
-    assert output =~ "0 failures"
+    refute output =~ "Failed:"
     refute output =~ "doctest"
   end
 
@@ -639,7 +639,7 @@ defmodule ExUnit.DocTestTest do
 
     output = capture_io(fn -> ExUnit.run() end)
 
-    assert output =~ "0 failures"
+    refute output =~ "Failed:"
     assert output =~ "2 skipped"
   end
 
@@ -763,7 +763,7 @@ defmodule ExUnit.DocTestTest do
     assert output =~
              "#{stack(starting_line + 28)}ExUnit.DocTestTest.Failure (module)"
 
-    assert output =~ "8 doctests, 8 failures"
+    assert output =~ "Failed: 8 doctests"
   end
 
   test "doctest invalid" do
@@ -969,7 +969,7 @@ defmodule ExUnit.DocTestTest do
                   #{stack(line)}ExUnit.DocTestTest.Invalid (module)
            """
 
-    assert output =~ "10 doctests, 10 failures"
+    assert output =~ "Failed: 10 doctests"
   end
 
   test "pattern matching assertions in doctests" do
@@ -1075,7 +1075,7 @@ defmodule ExUnit.DocTestTest do
                   (for doctest at) #{location}:#{starting_line + 19}: (test)
            """
 
-    assert output =~ "10 doctests, 8 failures"
+    assert output =~ "Failed: 8 doctests"
   end
 
   test "IEx prefix contains a number" do
@@ -1084,7 +1084,7 @@ defmodule ExUnit.DocTestTest do
       doctest ExUnit.DocTestTest.Numbered
     end
 
-    assert capture_io(fn -> ExUnit.run() end) =~ "1 doctest, 0 failures"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 1"
   end
 
   test "IEx prompt contains host" do
@@ -1253,7 +1253,7 @@ defmodule ExUnit.DocTestTest do
       end
 
       output = capture_io(fn -> ExUnit.run() end)
-      assert output =~ "2 doctests, 0 failures"
+      assert output =~ "Passed: 2"
     end
 
     test "failing" do

lib/ex_unit/test/ex_unit/register_test.exs

@@ -42,7 +42,7 @@ defmodule ExUnit.RegisterTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 0, skipped: 0, total: 2, excluded: 0}
-           end) =~ "1 property, 1 test, 0 failures"
+           end) =~ "Passed: 2 (1 property, 1 test)"
   end
 
   test "plural test types" do
@@ -96,6 +96,6 @@ defmodule ExUnit.RegisterTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 0, skipped: 0, total: 4, excluded: 0}
-           end) =~ "2 properties, 2 tests, 0 failures"
+           end) =~ "Passed: 4 (2 properties, 2 tests)"
   end
 end

lib/ex_unit/test/ex_unit_test.exs

@@ -26,14 +26,14 @@ defmodule ExUnitTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 2, skipped: 0, total: 2, excluded: 0}
-           end) =~ "\n2 tests, 2 failures\n"
+           end) =~ "Failed: 2 tests"
 
     ExUnit.Server.modules_loaded(false)
 
     assert capture_io(fn ->
              assert ExUnit.async_run() |> ExUnit.await_run() ==
                       %{failures: 0, skipped: 0, total: 0, excluded: 0}
-           end) =~ "\n0 tests, 0 failures\n"
+           end) =~ "\nPassed: 0\n"
   end
 
   test "supports rerunning given modules" do
@@ -70,7 +70,7 @@ defmodule ExUnitTest do
                       total: 1,
                       excluded: 0
                     }
-           end) =~ "\n1 test, 1 failure\n"
+           end) =~ "Failed: 1 test"
 
     sample = [SampleSyncTest, SampleAsyncTest]
 
@@ -81,7 +81,7 @@ defmodule ExUnitTest do
                       total: 2,
                       excluded: 0
                     }
-           end) =~ "\n2 tests, 2 failures\n"
+           end) =~ "Failed: 2 tests"
 
     assert capture_io(fn ->
              assert ExUnit.run(sample ++ sample) == %{
@@ -90,7 +90,7 @@ defmodule ExUnitTest do
                       total: 2,
                       excluded: 0
                     }
-           end) =~ "\n2 tests, 2 failures\n"
+           end) =~ "Failed: 2 tests"
   end
 
   test "prints aborted runs on sigquit", config do
@@ -138,11 +138,8 @@ defmodule ExUnitTest do
     assert result =~ ~r"\* ExUnitTest.SleepOnSetupAll \[.*test/ex_unit_test.exs\]"
     assert result =~ ~r"\* test true \[.*test/ex_unit_test.exs:#{line}\]"
 
-    assert result =~ """
-           Showing results so far...
-
-           0 tests, 0 failures
-           """
+    assert result =~ "Showing results so far..."
+    assert result =~ "Passed: 0"
   end
 
   test "doesn't hang on exits" do
@@ -162,7 +159,7 @@ defmodule ExUnitTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 1, skipped: 0, total: 1, excluded: 0}
-           end) =~ "\n1 test, 1 failure\n"
+           end) =~ "Failed: 1 test"
   end
 
   test "reports capture log crashes" do
@@ -180,7 +177,7 @@ defmodule ExUnitTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 1, skipped: 0, total: 1, excluded: 0}
-           end) =~ "\n1 test, 1 failure\n"
+           end) =~ "Failed: 1 test"
   end
 
   test "supports timeouts" do
@@ -309,7 +306,7 @@ defmodule ExUnitTest do
 
     {result, output} = run_with_filter([only_test_ids: test_ids], [])
     assert result == %{failures: 1, skipped: 0, excluded: 0, total: 1}
-    assert output =~ "\n1 test, 1 failure\n"
+    assert output =~ "Failed: 1 test"
   end
 
   test "filtering cases with tags" do
@@ -331,23 +328,23 @@ defmodule ExUnitTest do
     # Empty because it is already loaded
     {result, output} = run_with_filter([], [])
     assert result == %{failures: 1, skipped: 0, total: 4, excluded: 0}
-    assert output =~ "\n4 tests, 1 failure\n"
+    assert output =~ "Failed: 1 test"
 
     {result, output} = run_with_filter([exclude: [even: true]], [ParityTest])
     assert result == %{failures: 0, skipped: 0, excluded: 1, total: 4}
-    assert output =~ "\n3 tests, 0 failures (1 excluded)\n"
+    assert output =~ "Passed: 3"
 
     {result, output} = run_with_filter([exclude: :even], [ParityTest])
     assert result == %{failures: 0, skipped: 0, excluded: 3, total: 4}
-    assert output =~ "\n1 test, 0 failures (3 excluded)\n"
+    assert output =~ "Passed: 1"
 
     {result, output} = run_with_filter([exclude: :even, include: [even: true]], [ParityTest])
     assert result == %{failures: 1, skipped: 0, excluded: 2, total: 4}
-    assert output =~ "\n2 tests, 1 failure (2 excluded)\n"
+    assert output =~ "Failed: 1 test"
 
     {result, output} = run_with_filter([exclude: :test, include: [even: true]], [ParityTest])
     assert result == %{failures: 1, skipped: 0, excluded: 3, total: 4}
-    assert output =~ "\n1 test, 1 failure (3 excluded)\n"
+    assert output =~ "Failed: 1 test"
   end
 
   test "log capturing" do
@@ -414,7 +411,7 @@ defmodule ExUnitTest do
         assert ExUnit.run() == %{failures: 1, skipped: 0, total: 1, excluded: 0}
       end)
 
-    assert output =~ "\n1 test, 1 failure\n"
+    assert output =~ "Failed: 1 test"
     assert output =~ "\n  1) test multi (ExUnitTest.MultiTest)\n"
     assert output =~ "Failure #1\n"
     assert output =~ "Failure #2\n"
@@ -470,7 +467,7 @@ defmodule ExUnitTest do
       end)
 
     assert output =~ "Not implemented\n"
-    assert output =~ "\n1 test, 1 failure\n"
+    assert output =~ "Failed: 1 test"
   end
 
   test "skips tagged test with skip" do
@@ -496,7 +493,7 @@ defmodule ExUnitTest do
         assert ExUnit.run() == %{failures: 0, skipped: 2, total: 2, excluded: 0}
       end)
 
-    assert output =~ "\n2 tests, 0 failures, 2 skipped\n"
+    assert output =~ "Passed: 0/2"
   end
 
   test "filtering cases with :module tag" do
@@ -514,7 +511,7 @@ defmodule ExUnitTest do
     {result, output} = run_with_filter([exclude: :module], [])
 
     assert result == %{failures: 0, skipped: 0, excluded: 2, total: 2}
-    assert output =~ "\n0 tests, 0 failures (2 excluded)\n"
+    assert output =~ "Passed: 0"
 
     {result, output} =
       [exclude: :test, include: [module: "ExUnitTest.SecondTestModule"]]
@@ -522,7 +519,7 @@ defmodule ExUnitTest do
 
     assert result == %{failures: 1, skipped: 0, excluded: 1, total: 2}
     assert output =~ "\n  1) test false (ExUnitTest.SecondTestModule)\n"
-    assert output =~ "\n1 test, 1 failure (1 excluded)\n"
+    assert output =~ "Failed: 1 test"
   end
 
   test "raises on reserved tag :file in module" do
@@ -647,7 +644,7 @@ defmodule ExUnitTest do
 
     assert capture_io(fn ->
              assert ExUnit.run() == %{failures: 0, skipped: 0, total: 3, excluded: 0}
-           end) =~ "\n3 tests, 0 failures\n"
+           end) =~ "Passed: 3"
   end
 
   # Skipped and excluded tests should be included in the stats
@@ -685,7 +682,7 @@ defmodule ExUnitTest do
       end)
 
     refute output =~ max_failures_reached_msg()
-    assert output =~ "\n5 tests, 0 failures, 4 invalid, 1 skipped (1 excluded)\n"
+    assert output =~ "Passed: 0/5"
   end
 
   test "parameterized tests" do
@@ -730,7 +727,7 @@ defmodule ExUnitTest do
     end
 
     configure_and_reload_on_exit(trace: true)
-    assert capture_io(fn -> ExUnit.run() end) =~ "0 tests"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 0"
 
     defmodule EmptyGroupedParameterizedTests do
       use ExUnit.Case, async: true, parameterize: [], group: :example
@@ -741,7 +738,7 @@ defmodule ExUnitTest do
     end
 
     configure_and_reload_on_exit(trace: true)
-    assert capture_io(fn -> ExUnit.run() end) =~ "0 tests"
+    assert capture_io(fn -> ExUnit.run() end) =~ "Passed: 0"
   end
 
   describe "after_suite/1" do
@@ -816,7 +813,7 @@ defmodule ExUnitTest do
         end)
 
       assert output =~ max_failures_reached_msg()
-      assert output =~ "\n5 tests, 2 failures, 1 skipped (1 excluded)\n"
+      assert output =~ "Failed: 2 tests"
     end
 
     test ":max_failures is not reached" do
@@ -847,7 +844,7 @@ defmodule ExUnitTest do
         end)
 
       refute output =~ max_failures_reached_msg()
-      assert output =~ "\n6 tests, 2 failures, 1 skipped (2 excluded)\n"
+      assert output =~ "Failed: 2 tests"
     end
 
     test ":max_failures has been reached" do
@@ -881,7 +878,7 @@ defmodule ExUnitTest do
         end)
 
       assert output =~ max_failures_reached_msg()
-      assert output =~ "\n5 tests, 2 failures, 2 skipped (2 excluded)\n"
+      assert output =~ "Failed: 2 tests"
     end
 
     # Excluded and skipped tests are detected before setup_all
@@ -915,7 +912,7 @@ defmodule ExUnitTest do
         end)
 
       assert output =~ max_failures_reached_msg()
-      assert output =~ "\n3 tests, 0 failures, 2 invalid, 1 skipped (1 excluded)\n"
+      assert output =~ "Passed: 0/3"
     end
 
     test ":max_failures flushes all async/sync cases" do
@@ -942,7 +939,7 @@ defmodule ExUnitTest do
         end)
 
       assert output =~ max_failures_reached_msg()
-      assert output =~ "\n1 test, 1 failure\n"
+      assert output =~ "Failed: 1 test"
 
       capture_io(fn ->
         assert ExUnit.run() == %{total: 0, failures: 0, excluded: 0, skipped: 0}
@@ -1106,7 +1103,7 @@ defmodule ExUnitTest do
       end)
 
     assert output =~ "All tests have been excluded.\n"
-    assert output =~ "0 tests, 0 failures (2 excluded)\n"
+    assert output =~ "Passed: 0"
   end
 
   test "tests are run in compile order (FIFO)" do