From 44a6c2462a5148ebcb0a912ee90280580c8b9bc3 Mon Sep 17 00:00:00 2001 From: Tobias Boege Date: Sun, 7 Jun 2020 00:37:59 +0200 Subject: [PATCH] gb.test: Show bad plans in summary [GB.TEST] * NEW: Show bad plans in summary. * BUG: Set recently added TestAssertion properties from TapPrinter as well. --- .../test/gb.test/.src/Tap/TapPrinter.class | 2 ++ .../lib/test/gb.test/.src/TestAssertion.class | 2 +- .../gb.test/.src/TestMyself/TInternals.test | 2 +- .../gb.test/.src/TestMyself/TSummary.test | 11 ++++++- .../test/gb.test/.src/TestSuite/Test.module | 29 +++++++++++++++++-- main/lib/test/gb.test/.test | 6 +++- main/lib/test/gb.test/taskell.md | 4 +-- 7 files changed, 48 insertions(+), 8 deletions(-) diff --git a/main/lib/test/gb.test/.src/Tap/TapPrinter.class b/main/lib/test/gb.test/.src/Tap/TapPrinter.class index 991ab00b6..9e669295d 100644 --- a/main/lib/test/gb.test/.src/Tap/TapPrinter.class +++ b/main/lib/test/gb.test/.src/Tap/TapPrinter.class @@ -81,6 +81,8 @@ Public Sub Finish() ' Sum up this subtest .Finished = True + .Summary.SubPlanned = .Plan + .Summary.SubSkippedAll = .SkipAll .Summary.Ok = (.Plan > 0 Or .SkipAll) And (.TestsRun = .Plan) For Each hTest In .Summary.Subtests .Summary.Ok = .Summary.Ok And hTest.Success diff --git a/main/lib/test/gb.test/.src/TestAssertion.class b/main/lib/test/gb.test/.src/TestAssertion.class index e5b617122..d32429556 100644 --- a/main/lib/test/gb.test/.src/TestAssertion.class +++ b/main/lib/test/gb.test/.src/TestAssertion.class @@ -40,7 +40,7 @@ Private Function SubSuccess_Read() As Boolean If Not subtests.Count Then Return Me.Success - Dim bRes As Boolean = Not (Subtests.Count <> SubPlanned) + Dim bRes As Boolean = (Subtests.Count = SubPlanned) Dim iInd As Integer = 0 While bRes And (iInd < Subtests.Count) diff --git a/main/lib/test/gb.test/.src/TestMyself/TInternals.test b/main/lib/test/gb.test/.src/TestMyself/TInternals.test index b8da153f5..2d9391ffe 100644 --- a/main/lib/test/gb.test/.src/TestMyself/TInternals.test +++ b/main/lib/test/gb.test/.src/TestMyself/TInternals.test @@ -101,7 +101,7 @@ Public Sub ReflectTestsString() Test.Plan(2) - want = "TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TBailout.Bailout,TCrashes.DoACrash;ErrorTwo,TElse.NoMessage;NoMessageInbetween;Note;TestNoteGotAndExpected,TFailures.EqualsFailure;Error;LongFailure;LongTypeMismatchFailure;StringFailure,TInternals.CommandsToString;FindTestSuiteByName;InterpreterInterface;ParseTestCommands;ReflectTest;ReflectTestsString,TParser.Runner;SkipAll;Subtests,TSetup.A;B,TSkipAll.SkipAll,TSummary.DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan" + want = "TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TBailout.Bailout,TCrashes.DoACrash;ErrorTwo,TElse.NoMessage;NoMessageInbetween;Note;TestNoteGotAndExpected,TFailures.EqualsFailure;Error;LongFailure;LongTypeMismatchFailure;StringFailure,TInternals.CommandsToString;FindTestSuiteByName;InterpreterInterface;ParseTestCommands;ReflectTest;ReflectTestsString,TParser.Runner;SkipAll;Subtests,TSetup.A;B,TSkipAll.SkipAll,TSummary.DoBadPlan;DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan" got = Test.AllTests() Assert.Equals(got, want, "All Test names as string") diff --git a/main/lib/test/gb.test/.src/TestMyself/TSummary.test b/main/lib/test/gb.test/.src/TestMyself/TSummary.test index 750828af2..919e17d02 100644 --- a/main/lib/test/gb.test/.src/TestMyself/TSummary.test +++ b/main/lib/test/gb.test/.src/TestMyself/TSummary.test @@ -48,6 +48,15 @@ End '' Will fail Public Sub DoOneFailure() - Assert.Ok(False, "Intentional failure") + Test.IntendedFailure() + Assert.Ok(False, "Intentional failure") + +End + +Public Sub DoBadPlan() + + Test._Subtest("Bad plan will appear in the summary", 2) + Assert.Ok(True) + Test._Finish() End diff --git a/main/lib/test/gb.test/.src/TestSuite/Test.module b/main/lib/test/gb.test/.src/TestSuite/Test.module index 29760050f..9dde454aa 100644 --- a/main/lib/test/gb.test/.src/TestSuite/Test.module +++ b/main/lib/test/gb.test/.src/TestSuite/Test.module @@ -116,7 +116,10 @@ Private Sub PrintSummary() ShowTestCollection(("&1 todo:"), FindTodos(.Summary.Subtests, "")) ShowTestCollection(("&1 bonus:"), FindBonus(.Summary.Subtests, "")) - If Not .Summary.Success Then ShowTestCollection(("&1 tests failed:"), FindFailures(.Summary.Subtests, "")) + If Not .Summary.Success Then + ShowTestCollection(("&1 test plans failed:"), FindBadPlans(.Summary.Subtests, "")) + ShowTestCollection(("&1 tests failed:"), FindFailures(.Summary.Subtests, "")) + Endif Test.Note(IIf(.Summary.Success, "PASSED", "FAILED")) End With @@ -138,13 +141,35 @@ Private Sub ShowTestCollection(Description As String, TestCollection As Collecti Test.Note(Subst$(Description, TestCollection.Count)) For Each cTest In TestCollection Dim hTest As TestAssertion = cTest!Assertion - Test.Note(Subst$(("&2: &1 -- &3 &4"), cTest["Path"], hTest.Id, hTest.Description, IIf(hTest.Comment, "# " & hTest.Comment, ""))) + Dim sName, sNote As String + sName = hTest.Description + If cTest["Note"] Then + sNote = Subst$((": &1"), cTest["Note"]) + Else If hTest.Comment Then + sNote = Subst$(("# &1"), hTest.Comment) + Endif + Test.Note(Subst$(("&2: &1 -- &3&4"), cTest["Path"], hTest.Id, sName, IIf(sNote, " " & sNote, ""))) Next Test.Note(gb.Lf) Endif End +Private Function FindBadPlans(Tests As TestAssertion[], Prefix As String) As Collection[] + + Dim hTest As TestAssertion + Dim sName As String + Dim aRet As New Collection[] + + For Each hTest In Tests + sName = Prefix &/ hTest.Description + If hTest.Subtests.Count <> hTest.SubPlanned Then aRet.Add(["Path": Prefix, "Assertion": hTest, "Note": Subst$(("Planned &1 but ran &2"), hTest.SubPlanned, hTest.Subtests.Count)]) + aRet.Insert(FindBadPlans(hTest.Subtests, sName)) + Next + Return aRet + +End + Private Function FindFailures(Tests As TestAssertion[], Prefix As String) As Collection[] Dim hTest As TestAssertion diff --git a/main/lib/test/gb.test/.test b/main/lib/test/gb.test/.test index dc63640c8..3c0c23d94 100644 --- a/main/lib/test/gb.test/.test +++ b/main/lib/test/gb.test/.test @@ -1,5 +1,5 @@ [TestSuites] -Count=6 +Count=7 TestWith=4 Default="TSetup.A;B" @@ -24,6 +24,10 @@ Name="Keep this test suite, it is necessary for testing gb.test." Tests="TInternals.FindTestSuiteByName" [TestSuites/6] +Name="Summary" +Tests="TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TSummary.DoBadPlan;DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan" + +[TestSuites/7] Name="TAP Parser" Tests="TParser.Runner;SkipAll;Subtests" diff --git a/main/lib/test/gb.test/taskell.md b/main/lib/test/gb.test/taskell.md index 269b5f052..4f1156d9d 100644 --- a/main/lib/test/gb.test/taskell.md +++ b/main/lib/test/gb.test/taskell.md @@ -3,8 +3,6 @@ ## Work -- Plan - > If a plan failes it has to be reported in the TAP output as well as in the summary which is not the case at the moment. ## Done @@ -50,3 +48,5 @@ - Skip all > Parse `1..0 # SKIP` style TAPs indicating that a whole test was skipped. - Document Approximate and RelativeApproximate +- Plan + > If a plan failes it has to be reported in the TAP output as well as in the summary which is not the case at the moment.