gb.test: Show bad plans in summary

[GB.TEST]
* NEW: Show bad plans in summary.
* BUG: Set recently added TestAssertion properties from TapPrinter as well.
This commit is contained in:
Tobias Boege 2020-06-07 00:37:59 +02:00
parent 0eea48da06
commit 44a6c2462a
7 changed files with 48 additions and 8 deletions

View file

@ -81,6 +81,8 @@ Public Sub Finish()
' Sum up this subtest
.Finished = True
.Summary.SubPlanned = .Plan
.Summary.SubSkippedAll = .SkipAll
.Summary.Ok = (.Plan > 0 Or .SkipAll) And (.TestsRun = .Plan)
For Each hTest In .Summary.Subtests
.Summary.Ok = .Summary.Ok And hTest.Success

View file

@ -40,7 +40,7 @@ Private Function SubSuccess_Read() As Boolean
If Not subtests.Count Then Return Me.Success
Dim bRes As Boolean = Not (Subtests.Count <> SubPlanned)
Dim bRes As Boolean = (Subtests.Count = SubPlanned)
Dim iInd As Integer = 0
While bRes And (iInd < Subtests.Count)

View file

@ -101,7 +101,7 @@ Public Sub ReflectTestsString()
Test.Plan(2)
want = "TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TBailout.Bailout,TCrashes.DoACrash;ErrorTwo,TElse.NoMessage;NoMessageInbetween;Note;TestNoteGotAndExpected,TFailures.EqualsFailure;Error;LongFailure;LongTypeMismatchFailure;StringFailure,TInternals.CommandsToString;FindTestSuiteByName;InterpreterInterface;ParseTestCommands;ReflectTest;ReflectTestsString,TParser.Runner;SkipAll;Subtests,TSetup.A;B,TSkipAll.SkipAll,TSummary.DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan"
want = "TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TBailout.Bailout,TCrashes.DoACrash;ErrorTwo,TElse.NoMessage;NoMessageInbetween;Note;TestNoteGotAndExpected,TFailures.EqualsFailure;Error;LongFailure;LongTypeMismatchFailure;StringFailure,TInternals.CommandsToString;FindTestSuiteByName;InterpreterInterface;ParseTestCommands;ReflectTest;ReflectTestsString,TParser.Runner;SkipAll;Subtests,TSetup.A;B,TSkipAll.SkipAll,TSummary.DoBadPlan;DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan"
got = Test.AllTests()
Assert.Equals(got, want, "All Test names as string")

View file

@ -48,6 +48,15 @@ End
'' Will fail
Public Sub DoOneFailure()
Assert.Ok(False, "Intentional failure")
Test.IntendedFailure()
Assert.Ok(False, "Intentional failure")
End
Public Sub DoBadPlan()
Test._Subtest("Bad plan will appear in the summary", 2)
Assert.Ok(True)
Test._Finish()
End

View file

@ -116,7 +116,10 @@ Private Sub PrintSummary()
ShowTestCollection(("&1 todo:"), FindTodos(.Summary.Subtests, ""))
ShowTestCollection(("&1 bonus:"), FindBonus(.Summary.Subtests, ""))
If Not .Summary.Success Then ShowTestCollection(("&1 tests failed:"), FindFailures(.Summary.Subtests, ""))
If Not .Summary.Success Then
ShowTestCollection(("&1 test plans failed:"), FindBadPlans(.Summary.Subtests, ""))
ShowTestCollection(("&1 tests failed:"), FindFailures(.Summary.Subtests, ""))
Endif
Test.Note(IIf(.Summary.Success, "PASSED", "FAILED"))
End With
@ -138,13 +141,35 @@ Private Sub ShowTestCollection(Description As String, TestCollection As Collecti
Test.Note(Subst$(Description, TestCollection.Count))
For Each cTest In TestCollection
Dim hTest As TestAssertion = cTest!Assertion
Test.Note(Subst$(("&2: &1 -- &3 &4"), cTest["Path"], hTest.Id, hTest.Description, IIf(hTest.Comment, "# " & hTest.Comment, "")))
Dim sName, sNote As String
sName = hTest.Description
If cTest["Note"] Then
sNote = Subst$((": &1"), cTest["Note"])
Else If hTest.Comment Then
sNote = Subst$(("# &1"), hTest.Comment)
Endif
Test.Note(Subst$(("&2: &1 -- &3&4"), cTest["Path"], hTest.Id, sName, IIf(sNote, " " & sNote, "")))
Next
Test.Note(gb.Lf)
Endif
End
Private Function FindBadPlans(Tests As TestAssertion[], Prefix As String) As Collection[]
Dim hTest As TestAssertion
Dim sName As String
Dim aRet As New Collection[]
For Each hTest In Tests
sName = Prefix &/ hTest.Description
If hTest.Subtests.Count <> hTest.SubPlanned Then aRet.Add(["Path": Prefix, "Assertion": hTest, "Note": Subst$(("Planned &1 but ran &2"), hTest.SubPlanned, hTest.Subtests.Count)])
aRet.Insert(FindBadPlans(hTest.Subtests, sName))
Next
Return aRet
End
Private Function FindFailures(Tests As TestAssertion[], Prefix As String) As Collection[]
Dim hTest As TestAssertion

View file

@ -1,5 +1,5 @@
[TestSuites]
Count=6
Count=7
TestWith=4
Default="TSetup.A;B"
@ -24,6 +24,10 @@ Name="Keep this test suite, it is necessary for testing gb.test."
Tests="TInternals.FindTestSuiteByName"
[TestSuites/6]
Name="Summary"
Tests="TAllAsserts.Approximate;Equals;Error;ErrorCode;Fail;Greater;GreaterEqual;IsType;Less;LessEqual;Like;Match;NotNull;NotOk;Notequals;Noterror;Null;Ok;Pass;RelativeApproximate;StringEquals,TSummary.DoBadPlan;DoOneFailure;DoSkip;DoSomeAsserts;DoTodo;TodoErrorCode,TWrongPlan.IHaveAWrongPlan"
[TestSuites/7]
Name="TAP Parser"
Tests="TParser.Runner;SkipAll;Subtests"

View file

@ -3,8 +3,6 @@
## Work
- Plan
> If a plan failes it has to be reported in the TAP output as well as in the summary which is not the case at the moment.
## Done
@ -50,3 +48,5 @@
- Skip all
> Parse `1..0 # SKIP` style TAPs indicating that a whole test was skipped.
- Document Approximate and RelativeApproximate
- Plan
> If a plan failes it has to be reported in the TAP output as well as in the summary which is not the case at the moment.