Merge pull request #44 from nunit/issue-26

Create pseudo-results for ignored and skipped test cases for which NU…

demo/NUnitTestDemo/ExpetedOutcomeAttributes.cs → demo/NUnitTestDemo/ExpectedOutcomeAttributes.cs

@@ -23,6 +23,11 @@ public class ExpectIgnoreAttribute : PropertyAttribute
         public ExpectIgnoreAttribute() : base("Expect", "Ignore") { }
     }
 
+    public class ExpectSkipAttribute : PropertyAttribute
+    {
+        public ExpectSkipAttribute() : base("Expect", "Skipped") { }
+    }
+
     public class ExpectErrorAttribute : PropertyAttribute
     {
         public ExpectErrorAttribute() : base("Expect", "Error") { }

demo/NUnitTestDemo/NUnit3TestDemo.csproj

@@ -48,7 +48,7 @@
   <ItemGroup>
     <Compile Include="AsyncTests.cs" />
     <Compile Include="ConfigFileTests.cs" />
-    <Compile Include="ExpetedOutcomeAttributes.cs" />
+    <Compile Include="ExpectedOutcomeAttributes.cs" />
     <Compile Include="GenericTests.cs" />
     <Compile Include="TextOutputTests.cs" />
     <Compile Include="ParameterizedTests.cs" />

demo/NUnitTestDemo/ParameterizedTests.cs

@@ -61,6 +61,17 @@ public void TestCaseIsIgnored_Assert(int a, int b)
             Assert.Ignore("Ignoring this test case");
         }
 
+        [TestCase(31, 11, ExcludePlatform="NET"), ExpectSkip]
+        public void TestCaseIsSkipped_Property(int a, int b)
+        {
+        }
+
+        [Platform(Exclude = "NET"), ExpectSkip]
+        [TestCase(31, 11)]
+        public void TestCaseIsSkipped_Attribute(int a, int b)
+        {
+        }
+
         [TestCase(31, 11), ExpectError]
         public void TestCaseThrowsException(int a, int b)
         {

demo/NUnitTestDemo/SimpleTests.cs

@@ -50,6 +50,12 @@ public void TestIsIgnored_Assert()
             Assert.Ignore("Ignoring this test deliberately");
         }
 
+        // Since we only run under .NET, test is always excluded
+        [Test, ExpectSkip, Platform("Exclude=\"NET\"")]
+        public void TestIsSkipped_Platform()
+        {
+        }
+
         [Test, ExpectError]
         public void TestThrowsException()
         {

src/NUnitTestAdapter/NUnit3TestExecutor.cs

@@ -173,14 +173,16 @@ private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
                 {
                     TestLog.SendInformationalMessage(string.Format("Loading tests from {0}", assemblyName));
 
+                    var nunitTestCases = loadResult.SelectNodes("//test-case");
+
                     using (var testConverter = new TestConverter(TestLog, assemblyName))
                     {
                         var loadedTestCases = new List<TestCase>();
 
                         // As a side effect of calling TestConverter.ConvertTestCase, 
                         // the converter's cache of all test cases is populated as well. 
                         // All future calls to convert a test case may now use the cache.
-                        foreach (XmlNode testNode in loadResult.SelectNodes("//test-case"))
+                        foreach (XmlNode testNode in nunitTestCases)
                             loadedTestCases.Add(testConverter.ConvertTestCase(testNode));
 
                         // If we have a TFS Filter, convert it to an nunit filter
@@ -204,6 +206,8 @@ private void RunAssembly(string assemblyName, IFrameworkHandle frameworkHandle)
                                 TestLog.SendDebugMessage("Nullref caught");
                             }
                         }
+
+                        new NonEventTestProcessor(frameworkHandle, testConverter).ProcessTests(loadResult);
                     }
                 }
                 else
@@ -239,5 +243,63 @@ private static TestFilter MakeTestFilter(IEnumerable<TestCase> testCases)
         }
 
         #endregion
+
+        #region Nested NonEventTestProcessor Class
+
+        // This class implements an adhoc fix necessitated by the fact that NUnit 
+        // does not send events for tests that are not executed either due to the
+        // presence of an attribute that changes the runstate or to an error that
+        // makes the test NonRunnable. Consequently, no result is ever reported
+        // to VS for such tests by the NUnitEventListener.
+        //
+        // To make up for this, we go through the tests and create pseudo-results 
+        // for each one that is identified as not producing an event.
+        //
+        // TODO: Remove this after NUnit is modified to send events for all test cases
+        class NonEventTestProcessor
+        {
+            private IFrameworkHandle _frameworkHandle;
+            private TestConverter _testConverter;
+
+            public NonEventTestProcessor(IFrameworkHandle frameworkHandle, TestConverter testConverter)
+            {
+                _frameworkHandle = frameworkHandle;
+                _testConverter = testConverter;
+            }
+
+            public void ProcessTests(XmlNode node)
+            {
+                string name = node.Name;
+                string runstate = node.GetAttribute("runstate");
+
+                if (name == "test-suite" || name == "test-run")
+                {
+                    if (runstate == "Ignored" || runstate=="Skipped")
+                        ReportTestCases(node, TestOutcome.Skipped);
+                    else // Keep descending
+                        foreach (XmlNode childNode in node.ChildNodes)
+                            ProcessTests(childNode);
+                }
+            }
+
+            private void ReportTestCases(XmlNode node, TestOutcome outcome)
+            {
+                foreach (XmlNode childNode in node.ChildNodes)
+                {
+                    if (childNode.Name == "test-case")
+                    {
+                        var result = new TestResult(_testConverter.ConvertTestCase(childNode));
+                        result.Outcome = outcome;
+                        _frameworkHandle.RecordResult(result);
+                    }
+                    else if (childNode.Name == "test-suite" || childNode.Name == "test-run")
+                    {
+                        ReportTestCases(childNode, outcome);
+                    }
+                }
+            }
+        }
+
+        #endregion
     }
 }

src/NUnitTestAdapter/NUnitEventListener.cs

@@ -105,16 +105,19 @@ public void TestFinished(XmlNode resultNode)
 
         public void SuiteFinished(XmlNode resultNode)
         {
-            if (resultNode.GetAttribute("result") == "Failed")
+            var result = resultNode.GetAttribute("result");
+            var label = resultNode.GetAttribute("label");
+            var site = resultNode.GetAttribute("site");
+
+            if (result == "Failed")
             {
-                var site = resultNode.GetAttribute("site");
                 if (site == "SetUp" || site == "TearDown")
                 {
                     testLog.SendMessage(
                         TestMessageLevel.Error,
                         string.Format("{0} failed for test fixture {1}", site, resultNode.GetAttribute("fullname")));
 
-                    var messageNode = resultNode.SelectSingleNode("failure/messge");
+                    var messageNode = resultNode.SelectSingleNode("failure/message");
                     if (messageNode != null)
                         testLog.SendMessage(TestMessageLevel.Error, messageNode.InnerText);