configfailurepolicy=continue only works for BeforeTest when using TestNG XML file

Fixes testng-team#2731

Author: bj-9527

CHANGES.txt

@@ -1,5 +1,6 @@
 Current
 7.6.0
+Fixed: GITHUB-2731: configfailurepolicy=continue only works for BeforeTest when using TestNG XML file (Nan Liang)
 Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang)
 Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek)
 Fixed: GITHUB-2709: Testnames not working together with suites in suite (Martin Aldrin)

testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java

@@ -274,7 +274,8 @@ public void invokeConfigurations(ConfigMethodArguments arguments) {
                 tm.getGroups(),
                 arguments.getTestClass(),
                 arguments.getInstance())
-            && !alwaysRun) {
+            && !alwaysRun
+            && !m_continueOnFailedConfiguration) {
           log(3, "Skipping " + Utils.detailedMethodName(tm, true));
           InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), testResult);
           runConfigurationListeners(testResult, arguments.getTestMethod(), true /* before */);

testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java

@@ -597,7 +597,8 @@ private ITestResult invokeMethod(
             arguments.getTestMethod(),
             arguments.getTestMethod().getGroups(),
             arguments.getTestClass(),
-            arguments.getInstance())) {
+            arguments.getInstance())
+        && suite.getConfigFailurePolicy() == XmlSuite.FailurePolicy.SKIP) {
       Throwable exception =
           ExceptionUtils.getExceptionDetails(m_testContext, arguments.getInstance());
       ITestResult result =

testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java

@@ -10,6 +10,7 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 import org.testng.xml.XmlSuite;
+import test.configurationfailurepolicy.issue2731.ConfigFailTestSample;
 import testhelper.OutputDirectoryPatch;
 
 public class FailurePolicyTest {
@@ -26,32 +27,33 @@ public void setupClass(ITestContext testContext) {
   public Object[][] getData() {
     return new Object[][] {
       // params - confFail, confSkip, skippedTests
-      new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1},
-      new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1},
-      new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2},
+      new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
+      new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 0, 0},
+      new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 0},
       new Object[] {
         new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
         1,
-        3,
-        1
+        0,
+        0
       },
-      new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4},
-      new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2},
-      new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2},
+      new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 0},
+      new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 0, 0},
+      new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
       new Object[] {
         new Class[] {
           ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class
         },
         2,
-        3,
-        3
+        0,
+        0
       },
-      new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1},
-      new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2},
+      new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 0},
+      new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 0},
       new Object[] {
-        new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8
+        new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 0
       },
-      new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2},
+      new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 0, 0},
+      new Object[] {new Class[] {ConfigFailTestSample.class}, 4, 0, 0}
     };
   }
 
@@ -86,7 +88,7 @@ public void confFailureTestInvolvingGroups() {
     testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
     testng.setGroups("group1");
     testng.run();
-    verify(tla, 1, 3, 1);
+    verify(tla, 1, 0, 0);
   }
 
   @Test
@@ -124,7 +126,7 @@ public void commandLineTest_policyAsContinue() {
     TestListenerAdapter tla = new TestListenerAdapter();
     TestNG.privateMain(argv, tla);
 
-    verify(tla, 2, 0, 2);
+    verify(tla, 2, 0, 0);
   }
 
   @Test
@@ -160,7 +162,7 @@ public void commandLineTestWithXMLFile_policyAsContinue() {
     TestListenerAdapter tla = new TestListenerAdapter();
     TestNG.privateMain(argv, tla);
 
-    verify(tla, 2, 0, 2);
+    verify(tla, 2, 0, 0);
   }
 
   private void verify(

testng-core/src/test/java/test/configurationfailurepolicy/issue2731/ConfigFailTestSample.java

@@ -0,0 +1,33 @@
+package test.configurationfailurepolicy.issue2731;
+
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+public class ConfigFailTestSample {
+  @BeforeSuite
+  public void beforeSuite() {
+    Assert.fail("This before suite is fail");
+  }
+
+  @BeforeTest
+  public void beforeTest() {
+    Assert.fail("This before test is fail");
+  }
+
+  @BeforeClass
+  public void beforeClass() {
+    Assert.fail("This before class is fail");
+  }
+
+  @BeforeMethod
+  public void beforeMethod() {
+    Assert.fail("This before method is fail");
+  }
+
+  @Test
+  public void test() {}
+}

testng-core/src/test/java/test/listeners/github1602/IssueTest.java

@@ -35,19 +35,27 @@ public Object[][] getData() {
             "BeforeInvocation_afterMethod_STARTED",
             "AfterInvocation_afterMethod_SUCCESS");
     List<String> baseList =
+        Arrays.asList(
+            "BeforeInvocation_beforeMethod_STARTED",
+            "AfterInvocation_beforeMethod_FAILURE",
+            "BeforeInvocation_testMethod_STARTED",
+            "AfterInvocation_testMethod_SUCCESS",
+            "BeforeInvocation_afterMethod_STARTED");
+    List<String> commonSkipList =
         Arrays.asList(
             "BeforeInvocation_beforeMethod_STARTED",
             "AfterInvocation_beforeMethod_FAILURE",
             "BeforeInvocation_testMethod_SKIP",
             "AfterInvocation_testMethod_SKIP",
-            "BeforeInvocation_afterMethod_STARTED");
+            "BeforeInvocation_afterMethod_STARTED",
+            "AfterInvocation_afterMethod_SKIP");
     List<String> skipList = Lists.newArrayList(baseList);
     skipList.add("AfterInvocation_afterMethod_SKIP");
     List<String> failList = Lists.newArrayList(baseList);
     failList.add("AfterInvocation_afterMethod_FAILURE");
     return new Object[][] {
       {TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, passList},
-      {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, skipList},
+      {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.SKIP, commonSkipList},
       {TestClassWithPassingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, passList},
       {TestClassWithFailingConfigsSample.class, XmlSuite.FailurePolicy.CONTINUE, failList}
     };

testng-core/src/test/java/test/listeners/issue1777/IssueTest.java

@@ -28,7 +28,7 @@ public void testOnStartInvokedForSkippedTests() {
             "before_test_method: test1",
             "after_test_method: test1",
             "after_test_method: test1",
-            "testSkipped_test_method: test1",
+            "testSuccess_test_method: test1",
             "testStart_test_method: test2",
             "before_test_method: test2",
             "before_test_method: test2",