Skip to content

Commit a30dc42

Browse files
bj-9527bj-9527
committed
configfailurepolicy=continue only works for BeforeTest when using TestNG XML file
Fixes testng-team#2731
1 parent 0438740 commit a30dc42

File tree

9 files changed

+278
-87
lines changed

9 files changed

+278
-87
lines changed

‎CHANGES.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
Current
22
7.6.0
3+
Fixed: GITHUB-2731: configfailurepolicy=continue only works for BeforeTest when using TestNG XML file (Nan Liang)
34
Fixed: GITHUB-2729: beforeConfiguration() listener method should be invoked for skipped configurations as well(Nan Liang)
45
Fixed: assertEqualsNoOrder for Collection and Iterators size check was missing (Adam Kaczmarek)
56
Fixed: GITHUB-2709: Testnames not working together with suites in suite (Martin Aldrin)

‎testng-core/src/main/java/org/testng/internal/invokers/ConfigInvoker.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ public void invokeConfigurations(ConfigMethodArguments arguments) {
274274
tm.getGroups(),
275275
arguments.getTestClass(),
276276
arguments.getInstance())
277-
&& !alwaysRun) {
277+
&& !alwaysRun
278+
&& !m_continueOnFailedConfiguration) {
278279
log(3, "Skipping " + Utils.detailedMethodName(tm, true));
279280
InvokedMethod invokedMethod = new InvokedMethod(System.currentTimeMillis(), testResult);
280281
runConfigurationListeners(testResult, arguments.getTestMethod(), true /* before */);

‎testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -597,7 +597,8 @@ private ITestResult invokeMethod(
597597
arguments.getTestMethod(),
598598
arguments.getTestMethod().getGroups(),
599599
arguments.getTestClass(),
600-
arguments.getInstance())) {
600+
arguments.getInstance())
601+
&& suite.getConfigFailurePolicy() == XmlSuite.FailurePolicy.SKIP) {
601602
Throwable exception =
602603
ExceptionUtils.getExceptionDetails(m_testContext, arguments.getInstance());
603604
ITestResult result =

‎testng-core/src/test/java/test/configurationfailurepolicy/FailureContinuePolicyTest.java

+182
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package test.configurationfailurepolicy;
2+
3+
import org.testng.ITestContext;
4+
import org.testng.TestListenerAdapter;
5+
import org.testng.TestNG;
6+
import org.testng.annotations.BeforeClass;
7+
import org.testng.annotations.DataProvider;
8+
import org.testng.annotations.Test;
9+
import org.testng.xml.XmlSuite;
10+
import test.configurationfailurepolicy.issue2731.ConfigFailTestSample;
11+
import testhelper.OutputDirectoryPatch;
12+
13+
import static org.testng.Assert.assertEquals;
14+
import static test.SimpleBaseTest.getPathToResource;
15+
16+
public class FailureContinuePolicyTest {
17+
// only if this is run from an xml file that sets this on the suite
18+
@BeforeClass(enabled = false)
19+
public void setupClass(ITestContext testContext) {
20+
assertEquals(
21+
testContext.getSuite().getXmlSuite().getConfigFailurePolicy(),
22+
XmlSuite.FailurePolicy.CONTINUE);
23+
}
24+
25+
@DataProvider(name = "dp")
26+
public Object[][] getData() {
27+
return new Object[][] {
28+
// params - confFail, confSkip, skippedTests
29+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
30+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 0, 0},
31+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 0},
32+
new Object[] {
33+
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
34+
1,
35+
0,
36+
0
37+
},
38+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 0},
39+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 0, 0},
40+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 0, 0},
41+
new Object[] {
42+
new Class[] {
43+
ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class
44+
},
45+
2,
46+
0,
47+
0
48+
},
49+
new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 0},
50+
new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 0},
51+
new Object[] {
52+
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 0
53+
},
54+
new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 0, 0},
55+
new Object[] {new Class[] {ConfigFailTestSample.class}, 4, 0, 0}
56+
};
57+
}
58+
59+
@Test(dataProvider = "dp")
60+
public void confFailureTest(
61+
Class[] classesUnderTest,
62+
int configurationFailures,
63+
int configurationSkips,
64+
int skippedTests) {
65+
66+
TestListenerAdapter tla = new TestListenerAdapter();
67+
TestNG testng = new TestNG();
68+
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
69+
testng.setTestClasses(classesUnderTest);
70+
testng.addListener(tla);
71+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
72+
testng.run();
73+
74+
verify(tla, configurationFailures, configurationSkips, skippedTests);
75+
}
76+
77+
@Test
78+
public void confFailureTestInvolvingGroups() {
79+
Class[] classesUnderTest =
80+
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeGroupsAfterClassAfterGroups.class};
81+
82+
TestListenerAdapter tla = new TestListenerAdapter();
83+
TestNG testng = new TestNG();
84+
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
85+
testng.setTestClasses(classesUnderTest);
86+
testng.addListener(tla);
87+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
88+
testng.setGroups("group1");
89+
testng.run();
90+
verify(tla, 1, 0, 0);
91+
}
92+
93+
@Test
94+
public void commandLineTest_policyAsSkip() {
95+
String[] argv =
96+
new String[] {
97+
"-log",
98+
"0",
99+
"-d",
100+
OutputDirectoryPatch.getOutputDirectory(),
101+
"-configfailurepolicy",
102+
"skip",
103+
"-testclass",
104+
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
105+
};
106+
TestListenerAdapter tla = new TestListenerAdapter();
107+
TestNG.privateMain(argv, tla);
108+
109+
verify(tla, 1, 1, 2);
110+
}
111+
112+
@Test
113+
public void commandLineTest_policyAsContinue() {
114+
String[] argv =
115+
new String[] {
116+
"-log",
117+
"0",
118+
"-d",
119+
OutputDirectoryPatch.getOutputDirectory(),
120+
"-configfailurepolicy",
121+
"continue",
122+
"-testclass",
123+
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
124+
};
125+
TestListenerAdapter tla = new TestListenerAdapter();
126+
TestNG.privateMain(argv, tla);
127+
128+
verify(tla, 2, 0, 0);
129+
}
130+
131+
@Test
132+
public void commandLineTestWithXMLFile_policyAsSkip() {
133+
String[] argv =
134+
new String[] {
135+
"-log",
136+
"0",
137+
"-d",
138+
OutputDirectoryPatch.getOutputDirectory(),
139+
"-configfailurepolicy",
140+
"skip",
141+
getPathToResource("testng-configfailure.xml")
142+
};
143+
TestListenerAdapter tla = new TestListenerAdapter();
144+
TestNG.privateMain(argv, tla);
145+
146+
verify(tla, 1, 1, 2);
147+
}
148+
149+
@Test
150+
public void commandLineTestWithXMLFile_policyAsContinue() {
151+
String[] argv =
152+
new String[] {
153+
"-log",
154+
"0",
155+
"-d",
156+
OutputDirectoryPatch.getOutputDirectory(),
157+
"-configfailurepolicy",
158+
"continue",
159+
getPathToResource("testng-configfailure.xml")
160+
};
161+
TestListenerAdapter tla = new TestListenerAdapter();
162+
TestNG.privateMain(argv, tla);
163+
164+
verify(tla, 2, 0, 0);
165+
}
166+
167+
private void verify(
168+
TestListenerAdapter tla,
169+
int configurationFailures,
170+
int configurationSkips,
171+
int skippedTests) {
172+
assertEquals(
173+
tla.getConfigurationFailures().size(),
174+
configurationFailures,
175+
"wrong number of configuration failures");
176+
assertEquals(
177+
tla.getConfigurationSkips().size(),
178+
configurationSkips,
179+
"wrong number of configuration skips");
180+
assertEquals(tla.getSkippedTests().size(), skippedTests, "wrong number of skipped tests");
181+
}
182+
}

‎testng-core/src/test/java/test/configurationfailurepolicy/FailurePolicyTest.java

+46-82
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.testng.annotations.DataProvider;
1111
import org.testng.annotations.Test;
1212
import org.testng.xml.XmlSuite;
13+
import test.configurationfailurepolicy.issue2731.ConfigFailTestSample;
1314
import testhelper.OutputDirectoryPatch;
1415

1516
public class FailurePolicyTest {
@@ -19,39 +20,39 @@ public class FailurePolicyTest {
1920
public void setupClass(ITestContext testContext) {
2021
assertEquals(
2122
testContext.getSuite().getXmlSuite().getConfigFailurePolicy(),
22-
XmlSuite.FailurePolicy.CONTINUE);
23+
XmlSuite.FailurePolicy.SKIP);
2324
}
2425

2526
@DataProvider(name = "dp")
2627
public Object[][] getData() {
2728
return new Object[][] {
28-
// params - confFail, confSkip, skippedTests
29-
new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1},
30-
new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1},
31-
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2},
32-
new Object[] {
33-
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
34-
1,
35-
3,
36-
1
37-
},
38-
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4},
39-
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2},
40-
new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2},
41-
new Object[] {
42-
new Class[] {
43-
ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class
29+
// params - confFail, confSkip, skippedTests
30+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethod.class}, 1, 1, 1},
31+
new Object[] {new Class[] {ClassWithFailedBeforeClassMethodAndAfterClass.class}, 1, 1, 1},
32+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleTests.class}, 2, 0, 2},
33+
new Object[] {
34+
new Class[] {ClassWithFailedBeforeClassMethodAndBeforeMethodAfterMethodAfterClass.class},
35+
1,
36+
3,
37+
1
4438
},
45-
2,
46-
3,
47-
3
48-
},
49-
new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1},
50-
new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2},
51-
new Object[] {
52-
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8
53-
},
54-
new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2},
39+
new Object[] {new Class[] {ClassWithFailedBeforeMethodAndMultipleInvocations.class}, 4, 0, 4},
40+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeMethod.class}, 2, 2, 2},
41+
new Object[] {new Class[] {ExtendsClassWithFailedBeforeClassMethod.class}, 1, 2, 2},
42+
new Object[] {
43+
new Class[] {
44+
ClassWithFailedBeforeClassMethod.class, ExtendsClassWithFailedBeforeClassMethod.class
45+
},
46+
2,
47+
3,
48+
3
49+
},
50+
new Object[] {new Class[] {ClassWithSkippingBeforeMethod.class}, 0, 1, 1},
51+
new Object[] {new Class[] {FactoryClassWithFailedBeforeMethod.class}, 2, 0, 2},
52+
new Object[] {
53+
new Class[] {FactoryClassWithFailedBeforeMethodAndMultipleInvocations.class}, 8, 0, 8
54+
},
55+
new Object[] {new Class[] {FactoryClassWithFailedBeforeClassMethod.class}, 2, 2, 2},
5556
};
5657
}
5758

@@ -67,7 +68,7 @@ public void confFailureTest(
6768
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
6869
testng.setTestClasses(classesUnderTest);
6970
testng.addListener(tla);
70-
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
71+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP);
7172
testng.run();
7273

7374
verify(tla, configurationFailures, configurationSkips, skippedTests);
@@ -83,7 +84,7 @@ public void confFailureTestInvolvingGroups() {
8384
testng.setOutputDirectory(OutputDirectoryPatch.getOutputDirectory());
8485
testng.setTestClasses(classesUnderTest);
8586
testng.addListener(tla);
86-
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.CONTINUE);
87+
testng.setConfigFailurePolicy(XmlSuite.FailurePolicy.SKIP);
8788
testng.setGroups("group1");
8889
testng.run();
8990
verify(tla, 1, 3, 1);
@@ -93,76 +94,39 @@ public void confFailureTestInvolvingGroups() {
9394
public void commandLineTest_policyAsSkip() {
9495
String[] argv =
9596
new String[] {
96-
"-log",
97-
"0",
98-
"-d",
99-
OutputDirectoryPatch.getOutputDirectory(),
100-
"-configfailurepolicy",
101-
"skip",
102-
"-testclass",
103-
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
97+
"-log",
98+
"0",
99+
"-d",
100+
OutputDirectoryPatch.getOutputDirectory(),
101+
"-configfailurepolicy",
102+
"skip",
103+
"-testclass",
104+
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
104105
};
105106
TestListenerAdapter tla = new TestListenerAdapter();
106107
TestNG.privateMain(argv, tla);
107108

108109
verify(tla, 1, 1, 2);
109110
}
110111

111-
@Test
112-
public void commandLineTest_policyAsContinue() {
113-
String[] argv =
114-
new String[] {
115-
"-log",
116-
"0",
117-
"-d",
118-
OutputDirectoryPatch.getOutputDirectory(),
119-
"-configfailurepolicy",
120-
"continue",
121-
"-testclass",
122-
ClassWithFailedBeforeMethodAndMultipleTests.class.getCanonicalName()
123-
};
124-
TestListenerAdapter tla = new TestListenerAdapter();
125-
TestNG.privateMain(argv, tla);
126-
127-
verify(tla, 2, 0, 2);
128-
}
129-
130112
@Test
131113
public void commandLineTestWithXMLFile_policyAsSkip() {
132114
String[] argv =
133-
new String[] {
134-
"-log",
135-
"0",
136-
"-d",
137-
OutputDirectoryPatch.getOutputDirectory(),
138-
"-configfailurepolicy",
139-
"skip",
140-
getPathToResource("testng-configfailure.xml")
115+
new String[]{
116+
"-log",
117+
"0",
118+
"-d",
119+
OutputDirectoryPatch.getOutputDirectory(),
120+
"-configfailurepolicy",
121+
"skip",
122+
getPathToResource("testng-configfailure.xml")
141123
};
142124
TestListenerAdapter tla = new TestListenerAdapter();
143125
TestNG.privateMain(argv, tla);
144126

145127
verify(tla, 1, 1, 2);
146128
}
147129

148-
@Test
149-
public void commandLineTestWithXMLFile_policyAsContinue() {
150-
String[] argv =
151-
new String[] {
152-
"-log",
153-
"0",
154-
"-d",
155-
OutputDirectoryPatch.getOutputDirectory(),
156-
"-configfailurepolicy",
157-
"continue",
158-
getPathToResource("testng-configfailure.xml")
159-
};
160-
TestListenerAdapter tla = new TestListenerAdapter();
161-
TestNG.privateMain(argv, tla);
162-
163-
verify(tla, 2, 0, 2);
164-
}
165-
166130
private void verify(
167131
TestListenerAdapter tla,
168132
int configurationFailures,

0 commit comments

Comments
 (0)