Skip to content

Commit b8246dc

Browse files
andpabslawekjaranowski
andpab
authored and
slawekjaranowski
committed
[SUREFIRE-2143] Fix reporting of skipped parameterized test
- In executionSkipped handle ParameterizedTest which is a non-class test container the same as a regular test - In executionSkipped always remove the testIdentifier from the map testStartTime, regardless if it's a class or a test - In createReportEntry set methodName and methodText if the testExecutionResult is null, which is the case for skipped tests, otherwise the test name in the XML report is left empty - Add ITs also for the related previous issue SUREFIRE-2032 the fix of which caused this regression, to make sure that fix still works
1 parent 11bbdc9 commit b8246dc

File tree

10 files changed

+639
-4
lines changed

10 files changed

+639
-4
lines changed

‎surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2032NestedSkippedIT.java

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package org.apache.maven.surefire.its.jiras;
2+
3+
/*
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing,
15+
* software distributed under the License is distributed on an
16+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
* KIND, either express or implied. See the License for the
18+
* specific language governing permissions and limitations
19+
* under the License.
20+
*/
21+
22+
import static java.nio.charset.StandardCharsets.UTF_8;
23+
import static org.assertj.core.api.Assertions.assertThat;
24+
25+
import java.util.Arrays;
26+
import java.util.List;
27+
28+
import org.apache.maven.surefire.its.fixture.OutputValidator;
29+
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
30+
import org.junit.Test;
31+
32+
/**
33+
* Integration Test for SUREFIRE-2032
34+
*/
35+
@SuppressWarnings( "checkstyle:magicnumber" )
36+
public class Surefire2032NestedSkippedIT extends SurefireJUnit4IntegrationTestCase
37+
{
38+
@Test
39+
public void testXmlReport()
40+
{
41+
OutputValidator validator = unpack( "surefire-2032-nested-test-class-skipped" )
42+
.executeTest()
43+
.assertTestSuiteResults( 4, 0, 0, 2 );
44+
45+
String redXmlReport = validator
46+
.getSurefireReportsFile( "TEST-jira2032.DisabledNestedTest$RedTaggedEnabledTest.xml", UTF_8 )
47+
.readFileToString();
48+
49+
// Enabled nested subclass
50+
List<String> redTestCaseResults = Arrays.asList(
51+
redXmlReport.substring( redXmlReport.indexOf( "<testcase " ),
52+
redXmlReport.indexOf( "</testsuite>" ) )
53+
.split( ".(?=<testcase )" ) );
54+
55+
assertThat( redTestCaseResults ).hasSize( 2 )
56+
.filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
57+
.isEmpty();
58+
59+
// Disabled nested subclass
60+
String orangeXmlReport = validator
61+
.getSurefireReportsFile( "TEST-jira2032.DisabledNestedTest$OrangeTaggedDisabledTest.xml", UTF_8 )
62+
.readFileToString();
63+
64+
List<String> orangeTestCaseResults = Arrays.asList(
65+
orangeXmlReport.substring( orangeXmlReport.indexOf( "<testcase " ),
66+
orangeXmlReport.indexOf( "</testsuite>" ) )
67+
.split( ".(?=<testcase )" ) );
68+
69+
assertThat( orangeTestCaseResults )
70+
.hasSize( 2 )
71+
.filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
72+
.map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
73+
.containsExactlyInAnyOrder(
74+
"<testcase name=\"test1\" ",
75+
"<testcase name=\"test2\" " );
76+
}
77+
78+
}

‎surefire-its/src/test/java/org/apache/maven/surefire/its/jiras/Surefire2143ParameterizedSkippedIT.java

+102
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package org.apache.maven.surefire.its.jiras;
2+
3+
/*
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing,
15+
* software distributed under the License is distributed on an
16+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
* KIND, either express or implied. See the License for the
18+
* specific language governing permissions and limitations
19+
* under the License.
20+
*/
21+
22+
import static java.nio.charset.StandardCharsets.UTF_8;
23+
import static org.assertj.core.api.Assertions.assertThat;
24+
import static org.hamcrest.Matchers.matchesRegex;
25+
26+
import java.util.Arrays;
27+
import java.util.List;
28+
29+
import org.apache.maven.surefire.its.fixture.OutputValidator;
30+
import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
31+
import org.junit.Test;
32+
33+
/**
34+
* Integration Test for SUREFIRE-2143
35+
*/
36+
@SuppressWarnings( "checkstyle:magicnumber" )
37+
public class Surefire2143ParameterizedSkippedIT extends SurefireJUnit4IntegrationTestCase
38+
{
39+
@Test
40+
public void junit5ParameterizedSkipped()
41+
{
42+
OutputValidator validator = unpack( "surefire-2143-junit5-parameterized-test-skipped" )
43+
.executeTest()
44+
.assertTestSuiteResults( 5, 0, 0, 2 );
45+
46+
String xmlReport = validator
47+
.getSurefireReportsFile( "TEST-jira2143.DisabledParameterizedTest.xml", UTF_8 )
48+
.readFileToString();
49+
50+
List<String> testCaseResults = Arrays.asList(
51+
xmlReport.substring( xmlReport.indexOf( "<testcase " ),
52+
xmlReport.indexOf( "</testsuite>" ) )
53+
.split( ".(?=<testcase )" ) );
54+
55+
assertThat( testCaseResults )
56+
.hasSize( 5 )
57+
.filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
58+
.map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
59+
.containsExactlyInAnyOrder(
60+
"<testcase name=\"disabledParameterized(String)\" ",
61+
"<testcase name=\"disabledNonParameterized\" " );
62+
63+
validator.getSurefireReportsFile( "TEST-jira2143.DisabledParameterizedTest.xml", UTF_8 )
64+
.assertContainsText( "<testcase name=\"disabledParameterized(String)\" "
65+
+ "classname=\"jira2143.DisabledParameterizedTest\"" )
66+
.assertContainsText( "<testcase name=\"disabledNonParameterized\" "
67+
+ "classname=\"jira2143.DisabledParameterizedTest\"" )
68+
.assertContainsText( matchesRegex( ".*<skipped [^>]*disabledParameterized.*" ) )
69+
.assertContainsText( matchesRegex( ".*<skipped [^>]*disabledNonParameterized.*" ) )
70+
.assertContainsText( "<testcase name=\"enabledParameterized(String)[1]\"" )
71+
.assertContainsText( "<testcase name=\"enabledParameterized(String)[2]\"" )
72+
.assertContainsText( "<testcase name=\"enabledNonParameterized\"" )
73+
.assertNotContainsText( matchesRegex( ".*<skipped [^>]*enabledParameterized.*" ) )
74+
.assertNotContainsText( matchesRegex( ".*<skipped [^>]*enabledNonParameterized.*" ) );
75+
}
76+
77+
@Test
78+
public void junit4ParameterizedSkipped()
79+
{
80+
OutputValidator validator = unpack( "surefire-2143-junit4-parameterized-test-skipped" )
81+
.executeTest()
82+
.assertTestSuiteResults( 4, 0, 0, 2 );
83+
84+
String xmlReport = validator
85+
.getSurefireReportsFile( "TEST-jira2143.IgnoredParameterizedTest.xml", UTF_8 )
86+
.readFileToString();
87+
88+
List<String> testCaseResults = Arrays.asList(
89+
xmlReport.substring( xmlReport.indexOf( "<testcase " ),
90+
xmlReport.indexOf( "</testsuite>" ) )
91+
.split( ".(?=<testcase )" ) );
92+
93+
assertThat( testCaseResults )
94+
.hasSize( 4 )
95+
.filteredOn( testCaseResult -> testCaseResult.contains( "<skipped" ) )
96+
.map( testCaseResult -> testCaseResult.substring( 0, testCaseResult.indexOf( "classname" ) ) )
97+
.containsExactlyInAnyOrder(
98+
"<testcase name=\"ignoredParameterized[0]\" ",
99+
"<testcase name=\"ignoredParameterized[1]\" " );
100+
}
101+
102+
}

‎surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/pom.xml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns="http://maven.apache.org/POM/4.0.0"
22+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<groupId>org.apache.maven.plugins.surefire</groupId>
27+
<artifactId>surefire-2032-nested-test-class-skipped</artifactId>
28+
<version>1.0</version>
29+
<name>Test for: RunListenerAdapter, Nested and Disabled</name>
30+
31+
<properties>
32+
<maven.compiler.source>1.8</maven.compiler.source>
33+
<maven.compiler.target>1.8</maven.compiler.target>
34+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
35+
<junit5.version>5.9.1</junit5.version>
36+
</properties>
37+
38+
<!--
39+
Declare "junit-jupiter-engine" dependency because the
40+
Jupiter Engine is needed at test runtime. Artifacts
41+
needed for test compilation, like "junit-jupiter-api",
42+
are pulled-in via transitive dependency resolution.
43+
-->
44+
<dependencies>
45+
<dependency>
46+
<groupId>org.junit.jupiter</groupId>
47+
<artifactId>junit-jupiter-engine</artifactId>
48+
<version>${junit5.version}</version>
49+
<scope>test</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.junit.jupiter</groupId>
53+
<artifactId>junit-jupiter-params</artifactId>
54+
<version>${junit5.version}</version>
55+
<scope>test</scope>
56+
</dependency>
57+
</dependencies>
58+
59+
<build>
60+
<plugins>
61+
<plugin>
62+
<artifactId>maven-compiler-plugin</artifactId>
63+
<version>3.8.0</version>
64+
<configuration>
65+
<encoding>UTF-8</encoding>
66+
</configuration>
67+
</plugin>
68+
<plugin>
69+
<groupId>org.apache.maven.plugins</groupId>
70+
<artifactId>maven-surefire-plugin</artifactId>
71+
<version>${surefire.version}</version>
72+
<configuration>
73+
<forkCount>1.0C</forkCount>
74+
<redirectTestOutputToFile>true</redirectTestOutputToFile>
75+
</configuration>
76+
</plugin>
77+
</plugins>
78+
</build>
79+
80+
</project>

‎surefire-its/src/test/resources/surefire-2032-nested-test-class-skipped/src/test/java/jira2032/DisabledNestedTest.java

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
package jira2032;
2+
3+
import org.junit.jupiter.api.Disabled;
4+
import org.junit.jupiter.api.Nested;
5+
import org.junit.jupiter.api.Tag;
6+
import org.junit.jupiter.api.Test;
7+
import org.junit.jupiter.params.ParameterizedTest;
8+
import org.junit.jupiter.params.provider.ValueSource;
9+
10+
/*
11+
* Licensed to the Apache Software Foundation (ASF) under one
12+
* or more contributor license agreements. See the NOTICE file
13+
* distributed with this work for additional information
14+
* regarding copyright ownership. The ASF licenses this file
15+
* to you under the Apache License, Version 2.0 (the
16+
* "License"); you may not use this file except in compliance
17+
* with the License. You may obtain a copy of the License at
18+
*
19+
* http://www.apache.org/licenses/LICENSE-2.0
20+
*
21+
* Unless required by applicable law or agreed to in writing,
22+
* software distributed under the License is distributed on an
23+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
24+
* KIND, either express or implied. See the License for the
25+
* specific language governing permissions and limitations
26+
* under the License.
27+
*/
28+
29+
class DisabledNestedTest
30+
{
31+
@Tag("red")
32+
@Nested
33+
public class RedTaggedEnabledTest extends TagTest {
34+
}
35+
36+
@Disabled
37+
@Tag("orange")
38+
@Nested
39+
public class OrangeTaggedDisabledTest extends TagTest {
40+
}
41+
42+
abstract class TagTest {
43+
44+
@Test
45+
public void test1() {
46+
// Do Nothing
47+
}
48+
49+
@Test
50+
public void test2() {
51+
// Do Nothing
52+
}
53+
54+
}
55+
56+
}

‎surefire-its/src/test/resources/surefire-2143-junit4-parameterized-test-skipped/pom.xml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one
4+
~ or more contributor license agreements. See the NOTICE file
5+
~ distributed with this work for additional information
6+
~ regarding copyright ownership. The ASF licenses this file
7+
~ to you under the Apache License, Version 2.0 (the
8+
~ "License"); you may not use this file except in compliance
9+
~ with the License. You may obtain a copy of the License at
10+
~
11+
~ http://www.apache.org/licenses/LICENSE-2.0
12+
~
13+
~ Unless required by applicable law or agreed to in writing,
14+
~ software distributed under the License is distributed on an
15+
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
~ KIND, either express or implied. See the License for the
17+
~ specific language governing permissions and limitations
18+
~ under the License.
19+
-->
20+
21+
<project xmlns="http://maven.apache.org/POM/4.0.0"
22+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
23+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
24+
<modelVersion>4.0.0</modelVersion>
25+
26+
<groupId>org.apache.maven.plugins.surefire</groupId>
27+
<artifactId>surefire-2143-parameterized-test-skipped</artifactId>
28+
<version>1.0</version>
29+
<name>Test for: RunListenerAdapter, Parameterized and Ignore</name>
30+
31+
<properties>
32+
<maven.compiler.source>1.8</maven.compiler.source>
33+
<maven.compiler.target>1.8</maven.compiler.target>
34+
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
35+
<junit5.version>5.9.1</junit5.version>
36+
</properties>
37+
38+
<!--
39+
Declare "junit-jupiter-engine" dependency because the
40+
Jupiter Engine is needed at test runtime. Artifacts
41+
needed for test compilation, like "junit-jupiter-api",
42+
are pulled-in via transitive dependency resolution.
43+
-->
44+
<dependencies>
45+
<dependency>
46+
<groupId>org.junit.vintage</groupId>
47+
<artifactId>junit-vintage-engine</artifactId>
48+
<version>${junit5.version}</version>
49+
<scope>test</scope>
50+
</dependency>
51+
</dependencies>
52+
53+
<build>
54+
<plugins>
55+
<plugin>
56+
<artifactId>maven-compiler-plugin</artifactId>
57+
<version>3.8.0</version>
58+
<configuration>
59+
<encoding>UTF-8</encoding>
60+
</configuration>
61+
</plugin>
62+
<plugin>
63+
<groupId>org.apache.maven.plugins</groupId>
64+
<artifactId>maven-surefire-plugin</artifactId>
65+
<version>${surefire.version}</version>
66+
<configuration>
67+
<forkCount>1.0C</forkCount>
68+
<redirectTestOutputToFile>true</redirectTestOutputToFile>
69+
</configuration>
70+
</plugin>
71+
</plugins>
72+
</build>
73+
74+
</project>

0 commit comments

Comments
 (0)