From 78aff0dded23242aaedee01f24f6a1b6b3943547 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Tue, 23 Apr 2024 16:19:19 +0200 Subject: [PATCH 1/3] Add func tests for retrying Cucumber tests using JUnit4 runner Signed-off-by: Pavlo Shevchenko --- .../CucumberJUnit4FuncTest.groovy | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy new file mode 100644 index 00000000..a5b535dc --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy @@ -0,0 +1,118 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework + +import org.gradle.testretry.AbstractFrameworkFuncTest + +class CucumberJUnit4FuncTest extends AbstractFrameworkFuncTest { + + def setup() { + buildFile << """ + dependencies { + testImplementation 'io.cucumber:cucumber-java:7.0.0' + testImplementation 'io.cucumber:cucumber-junit:7.0.0' + } + + test.retry.maxRetries = 1 + """ + } + + def "retries scenarios independently from each other (gradle version #gradleVersion)"(String gradleVersion) { + given: + writeFlakyFeatureFile() + writeFlakyStepDefinitions() + writeCucumberEntrypoint() + + when: + def runner = gradleRunner(gradleVersion as String) + def result = runner.build() + + then: + with(result.output) { + it.count("Passing scenario PASSED") == 1 + it.count("Flaky scenario FAILED") == 1 + it.count("Flaky scenario PASSED") == 1 + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } + + private void writeFlakyFeatureFile() { + writeFeatureFile("flaky_and_passing_scenarios", """ + Feature: Retry feature + + Scenario: Passing scenario + Given a passing setup + When a passing action + Then a passing verification + + Scenario: Flaky scenario + Given a passing setup + When a passing action + Then a flaky verification + """) + } + + private void writeFeatureFile(String name, String content) { + def featuresDir = "src/test/resources/features" + new File(testProjectDir.root, featuresDir).mkdirs() + testProjectDir.newFile("$featuresDir/${name}.feature") << content + } + + private void writeFlakyStepDefinitions() { + writeJavaTestSource """ + package acme; + + import io.cucumber.java.en.Given; + import io.cucumber.java.en.Then; + import io.cucumber.java.en.When; + + public class RetryFeatureStepDefinitions { + @Given("a passing setup") + public void passingSetup() { + } + + @When("a passing action") + public void passingAction() { + } + + @Then("a passing verification") + public void passingVerification() { + } + + @Then("a flaky verification") + public void flakyVerification() { + ${flakyAssert("cucumber")} + } + } + """ + } + + private writeCucumberEntrypoint() { + writeJavaTestSource """ + package acme; + + @org.junit.runner.RunWith(io.cucumber.junit.Cucumber.class) + @io.cucumber.junit.CucumberOptions( + features = "src/test/resources/features", + glue = "acme" + ) + public class RetryFeatureTest { + } + """ + } +} From f062222efdcd8e0bd7a4a06c1853710c2d9d83e7 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Tue, 23 Apr 2024 21:59:52 +0200 Subject: [PATCH 2/3] Add func tests for retrying Cucumber tests using TestNG runner Signed-off-by: Pavlo Shevchenko --- .../AbstractCucumberFuncTest.groovy} | 46 ++---------- .../cucumber/CucumberJUnit4FuncTest.groovy | 62 ++++++++++++++++ .../cucumber/CucumberTestNGFuncTest.groovy | 72 +++++++++++++++++++ 3 files changed, 139 insertions(+), 41 deletions(-) rename plugin/src/test/groovy/org/gradle/testretry/testframework/{CucumberJUnit4FuncTest.groovy => cucumber/AbstractCucumberFuncTest.groovy} (63%) create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit4FuncTest.groovy create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberTestNGFuncTest.groovy diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/AbstractCucumberFuncTest.groovy similarity index 63% rename from plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy rename to plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/AbstractCucumberFuncTest.groovy index a5b535dc..72963c2c 100644 --- a/plugin/src/test/groovy/org/gradle/testretry/testframework/CucumberJUnit4FuncTest.groovy +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/AbstractCucumberFuncTest.groovy @@ -13,45 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.gradle.testretry.testframework +package org.gradle.testretry.testframework.cucumber import org.gradle.testretry.AbstractFrameworkFuncTest -class CucumberJUnit4FuncTest extends AbstractFrameworkFuncTest { +abstract class AbstractCucumberFuncTest extends AbstractFrameworkFuncTest { def setup() { buildFile << """ dependencies { testImplementation 'io.cucumber:cucumber-java:7.0.0' - testImplementation 'io.cucumber:cucumber-junit:7.0.0' } test.retry.maxRetries = 1 """ } - def "retries scenarios independently from each other (gradle version #gradleVersion)"(String gradleVersion) { - given: - writeFlakyFeatureFile() - writeFlakyStepDefinitions() - writeCucumberEntrypoint() - - when: - def runner = gradleRunner(gradleVersion as String) - def result = runner.build() - - then: - with(result.output) { - it.count("Passing scenario PASSED") == 1 - it.count("Flaky scenario FAILED") == 1 - it.count("Flaky scenario PASSED") == 1 - } - - where: - gradleVersion << GRADLE_VERSIONS_UNDER_TEST - } - - private void writeFlakyFeatureFile() { + protected void writeFlakyFeatureFile() { writeFeatureFile("flaky_and_passing_scenarios", """ Feature: Retry feature @@ -67,13 +45,13 @@ class CucumberJUnit4FuncTest extends AbstractFrameworkFuncTest { """) } - private void writeFeatureFile(String name, String content) { + protected void writeFeatureFile(String name, String content) { def featuresDir = "src/test/resources/features" new File(testProjectDir.root, featuresDir).mkdirs() testProjectDir.newFile("$featuresDir/${name}.feature") << content } - private void writeFlakyStepDefinitions() { + protected void writeFlakyStepDefinitions() { writeJavaTestSource """ package acme; @@ -101,18 +79,4 @@ class CucumberJUnit4FuncTest extends AbstractFrameworkFuncTest { } """ } - - private writeCucumberEntrypoint() { - writeJavaTestSource """ - package acme; - - @org.junit.runner.RunWith(io.cucumber.junit.Cucumber.class) - @io.cucumber.junit.CucumberOptions( - features = "src/test/resources/features", - glue = "acme" - ) - public class RetryFeatureTest { - } - """ - } } diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit4FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit4FuncTest.groovy new file mode 100644 index 00000000..a3c972f4 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit4FuncTest.groovy @@ -0,0 +1,62 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework.cucumber + +class CucumberJUnit4FuncTest extends AbstractCucumberFuncTest { + + def setup() { + buildFile << """ + dependencies { + testImplementation 'io.cucumber:cucumber-junit:7.0.0' + } + """ + } + + def "retries scenarios independently from each other (gradle version #gradleVersion)"(String gradleVersion) { + given: + writeFlakyFeatureFile() + writeFlakyStepDefinitions() + writeCucumberEntrypoint() + + when: + def runner = gradleRunner(gradleVersion as String) + def result = runner.build() + + then: + with(result.output) { + it.count("Passing scenario PASSED") == 1 + it.count("Flaky scenario FAILED") == 1 + it.count("Flaky scenario PASSED") == 1 + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } + + private writeCucumberEntrypoint() { + writeJavaTestSource """ + package acme; + + @org.junit.runner.RunWith(io.cucumber.junit.Cucumber.class) + @io.cucumber.junit.CucumberOptions( + features = "src/test/resources/features", + glue = "acme" + ) + public class RetryFeatureTest { + } + """ + } +} diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberTestNGFuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberTestNGFuncTest.groovy new file mode 100644 index 00000000..659c85b7 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberTestNGFuncTest.groovy @@ -0,0 +1,72 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework.cucumber + +import spock.lang.PendingFeature + +class CucumberTestNGFuncTest extends AbstractCucumberFuncTest { + + def setup() { + buildFile << """ + dependencies { + testImplementation 'org.testng:testng:7.5' + testImplementation 'io.cucumber:cucumber-testng:7.0.0' + } + + test.useTestNG() + """ + } + + @PendingFeature + def "retries scenarios independently from each other (gradle version #gradleVersion)"(String gradleVersion) { + given: + writeFlakyFeatureFile() + writeFlakyStepDefinitions() + writeCucumberEntrypoint() + + when: + def runner = gradleRunner(gradleVersion as String) + def result = runner.build() + + then: + with(result.output) { + it.count('runScenario[0]("Passing scenario", "Optional[Retry feature]") PASSED') == 1 + it.count('runScenario[1]("Flaky scenario", "Optional[Retry feature]") FAILED') == 1 + it.count('runScenario[1]("Flaky scenario", "Optional[Retry feature]") PASSED') == 1 + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } + + private writeCucumberEntrypoint() { + writeJavaTestSource """ + package acme; + + import io.cucumber.testng.AbstractTestNGCucumberTests; + import io.cucumber.testng.CucumberOptions; + import org.testng.annotations.Test; + + @CucumberOptions( + features = "src/test/resources/features", + glue = "acme" + ) + @Test + public class RetryFeatureTest extends AbstractTestNGCucumberTests { + } + """ + } +} From c5d9734d96a8c4493b94291c79feebfc50bf11f3 Mon Sep 17 00:00:00 2001 From: Pavlo Shevchenko Date: Tue, 23 Apr 2024 22:08:11 +0200 Subject: [PATCH 3/3] Add func tests for retrying Cucumber tests using JUnit Platform runner Signed-off-by: Pavlo Shevchenko --- .../cucumber/CucumberJUnit5FuncTest.groovy | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit5FuncTest.groovy diff --git a/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit5FuncTest.groovy b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit5FuncTest.groovy new file mode 100644 index 00000000..a268f063 --- /dev/null +++ b/plugin/src/test/groovy/org/gradle/testretry/testframework/cucumber/CucumberJUnit5FuncTest.groovy @@ -0,0 +1,62 @@ +/* + * Copyright 2023 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.gradle.testretry.testframework.cucumber + +import spock.lang.PendingFeature + +class CucumberJUnit5FuncTest extends AbstractCucumberFuncTest { + + def setup() { + buildFile.text = """\ + plugins { + id("com.gradle.cucumber.companion") version "1.0.1" + } + ${buildFile.text} + """ + buildFile << """ + dependencies { + testImplementation platform('org.junit:junit-bom:5.10.2') + testImplementation 'io.cucumber:cucumber-java:7.0.0' + testImplementation 'io.cucumber:cucumber-junit-platform-engine:7.0.0' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.junit.platform:junit-platform-suite' + } + + test.useJUnitPlatform() + """ + } + + @PendingFeature + def "retries scenarios independently from each other (gradle version #gradleVersion)"(String gradleVersion) { + given: + writeFlakyFeatureFile() + writeFlakyStepDefinitions() + + when: + def runner = gradleRunner(gradleVersion as String) + def result = runner.build() + + then: + with(result.output) { + it.count("Passing scenario PASSED") == 1 + it.count("Flaky scenario FAILED") == 1 + it.count("Flaky scenario PASSED") == 1 + } + + where: + gradleVersion << GRADLE_VERSIONS_UNDER_TEST + } +}